From 3c0a70ac4fcb751d42049349863da6aed0f7e2b9 Mon Sep 17 00:00:00 2001 From: jphillips Date: Sat, 31 Aug 2024 16:26:15 -0500 Subject: [PATCH] Add container for api Signed-off-by: jphillips --- .env.template | 3 +- Taskfile.yml | 30 ++++++---- docker-compose.yml | 51 +++++++++++++++++ modules/odr_api/Taskfile.api.yml | 55 +++++++++++++++++++ modules/odr_api/Taskfile.yml | 17 ------ modules/odr_api/docker/Dockerfile.api | 34 ++++++++++++ modules/odr_api/odr_api/app.py | 37 +++++++++++++ modules/odr_api/odr_api/main.py | 47 +++++----------- ...Taskfile.api.yml => Taskfile.api.test.yml} | 0 modules/odr_core/odr_core/config.py | 2 +- modules/odr_core/requirements.txt | 4 +- 11 files changed, 214 insertions(+), 66 deletions(-) create mode 100644 docker-compose.yml create mode 100644 modules/odr_api/Taskfile.api.yml delete mode 100644 modules/odr_api/Taskfile.yml create mode 100644 modules/odr_api/docker/Dockerfile.api create mode 100644 modules/odr_api/odr_api/app.py rename modules/odr_api/tests/{Taskfile.api.yml => Taskfile.api.test.yml} (100%) diff --git a/.env.template b/.env.template index 2fdc364..20df8c2 100644 --- a/.env.template +++ b/.env.template @@ -2,10 +2,11 @@ ROOT_DIR= MODEL_CACHE_DIR= ## Postgres +POSTGRES_HOST=postgres POSTGRES_DB=opendatarepository POSTGRES_USER=opendatarepository POSTGRES_PASSWORD=opendatarepository -POSTGRES_PORT=35432 +POSTGRES_PORT=5432 TEST_POSTGRES_DB=test_opendatarepository # PGADMIN diff --git a/Taskfile.yml b/Taskfile.yml index 2021635..160b9b4 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -2,12 +2,12 @@ version: '3' includes: - api: ./modules/odr_api/Taskfile.yml + api: ./modules/odr_api/Taskfile.api.yml db: ./modules/odr_database/Taskfile.yml core: ./modules/odr_core/Taskfile.yml data: ./modules/odr_datamodel/Taskfile.yml frontend: ./modules/odr_frontend/Taskfile.yml - api-test: ./modules/odr_api/tests/Taskfile.yml + api-test: ./modules/odr_api/tests/Taskfile.api.test.yml monitoring: ./modules/odr_monitoring/Taskfile.yml dotenv: ['.env', '{{.ENV}}/.env', '{{.HOME}}/.env'] @@ -56,9 +56,7 @@ tasks: cmds: - echo "Starting all services..." - | - task db:start-postgres & - task api:start & - task frontend:dev & + task start-containers wait - echo "All services started" stop-all: @@ -66,8 +64,7 @@ tasks: cmds: - echo "Stopping all services..." - | - task db:stop-postgres & - wait + task stop-containers - echo "All services stopped" test-all: @@ -87,14 +84,23 @@ tasks: desc: Watch all service logs cmds: - echo "Watching all logs..." - - | - task db:watch-postgres-logs & - task api:watch-api-logs & - wait - - echo "All logs watched" + - docker compose logs -f smoke_tests: desc: Run smoke tests for both API and PostgreSQL cmds: - task api-test:health-check - task db:test-postgres + + build-all: + desc: Build all services + cmds: + - docker compose build + start-containers: + desc: Start all services + cmds: + - docker compose up -d + stop-containers: + desc: Stop all running containers + cmds: + - docker compose down diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6017dff --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,51 @@ +name: omi-postgres +services: + postgres: + image: pgvector/pgvector:pg16 + environment: + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "${POSTGRES_PORT:-35432}:5432" + networks: + - omi-network + + pgadmin: + image: dpage/pgadmin4 + environment: + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-admin@example.com} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin} + PGADMIN_LISTEN_ADDRESS: 0.0.0.0 + volumes: + - pgadmin_data:/var/lib/pgadmin + ports: + - "${PGADMIN_PORT:-35050}:80" + networks: + - omi-network + depends_on: + - postgres + + odr-api: + build: + context: . + dockerfile: modules/odr_api/docker/Dockerfile.api + env_file: + - .env + ports: + - "31100:31100" + networks: + - omi-network + depends_on: + - postgres + +volumes: + postgres_data: + pgadmin_data: + +networks: + omi-network: + external: true + driver: bridge diff --git a/modules/odr_api/Taskfile.api.yml b/modules/odr_api/Taskfile.api.yml new file mode 100644 index 0000000..58aa353 --- /dev/null +++ b/modules/odr_api/Taskfile.api.yml @@ -0,0 +1,55 @@ +version: 3 + +tasks: + start: + desc: Start the api server + cmds: + - cmd: powershell.exe -ExecutionPolicy ByPass ./modules/odr_api/scripts/run_server.ps1 -debug + platforms: [windows] + - cmd: ./modules/odr_api/scripts/start_server.sh --debug + platforms: [linux, darwin] + - echo 'Start server stub' + + debug: + desc: Start the api server in debug mode + cmds: + - cmd: python -m debugpy --listen 0.0.0.0:5678 --wait-for-client ./modules/odr_api/odr_api/main.py + platforms: [windows, linux, darwin] + build: + dir: '{{.ROOT_DIR}}' + cmds: + - docker build -t odr-api -f modules/odr_api/docker/Dockerfile.api . + + run: + desc: Run the odr-api Docker container + dir: '{{.ROOT_DIR}}' + cmds: + - > + docker run -d + --name odr-api + -p 31100:31100 + --env-file .env + -e POSTGRES_URL={{.POSTGRES_URL | default "postgres"}} + odr-api + vars: + POSTGRES_URL: + sh: echo $POSTGRES_URL + + stop: + cmds: + - docker stop odr-api + - docker rm odr-api + + logs: + cmds: + - docker logs odr-api -f + + shell: + cmds: + - docker exec -it odr-api /bin/sh + + redeploy: + cmds: + - task: stop + - task: build + - task: run diff --git a/modules/odr_api/Taskfile.yml b/modules/odr_api/Taskfile.yml deleted file mode 100644 index bc46042..0000000 --- a/modules/odr_api/Taskfile.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: 3 - -tasks: - start: - desc: Start the api server - cmds: - - cmd: powershell.exe -ExecutionPolicy ByPass ./modules/odr_api/scripts/run_server.ps1 -debug - platforms: [windows] - - cmd: ./modules/odr_api/scripts/start_server.sh --debug - platforms: [linux, darwin] - - echo 'Start server stub' - - debug: - desc: Start the api server in debug mode - cmds: - - cmd: python -m debugpy --listen 0.0.0.0:5678 --wait-for-client ./modules/odr_api/odr_api/main.py - platforms: [windows, linux, darwin] diff --git a/modules/odr_api/docker/Dockerfile.api b/modules/odr_api/docker/Dockerfile.api new file mode 100644 index 0000000..fed2068 --- /dev/null +++ b/modules/odr_api/docker/Dockerfile.api @@ -0,0 +1,34 @@ +# Build stage +FROM python:3.11-slim-bookworm + +RUN apt-get update && apt-get install -y gcc && rm -rf /var/lib/apt/lists/* + +ENV PYTHONUNBUFFERED=1 \ + DEBIAN_FRONTEND=noninteractive \ + PIP_NO_CACHE_DIR=off \ + PIP_DISABLE_PIP_VERSION_CHECK=on \ + PIP_DEFAULT_TIMEOUT=100 + +WORKDIR /app + +# Copy requirements file +COPY modules/odr_api/requirements.txt . + +# Install dependencies +RUN pip install --no-cache-dir -r requirements.txt + +COPY modules/odr_core /app/modules/odr_core +RUN pip install --no-cache-dir -e /app/modules/odr_core + +COPY modules/odr_api /app +RUN pip install --no-cache-dir . + +# Set Python path to include the modules +ENV PYTHONPATH=/app:/app/modules:$PYTHONPATH + +# Expose the port the app runs on +EXPOSE 31100 +# Set the entrypoint +# CMD ["tail", "-f", "/dev/null"] +ENTRYPOINT ["uvicorn"] +CMD ["odr_api.app:app", "--host", "0.0.0.0", "--port", "31100"] diff --git a/modules/odr_api/odr_api/app.py b/modules/odr_api/odr_api/app.py new file mode 100644 index 0000000..6b632a3 --- /dev/null +++ b/modules/odr_api/odr_api/app.py @@ -0,0 +1,37 @@ +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from odr_api.api.endpoints import user_router, team_router, content_router, annotation_router, auth_router, embedding_router, health_router +from odr_core.config import settings +import uvicorn + +app = FastAPI(title=settings.PROJECT_NAME) +# Allow CORS +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], # Allows all origins + allow_credentials=True, + allow_methods=["*"], # Allows all methods + allow_headers=["*"], # Allows all headers +) + + +@app.get("/") +def read_root(): + return {"message": "Hello, World!"} + + +@app.get("/test") +def test_communication(): + return {"message": "Communication successful!"} + + +app.include_router(team_router, prefix=settings.API_V1_STR) +app.include_router(user_router, prefix=settings.API_V1_STR) +app.include_router(content_router, prefix=settings.API_V1_STR) +app.include_router(annotation_router, prefix=settings.API_V1_STR) +app.include_router(auth_router, prefix=settings.API_V1_STR) +app.include_router(embedding_router, prefix=settings.API_V1_STR) +app.include_router(health_router, prefix=settings.API_V1_STR) + +if __name__ == "__main__": + uvicorn.run("main:app", host="0.0.0.0", port=31100, reload=True) diff --git a/modules/odr_api/odr_api/main.py b/modules/odr_api/odr_api/main.py index 6b632a3..72b1558 100644 --- a/modules/odr_api/odr_api/main.py +++ b/modules/odr_api/odr_api/main.py @@ -1,37 +1,18 @@ -from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware -from odr_api.api.endpoints import user_router, team_router, content_router, annotation_router, auth_router, embedding_router, health_router -from odr_core.config import settings -import uvicorn +import argparse +from loguru import logger -app = FastAPI(title=settings.PROJECT_NAME) -# Allow CORS -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], # Allows all origins - allow_credentials=True, - allow_methods=["*"], # Allows all methods - allow_headers=["*"], # Allows all headers -) - - -@app.get("/") -def read_root(): - return {"message": "Hello, World!"} - - -@app.get("/test") -def test_communication(): - return {"message": "Communication successful!"} +if __name__ == "__main__": + import uvicorn + parser = argparse.ArgumentParser(description="Open Data Repository API") + parser.add_argument("--dev", action="store_true", help="Run in development mode with hot reloading") + args = parser.parse_args() -app.include_router(team_router, prefix=settings.API_V1_STR) -app.include_router(user_router, prefix=settings.API_V1_STR) -app.include_router(content_router, prefix=settings.API_V1_STR) -app.include_router(annotation_router, prefix=settings.API_V1_STR) -app.include_router(auth_router, prefix=settings.API_V1_STR) -app.include_router(embedding_router, prefix=settings.API_V1_STR) -app.include_router(health_router, prefix=settings.API_V1_STR) + logger.info("Starting Open Data Repository API from main") -if __name__ == "__main__": - uvicorn.run("main:app", host="0.0.0.0", port=31100, reload=True) + if args.dev: + logger.info("Running in development mode with hot reloading") + uvicorn.run("odr_api.api.app:app", host="0.0.0.0", port=31100, reload=True) + else: + logger.info("Running in production mode with 4 workers") + uvicorn.run("odr_api.api.app:app", host="0.0.0.0", port=31100, workers=8) diff --git a/modules/odr_api/tests/Taskfile.api.yml b/modules/odr_api/tests/Taskfile.api.test.yml similarity index 100% rename from modules/odr_api/tests/Taskfile.api.yml rename to modules/odr_api/tests/Taskfile.api.test.yml diff --git a/modules/odr_core/odr_core/config.py b/modules/odr_core/odr_core/config.py index a1fa0b2..a38ba2b 100644 --- a/modules/odr_core/odr_core/config.py +++ b/modules/odr_core/odr_core/config.py @@ -6,7 +6,7 @@ class Settings(BaseSettings): PROJECT_NAME: str = "OMI-DataModel" # Postgres - POSTGRES_HOST: str = "localhost" + POSTGRES_HOST: str POSTGRES_DB: str POSTGRES_USER: str POSTGRES_PASSWORD: str diff --git a/modules/odr_core/requirements.txt b/modules/odr_core/requirements.txt index 73ff382..e197292 100644 --- a/modules/odr_core/requirements.txt +++ b/modules/odr_core/requirements.txt @@ -10,5 +10,5 @@ argon2-cffi==23.1.0 pyjwt==2.9.0 pgvector==0.1.6 -# fastembed==0.3.4 replace with version when this commit is released - https://github.com/qdrant/fastembed/commit/9c72d2f59f91f87753da07c12a6f1082e233ecb3 -fastembed @ git+https://github.com/qdrant/fastembed@9c72d2f59f91f87753da07c12a6f1082e233ecb3 +fastembed==0.3.4 +# fastembed @ git+https://github.com/qdrant/fastembed@9c72d2f59f91f87753da07c12a6f1082e233ecb3