2024 refactor

.github/workflows/build_dataapi.yml

@@ -0,0 +1,48 @@
+name: Build and Deploy Docker Image for dataapi
+
+on:
+  push:
+    branches:
+      - 2024-refactor
+    #paths:
+    #  - "api/fastapi/**"
+
+env:
+  IMAGE_NAME: ooni/dataapi
+  DOCKERFILE_PATH: ./api/fastapi/
+
+jobs:
+  build_and_push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Format version number
+        id: version
+        run: |
+          DATE=$(date +'%Y%m%d')
+          SHORT_SHA=$(echo ${{ github.sha }} | cut -c1-4)
+          TAG="v${DATE}-${SHORT_SHA}"
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+
+      - name: Build and Push Docker Image
+        run: |
+          BUILD_LABEL=${{ steps.version.outputs.tag }}
+          TAG_LATEST=$IMAGE_NAME:latest
+          TAG_ENVIRONMENT=$IMAGE_NAME:production
+          TAG_VERSION=$IMAGE_NAME:$BUILD_LABEL
+
+          # Build Docker image with multiple tags
+          docker build --build-arg BUILD_LABEL=$BUILD_LABEL -t $TAG_LATEST -t $TAG_VERSION -t $TAG_ENVIRONMENT $DOCKERFILE_PATH
+
+          # Push all tags
+          docker push $TAG_LATEST
+          docker push $TAG_VERSION
+          docker push $TAG_ENVIRONMENT

api/api.conf.example

@@ -45,3 +45,5 @@ GITHUB_WORKDIR = "/var/lib/ooniapi/citizenlab"
 MSMT_SPOOL_DIR = "/tmp/oonispool"
 GEOIP_ASN_DB = "/var/lib/ooniapi/asn.mmdb"
 GEOIP_CC_DB = "/var/lib/ooniapi/cc.mmdb"
+
+LMDB_DIR = "/tmp/llmdb"

api/fastapi/Dockerfile

@@ -0,0 +1,30 @@
+# Python builder
+# Produced a virtualenv with all the deps good to go
+
+# TODO(art): upgrade to bookworm (blocked by this
+# https://github.com/actions/setup-python/issues/721) or stop using debian to
+# have more recent base OS
+FROM python:3.11-bullseye as py-build
+ARG BUILD_LABEL=docker
+
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+     && apt-get -y install --no-install-recommends python3 python3-dev
+
+RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/opt/poetry python3 -
+
+COPY . /app
+WORKDIR /app
+ENV PATH=/opt/poetry/bin:$PATH
+RUN echo "$BUILD_LABEL" > /app/dataapi/BUILD_LABEL
+RUN poetry config virtualenvs.in-project true && poetry install --no-interaction --no-ansi
+
+### Actual image running on the host
+FROM python:3.11-bullseye
+
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+COPY --from=py-build /app /app
+WORKDIR /app
+CMD ["/app/.venv/bin/uvicorn", "dataapi.main:app", "--host", "0.0.0.0", "--port", "80"]
+EXPOSE 80

api/fastapi/Readme.md

@@ -0,0 +1,6 @@
+Running:
+
+```
+poetry install
+poetry run uvicorn dataapi.main:app
+```

api/fastapi/dataapi/config.py

@@ -0,0 +1,22 @@
+import statsd
+
+from typing import List
+from pydantic_settings import BaseSettings
+
+
+class Settings(BaseSettings):
+    app_name: str = "OONI Data API"
+    base_url: str = "https://api.ooni.io"
+    clickhouse_url: str = "clickhouse://localhost"
+    log_level: str = "info"
+    s3_bucket_name: str = "oonidata-eufra"
+    other_collectors: List[str] = []
+    statsd_host: str = "localhost"
+    statsd_port: int = 8125
+    statsd_prefix: str = "ooniapi"
+
+
+settings = Settings()
+metrics = statsd.StatsClient(
+    settings.statsd_host, settings.statsd_port, prefix=settings.statsd_prefix
+)

api/fastapi/dataapi/dependencies.py

@@ -0,0 +1,7 @@
+from clickhouse_driver import Client as ClickhouseClient
+
+from .config import settings
+
+
+def get_clickhouse_client() -> ClickhouseClient:
+    return ClickhouseClient.from_url(settings.clickhouse_url)

api/fastapi/dataapi/main.py

@@ -0,0 +1,34 @@
+from fastapi import FastAPI
+
+from .routers import measurements
+from .routers import aggregation
+
+from .config import settings
+
+import logging
+
+logging.basicConfig(level=getattr(logging, settings.log_level.upper()))
+
+app = FastAPI()
+app.include_router(measurements.router, prefix="/api")
+app.include_router(aggregation.router, prefix="/api")
+
+from importlib.metadata import version as importlib_version
+from importlib.resources import files as importlib_files
+
+pkg_name = "dataapi"
+
+pkg_version = importlib_version(pkg_name)
+try:
+    with importlib_files("dataapi").joinpath("BUILD_LABEL").open('r') as in_file:
+        build_label = in_file.read().strip()
+except:
+    build_label = None
+
+@app.get("/version")
+async def version():
+    return {"version": pkg_version, "build_label": build_label}
+
+@app.get("/")
+async def root():
+    return {"message": "Hello OONItarian!"}