Create new ooniprobe service to support OpenVPN experiment

Add experimental entrypoint to provision credentials to OpenVPN experiment.

Single provider (riseup) for the MVP.
Configurable "freshness" parameter to limit the query.
Stub for inserting new credentials periodically.
This superseeds: #830

More detailed changelog:

* add experimental openvpn-config endpoint

* Setup boilerplate for ooniprobe services

* Move alembic migrations into common

* Move vpn_service into ooniprobe service

* No target_metadata for almebic

* Implement basic MVP for probe services with openvpn config

* Add ooniprobe service test to CI

* Add support for running DB migration and simple tests

* Create empty readme files

* Remove changes to legacy API

* Import postgresql from common

* Rollback poetry.lock changes

* Rollback probe_services diff

* Change pydantic based model to make use of the more modern ConfigDict

Add comment on future work needed to upgrade to newer pattern

* Reach 100% code coverage

* minor comments

* test for updated

* constant, raise error

* add test for exception while fetching

* copy alembic ini & folder

* add target to migrate local db

* use a shorter label for provider (omit vpn suffix)

* use base64 prefix, hardcode riseup endpoints

* add todo

* update tests

* rever symlinks for alembic

* revert base64 encoding

* Implement separate table for storing endpoints

* Invert order of table dropping to allow DB downgrade

* Add support for fetching endpoint list from riseupvpn

* Move refresh interval to settings

* Make lookup more robust to failures

* Add checks for serving stale data in case of error

Reach 100% test coverage

---------

Co-authored-by: ain ghazal <[email protected]>

.github/workflows/test_ooniapi_ooniprobe.yml

@@ -0,0 +1,25 @@
+name: test ooniapi/ooniprobe
+on: push
+jobs:
+  run_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install hatch
+        run: pip install hatch
+
+      - name: Run all tests
+        run: make test-cov
+        working-directory: ./ooniapi/services/ooniprobe/
+
+      - name: Upload coverage to codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: ooniprobe
+          working-directory: ./ooniapi/services/ooniprobe/

ooniapi/common/src/common/alembic/README.md

@@ -0,0 +1,18 @@
+## Alembic database migrations
+
+When you make changes to the DB schema you will have to run the alembic scripts for generating an appropriate migration file.
+
+This is how you do it:
+
+1. Create the template migration script
+
+```
+alembic revision -m "name of the revision"
+```
+
+2. Edit the newly created python file and fill out the `upgrade()` and `downgrade()` function with the relevant code bits
+3. You can now run the migration like so:
+
+```
+OONI_PG_URL=postgresql://oonipg:oonipg@localhost/oonipg hatch run alembic upgrade head
+```

ooniapi/services/oonirun/alembic/env.py → ooniapi/common/src/common/alembic/env.py

@@ -16,13 +16,8 @@
 if config.config_file_name is not None:  # no cov
     fileConfig(config.config_file_name)
 
-# add your model's MetaData object here
-# for 'autogenerate' support
-# from myapp import mymodel
-# target_metadata = mymodel.Base.metadata
-from oonirun import models
-
-target_metadata = models.Base.metadata
+# we have no interest in 'autogenerate' support
+target_metadata = None
 
 section = config.config_ini_section
 config.set_section_option(

ooniapi/common/src/common/alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

ooniapi/common/src/common/alembic/versions/c9119c05cf42_ooniprobe_services.py

@@ -0,0 +1,69 @@
+"""ooniprobe services
+
+Revision ID: c9119c05cf42
+Revises: 981d92cf8790
+Create Date: 2024-03-22 20:41:51.940695
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.schema import Sequence, CreateSequence
+
+# revision identifiers, used by Alembic.
+revision: str = "c9119c05cf42"
+down_revision: Union[str, None] = "981d92cf8790"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    ooniprobe_vpn_provider_id_seq = Sequence("ooniprobe_vpn_provider_id_seq", start=1)
+    op.execute(CreateSequence(ooniprobe_vpn_provider_id_seq))
+
+    op.create_table(
+        "ooniprobe_vpn_provider",
+        sa.Column(
+            "id",
+            sa.String(),
+            nullable=False,
+            server_default=ooniprobe_vpn_provider_id_seq.next_value(),
+            primary_key=True,
+        ),
+        sa.Column("date_created", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("date_updated", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("provider_name", sa.String(), nullable=False),
+        sa.Column("openvpn_cert", sa.String(), nullable=True),
+        sa.Column("openvpn_ca", sa.String(), nullable=False),
+        sa.Column("openvpn_key", sa.String(), nullable=False),
+    )
+
+    ooniprobe_vpn_provider_endpoint_id_seq = Sequence("ooniprobe_vpn_provider_endpoint_id_seq", start=1)
+    op.execute(CreateSequence(ooniprobe_vpn_provider_endpoint_id_seq))
+
+    op.create_table(
+        "ooniprobe_vpn_provider_endpoint",
+        sa.Column(
+            "id",
+            sa.String(),
+            nullable=False,
+            server_default=ooniprobe_vpn_provider_endpoint_id_seq.next_value(),
+            primary_key=True,
+        ),
+        sa.Column("date_created", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("date_updated", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("address", sa.String(), nullable=False),
+        sa.Column("protocol", sa.String(), nullable=True),
+        sa.Column("transport", sa.String(), nullable=True),
+        sa.Column("provider_id", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["provider_id"],
+            ["ooniprobe_vpn_provider.id"],
+        ),
+    )
+
+def downgrade() -> None:
+    op.drop_table("ooniprobe_vpn_provider_endpoint")
+    op.drop_table("ooniprobe_vpn_provider")

ooniapi/common/src/common/config.py

@@ -28,3 +28,5 @@ class Settings(BaseSettings):
     aws_access_key_id: str = ""
     aws_secret_access_key: str = ""
     email_source_address: str = "[email protected]"
+
+    vpn_credential_refresh_hours: int = 24

ooniapi/common/src/common/models.py

@@ -0,0 +1,38 @@
+from datetime import datetime, timezone
+from typing import List, Dict, Any
+from sqlalchemy.types import DateTime, TypeDecorator
+
+
+class UtcDateTime(TypeDecorator):
+    """
+    Taken from: https://github.com/spoqa/sqlalchemy-utc/blob/8409688000ba0f52c928cc38d34069e521c24bae/sqlalchemy_utc/sqltypes.py
+    Almost equivalent to :class:`~sqlalchemy.types.DateTime` with
+    ``timezone=True`` option, but it differs from that by:
+
+    - Never silently take naive :class:`~datetime.datetime`, instead it
+      always raise :exc:`ValueError` unless time zone aware value.
+    - :class:`~datetime.datetime` value's :attr:`~datetime.datetime.tzinfo`
+      is always converted to UTC.
+    - Unlike SQLAlchemy's built-in :class:`~sqlalchemy.types.DateTime`,
+      it never return naive :class:`~datetime.datetime`, but time zone
+      aware value, even with SQLite or MySQL.
+    """
+
+    impl = DateTime(timezone=True)
+    cache_ok = True
+
+    def process_bind_param(self, value, dialect):
+        if value is not None:
+            if not isinstance(value, datetime):
+                raise TypeError("expected datetime.datetime, not " + repr(value))
+            elif value.tzinfo is None:
+                raise ValueError("naive datetime is disallowed")
+            return value.astimezone(timezone.utc)
+
+    def process_result_value(self, value, dialect):
+        if value is not None:  # no cov
+            if value.tzinfo is None:
+                value = value.replace(tzinfo=timezone.utc)
+            else:
+                value = value.astimezone(timezone.utc)
+        return value

ooniapi/common/src/common/routers.py

@@ -1,14 +1,20 @@
 from datetime import date, datetime
 from pydantic import BaseModel as PydandicBaseModel
+from pydantic import ConfigDict
 
 
 ISO_FORMAT_DATETIME = "%Y-%m-%dT%H:%M:%S.%fZ"
 ISO_FORMAT_DATE = "%Y-%m-%d"
 
 
 class BaseModel(PydandicBaseModel):
-    class Config:
-        json_encoders = {
+    model_config = ConfigDict(
+        # TODO(art): this should be ported over to the functional serializer
+        # pattern (https://docs.pydantic.dev/latest/api/functional_serializers/)
+        # since json_encoders is deprecated, see:
+        # https://docs.pydantic.dev/2.6/api/config/#pydantic.config.ConfigDict.json_encoders
+        json_encoders={
             datetime: lambda v: v.strftime(ISO_FORMAT_DATETIME),
             date: lambda v: v.strftime(ISO_FORMAT_DATE),
         }
+    )

ooniapi/services/ooniprobe/.dockerignore

@@ -0,0 +1,10 @@
+.DS_Store
+*.log
+*.pyc
+*.swp
+*.env
+.coverage
+coverage.xml
+dist/
+.venv/
+__pycache__/

ooniapi/services/ooniprobe/.gitignore

@@ -0,0 +1,3 @@
+/dist
+/coverage_html
+*.coverage*

ooniapi/services/ooniprobe/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "python.defaultInterpreterPath": "${workspaceFolder}/.venv",
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

ooniapi/services/ooniprobe/Dockerfile

@@ -0,0 +1,33 @@
+# Python builder
+FROM python:3.11-bookworm as builder
+ARG BUILD_LABEL=docker
+
+WORKDIR /build
+
+RUN python -m pip install hatch
+
+COPY . /build
+
+# When you build stuff on macOS you end up with ._ files
+# https://apple.stackexchange.com/questions/14980/why-are-dot-underscore-files-created-and-how-can-i-avoid-them
+RUN find /build -type f -name '._*' -delete
+
+RUN echo "$BUILD_LABEL" > /build/src/ooniprobe/BUILD_LABEL
+
+RUN hatch build
+
+### Actual image running on the host
+FROM python:3.11-bookworm as runner
+
+WORKDIR /app
+
+COPY --from=builder /build/README.md /app/
+COPY --from=builder /build/dist/*.whl /app/
+RUN pip install /app/*whl && rm /app/*whl
+
+COPY --from=builder /build/src/ooniprobe/common/alembic/ /app/alembic/
+COPY --from=builder /build/src/ooniprobe/common/alembic.ini /app/
+RUN rm -rf /app/alembic/__pycache__
+
+CMD ["uvicorn", "ooniprobe.main:app", "--host", "0.0.0.0", "--port", "80"]
+EXPOSE 80

ooniapi/services/ooniprobe/LICENSE.txt

@@ -0,0 +1,26 @@
+Copyright 2022-present Open Observatory of Network Interference Foundation (OONI) ETS
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ooniapi/services/ooniprobe/Makefile

@@ -0,0 +1,64 @@
+SERVICE_NAME ?= ooniprobe
+
+ECS_CONTAINER_NAME ?= ooniapi-service-$(SERVICE_NAME)
+IMAGE_NAME ?= ooni/api-$(SERVICE_NAME)
+DATE := $(shell python3 -c "import datetime;print(datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d'))")
+GIT_FULL_SHA ?= $(shell git rev-parse HEAD)
+SHORT_SHA := $(shell echo ${GIT_FULL_SHA} | cut -c1-8)
+PKG_VERSION := $(shell hatch version)
+
+BUILD_LABEL := $(DATE)-$(SHORT_SHA)
+VERSION_LABEL = v$(PKG_VERSION)
+ENV_LABEL ?= latest
+
+print-labels:
+	echo "ECS_CONTAINER_NAME=${ECS_CONTAINER_NAME}"
+	echo "PKG_VERSION=${PKG_VERSION}"
+	echo "BUILD_LABEL=${BUILD_LABEL}"
+	echo "VERSION_LABEL=${VERSION_LABEL}"
+	echo "ENV_LABEL=${ENV_LABEL}"
+
+init:
+	hatch env create
+
+docker-build:
+	# We need to use tar -czh to resolve the common dir symlink
+	tar -czh . | docker build \
+		--build-arg BUILD_LABEL=${BUILD_LABEL} \
+		-t ${IMAGE_NAME}:${BUILD_LABEL} \
+		-t ${IMAGE_NAME}:${VERSION_LABEL} \
+		-t ${IMAGE_NAME}:${ENV_LABEL} \
+		-
+	echo "built image: ${IMAGE_NAME}:${BUILD_LABEL} (${IMAGE_NAME}:${VERSION_LABEL} ${IMAGE_NAME}:${ENV_LABEL})"
+
+docker-push:
+	# We need to use tar -czh to resolve the common dir symlink
+	docker push ${IMAGE_NAME}:${BUILD_LABEL}
+	docker push ${IMAGE_NAME}:${VERSION_LABEL}
+	docker push ${IMAGE_NAME}:${ENV_LABEL}
+
+docker-smoketest:
+	./scripts/docker-smoketest.sh ${IMAGE_NAME}:${BUILD_LABEL}
+
+imagedefinitions.json:
+	echo '[{"name":"${ECS_CONTAINER_NAME}","imageUri":"${IMAGE_NAME}:${BUILD_LABEL}"}]' > imagedefinitions.json
+
+test:
+	hatch run test
+
+test-cov:
+	hatch run test-cov
+
+build:
+	hatch build
+
+clean:
+	hatch clean
+	rm -f imagedefinitions.json
+	rm -rf build dist *eggs *.egg-info
+	rm -rf .venv
+
+run:
+	hatch run uvicorn $(SERVICE_NAME).main:app
+
+.PHONY: init test build clean docker print-labels

ooniapi/services/ooniprobe/README.md

@@ -0,0 +1 @@
+