From 33ddfd9dcd80d4037096a652ea8b9cadb0ddcdf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 8 Apr 2024 06:27:29 -0400 Subject: [PATCH] Create new ooniprobe service to support OpenVPN experiment Add experimental entrypoint to provision credentials to OpenVPN experiment. Single provider (riseup) for the MVP. Configurable "freshness" parameter to limit the query. Stub for inserting new credentials periodically. This superseeds: #830 More detailed changelog: * add experimental openvpn-config endpoint * Setup boilerplate for ooniprobe services * Move alembic migrations into common * Move vpn_service into ooniprobe service * No target_metadata for almebic * Implement basic MVP for probe services with openvpn config * Add ooniprobe service test to CI * Add support for running DB migration and simple tests * Create empty readme files * Remove changes to legacy API * Import postgresql from common * Rollback poetry.lock changes * Rollback probe_services diff * Change pydantic based model to make use of the more modern ConfigDict Add comment on future work needed to upgrade to newer pattern * Reach 100% code coverage * minor comments * test for updated * constant, raise error * add test for exception while fetching * copy alembic ini & folder * add target to migrate local db * use a shorter label for provider (omit vpn suffix) * use base64 prefix, hardcode riseup endpoints * add todo * update tests * rever symlinks for alembic * revert base64 encoding * Implement separate table for storing endpoints * Invert order of table dropping to allow DB downgrade * Add support for fetching endpoint list from riseupvpn * Move refresh interval to settings * Make lookup more robust to failures * Add checks for serving stale data in case of error Reach 100% test coverage --------- Co-authored-by: ain ghazal --- .github/workflows/test_ooniapi_ooniprobe.yml | 25 +++ .../oonirun => common/src/common}/alembic.ini | 0 ooniapi/common/src/common/alembic/README.md | 18 +++ .../src/common}/alembic/__init__.py | 0 .../src/common}/alembic/env.py | 9 +- .../common/src/common/alembic/script.py.mako | 26 +++ .../versions/981d92cf8790_init_tables.py | 0 .../c9119c05cf42_ooniprobe_services.py | 69 ++++++++ ooniapi/common/src/common/config.py | 2 + ooniapi/common/src/common/models.py | 38 +++++ .../src/common}/postgresql.py | 0 ooniapi/common/src/common/routers.py | 10 +- ooniapi/services/ooniprobe/.dockerignore | 10 ++ ooniapi/services/ooniprobe/.gitignore | 3 + .../services/ooniprobe/.vscode/settings.json | 5 + ooniapi/services/ooniprobe/Dockerfile | 33 ++++ ooniapi/services/ooniprobe/LICENSE.txt | 26 +++ ooniapi/services/ooniprobe/Makefile | 64 ++++++++ ooniapi/services/ooniprobe/README.md | 1 + ooniapi/services/ooniprobe/buildspec.yml | 29 ++++ ooniapi/services/ooniprobe/pyproject.toml | 93 +++++++++++ .../ooniprobe/scripts/docker-smoketest.sh | 34 ++++ .../ooniprobe/src/ooniprobe/__about__.py | 1 + .../ooniprobe/src/ooniprobe/__init__.py | 0 .../services/ooniprobe/src/ooniprobe/common | 1 + .../ooniprobe/src/ooniprobe/dependencies.py | 21 +++ .../services/ooniprobe/src/ooniprobe/main.py | 97 +++++++++++ .../ooniprobe/src/ooniprobe/models.py | 51 ++++++ .../ooniprobe/src/ooniprobe/routers/v2.py | 152 ++++++++++++++++++ .../services/ooniprobe/src/ooniprobe/utils.py | 96 +++++++++++ ooniapi/services/ooniprobe/tests/__init__.py | 0 ooniapi/services/ooniprobe/tests/conftest.py | 80 +++++++++ ooniapi/services/ooniprobe/tests/test_main.py | 35 ++++ .../services/ooniprobe/tests/test_models.py | 97 +++++++++++ ooniapi/services/ooniprobe/tests/test_v2.py | 118 ++++++++++++++ ooniapi/services/oonirun/README.md | 14 -- .../services/oonirun/src/oonirun/models.py | 41 +---- ooniapi/services/oonirun/tests/conftest.py | 4 +- .../services/oonirun/tests/test_database.py | 4 +- 39 files changed, 1244 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/test_ooniapi_ooniprobe.yml rename ooniapi/{services/oonirun => common/src/common}/alembic.ini (100%) create mode 100644 ooniapi/common/src/common/alembic/README.md rename ooniapi/{services/oonirun => common/src/common}/alembic/__init__.py (100%) rename ooniapi/{services/oonirun => common/src/common}/alembic/env.py (91%) create mode 100644 ooniapi/common/src/common/alembic/script.py.mako rename ooniapi/{services/oonirun => common/src/common}/alembic/versions/981d92cf8790_init_tables.py (100%) create mode 100644 ooniapi/common/src/common/alembic/versions/c9119c05cf42_ooniprobe_services.py create mode 100644 ooniapi/common/src/common/models.py rename ooniapi/{services/oonirun/src/oonirun => common/src/common}/postgresql.py (100%) create mode 100644 ooniapi/services/ooniprobe/.dockerignore create mode 100644 ooniapi/services/ooniprobe/.gitignore create mode 100644 ooniapi/services/ooniprobe/.vscode/settings.json create mode 100644 ooniapi/services/ooniprobe/Dockerfile create mode 100644 ooniapi/services/ooniprobe/LICENSE.txt create mode 100644 ooniapi/services/ooniprobe/Makefile create mode 100644 ooniapi/services/ooniprobe/README.md create mode 100644 ooniapi/services/ooniprobe/buildspec.yml create mode 100644 ooniapi/services/ooniprobe/pyproject.toml create mode 100755 ooniapi/services/ooniprobe/scripts/docker-smoketest.sh create mode 100644 ooniapi/services/ooniprobe/src/ooniprobe/__about__.py create mode 100644 ooniapi/services/ooniprobe/src/ooniprobe/__init__.py create mode 120000 ooniapi/services/ooniprobe/src/ooniprobe/common create mode 100644 ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py create mode 100644 ooniapi/services/ooniprobe/src/ooniprobe/main.py create mode 100644 ooniapi/services/ooniprobe/src/ooniprobe/models.py create mode 100644 ooniapi/services/ooniprobe/src/ooniprobe/routers/v2.py create mode 100644 ooniapi/services/ooniprobe/src/ooniprobe/utils.py create mode 100644 ooniapi/services/ooniprobe/tests/__init__.py create mode 100644 ooniapi/services/ooniprobe/tests/conftest.py create mode 100644 ooniapi/services/ooniprobe/tests/test_main.py create mode 100644 ooniapi/services/ooniprobe/tests/test_models.py create mode 100644 ooniapi/services/ooniprobe/tests/test_v2.py diff --git a/.github/workflows/test_ooniapi_ooniprobe.yml b/.github/workflows/test_ooniapi_ooniprobe.yml new file mode 100644 index 00000000..2504d81b --- /dev/null +++ b/.github/workflows/test_ooniapi_ooniprobe.yml @@ -0,0 +1,25 @@ +name: test ooniapi/ooniprobe +on: push +jobs: + run_tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install hatch + run: pip install hatch + + - name: Run all tests + run: make test-cov + working-directory: ./ooniapi/services/ooniprobe/ + + - name: Upload coverage to codecov + uses: codecov/codecov-action@v3 + with: + flags: ooniprobe + working-directory: ./ooniapi/services/ooniprobe/ diff --git a/ooniapi/services/oonirun/alembic.ini b/ooniapi/common/src/common/alembic.ini similarity index 100% rename from ooniapi/services/oonirun/alembic.ini rename to ooniapi/common/src/common/alembic.ini diff --git a/ooniapi/common/src/common/alembic/README.md b/ooniapi/common/src/common/alembic/README.md new file mode 100644 index 00000000..ebfa9859 --- /dev/null +++ b/ooniapi/common/src/common/alembic/README.md @@ -0,0 +1,18 @@ +## Alembic database migrations + +When you make changes to the DB schema you will have to run the alembic scripts for generating an appropriate migration file. + +This is how you do it: + +1. Create the template migration script + +``` +alembic revision -m "name of the revision" +``` + +2. Edit the newly created python file and fill out the `upgrade()` and `downgrade()` function with the relevant code bits +3. You can now run the migration like so: + +``` +OONI_PG_URL=postgresql://oonipg:oonipg@localhost/oonipg hatch run alembic upgrade head +``` diff --git a/ooniapi/services/oonirun/alembic/__init__.py b/ooniapi/common/src/common/alembic/__init__.py similarity index 100% rename from ooniapi/services/oonirun/alembic/__init__.py rename to ooniapi/common/src/common/alembic/__init__.py diff --git a/ooniapi/services/oonirun/alembic/env.py b/ooniapi/common/src/common/alembic/env.py similarity index 91% rename from ooniapi/services/oonirun/alembic/env.py rename to ooniapi/common/src/common/alembic/env.py index 4b9c0076..0e985afe 100644 --- a/ooniapi/services/oonirun/alembic/env.py +++ b/ooniapi/common/src/common/alembic/env.py @@ -16,13 +16,8 @@ if config.config_file_name is not None: # no cov fileConfig(config.config_file_name) -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -from oonirun import models - -target_metadata = models.Base.metadata +# we have no interest in 'autogenerate' support +target_metadata = None section = config.config_ini_section config.set_section_option( diff --git a/ooniapi/common/src/common/alembic/script.py.mako b/ooniapi/common/src/common/alembic/script.py.mako new file mode 100644 index 00000000..fbc4b07d --- /dev/null +++ b/ooniapi/common/src/common/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/ooniapi/services/oonirun/alembic/versions/981d92cf8790_init_tables.py b/ooniapi/common/src/common/alembic/versions/981d92cf8790_init_tables.py similarity index 100% rename from ooniapi/services/oonirun/alembic/versions/981d92cf8790_init_tables.py rename to ooniapi/common/src/common/alembic/versions/981d92cf8790_init_tables.py diff --git a/ooniapi/common/src/common/alembic/versions/c9119c05cf42_ooniprobe_services.py b/ooniapi/common/src/common/alembic/versions/c9119c05cf42_ooniprobe_services.py new file mode 100644 index 00000000..af86a989 --- /dev/null +++ b/ooniapi/common/src/common/alembic/versions/c9119c05cf42_ooniprobe_services.py @@ -0,0 +1,69 @@ +"""ooniprobe services + +Revision ID: c9119c05cf42 +Revises: 981d92cf8790 +Create Date: 2024-03-22 20:41:51.940695 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.schema import Sequence, CreateSequence + +# revision identifiers, used by Alembic. +revision: str = "c9119c05cf42" +down_revision: Union[str, None] = "981d92cf8790" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + ooniprobe_vpn_provider_id_seq = Sequence("ooniprobe_vpn_provider_id_seq", start=1) + op.execute(CreateSequence(ooniprobe_vpn_provider_id_seq)) + + op.create_table( + "ooniprobe_vpn_provider", + sa.Column( + "id", + sa.String(), + nullable=False, + server_default=ooniprobe_vpn_provider_id_seq.next_value(), + primary_key=True, + ), + sa.Column("date_created", sa.DateTime(timezone=True), nullable=False), + sa.Column("date_updated", sa.DateTime(timezone=True), nullable=False), + sa.Column("provider_name", sa.String(), nullable=False), + sa.Column("openvpn_cert", sa.String(), nullable=True), + sa.Column("openvpn_ca", sa.String(), nullable=False), + sa.Column("openvpn_key", sa.String(), nullable=False), + ) + + ooniprobe_vpn_provider_endpoint_id_seq = Sequence("ooniprobe_vpn_provider_endpoint_id_seq", start=1) + op.execute(CreateSequence(ooniprobe_vpn_provider_endpoint_id_seq)) + + op.create_table( + "ooniprobe_vpn_provider_endpoint", + sa.Column( + "id", + sa.String(), + nullable=False, + server_default=ooniprobe_vpn_provider_endpoint_id_seq.next_value(), + primary_key=True, + ), + sa.Column("date_created", sa.DateTime(timezone=True), nullable=False), + sa.Column("date_updated", sa.DateTime(timezone=True), nullable=False), + sa.Column("address", sa.String(), nullable=False), + sa.Column("protocol", sa.String(), nullable=True), + sa.Column("transport", sa.String(), nullable=True), + sa.Column("provider_id", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["provider_id"], + ["ooniprobe_vpn_provider.id"], + ), + ) + +def downgrade() -> None: + op.drop_table("ooniprobe_vpn_provider_endpoint") + op.drop_table("ooniprobe_vpn_provider") diff --git a/ooniapi/common/src/common/config.py b/ooniapi/common/src/common/config.py index 47194843..431b6213 100644 --- a/ooniapi/common/src/common/config.py +++ b/ooniapi/common/src/common/config.py @@ -28,3 +28,5 @@ class Settings(BaseSettings): aws_access_key_id: str = "" aws_secret_access_key: str = "" email_source_address: str = "contact+dev@ooni.io" + + vpn_credential_refresh_hours: int = 24 diff --git a/ooniapi/common/src/common/models.py b/ooniapi/common/src/common/models.py new file mode 100644 index 00000000..2f74f40b --- /dev/null +++ b/ooniapi/common/src/common/models.py @@ -0,0 +1,38 @@ +from datetime import datetime, timezone +from typing import List, Dict, Any +from sqlalchemy.types import DateTime, TypeDecorator + + +class UtcDateTime(TypeDecorator): + """ + Taken from: https://github.com/spoqa/sqlalchemy-utc/blob/8409688000ba0f52c928cc38d34069e521c24bae/sqlalchemy_utc/sqltypes.py + Almost equivalent to :class:`~sqlalchemy.types.DateTime` with + ``timezone=True`` option, but it differs from that by: + + - Never silently take naive :class:`~datetime.datetime`, instead it + always raise :exc:`ValueError` unless time zone aware value. + - :class:`~datetime.datetime` value's :attr:`~datetime.datetime.tzinfo` + is always converted to UTC. + - Unlike SQLAlchemy's built-in :class:`~sqlalchemy.types.DateTime`, + it never return naive :class:`~datetime.datetime`, but time zone + aware value, even with SQLite or MySQL. + """ + + impl = DateTime(timezone=True) + cache_ok = True + + def process_bind_param(self, value, dialect): + if value is not None: + if not isinstance(value, datetime): + raise TypeError("expected datetime.datetime, not " + repr(value)) + elif value.tzinfo is None: + raise ValueError("naive datetime is disallowed") + return value.astimezone(timezone.utc) + + def process_result_value(self, value, dialect): + if value is not None: # no cov + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + else: + value = value.astimezone(timezone.utc) + return value diff --git a/ooniapi/services/oonirun/src/oonirun/postgresql.py b/ooniapi/common/src/common/postgresql.py similarity index 100% rename from ooniapi/services/oonirun/src/oonirun/postgresql.py rename to ooniapi/common/src/common/postgresql.py diff --git a/ooniapi/common/src/common/routers.py b/ooniapi/common/src/common/routers.py index 80868140..305d3079 100644 --- a/ooniapi/common/src/common/routers.py +++ b/ooniapi/common/src/common/routers.py @@ -1,5 +1,6 @@ from datetime import date, datetime from pydantic import BaseModel as PydandicBaseModel +from pydantic import ConfigDict ISO_FORMAT_DATETIME = "%Y-%m-%dT%H:%M:%S.%fZ" @@ -7,8 +8,13 @@ class BaseModel(PydandicBaseModel): - class Config: - json_encoders = { + model_config = ConfigDict( + # TODO(art): this should be ported over to the functional serializer + # pattern (https://docs.pydantic.dev/latest/api/functional_serializers/) + # since json_encoders is deprecated, see: + # https://docs.pydantic.dev/2.6/api/config/#pydantic.config.ConfigDict.json_encoders + json_encoders={ datetime: lambda v: v.strftime(ISO_FORMAT_DATETIME), date: lambda v: v.strftime(ISO_FORMAT_DATE), } + ) diff --git a/ooniapi/services/ooniprobe/.dockerignore b/ooniapi/services/ooniprobe/.dockerignore new file mode 100644 index 00000000..4f7a82b5 --- /dev/null +++ b/ooniapi/services/ooniprobe/.dockerignore @@ -0,0 +1,10 @@ +.DS_Store +*.log +*.pyc +*.swp +*.env +.coverage +coverage.xml +dist/ +.venv/ +__pycache__/ diff --git a/ooniapi/services/ooniprobe/.gitignore b/ooniapi/services/ooniprobe/.gitignore new file mode 100644 index 00000000..9a1b4f54 --- /dev/null +++ b/ooniapi/services/ooniprobe/.gitignore @@ -0,0 +1,3 @@ +/dist +/coverage_html +*.coverage* diff --git a/ooniapi/services/ooniprobe/.vscode/settings.json b/ooniapi/services/ooniprobe/.vscode/settings.json new file mode 100644 index 00000000..07ea54eb --- /dev/null +++ b/ooniapi/services/ooniprobe/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python.defaultInterpreterPath": "${workspaceFolder}/.venv", + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} diff --git a/ooniapi/services/ooniprobe/Dockerfile b/ooniapi/services/ooniprobe/Dockerfile new file mode 100644 index 00000000..42fae6d0 --- /dev/null +++ b/ooniapi/services/ooniprobe/Dockerfile @@ -0,0 +1,33 @@ +# Python builder +FROM python:3.11-bookworm as builder +ARG BUILD_LABEL=docker + +WORKDIR /build + +RUN python -m pip install hatch + +COPY . /build + +# When you build stuff on macOS you end up with ._ files +# https://apple.stackexchange.com/questions/14980/why-are-dot-underscore-files-created-and-how-can-i-avoid-them +RUN find /build -type f -name '._*' -delete + +RUN echo "$BUILD_LABEL" > /build/src/ooniprobe/BUILD_LABEL + +RUN hatch build + +### Actual image running on the host +FROM python:3.11-bookworm as runner + +WORKDIR /app + +COPY --from=builder /build/README.md /app/ +COPY --from=builder /build/dist/*.whl /app/ +RUN pip install /app/*whl && rm /app/*whl + +COPY --from=builder /build/src/ooniprobe/common/alembic/ /app/alembic/ +COPY --from=builder /build/src/ooniprobe/common/alembic.ini /app/ +RUN rm -rf /app/alembic/__pycache__ + +CMD ["uvicorn", "ooniprobe.main:app", "--host", "0.0.0.0", "--port", "80"] +EXPOSE 80 diff --git a/ooniapi/services/ooniprobe/LICENSE.txt b/ooniapi/services/ooniprobe/LICENSE.txt new file mode 100644 index 00000000..3ec29c80 --- /dev/null +++ b/ooniapi/services/ooniprobe/LICENSE.txt @@ -0,0 +1,26 @@ +Copyright 2022-present Open Observatory of Network Interference Foundation (OONI) ETS + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/ooniapi/services/ooniprobe/Makefile b/ooniapi/services/ooniprobe/Makefile new file mode 100644 index 00000000..afa98949 --- /dev/null +++ b/ooniapi/services/ooniprobe/Makefile @@ -0,0 +1,64 @@ +SERVICE_NAME ?= ooniprobe + +ECS_CONTAINER_NAME ?= ooniapi-service-$(SERVICE_NAME) +IMAGE_NAME ?= ooni/api-$(SERVICE_NAME) +DATE := $(shell python3 -c "import datetime;print(datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d'))") +GIT_FULL_SHA ?= $(shell git rev-parse HEAD) +SHORT_SHA := $(shell echo ${GIT_FULL_SHA} | cut -c1-8) +PKG_VERSION := $(shell hatch version) + +BUILD_LABEL := $(DATE)-$(SHORT_SHA) +VERSION_LABEL = v$(PKG_VERSION) +ENV_LABEL ?= latest + +print-labels: + echo "ECS_CONTAINER_NAME=${ECS_CONTAINER_NAME}" + echo "PKG_VERSION=${PKG_VERSION}" + echo "BUILD_LABEL=${BUILD_LABEL}" + echo "VERSION_LABEL=${VERSION_LABEL}" + echo "ENV_LABEL=${ENV_LABEL}" + +init: + hatch env create + +docker-build: + # We need to use tar -czh to resolve the common dir symlink + tar -czh . | docker build \ + --build-arg BUILD_LABEL=${BUILD_LABEL} \ + -t ${IMAGE_NAME}:${BUILD_LABEL} \ + -t ${IMAGE_NAME}:${VERSION_LABEL} \ + -t ${IMAGE_NAME}:${ENV_LABEL} \ + - + echo "built image: ${IMAGE_NAME}:${BUILD_LABEL} (${IMAGE_NAME}:${VERSION_LABEL} ${IMAGE_NAME}:${ENV_LABEL})" + +docker-push: + # We need to use tar -czh to resolve the common dir symlink + docker push ${IMAGE_NAME}:${BUILD_LABEL} + docker push ${IMAGE_NAME}:${VERSION_LABEL} + docker push ${IMAGE_NAME}:${ENV_LABEL} + +docker-smoketest: + ./scripts/docker-smoketest.sh ${IMAGE_NAME}:${BUILD_LABEL} + +imagedefinitions.json: + echo '[{"name":"${ECS_CONTAINER_NAME}","imageUri":"${IMAGE_NAME}:${BUILD_LABEL}"}]' > imagedefinitions.json + +test: + hatch run test + +test-cov: + hatch run test-cov + +build: + hatch build + +clean: + hatch clean + rm -f imagedefinitions.json + rm -rf build dist *eggs *.egg-info + rm -rf .venv + +run: + hatch run uvicorn $(SERVICE_NAME).main:app + +.PHONY: init test build clean docker print-labels diff --git a/ooniapi/services/ooniprobe/README.md b/ooniapi/services/ooniprobe/README.md new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/ooniapi/services/ooniprobe/README.md @@ -0,0 +1 @@ + diff --git a/ooniapi/services/ooniprobe/buildspec.yml b/ooniapi/services/ooniprobe/buildspec.yml new file mode 100644 index 00000000..07a5d601 --- /dev/null +++ b/ooniapi/services/ooniprobe/buildspec.yml @@ -0,0 +1,29 @@ +version: 0.2 +env: + variables: + OONI_CODE_PATH: ooniapi/services/ooniprobe + DOCKERHUB_SECRET_ID: oonidevops/dockerhub/access_token + +phases: + install: + runtime-versions: + python: 3.11 + + pre_build: + commands: + - echo "Logging in to dockerhub" + - DOCKER_SECRET=$(aws secretsmanager get-secret-value --secret-id $DOCKERHUB_SECRET_ID --query SecretString --output text) + - echo $DOCKER_SECRET | docker login --username ooni --password-stdin + + build: + commands: + - export GIT_FULL_SHA=${CODEBUILD_RESOLVED_SOURCE_VERSION} + - cd $OONI_CODE_PATH + - make docker-build + - make docker-smoketest + - make docker-push + - make imagedefinitions.json + - cat imagedefinitions.json | tee ${CODEBUILD_SRC_DIR}/imagedefinitions.json + +artifacts: + files: imagedefinitions.json diff --git a/ooniapi/services/ooniprobe/pyproject.toml b/ooniapi/services/ooniprobe/pyproject.toml new file mode 100644 index 00000000..ca22cdd2 --- /dev/null +++ b/ooniapi/services/ooniprobe/pyproject.toml @@ -0,0 +1,93 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "ooniprobe" +description = '' +dynamic = ["version"] + +dependencies = [ + "fastapi ~= 0.108.0", + "clickhouse-driver ~= 0.2.6", + "sqlalchemy ~= 2.0.27", + "ujson ~= 5.9.0", + "urllib3 ~= 2.1.0", + "python-dateutil ~= 2.8.2", + "pydantic-settings ~= 2.1.0", + "statsd ~= 4.0.1", + "uvicorn ~= 0.25.0", + "psycopg2 ~= 2.9.9", + "httpx ~= 0.26.0", + "pyjwt ~= 2.8.0", + "alembic ~= 1.13.1", + "prometheus-fastapi-instrumentator ~= 6.1.0", + "prometheus-client", + "pem ~= 23.1.0", +] + +readme = "README.md" +requires-python = ">=3.11" +license = "MPL-2.0" +keywords = [] +authors = [{ name = "OONI", email = "contact@ooni.org" }] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + +[project.urls] +Documentation = "https://github.com/ooni/backend" +Issues = "https://github.com/ooni/backend/issues" +Source = "https://github.com/ooni/backend" + +[tool.hatch.version] +path = "src/ooniprobe/__about__.py" + +[tool.hatch.build.targets.sdist] +include = ["BUILD_LABEL"] + +[tool.hatch.build.targets.wheel] +packages = ["src/ooniprobe"] +artifacts = ["BUILD_LABEL"] + +[tool.hatch.envs.default] +dependencies = [ + "pytest", + "pytest-cov", + "click", + "black", + "pytest-postgresql", + "pytest-asyncio", + "freezegun", +] +path = ".venv/" + +[tool.hatch.envs.default.scripts] +test = "pytest {args:tests}" +test-cov = "pytest -s --full-trace --log-level=INFO --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=html --cov-report=term {args:tests}" +cov-report = ["coverage report"] +cov = ["test-cov", "cov-report"] + +[tool.pytest.ini_options] +addopts = ["--import-mode=importlib"] + +[tool.coverage.run] +branch = true +parallel = true +source_pkgs = ["ooniprobe", "tests"] +omit = ["src/ooniprobe/common/*", "src/ooniprobe/__about__.py"] + +[tool.coverage.paths] +ooniprobe = ["src/ooniprobe"] +tests = ["tests"] + +[tool.coverage.report] +exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"] diff --git a/ooniapi/services/ooniprobe/scripts/docker-smoketest.sh b/ooniapi/services/ooniprobe/scripts/docker-smoketest.sh new file mode 100755 index 00000000..fdc964cc --- /dev/null +++ b/ooniapi/services/ooniprobe/scripts/docker-smoketest.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -ex + +if [ $# -eq 0 ]; then + echo "Error: No Docker image name provided." + echo "Usage: $0 [IMAGE_NAME]" + exit 1 +fi + +IMAGE=$1 +CONTAINER_NAME=ooniapi-smoketest-$RANDOM +PORT=$((RANDOM % 10001 + 30000)) + +cleanup() { + echo "cleaning up" + docker logs $CONTAINER_NAME + docker stop $CONTAINER_NAME >/dev/null 2>&1 + docker rm $CONTAINER_NAME >/dev/null 2>&1 +} + +echo "[+] Running smoketest of ${IMAGE}" +docker run -d --name $CONTAINER_NAME -p $PORT:80 ${IMAGE} + +trap cleanup INT TERM EXIT + +sleep 2 +response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:$PORT/health) +if [ "${response}" -eq 200 ]; then + echo "Smoke test passed: Received 200 OK from /health endpoint." +else + echo "Smoke test failed: Did not receive 200 OK from /health endpoint. Received: $response" + exit 1 +fi diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/__about__.py b/ooniapi/services/ooniprobe/src/ooniprobe/__about__.py new file mode 100644 index 00000000..674608cc --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/__about__.py @@ -0,0 +1 @@ +VERSION = "0.1.0rc0" diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/__init__.py b/ooniapi/services/ooniprobe/src/ooniprobe/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/common b/ooniapi/services/ooniprobe/src/ooniprobe/common new file mode 120000 index 00000000..3f599f25 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/common @@ -0,0 +1 @@ +../../../../common/src/common \ No newline at end of file diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py b/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py new file mode 100644 index 00000000..84a8c8a5 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py @@ -0,0 +1,21 @@ +from functools import lru_cache +from typing import Annotated + +from fastapi import Depends + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from .common.config import Settings +from .common.dependencies import get_settings + + +def get_postgresql_session(settings: Annotated[Settings, Depends(get_settings)]): + engine = create_engine(settings.postgresql_url) + SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + db = SessionLocal() + try: + yield db + finally: + db.close() diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/main.py b/ooniapi/services/ooniprobe/src/ooniprobe/main.py new file mode 100644 index 00000000..44a53544 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/main.py @@ -0,0 +1,97 @@ +import logging +from contextlib import asynccontextmanager + +from fastapi import Depends, FastAPI +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import RedirectResponse + +from pydantic import BaseModel + +from prometheus_fastapi_instrumentator import Instrumentator + +from . import models +from .routers import v2 + +from .dependencies import get_postgresql_session +from .common.dependencies import get_settings +from .common.version import get_build_label +from .common.metrics import mount_metrics +from .__about__ import VERSION + +pkg_name = "ooniprobe" + +build_label = get_build_label(pkg_name) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings = get_settings() + logging.basicConfig(level=getattr(logging, settings.log_level.upper())) + mount_metrics(app, instrumentor.registry) + yield + + +app = FastAPI(lifespan=lifespan) + +instrumentor = Instrumentator().instrument( + app, metric_namespace="ooniapi", metric_subsystem="ooniprobe" +) + +# TODO: temporarily enable all +origins = ["*"] +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +app.include_router(v2.router, prefix="/api") + + +@app.get("/version") +async def version(): + return {"version": VERSION, "build_label": build_label} + + +class HealthStatus(BaseModel): + status: str + errors: list[str] = [] + version: str + build_label: str + + +@app.get("/health") +async def health( + settings=Depends(get_settings), + db=Depends(get_postgresql_session), +): + errors = [] + try: + db.query(models.OONIProbeVPNProvider).limit(1).all() + except Exception as exc: + print(exc) + errors.append("db_error") + + if settings.jwt_encryption_key == "CHANGEME": + errors.append("bad_jwt_secret") + + if settings.prometheus_metrics_password == "CHANGEME": + errors.append("bad_prometheus_password") + + status = "ok" + if len(errors) > 0: + status = "fail" + + return { + "status": status, + "errors": errors, + "version": VERSION, + "build_label": build_label, + } + + +@app.get("/") +async def root(): + return RedirectResponse("/docs") diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/models.py b/ooniapi/services/ooniprobe/src/ooniprobe/models.py new file mode 100644 index 00000000..1cfc2e10 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/models.py @@ -0,0 +1,51 @@ +from datetime import datetime +from typing import Dict +from .common.models import UtcDateTime +from .common.postgresql import Base +from sqlalchemy import ForeignKey, Sequence, String, Integer +from sqlalchemy.orm import Mapped +from sqlalchemy.orm import mapped_column, relationship + + +class OONIProbeVPNProvider(Base): + __tablename__ = "ooniprobe_vpn_provider" + + id: Mapped[str] = mapped_column( + String, + Sequence("ooniprobe_vpn_provider_id_seq", start=1), + primary_key=True, + nullable=False, + ) + date_updated: Mapped[datetime] = mapped_column(UtcDateTime()) + date_created: Mapped[datetime] = mapped_column(UtcDateTime()) + + provider_name: Mapped[str] = mapped_column() + + openvpn_cert: Mapped[str] = mapped_column(nullable=True) + openvpn_ca: Mapped[str] = mapped_column(nullable=True) + openvpn_key: Mapped[str] = mapped_column(nullable=True) + + endpoints = relationship("OONIProbeVPNProviderEndpoint", back_populates="provider") + + +class OONIProbeVPNProviderEndpoint(Base): + __tablename__ = "ooniprobe_vpn_provider_endpoint" + + id: Mapped[str] = mapped_column( + String, + Sequence("ooniprobe_vpn_provider_endpoint_id_seq", start=1), + primary_key=True, + nullable=False, + ) + date_updated: Mapped[datetime] = mapped_column(UtcDateTime()) + date_created: Mapped[datetime] = mapped_column(UtcDateTime()) + + protocol: Mapped[str] = mapped_column() + address: Mapped[str] = mapped_column() + transport: Mapped[str] = mapped_column() + # TODO: maybe we want this in the future to store location and other + # metadata about an endpoint + # metadata: Mapped[Dict[str, str]] = mapped_column(nullable=True) + + provider_id = mapped_column(ForeignKey("ooniprobe_vpn_provider.id")) + provider = relationship("OONIProbeVPNProvider", back_populates="endpoints") diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v2.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v2.py new file mode 100644 index 00000000..74cd05b7 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v2.py @@ -0,0 +1,152 @@ +from datetime import datetime, timedelta, timezone, date +import random +from typing import Dict, List +import logging + +import sqlalchemy as sa +from sqlalchemy.orm import Session +from fastapi import APIRouter, Depends, HTTPException + +from .. import models + +from ..utils import ( + fetch_openvpn_config, + fetch_openvpn_endpoints, + format_endpoint, + upsert_endpoints, +) +from ..common.routers import BaseModel +from ..common.dependencies import get_settings +from ..dependencies import get_postgresql_session + + +log = logging.getLogger(__name__) + +router = APIRouter() + + +class VPNConfig(BaseModel): + provider: str + protocol: str + config: Dict[str, str] + # date_updated is when the credentials or other config has been updated; + # inputs will follow a different lifecycle. + date_updated: str + endpoints: List[str] + + +def update_vpn_provider(db: Session, provider_name: str) -> models.OONIProbeVPNProvider: + """Fetch a fresh config for a given provider and update the database entry""" + # we are only handling a single provider for the time being (riseup). + # TODO: manage an inventory of known providers. + vpn_cert = fetch_openvpn_config() + + try: + provider = ( + db.query(models.OONIProbeVPNProvider) + .filter( + models.OONIProbeVPNProvider.provider_name == provider_name, + ) + .one() + ) + vpn_endpoints = fetch_openvpn_endpoints() + + provider.openvpn_ca = vpn_cert["ca"] + provider.openvpn_cert = vpn_cert["cert"] + provider.openvpn_key = vpn_cert["key"] + provider.date_updated = datetime.now(timezone.utc) + upsert_endpoints(db, vpn_endpoints, provider) + db.commit() + + except sa.orm.exc.NoResultFound: + provider = models.OONIProbeVPNProvider( + provider_name=provider_name, + date_updated=datetime.now(timezone.utc), + date_created=datetime.now(timezone.utc), + openvpn_ca=vpn_cert["ca"], + openvpn_cert=vpn_cert["cert"], + openvpn_key=vpn_cert["key"], + ) + db.add(provider) + vpn_endpoints = fetch_openvpn_endpoints() + upsert_endpoints(db, vpn_endpoints, provider) + db.commit() + + return provider + + +def get_or_update_riseupvpn( + db: Session, provider_name: str, vpn_credential_refresh_hours: int +) -> models.OONIProbeVPNProvider: + """Get a configuration entry for the given provider, or fetch a fresh one if None found""" + provider = ( + db.query(models.OONIProbeVPNProvider) + .filter( + models.OONIProbeVPNProvider.provider_name == provider_name, + models.OONIProbeVPNProvider.date_updated + > datetime.now(timezone.utc) + - timedelta(hours=vpn_credential_refresh_hours), + ) + .first() + ) + if provider: + return provider + + try: + provider = update_vpn_provider(db, provider_name) + return provider + except: + log.error(f"failed to update vpn provider {provider_name}") + + try: + # In this case we at least serve a stale version of the provider instead + # of just failing. + provider = ( + db.query(models.OONIProbeVPNProvider) + .filter( + models.OONIProbeVPNProvider.provider_name == provider_name, + ) + .one() + ) + return provider + except sa.orm.exc.NoResultFound: + raise HTTPException(status_code=500, detail="error updating provider") + + +@router.get("/v2/ooniprobe/vpn-config/{provider_name}", tags=["ooniprobe"]) +def get_vpn_config( + provider_name: str, + db=Depends(get_postgresql_session), + settings=Depends(get_settings), +) -> VPNConfig: + """GET VPN config parameters for a given provider, including authentication""" + log.debug(f"GET vpn config for {provider_name}") + + if provider_name != "riseupvpn": + raise HTTPException(status_code=404, detail="provider not found") + + try: + provider = get_or_update_riseupvpn( + db=db, + provider_name=provider_name, + vpn_credential_refresh_hours=settings.vpn_credential_refresh_hours, + ) + except Exception as exc: + log.error("Error while fetching credentials for riseup: %s", exc) + raise HTTPException(status_code=500, detail="could not fetch credentials") + + endpoints = [ + format_endpoint(provider.provider_name, ep) for ep in provider.endpoints + ] + return VPNConfig( + provider=provider.provider_name, + protocol="openvpn", + config={ + "ca": provider.openvpn_ca, + "cert": provider.openvpn_cert, + "key": provider.openvpn_key, + }, + # Pick 4 random endpoints to serve to the client + endpoints=random.sample(endpoints, min(len(endpoints), 4)), + date_updated=provider.date_updated.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + ) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/utils.py b/ooniapi/services/ooniprobe/src/ooniprobe/utils.py new file mode 100644 index 00000000..532633a3 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/utils.py @@ -0,0 +1,96 @@ +""" +VPN Services + +Insert VPN credentials into database. +""" +import base64 +from datetime import datetime, timezone +import itertools +import logging +from typing import Dict, List, Mapping, TypedDict + +from sqlalchemy.orm import Session +import pem +import httpx + +from ooniprobe.models import OONIProbeVPNProvider, OONIProbeVPNProviderEndpoint + +RISEUP_CA_URL = "https://api.black.riseup.net/ca.crt" +RISEUP_CERT_URL = "https://api.black.riseup.net/3/cert" +RISEUP_ENDPOINT_URL = "https://api.black.riseup.net/3/config/eip-service.json" + +log = logging.getLogger(__name__) + + +class OpenVPNConfig(TypedDict): + ca: str + cert: str + key: str + +class OpenVPNEndpoint(TypedDict): + address: str + protocol: str + transport: str + +def fetch_riseup_ca() -> str: + r = httpx.get(RISEUP_CA_URL) + r.raise_for_status() + return r.text.strip() + + +def fetch_riseup_cert() -> str: + r = httpx.get(RISEUP_CERT_URL) + r.raise_for_status() + return r.text.strip() + + +def fetch_openvpn_config() -> OpenVPNConfig: + ca = fetch_riseup_ca() + pem_cert = fetch_riseup_cert() + key, cert = pem.parse(pem_cert) + return OpenVPNConfig(ca=ca, cert=cert.as_text(), key=key.as_text()) + +def fetch_openvpn_endpoints() -> List[OpenVPNEndpoint]: + endpoints = [] + + r = httpx.get(RISEUP_ENDPOINT_URL) + r.raise_for_status() + j = r.json() + for ep in j["gateways"]: + ip = ep["ip_address"] + # TODO(art): do we want to store this metadata somewhere? + #location = ep["location"] + #hostname = ep["host"] + for t in ep["capabilities"]["transport"]: + if t["type"] != "openvpn": + continue + for transport, port in itertools.product(t["protocols"], t["ports"]): + endpoints.append(OpenVPNEndpoint( + address=f"{ip}:{port}", + protocol="openvpn", + transport=transport + )) + return endpoints + +def format_endpoint(provider_name: str, ep: OONIProbeVPNProviderEndpoint) -> str: + return f"{ep.protocol}://{provider_name}.corp/?address={ep.address}&transport={ep.transport}" + +def upsert_endpoints(db: Session, new_endpoints: List[OpenVPNEndpoint], provider: OONIProbeVPNProvider): + new_endpoints_map = {f'{ep["address"]}-{ep["protocol"]}-{ep["transport"]}': ep for ep in new_endpoints} + for endpoint in provider.endpoints: + key = f'{endpoint.address}-{endpoint.protocol}-{endpoint.transport}' + if key in new_endpoints_map: + endpoint.date_updated = datetime.now(timezone.utc) + new_endpoints_map.pop(key) + else: + db.delete(endpoint) + + for ep in new_endpoints_map.values(): + db.add(OONIProbeVPNProviderEndpoint( + date_created=datetime.now(timezone.utc), + date_updated=datetime.now(timezone.utc), + protocol=ep["protocol"], + address=ep["address"], + transport=ep["transport"], + provider=provider + )) \ No newline at end of file diff --git a/ooniapi/services/ooniprobe/tests/__init__.py b/ooniapi/services/ooniprobe/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ooniapi/services/ooniprobe/tests/conftest.py b/ooniapi/services/ooniprobe/tests/conftest.py new file mode 100644 index 00000000..c51b6004 --- /dev/null +++ b/ooniapi/services/ooniprobe/tests/conftest.py @@ -0,0 +1,80 @@ +import pathlib +import pytest + +import time +import jwt + +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from ooniprobe.common.config import Settings +from ooniprobe.common.dependencies import get_settings +from ooniprobe.main import app + + +def make_override_get_settings(**kw): + def override_get_settings(): + return Settings(**kw) + + return override_get_settings + + +@pytest.fixture +def pg_url(postgresql): + return f"postgresql://{postgresql.info.user}:@{postgresql.info.host}:{postgresql.info.port}/{postgresql.info.dbname}" + + +@pytest.fixture +def db(pg_url): + engine = create_engine(pg_url) + SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + db = SessionLocal() + try: + yield db + finally: + db.close() + + +@pytest.fixture +def alembic_migration(pg_url): + from alembic import command + from alembic.config import Config + + migrations_path = ( + pathlib.Path(__file__).parent.parent + / "src" + / "ooniprobe" + / "common" + / "alembic" + ).resolve() + + alembic_cfg = Config() + alembic_cfg.set_main_option("script_location", str(migrations_path)) + alembic_cfg.set_main_option("sqlalchemy.url", pg_url) + + command.upgrade(alembic_cfg, "head") + yield pg_url + + +@pytest.fixture +def client_with_bad_settings(): + app.dependency_overrides[get_settings] = make_override_get_settings( + postgresql_url="postgresql://bad:bad@localhost/bad" + ) + + client = TestClient(app) + yield client + + +@pytest.fixture +def client(alembic_migration): + app.dependency_overrides[get_settings] = make_override_get_settings( + postgresql_url=alembic_migration, + jwt_encryption_key="super_secure", + prometheus_metrics_password="super_secure", + ) + + client = TestClient(app) + yield client diff --git a/ooniapi/services/ooniprobe/tests/test_main.py b/ooniapi/services/ooniprobe/tests/test_main.py new file mode 100644 index 00000000..1f88488e --- /dev/null +++ b/ooniapi/services/ooniprobe/tests/test_main.py @@ -0,0 +1,35 @@ +import pytest + +import httpx +from fastapi.testclient import TestClient +from ooniprobe.main import lifespan, app + + +def test_health_good(client): + r = client.get("health") + j = r.json() + assert j["status"] == "ok", j + assert len(j["errors"]) == 0, j + + +def test_health_bad(client_with_bad_settings): + r = client_with_bad_settings.get("health") + j = r.json() + assert j["status"] == "fail", j + assert len(j["errors"]) > 0, j + + +def test_metrics(client): + r = client.get("/metrics") + + +@pytest.mark.asyncio +async def test_lifecycle(): + async with lifespan(app) as ls: + client = TestClient(app) + r = client.get("/metrics") + assert r.status_code == 401 + + auth = httpx.BasicAuth(username="prom", password="super_secure") + r = client.get("/metrics", auth=auth) + assert r.status_code == 200, r.text diff --git a/ooniapi/services/ooniprobe/tests/test_models.py b/ooniapi/services/ooniprobe/tests/test_models.py new file mode 100644 index 00000000..092238b6 --- /dev/null +++ b/ooniapi/services/ooniprobe/tests/test_models.py @@ -0,0 +1,97 @@ +from datetime import datetime, timedelta, timezone +from ooniprobe.models import OONIProbeVPNProvider, OONIProbeVPNProviderEndpoint +from ooniprobe.utils import OpenVPNEndpoint, upsert_endpoints + + +defaultRiseupTargets = [ + "openvpn://riseup.corp/?address=51.15.187.53:1194&transport=tcp", + "openvpn://riseup.corp/?address=51.15.187.53:1194&transport=udp", +] + +def test_create_providers(db, alembic_migration): + provider = OONIProbeVPNProvider( + provider_name="riseupvpn", + date_created=datetime.now(timezone.utc), + date_updated=datetime.now(timezone.utc), + openvpn_cert="OPENVPN_CERT", + openvpn_ca="OPENVPN_CA", + openvpn_key="OPENVPN_KEY" + ) + db.add(provider) + db.add(OONIProbeVPNProviderEndpoint( + date_created=datetime.now(timezone.utc)-timedelta(hours=1), + date_updated=datetime.now(timezone.utc)-timedelta(hours=1), + protocol="openvpn", + address="51.15.187.53:1194", + transport="tcp", + provider=provider + )) + db.add(OONIProbeVPNProviderEndpoint( + date_created=datetime.now(timezone.utc)-timedelta(hours=1), + date_updated=datetime.now(timezone.utc)-timedelta(hours=1), + protocol="openvpn", + address="51.15.187.53:1194", + transport="udp", + provider=provider + )) + db.add(OONIProbeVPNProviderEndpoint( + date_created=datetime.now(timezone.utc)-timedelta(hours=1), + date_updated=datetime.now(timezone.utc)-timedelta(hours=1), + protocol="openvpn", + address="1.1.1.1:1194", + transport="udp", + provider=provider + )) + db.commit() + + all_endpoints = db.query(OONIProbeVPNProviderEndpoint).all() + assert len(all_endpoints) == 3 + addresses = set() + for endpoint in all_endpoints: + addresses.add(endpoint.address) + assert endpoint.protocol + assert endpoint.address + assert endpoint.transport + assert endpoint.provider.provider_name == "riseupvpn" + assert addresses == set(["51.15.187.53:1194", "1.1.1.1:1194"]) + + provider = db.query(OONIProbeVPNProvider).filter( + OONIProbeVPNProvider.provider_name == "riseupvpn", + OONIProbeVPNProvider.date_updated + > datetime.now(timezone.utc) + - timedelta(days=7), + ).one() + assert len(provider.endpoints) == 3 + + new_endpoints = [ + OpenVPNEndpoint( + address="51.15.187.53:1194", + protocol="openvpn", + transport="udp" + ), + OpenVPNEndpoint( + address="51.15.187.53:1194", + protocol="openvpn", + transport="tcp" + ), + OpenVPNEndpoint( + address="3.2.1.3:1194", + protocol="openvpn", + transport="udp" + ), + ] + + upsert_endpoints(db, new_endpoints, provider) + db.commit() + + all_endpoints = db.query(OONIProbeVPNProviderEndpoint).all() + assert len(all_endpoints) == 3 + addresses = set() + for endpoint in all_endpoints: + addresses.add(endpoint.address) + assert endpoint.protocol + assert endpoint.address + assert endpoint.transport + assert endpoint.provider.provider_name == "riseupvpn" + assert endpoint.date_updated > datetime.now(timezone.utc) - timedelta(minutes=1) + assert addresses == set(["51.15.187.53:1194", "3.2.1.3:1194"]) diff --git a/ooniapi/services/ooniprobe/tests/test_v2.py b/ooniapi/services/ooniprobe/tests/test_v2.py new file mode 100644 index 00000000..887f7770 --- /dev/null +++ b/ooniapi/services/ooniprobe/tests/test_v2.py @@ -0,0 +1,118 @@ +""" +Integration test for OONIProbe API +""" + +from datetime import datetime, timedelta, timezone +from unittest.mock import patch + +from httpx import HTTPError +from freezegun import freeze_time +import pytest + +from ooniprobe.utils import OpenVPNConfig +from ooniprobe import models +from ooniprobe.routers import v2 + +DUMMY_VPN_CERT = OpenVPNConfig( + ca="-----BEGIN CERTIFICATE-----\nSAMPLE CERTIFICATE\n-----END CERTIFICATE-----\n", + cert="-----BEGIN CERTIFICATE-----\nSAMPLE CERTIFICATE\n-----END CERTIFICATE-----\n", + key="-----BEGIN RSA PRIVATE KEY-----\nSAMPLE KEY\n-----END RSA PRIVATE KEY-----\n", +) + + +def test_get_version(client): + r = client.get("/version") + j = r.json() + assert "version" in j + assert "build_label" in j + + +def test_get_root(client): + r = client.get("/") + assert r.status_code == 200 + + +def test_get_config(client): + r = client.get("/api/v2/ooniprobe/vpn-config/riseupvpn") + assert r.status_code == 200 + j = r.json() + assert j["provider"] == "riseupvpn" + assert j["protocol"] == "openvpn" + assert j["config"]["cert"].startswith("-----BEGIN CERTIFICATE") + assert j["config"]["ca"].startswith("-----BEGIN CERTIFICATE") + assert j["config"]["key"].startswith("-----BEGIN RSA PRIVATE KEY") + date_updated = j["date_updated"] + + r = client.get("/api/v2/ooniprobe/vpn-config/riseupvpn") + assert r.status_code == 200 + j = r.json() + assert j["date_updated"] == date_updated + + +def test_invalid_provider_name(client, db): + # we probably aren't going to add NSA VPN to our provider list anytime soon :D + r = client.get("/api/v2/ooniprobe/vpn-config/nsavpn") + assert r.status_code != 200 + + +def test_config_updated(client, db): + with freeze_time("1984-01-01"): + provider = models.OONIProbeVPNProvider( + provider_name="riseupvpn", + date_updated=datetime.now(timezone.utc), + date_created=datetime.now(timezone.utc), + openvpn_ca=DUMMY_VPN_CERT["ca"], + openvpn_cert=DUMMY_VPN_CERT["cert"], + openvpn_key=DUMMY_VPN_CERT["key"], + ) + db.add(provider) + db.commit() + + r = client.get("/api/v2/ooniprobe/vpn-config/riseupvpn") + assert r.status_code == 200 + j = r.json() + assert j["provider"] == "riseupvpn" + assert j["protocol"] == "openvpn" + assert j["config"]["cert"] == DUMMY_VPN_CERT["cert"] + assert j["config"]["ca"] == DUMMY_VPN_CERT["ca"] + assert j["config"]["key"] == DUMMY_VPN_CERT["key"] + + # Check to see if the cert got updated + with freeze_time("1984-04-01"): + r = client.get("/api/v2/ooniprobe/vpn-config/riseupvpn") + assert r.status_code == 200 + j = r.json() + assert j["provider"] == "riseupvpn" + assert j["protocol"] == "openvpn" + assert j["config"]["cert"] != DUMMY_VPN_CERT["cert"] + assert j["config"]["ca"] != DUMMY_VPN_CERT["ca"] + assert j["config"]["key"] != DUMMY_VPN_CERT["key"] + assert j["date_updated"].startswith("1984-04-01") + + +@pytest.mark.parametrize("error", [HTTPError, Exception]) +def test_get_config_fails_if_exception_while_fetching_credentials(client, db, error): + # no previous credential; when forcing any exception on the fetch code the http client should get a 500 + with patch.object(v2, "get_or_update_riseupvpn", side_effect=error("err")): + r = client.get("/api/v2/ooniprobe/vpn-config/riseupvpn") + assert r.status_code == 500 + + with patch.object(v2, "update_vpn_provider", side_effect=error("err")): + r = client.get("/api/v2/ooniprobe/vpn-config/riseupvpn") + assert r.status_code == 500 + + # Check that we get stale data if we have it and it's failing to fetch the data + provider = models.OONIProbeVPNProvider( + provider_name="riseupvpn", + date_updated=datetime.now(timezone.utc) - timedelta(days=20), + date_created=datetime.now(timezone.utc) - timedelta(days=20), + openvpn_ca=DUMMY_VPN_CERT["ca"], + openvpn_cert=DUMMY_VPN_CERT["cert"], + openvpn_key=DUMMY_VPN_CERT["key"], + ) + db.add(provider) + db.commit() + + with patch.object(v2, "update_vpn_provider", side_effect=error("err")): + r = client.get("/api/v2/ooniprobe/vpn-config/riseupvpn") + assert r.status_code == 200 diff --git a/ooniapi/services/oonirun/README.md b/ooniapi/services/oonirun/README.md index 6041f8df..8b137891 100644 --- a/ooniapi/services/oonirun/README.md +++ b/ooniapi/services/oonirun/README.md @@ -1,15 +1 @@ -## Alembic database migrations -When you make changes to the DB schema you will have to run the alembic scripts for generating an appropriate migration file. - -This is how you do it: - -1. Create the template migration script -``` -poetry run alembic revision -m "name of the revision" -``` -2. Edit the newly created python file and fill out the `upgrade()` and `downgrade()` function with the relevant code bits -3. You can now run the migration like so: -``` -OONI_PG_URL=postgresql://oonipg:oonipg@localhost/oonipg hatch run alembic upgrade head -``` diff --git a/ooniapi/services/oonirun/src/oonirun/models.py b/ooniapi/services/oonirun/src/oonirun/models.py index 9b055b4a..efadc5fc 100644 --- a/ooniapi/services/oonirun/src/oonirun/models.py +++ b/ooniapi/services/oonirun/src/oonirun/models.py @@ -1,48 +1,13 @@ -from datetime import datetime, timezone +from datetime import datetime from typing import List, Dict, Any import sqlalchemy as sa from sqlalchemy import ForeignKey, Sequence, String from sqlalchemy.orm import Mapped from sqlalchemy.orm import mapped_column from sqlalchemy.orm import relationship -from sqlalchemy.types import DateTime, TypeDecorator -from .postgresql import Base - - -class UtcDateTime(TypeDecorator): - """ - Taken from: https://github.com/spoqa/sqlalchemy-utc/blob/8409688000ba0f52c928cc38d34069e521c24bae/sqlalchemy_utc/sqltypes.py - Almost equivalent to :class:`~sqlalchemy.types.DateTime` with - ``timezone=True`` option, but it differs from that by: - - - Never silently take naive :class:`~datetime.datetime`, instead it - always raise :exc:`ValueError` unless time zone aware value. - - :class:`~datetime.datetime` value's :attr:`~datetime.datetime.tzinfo` - is always converted to UTC. - - Unlike SQLAlchemy's built-in :class:`~sqlalchemy.types.DateTime`, - it never return naive :class:`~datetime.datetime`, but time zone - aware value, even with SQLite or MySQL. - """ - - impl = DateTime(timezone=True) - cache_ok = True - - def process_bind_param(self, value, dialect): - if value is not None: - if not isinstance(value, datetime): - raise TypeError("expected datetime.datetime, not " + repr(value)) - elif value.tzinfo is None: - raise ValueError("naive datetime is disallowed") - return value.astimezone(timezone.utc) - - def process_result_value(self, value, dialect): - if value is not None: # no cov - if value.tzinfo is None: - value = value.replace(tzinfo=timezone.utc) - else: - value = value.astimezone(timezone.utc) - return value +from .common.models import UtcDateTime +from .common.postgresql import Base class OONIRunLink(Base): diff --git a/ooniapi/services/oonirun/tests/conftest.py b/ooniapi/services/oonirun/tests/conftest.py index 3bf337e3..aab2193f 100644 --- a/ooniapi/services/oonirun/tests/conftest.py +++ b/ooniapi/services/oonirun/tests/conftest.py @@ -25,7 +25,9 @@ def alembic_migration(postgresql): db_url = f"postgresql://{postgresql.info.user}:@{postgresql.info.host}:{postgresql.info.port}/{postgresql.info.dbname}" - migrations_path = (pathlib.Path(__file__).parent.parent / "alembic").resolve() + migrations_path = ( + pathlib.Path(__file__).parent.parent / "src" / "oonirun" / "common" / "alembic" + ).resolve() alembic_cfg = Config() alembic_cfg.set_main_option("script_location", str(migrations_path)) diff --git a/ooniapi/services/oonirun/tests/test_database.py b/ooniapi/services/oonirun/tests/test_database.py index 0606a441..722f41a2 100644 --- a/ooniapi/services/oonirun/tests/test_database.py +++ b/ooniapi/services/oonirun/tests/test_database.py @@ -52,7 +52,9 @@ def config_alembic(db_url): from alembic.config import Config - migrations_path = (pathlib.Path(__file__).parent.parent / "alembic").resolve() + migrations_path = ( + pathlib.Path(__file__).parent.parent / "src" / "oonirun" / "common" / "alembic" + ).resolve() alembic_cfg = Config() alembic_cfg.set_main_option("script_location", str(migrations_path))