Monkeypatch env in tests

.gitignore

@@ -33,3 +33,4 @@ __pypackages__/
 
 # tad tool
 tad.log*
+database.sqlite3

compose.yml

@@ -9,7 +9,8 @@ services:
       db:
         condition: service_healthy
     env_file:
-      - .env
+      - path: .env
+        required: true
     environment:
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?Variable not set}
     ports:
@@ -22,7 +23,8 @@ services:
       - app-db-data:/var/lib/postgresql/data/pgdata
       - ./database/:/docker-entrypoint-initdb.d/:cached
     env_file:
-      - .env
+      - path: .env
+        required: true
     environment:
       - PGDATA=/var/lib/postgresql/data/pgdata
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?Variable not set}
@@ -38,7 +40,7 @@ services:
     ports:
       - 8080:8080
     environment:
-      - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL:-berry.hartog@minbzk.nl}
+      - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL:-tad@minbzk.nl}
       - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD:?Variable not set}
       - PGADMIN_LISTEN_PORT=${PGADMIN_LISTEN_PORT:-8080}
     depends_on:

pyproject.toml

@@ -24,12 +24,12 @@ packages = [
 python = "^3.10"
 fastapi = "^0.110.3"
 sqlmodel = "^0.0.18"
-psycopg = {extras = ["binary"], version = "^3.1.18"}
 alembic = "^1.13.1"
 pydantic = "^2.7.1"
 jinja2 = "^3.1.4"
 pydantic-settings = "^2.2.1"
 granian = {extras = ["reload"], version = "^1.3.1"}
+psycopg2-binary = "^2.9.9"
 
 [tool.poetry.group.test.dependencies]
 pytest = "^8.1.1"
@@ -76,11 +76,18 @@ typeCheckingMode = "strict"
 reportMissingImports = true
 reportMissingTypeStubs = true
 reportUnnecessaryIsInstance = false
+exclude = [
+    "tad/migrations"
+]
 
 [tool.coverage.run]
 branch = true
 command_line = "-m pytest"
 relative_files = true # needed for sonarcloud code coverage
+omit = [
+    "tests/*"
+]
+
 
 [tool.coverage.report]
 fail_under = 95
@@ -94,6 +101,7 @@ title = "tad"
 testpaths = [
     "tests"
 ]
+addopts = "--strict-markers"
 filterwarnings = [
     "ignore::UserWarning"
 ]

tad/core/config.py

@@ -1,5 +1,5 @@
+import logging
 import secrets
-import warnings
 from typing import Any, TypeVar
 
 from pydantic import (
@@ -18,7 +18,10 @@
 
 class Settings(BaseSettings):
     # todo(berry): investigate yaml, toml or json file support for SettingsConfigDict
-    model_config = SettingsConfigDict(env_file=".env", env_ignore_empty=True, extra="ignore")
+    # todo(berry): investigate multiple .env files support for SettingsConfigDict
+    model_config = SettingsConfigDict(
+        env_file=(".env", ".env.test", ".env.prod"), env_ignore_empty=True, extra="ignore"
+    )
     SECRET_KEY: str = secrets.token_urlsafe(32)
 
     DOMAIN: str = "localhost"
@@ -43,23 +46,33 @@ def server_host(self) -> str:
 
     # todo(berry): create submodel for database settings
     APP_DATABASE_SCHEME: DatabaseSchemaType = "sqlite"
+    APP_DATABASE_DRIVER: str | None = None
+
     APP_DATABASE_SERVER: str = "db"
     APP_DATABASE_PORT: int = 5432
     APP_DATABASE_USER: str = "tad"
-    APP_DATABASE_PASSWORD: str
+    APP_DATABASE_PASSWORD: str | None = None
     APP_DATABASE_DB: str = "tad"
 
-    SQLITE_FILE: str = "//./database"
+    APP_DATABASE_FILE: str = "database.sqlite3"
 
     @computed_field  # type: ignore[misc]
     @property
     def SQLALCHEMY_DATABASE_URI(self) -> str:
+        logging.info(f"test: {self.APP_DATABASE_SCHEME}")
+
         if self.APP_DATABASE_SCHEME == "sqlite":
-            return str(MultiHostUrl.build(scheme=self.APP_DATABASE_SCHEME, host="", path=self.SQLITE_FILE))
+            return str(MultiHostUrl.build(scheme=self.APP_DATABASE_SCHEME, host="", path=self.APP_DATABASE_FILE))
+
+        scheme: str = (
+            f"{self.APP_DATABASE_SCHEME}+{self.APP_DATABASE_DRIVER}"
+            if isinstance(self.APP_DATABASE_DRIVER, str)
+            else self.APP_DATABASE_SCHEME
+        )
 
         return str(
             MultiHostUrl.build(
-                scheme=self.APP_DATABASE_SCHEME,
+                scheme=scheme,
                 username=self.APP_DATABASE_USER,
                 password=self.APP_DATABASE_PASSWORD,
                 host=self.APP_DATABASE_SERVER,
@@ -68,21 +81,6 @@ def SQLALCHEMY_DATABASE_URI(self) -> str:
             )
         )
 
-    def _check_default_secret(self, var_name: str, value: str | None) -> None:
-        if value == "changethis":
-            message = f'The value of {var_name} is "changethis", ' "for security, please change it"
-            if self.ENVIRONMENT == "local":
-                warnings.warn(message, stacklevel=1)
-            else:
-                raise SettingsError(message)
-
-    @model_validator(mode="after")
-    def _enforce_non_default_secrets(self: SelfSettings) -> SelfSettings:
-        self._check_default_secret("SECRET_KEY", self.SECRET_KEY)
-        self._check_default_secret("APP_DATABASE_PASSWORD", self.APP_DATABASE_PASSWORD)
-
-        return self
-
     @model_validator(mode="after")
     def _enforce_database_rules(self: SelfSettings) -> SelfSettings:
         if self.ENVIRONMENT != "local" and self.APP_DATABASE_SCHEME == "sqlite":

tad/core/db.py

@@ -1,5 +1,10 @@
-from sqlmodel import create_engine
+from sqlmodel import Session, create_engine, select
 
 from tad.core.config import settings
 
-engine = create_engine(settings.SQLALCHEMY_DATABASE_URI, echo=True)
+engine = create_engine(settings.SQLALCHEMY_DATABASE_URI)
+
+
+async def check_db():
+    with Session(engine) as session:
+        session.exec(select(1))

tad/main.py

@@ -10,6 +10,7 @@
 
 from tad.api.main import api_router
 from tad.core.config import settings
+from tad.core.db import check_db
 from tad.core.exception_handlers import (
     http_exception_handler as tad_http_exception_handler,
 )
@@ -32,6 +33,7 @@ async def lifespan(app: FastAPI):
     logger.info(f"Starting {settings.PROJECT_NAME} version {settings.VERSION}")
     logger.info(f"Settings: {mask.secrets(settings.model_dump())}")
     # todo(berry): setup database connection
+    await check_db()
     yield
     logger.info(f"Stopping application {settings.PROJECT_NAME} version {settings.VERSION}")
     logging.shutdown()

tad/models/__init__.py

@@ -0,0 +1,3 @@
+from .hero import Hero
+
+__all__ = ["Hero"]

tests/conftest.py

@@ -1,23 +1,28 @@
-import os
 from collections.abc import Generator
 
 import pytest
 from fastapi.testclient import TestClient
+from sqlmodel import Session, SQLModel, create_engine
+from sqlmodel.pool import StaticPool
 from tad.main import app
 
+# needed to make sure create_all knows about the models
+from tad.models import *  # noqa: F403
+
 
-# todo(berry): add database fixtures
 @pytest.fixture(scope="module")
-def client() -> Generator[TestClient, None, None]:
+def client(db: Session) -> Generator[TestClient, None, None]:
     with TestClient(app, raise_server_exceptions=True) as c:
-        c.timeout = 3
+        c.timeout = 5
         yield c
 
 
-@pytest.fixture(autouse=True)
-def setup_basic_environmental_variables(monkeypatch: pytest.MonkeyPatch) -> Generator[None, None, None]:  # noqa: PT004
-    original_environ = dict(os.environ)
-    monkeypatch.setenv("APP_DATABASE_PASSWORD", "changethis")
-    yield
-    os.environ.clear()
-    os.environ.update(original_environ)
+@pytest.fixture(scope="module")
+def db() -> Generator[Session, None, None]:
+    engine = create_engine("sqlite://", poolclass=StaticPool)
+    SQLModel.metadata.create_all(engine)
+
+    with Session(engine) as session:
+        yield session
+
+    SQLModel.metadata.drop_all(engine)

tests/core/test_config.py

@@ -1,5 +1,3 @@
-import os
-
 import pytest
 from tad.core.config import Settings
 from tad.core.exceptions import SettingsError
@@ -16,23 +14,18 @@ def test_default_settings():
     assert settings.PROJECT_NAME == "TAD"
     assert settings.PROJECT_DESCRIPTION == "Transparency of Algorithmic Decision making"
     assert settings.APP_DATABASE_SCHEME == "sqlite"
-    assert settings.APP_DATABASE_SERVER == "db"
-    assert settings.APP_DATABASE_PORT == 5432
-    assert settings.APP_DATABASE_USER == "tad"
-    assert settings.APP_DATABASE_DB == "tad"
-    assert settings.SQLITE_FILE == "//./database"
-    assert settings.SQLALCHEMY_DATABASE_URI == "sqlite://///database"
-
-
-def test_environment_settings():
-    os.environ["DOMAIN"] = "google.com"
-    os.environ["ENVIRONMENT"] = "production"
-    os.environ["PROJECT_NAME"] = "TAD2"
-    os.environ["SECRET_KEY"] = "mysecret"  # noqa: S105
-    os.environ["APP_DATABASE_SCHEME"] = "postgresql"
-    os.environ["APP_DATABASE_USER"] = "tad2"
-    os.environ["APP_DATABASE_DB"] = "tad2"
-    os.environ["APP_DATABASE_PASSWORD"] = "mypassword"  # noqa: S105
+    assert settings.SQLALCHEMY_DATABASE_URI == "sqlite:///database.sqlite3"
+
+
+def test_environment_settings(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setenv("DOMAIN", "google.com")
+    monkeypatch.setenv("ENVIRONMENT", "production")
+    monkeypatch.setenv("PROJECT_NAME", "TAD2")
+    monkeypatch.setenv("SECRET_KEY", "mysecret")
+    monkeypatch.setenv("APP_DATABASE_SCHEME", "postgresql")
+    monkeypatch.setenv("APP_DATABASE_USER", "tad2")
+    monkeypatch.setenv("APP_DATABASE_DB", "tad2")
+    monkeypatch.setenv("APP_DATABASE_PASSWORD", "mypassword")
     settings = Settings(_env_file="nonexisitingfile")  # type: ignore
 
     assert settings.SECRET_KEY == "mysecret"  # noqa: S105
@@ -49,22 +42,21 @@ def test_environment_settings():
     assert settings.APP_DATABASE_USER == "tad2"
     assert settings.APP_DATABASE_PASSWORD == "mypassword"  # noqa: S105
     assert settings.APP_DATABASE_DB == "tad2"
-    assert settings.SQLITE_FILE == "//./database"
     assert settings.SQLALCHEMY_DATABASE_URI == "postgresql://tad2:mypassword@db:5432/tad2"
 
 
-def test_environment_settings_production_sqlite_error():
-    os.environ["ENVIRONMENT"] = "production"
-    os.environ["APP_DATABASE_SCHEME"] = "sqlite"
-    os.environ["APP_DATABASE_PASSWORD"] = "32452345432"  # noqa: S105
+def test_environment_settings_production_sqlite_error(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setenv("ENVIRONMENT", "production")
+    monkeypatch.setenv("APP_DATABASE_SCHEME", "sqlite")
+    monkeypatch.setenv("APP_DATABASE_PASSWORD", "32452345432")
     with pytest.raises(SettingsError) as e:
         _settings = Settings(_env_file="nonexisitingfile")  # type: ignore
 
     assert e.value.message == "SQLite is not supported in production"
 
 
-def test_environment_settings_production_nopassword_error():
-    os.environ["ENVIRONMENT"] = "production"
+def test_environment_settings_production_nopassword_error(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setenv("ENVIRONMENT", "production")
 
     with pytest.raises(SettingsError):
         _settings = Settings(_env_file="nonexisitingfile")  # type: ignore

tests/core/test_db.py

@@ -0,0 +1,15 @@
+from unittest.mock import Mock, patch
+
+import pytest
+from sqlmodel import Session, select
+from tad.core.db import check_db
+
+
+@pytest.mark.skip(reason="not working yet")
+async def test_check_dabase():
+    mock_session = Mock(spec=Session)
+
+    with patch("sqlmodel.Session", return_value=mock_session):
+        await check_db()
+
+    assert mock_session.exec.assert_called_once_with(select(1))

tests/core/test_exception_handlers.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 import pytest
 from fastapi import status
 from fastapi.exceptions import HTTPException, RequestValidationError
@@ -6,28 +8,26 @@
 
 
 @pytest.fixture()
-def _mock_error_500():
+def mock_error_500() -> Any:  # noqa: PT004
     """Add route to raise ValueError"""
 
     @app.get("/raise-http-exception")
-    async def raise_http_error():
+    async def _raise_http_error():  # type: ignore
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
 
     @app.get("/raise-request-validation-exception")
-    async def raise_request_error():
-        raise RequestValidationError(errors=None)
+    async def _raise_request_error():  # type: ignore
+        raise RequestValidationError(errors="None")
 
 
-@pytest.mark.usefixtures("_mock_error_500")
-def test_http_exception_handler(client: TestClient):
+def test_http_exception_handler(client: TestClient, mock_error_500: Any):
     response = client.get("/raise-http-exception")
 
     assert response.status_code == status.HTTP_404_NOT_FOUND
     assert response.headers["content-type"] == "text/html; charset=utf-8"
 
 
-@pytest.mark.usefixtures("_mock_error_500")
-def test_request_validation_exception_handler(client: TestClient):
+def test_request_validation_exception_handler(client: TestClient, mock_error_500: Any):
     response = client.get("/raise-request-validation-exception")
 
     assert response.status_code == status.HTTP_400_BAD_REQUEST