Merge pull request #86 from metno/map-overlay

Implements API for map overlay images.

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = aerovaldb
-version = 0.1.0rc1
+version = 0.1.0rc2
 author = Augustin Mortier, Thorbjørn Lundin, Heiko Klein
 author_email = [email protected]
 description = aeroval database to communicate between pyaerocom and aeroval

src/aerovaldb/aerovaldb.py

@@ -1263,3 +1263,46 @@ async def put_report_image(self, obj, project: str, experiment: str, path: str):
             image data
         """
         raise NotImplementedError
+
+    @async_and_sync
+    @get_method(ROUTE_MAP_OVERLAY)
+    async def get_map_overlay(
+        self,
+        project: str,
+        experiment: str,
+        source: str,
+        variable: str,
+        date: str,
+        access_type: str | AccessType = AccessType.BLOB,
+    ):
+        """Getter for map overlay images.
+
+        :param project : Project ID.
+        :param experiment : Experiment ID.
+        :param source : Data source. Can be either an observation network or a model ID.
+        :param variable : Variable name.
+        :param date : Date.
+        """
+        raise NotImplementedError
+
+    @async_and_sync
+    @put_method(ROUTE_MAP_OVERLAY)
+    async def put_map_overlay(
+        self,
+        obj,
+        project: str,
+        experiment: str,
+        source: str,
+        variable: str,
+        date: str,
+    ):
+        """Putter for map overlay images.
+
+        :param obj : The object to be stored.
+        :param project : Project ID.
+        :param experiment : Experiment ID.
+        :param source : Data source. Can be either an observation network or a model ID.
+        :param variable : Variable name.
+        :param date : Date.
+        """
+        raise NotImplementedError

src/aerovaldb/jsondb/jsonfiledb.py

@@ -4,15 +4,14 @@
 import os
 import shutil
 from pathlib import Path
-from typing import Callable, Awaitable, Any, Generator
+from typing import Callable, Awaitable, Any
 import importlib.metadata
 
 from async_lru import alru_cache
 from packaging.version import Version
 
 from aerovaldb.aerovaldb import AerovalDB
 from aerovaldb.const import IMG_FILE_EXTS
-from aerovaldb.exceptions import UnusedArguments
 from aerovaldb.types import AccessType
 from ..utils.string_mapper import StringMapper, VersionConstraintMapper
 
@@ -116,6 +115,7 @@ def __init__(self, basedir: str | Path, /, use_async: bool = False):
                 ROUTE_GRIDDED_MAP: "./{project}/{experiment}/contour/{obsvar}_{model}.json",
                 ROUTE_REPORT: "./reports/{project}/{experiment}/{title}.json",
                 ROUTE_REPORT_IMAGE: "./reports/{project}/{experiment}/{path}",
+                ROUTE_MAP_OVERLAY: "./{project}/{experiment}/contour/overlay/{source}_{variable}_{date}.png",
             },
             version_provider=self._get_version,
         )
@@ -349,7 +349,7 @@ async def _get_uri_for_file(self, file_path: str) -> str:
 
         _, ext = os.path.splitext(file_path)
 
-        if ext.lower() in IMG_FILE_EXTS:
+        if file_path.startswith("reports/") and ext.lower() in IMG_FILE_EXTS:
             # TODO: Fix this.
             # The image endpoint is the only endpoint which needs to accept an arbitrary path
             # under the experiment directory. Treating it as a special case for now.
@@ -554,6 +554,16 @@ async def get_by_uri(
                 access_type=access_type,
             )
 
+        if route.startswith("/v0/map-overlay/"):
+            return await self.get_map_overlay(
+                route_args["project"],
+                route_args["experiment"],
+                route_args["source"],
+                route_args["variable"],
+                route_args["date"],
+                access_type=access_type,
+            )
+
         return await self._get(
             route,
             route_args,
@@ -573,6 +583,17 @@ async def put_by_uri(self, obj, uri: str):
             )
             return
 
+        if route.startswith("/v0/map-overlay/"):
+            await self.put_map_overlay(
+                obj,
+                route_args["project"],
+                route_args["experiment"],
+                route_args["source"],
+                route_args["variable"],
+                route_args["date"],
+            )
+            return
+
         await self._put(obj, route, route_args, **kwargs)
 
     def _get_lock_file(self) -> str:
@@ -658,3 +679,75 @@ async def put_report_image(self, obj, project: str, experiment: str, path: str):
         os.makedirs(Path(file_path).parent, exist_ok=True)
         with open(file_path, "wb") as f:
             f.write(obj)
+
+    @async_and_sync
+    async def get_map_overlay(
+        self,
+        project: str,
+        experiment: str,
+        source: str,
+        variable: str,
+        date: str,
+        access_type: str | AccessType = AccessType.BLOB,
+    ):
+        access_type = self._normalize_access_type(access_type)
+
+        if access_type not in (AccessType.FILE_PATH, AccessType.BLOB):
+            raise UnsupportedOperation(
+                f"The report image endpoint does not support access type {access_type}."
+            )
+
+        file_path = await self._get(
+            route=ROUTE_MAP_OVERLAY,
+            route_args={
+                "project": project,
+                "experiment": experiment,
+                "source": source,
+                "variable": variable,
+                "date": date,
+            },
+            access_type=AccessType.FILE_PATH,
+        )
+        logger.debug(f"Fetching image with path '{file_path}'")
+
+        if access_type == AccessType.FILE_PATH:
+            return file_path
+
+        with open(file_path, "rb") as f:
+            return f.read()
+
+    @async_and_sync
+    async def put_map_overlay(
+        self,
+        obj,
+        project: str,
+        experiment: str,
+        source: str,
+        variable: str,
+        date: str,
+    ):
+        """Putter for map overlay images.
+
+        :param obj : The object to be stored.
+        :param project : Project ID.
+        :param experiment : Experiment ID.
+        :param source : Data source. Can be either an observation network or a model ID.
+        :param variable : Variable name.
+        :param date : Date.
+        """
+        template = await self._get_template(ROUTE_MAP_OVERLAY, {})
+
+        file_path = os.path.join(
+            self._basedir,
+            template.format(
+                project=project,
+                experiment=experiment,
+                source=source,
+                variable=variable,
+                date=date,
+            ),
+        )
+
+        os.makedirs(Path(file_path).parent, exist_ok=True)
+        with open(file_path, "wb") as f:
+            f.write(obj)

src/aerovaldb/routes.py

@@ -46,6 +46,8 @@
 
 ROUTE_REPORT_IMAGE = "/v0/report-image/{project}/{experiment}/{path}"
 
+ROUTE_MAP_OVERLAY = "/v0/map-overlay/{project}/{experiment}/{source}/{variable}/{date}"
+
 ALL_ROUTES = [
     ROUTE_GLOB_STATS,
     ROUTE_REG_STATS,
@@ -68,4 +70,5 @@
     ROUTE_GRIDDED_MAP,
     ROUTE_REPORT,
     ROUTE_REPORT_IMAGE,
+    ROUTE_MAP_OVERLAY,
 ]

src/aerovaldb/sqlitedb/sqlitedb.py

@@ -103,6 +103,7 @@ class AerovalSqliteDB(AerovalDB):
         "gridded_map": extract_substitutions(ROUTE_GRIDDED_MAP),
         "report": extract_substitutions(ROUTE_REPORT),
         "reportimages": extract_substitutions(ROUTE_REPORT_IMAGE),
+        "mapoverlay": extract_substitutions(ROUTE_MAP_OVERLAY),
     }
 
     TABLE_NAME_TO_ROUTE = {
@@ -130,6 +131,7 @@ class AerovalSqliteDB(AerovalDB):
         "gridded_map": ROUTE_GRIDDED_MAP,
         "report": ROUTE_REPORT,
         "reportimages": ROUTE_REPORT_IMAGE,
+        "mapoverlay": ROUTE_MAP_OVERLAY,
     }
 
     def __init__(self, database: str, /, **kwargs):
@@ -211,6 +213,7 @@ def __init__(self, database: str, /, **kwargs):
                 ROUTE_GRIDDED_MAP: "gridded_map",
                 ROUTE_REPORT: "report",
                 ROUTE_REPORT_IMAGE: "reportimages",
+                ROUTE_MAP_OVERLAY: "mapoverlay",
             },
             version_provider=self._get_version,
         )
@@ -307,7 +310,7 @@ def _initialize_db(self):
             args = AerovalSqliteDB.TABLE_COLUMN_NAMES[table_name]
 
             column_names = ",".join(args)
-            if table_name == "reportimages":
+            if table_name in ("reportimages", "mapoverlay"):
                 cur.execute(
                     f"""
                     CREATE TABLE IF NOT EXISTS {table_name}(
@@ -497,6 +500,15 @@ async def get_by_uri(
                 route_args["project"], route_args["experiment"], route_args["path"]
             )
 
+        if route == ROUTE_MAP_OVERLAY:
+            return await self.get_map_overlay(
+                route_args["project"],
+                route_args["experiment"],
+                route_args["source"],
+                route_args["variable"],
+                route_args["date"],
+            )
+
         return await self._get(
             route,
             route_args,
@@ -515,6 +527,17 @@ async def put_by_uri(self, obj, uri: str):
             )
             return
 
+        if route == ROUTE_MAP_OVERLAY:
+            await self.put_map_overlay(
+                obj,
+                route_args["project"],
+                route_args["experiment"],
+                route_args["source"],
+                route_args["variable"],
+                route_args["date"],
+            )
+            return
+
         await self._put(obj, route, route_args, **kwargs)
 
     @async_and_sync
@@ -674,6 +697,7 @@ def rm_experiment_data(self, project: str, experiment: str) -> None:
             "heatmap_timeseries2",
             "forecast",
             "gridded_map",
+            "mapoverlay",
         ]:
             cur.execute(
                 f"""
@@ -727,3 +751,63 @@ async def put_report_image(self, obj, project: str, experiment: str, path: str):
             """,
             (project, experiment, path, obj),
         )
+        self._con.commit()
+
+    @async_and_sync
+    async def get_map_overlay(
+        self,
+        project: str,
+        experiment: str,
+        source: str,
+        variable: str,
+        date: str,
+        access_type: str | AccessType = AccessType.BLOB,
+    ):
+        access_type = self._normalize_access_type(access_type)
+
+        if access_type != AccessType.BLOB:
+            raise UnsupportedOperation(
+                f"Sqlitedb does not support accesstype {access_type}."
+            )
+
+        cur = self._con.cursor()
+        cur.execute(
+            """
+            SELECT * FROM mapoverlay
+            WHERE
+                (project, experiment, source, variable, date) = (?, ?, ?, ?, ?)
+            """,
+            (project, experiment, source, variable, date),
+        )
+        fetched = cur.fetchone()
+
+        if fetched is None:
+            raise FileNotFoundError(
+                f"Object not found. {project, experiment, source, variable, date}"
+            )
+
+        return fetched["blob"]
+
+    @async_and_sync
+    async def put_map_overlay(
+        self,
+        obj,
+        project: str,
+        experiment: str,
+        source: str,
+        variable: str,
+        date: str,
+    ):
+        cur = self._con.cursor()
+
+        if not isinstance(obj, bytes):
+            raise TypeError(f"Expected bytes. Got {type(obj)}")
+
+        cur.execute(
+            """
+            INSERT OR REPLACE INTO mapoverlay(project, experiment, source, variable, date, blob)
+            VALUES(?, ?, ?, ?, ?, ?)
+            """,
+            (project, experiment, source, variable, date, obj),
+        )
+        self._con.commit()

src/aerovaldb/utils/copy.py

@@ -34,7 +34,7 @@ async def copy_db_contents(source: str | AerovalDB, dest: str | AerovalDB):
     for i, uri in enumerate(await source.list_all()):
         logger.info(f"Processing item {i} of {len(await source.list_all())}")
         access = AccessType.JSON_STR
-        if uri.startswith("/v0/report-image/"):
+        if uri.startswith("/v0/report-image/") or uri.startswith("/v0/map-overlay/"):
             access = AccessType.BLOB
         data = await source.get_by_uri(uri, access_type=access)
 

tests/jsondb/test_jsonfiledb.py

@@ -26,3 +26,17 @@ def test_with_symlink():
         data = db.get_config("linked-json-project", "experiment")
 
         assert data["path"] == "link"
+
+
+def test_get_map_overlay():
+    with aerovaldb.open("json_files:./tests/test-db/json") as db:
+        path = db.get_map_overlay(
+            "project",
+            "experiment",
+            "source",
+            "variable",
+            "date",
+            access_type=aerovaldb.AccessType.FILE_PATH,
+        )
+
+        assert os.path.exists(path)

tests/test_aerovaldb.py

@@ -487,7 +487,7 @@ def test_list_glob_stats(testdb):
 @TESTDB_PARAMETRIZATION
 def test_list_all(testdb):
     with aerovaldb.open(testdb) as db:
-        assert len(db.list_all()) == 46
+        assert len(db.list_all()) == 47
 
 
 @TESTDB_PARAMETRIZATION