From bf5a5227381f6473eea9cf1799c7e2b1edf69fd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20No=C3=A9?= <nicolas@niconoe.eu>
Date: Fri, 14 Jun 2024 11:19:54 +0200
Subject: [PATCH 01/12] Adjusted Poetry options

---
 pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pyproject.toml b/pyproject.toml
index 1aaaf4a..aa2712e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,6 +3,7 @@ name = "gbif-alert"
 version = "1.7.4"
 description = "A GBIF-based early alert system"
 authors = ["Nicolas Noé <nicolas@niconoe.eu>"]
+package-mode = false
 
 [tool.poetry.dependencies]
 python = "^3.11"

From 5a0ebdd11143d544c2e12abce2f152a576d43a0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20No=C3=A9?= <nicolas@niconoe.eu>
Date: Tue, 23 Jul 2024 11:22:43 +0200
Subject: [PATCH 02/12] The min-max-per-hexagon endpoint now uses materialized
 views to speed up its process.

---
 CHANGELOG.md                                  |   3 +-
 .../commands/import_observations.py           |  85 +++++++-----
 dashboard/tests/selenium/test_integration.py  |   2 +
 dashboard/tests/views/test_maps.py            |  42 ++++--
 dashboard/views/helpers.py                    |  47 +++++++
 dashboard/views/maps.py                       | 123 ++++++++----------
 djangoproject/settings.py                     |  27 ++++
 7 files changed, 218 insertions(+), 111 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 87a7246..754b103 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,7 @@
 # Current (unreleased)
 
-- Fix a compatibility issue with Windows platform (data import script). Thanks @sronveaux!
+- Fix a compatibility issue with Windows platform (data import script). Thanks, @sronveaux!
+- Major improvements under the hood to map performances (Thanks for the suggestion, @sronveaux and @silenius!)
 
 # v1.7.4 (2024-05-24)
 
diff --git a/dashboard/management/commands/import_observations.py b/dashboard/management/commands/import_observations.py
index 419dd0d..87eeee1 100644
--- a/dashboard/management/commands/import_observations.py
+++ b/dashboard/management/commands/import_observations.py
@@ -1,5 +1,6 @@
 import argparse
 import datetime
+import logging
 import os
 import tempfile
 import time
@@ -18,6 +19,7 @@
 
 from dashboard.management.commands.helpers import get_dataset_name_from_gbif_api
 from dashboard.models import Species, Observation, DataImport, Dataset
+from dashboard.views.helpers import create_or_refresh_all_materialized_views
 
 BULK_CREATE_CHUNK_SIZE = 10000
 
@@ -199,6 +201,9 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.transaction_was_successful = False
 
+    def log_with_time(self, message: str):
+        self.stdout.write(f"{time.ctime()}: {message}")
+
     def _import_all_observations_from_dwca(
         self,
         dwca: DwCAReader,
@@ -227,7 +232,7 @@ def _import_all_observations_from_dwca(
                 self.stdout.write("x", ending="")
 
             if index % BULK_CREATE_CHUNK_SIZE == 0:
-                self.stdout.write(f"{time.ctime()}: Bulk size reached...")
+                self.log_with_time("Bulk size reached...")
                 self.batch_insert_observations(observations_to_insert)
                 observations_to_insert = []
 
@@ -237,9 +242,9 @@ def _import_all_observations_from_dwca(
         return skipped_observations_counter
 
     def batch_insert_observations(self, observations_to_insert: list[Observation]):
-        self.stdout.write(f"{time.ctime()}: Bulk creation")
+        self.log_with_time("Bulk creation")
         inserted_observations = Observation.objects.bulk_create(observations_to_insert)
-        self.stdout.write(f"{time.ctime()}: Migrating linked entities")
+        self.log_with_time("Migrating linked entities")
         for obs in inserted_observations:
             obs.migrate_linked_entities()
 
@@ -254,21 +259,28 @@ def flag_transaction_as_successful(self):
         self.transaction_was_successful = True
 
     def handle(self, *args, **options) -> None:
-        self.stdout.write(f"{time.ctime()}: (Re)importing all observations")
+        # Allow the verbosity option for our custom logging
+        # (see https://reinout.vanrees.org/weblog/2017/03/08/logging-verbosity-managment-commands.html)
+        verbosity = int(options["verbosity"])
+        root_logger = logging.getLogger("")
+        if verbosity > 1:
+            root_logger.setLevel(logging.DEBUG)
+
+        self.log_with_time("(Re)importing all observations")
 
         # 1. Data preparation / download
         gbif_predicate = None
         if options["source_dwca"]:
-            self.stdout.write(f"{time.ctime()}: Using a user-provided DWCA file")
+            self.log_with_time("Using a user-provided DWCA file")
             source_data_path = options["source_dwca"].name
         else:
-            self.stdout.write(
-                f"{time.ctime()}: No DWCA file provided, we'll generate and get a new GBIF download"
+            self.log_with_time(
+                "No DWCA file provided, we'll generate and get a new GBIF download"
             )
-
-            self.stdout.write(
-                f"{time.ctime()}: Triggering a GBIF download and waiting for it - this can be long..."
+            self.log_with_time(
+                "Triggering a GBIF download and waiting for it - this can be long..."
             )
+
             tmp_file = tempfile.NamedTemporaryFile(delete=False)
             source_data_path = tmp_file.name
             tmp_file.close()
@@ -283,11 +295,10 @@ def handle(self, *args, **options) -> None:
                 password=settings.GBIF_ALERT["GBIF_DOWNLOAD_CONFIG"]["PASSWORD"],
                 output_path=source_data_path,
             )
-            self.stdout.write(f"{time.ctime()}: Observations downloaded")
+            self.log_with_time("Observations downloaded")
 
-        self.stdout.write(
-            f"{time.ctime()}: We now have a (locally accessible) source dwca, real import is starting. We'll use a transaction and put "
-            "the website in maintenance mode"
+        self.log_with_time(
+            "We now have a (locally accessible) source dwca, real import is starting. We'll use a transaction and put the website in maintenance mode"
         )
 
         set_maintenance_mode(True)
@@ -298,12 +309,12 @@ def handle(self, *args, **options) -> None:
             current_data_import = DataImport.objects.create(
                 start=timezone.now(), gbif_predicate=gbif_predicate
             )
-            self.stdout.write(
-                f"{time.ctime()}: Created a new DataImport object: #{current_data_import.pk}"
+            self.log_with_time(
+                f"Created a new DataImport object: #{current_data_import.pk}"
             )
 
             # 3. Pre-import all the datasets (better here than in a loop that goes over each observation)
-            self.stdout.write(f"{time.ctime()}: Pre-importing all datasets")
+            self.log_with_time("Pre-importing all datasets")
             # 3.1 Get all the dataset keys / names from the DwCA
             datasets_referenced_in_dwca = dict()
             with DwCAReader(source_data_path) as dwca:
@@ -344,7 +355,7 @@ def handle(self, *args, **options) -> None:
                 current_data_import.set_gbif_download_id(
                     extract_gbif_download_id_from_dwca(dwca)
                 )
-                self.stdout.write(f"{time.ctime()}: Importing all rows")
+                self.log_with_time("Importing all rows")
                 current_data_import.skipped_observations_counter = (
                     self._import_all_observations_from_dwca(
                         dwca,
@@ -354,41 +365,53 @@ def handle(self, *args, **options) -> None:
                     )
                 )
 
-            self.stdout.write(
-                f"{time.ctime()}: All observations imported, now deleting observations linked to previous data imports..."
+            self.log_with_time(
+                "All observations imported, now deleting observations linked to previous data imports..."
             )
 
             # 6. Remove previous observations
             Observation.objects.exclude(data_import=current_data_import).delete()
+            self.log_with_time("Previous observations deleted")
+
+            self.log_with_time(
+                "We'll now create or refresh the materialized views. This can take a while."
+            )
 
-            # 7. Remove unused Dataset entries (and edit related alerts)
+            # 7. Create or refresh the materialized views (for the map)
+            create_or_refresh_all_materialized_views()
+
+            # 8. Remove unused Dataset entries (and edit related alerts)
             for dataset in Dataset.objects.all():
                 if dataset.observation_set.count() == 0:
-                    self.stdout.write(
-                        f"{time.ctime()}: Deleting (no longer used) dataset {dataset}"
-                    )
+                    self.log_with_time(f"Deleting (no longer used) dataset {dataset}")
+
                     alerts_referencing_dataset = dataset.alert_set.all()
                     if alerts_referencing_dataset.count() > 0:
                         for alert in alerts_referencing_dataset:
-                            self.stdout.write(
-                                f"{time.ctime()}: We'll first need to un-reference this dataset from alert #{alert}"
+                            self.log_with_time(
+                                f"We'll first need to un-reference this dataset from alert #{alert}"
                             )
                             alert.datasets.remove(dataset)
 
                     dataset.delete()
 
-            # 6. Finalize the DataImport object
-            self.stdout.write(f"{time.ctime()}: Updating the DataImport object")
+            # 9. Finalize the DataImport object
+            self.log_with_time("Updating the DataImport object")
+
             current_data_import.complete()
             if options["source_dwca"] is None:
+                self.log_with_time("Deleting the (temporary) source DWCA file")
                 os.unlink(source_data_path)
-            self.stdout.write("Done.")
+            self.log_with_time("Committing the transaction")
 
-        self.stdout.write(f"{time.ctime()}: Leaving maintenance mode.")
+        self.log_with_time("Transaction committed")
+        self.log_with_time("Leaving maintenance mode.")
         set_maintenance_mode(False)
 
-        self.stdout.write(f"{time.ctime()}: Sending email report")
+        self.log_with_time("Sending email report")
         if self.transaction_was_successful:
             send_successful_import_email()
         else:
             send_error_import_email()
+
+        self.log_with_time("Import observations process successfully completed")
diff --git a/dashboard/tests/selenium/test_integration.py b/dashboard/tests/selenium/test_integration.py
index 8887a18..4aa0d65 100644
--- a/dashboard/tests/selenium/test_integration.py
+++ b/dashboard/tests/selenium/test_integration.py
@@ -27,6 +27,7 @@
     ObservationView,
     Alert,
 )
+from dashboard.views.helpers import create_or_refresh_all_materialized_views
 
 
 def _get_webdriver() -> WebDriver:
@@ -142,6 +143,7 @@ def setUp(self):
         )
 
         alert.species.add(self.first_species)
+        create_or_refresh_all_materialized_views()
 
 
 class SeleniumAlertTests(SeleniumTestsCommon):
diff --git a/dashboard/tests/views/test_maps.py b/dashboard/tests/views/test_maps.py
index 38f865b..392522c 100644
--- a/dashboard/tests/views/test_maps.py
+++ b/dashboard/tests/views/test_maps.py
@@ -15,6 +15,10 @@
     Area,
     ObservationView,
 )
+from dashboard.views.helpers import (
+    create_or_refresh_all_materialized_views,
+    create_or_refresh_materialized_views,
+)
 
 
 class MapsTestDataMixin(object):
@@ -105,6 +109,8 @@ def setUpTestData(cls):
 
         ObservationView.objects.create(observation=second_obs, user=cls.user)
 
+        create_or_refresh_all_materialized_views()
+
 
 @override_settings(
     STATICFILES_STORAGE="django.contrib.staticfiles.storage.StaticFilesStorage"
@@ -135,6 +141,8 @@ def test_min_max_per_hexagon(self):
             location=Point(4.36229, 50.64628, srid=4326),  # Lillois, bakkerij
         )
 
+        create_or_refresh_materialized_views(zoom_levels=[8, 1, 13])
+
         # Now, at zoom level 8 we should have a hexagon with count=1 and another one with count=2
         response = self.client.get(
             reverse("dashboard:internal-api:maps:mvt-min-max-per-hexagon"),
@@ -154,7 +162,7 @@ def test_min_max_per_hexagon(self):
         # At zoom level 17, there's no hexagons that cover more than 1 observation
         response = self.client.get(
             reverse("dashboard:internal-api:maps:mvt-min-max-per-hexagon"),
-            data={"zoom": 17},
+            data={"zoom": 13},
         )
         self.assertEqual(response.json()["min"], 1)
         self.assertEqual(response.json()["max"], 1)
@@ -172,6 +180,8 @@ def test_min_max_per_hexagon_with_species_filter(self):
             location=Point(4.36229, 50.64628, srid=4326),  # Lillois, bakkerij
         )
 
+        create_or_refresh_materialized_views(zoom_levels=[8])
+
         response = self.client.get(
             reverse("dashboard:internal-api:maps:mvt-min-max-per-hexagon"),
             data={"zoom": 8, "speciesIds[]": self.first_species.pk},
@@ -203,6 +213,8 @@ def test_min_max_per_hexagon_with_species_filter(self):
             location=Point(5.095610, 50.48800, srid=4326),
         )
 
+        create_or_refresh_materialized_views(zoom_levels=[8])
+
         response = self.client.get(
             reverse("dashboard:internal-api:maps:mvt-min-max-per-hexagon"),
             data={"zoom": 8, "speciesIds[]": self.second_species.pk},
@@ -224,6 +236,8 @@ def test_min_max_per_hexagon_with_dataset_filter(self):
             location=Point(4.36229, 50.64628, srid=4326),  # Lillois, bakkerij
         )
 
+        create_or_refresh_materialized_views(zoom_levels=[8])
+
         response = self.client.get(
             reverse("dashboard:internal-api:maps:mvt-min-max-per-hexagon"),
             data={"zoom": 8, "datasetsIds[]": self.first_dataset.pk},
@@ -255,6 +269,8 @@ def test_min_max_per_hexagon_with_dataset_filter(self):
             location=Point(5.095610, 50.48800, srid=4326),
         )
 
+        create_or_refresh_materialized_views(zoom_levels=[8])
+
         response = self.client.get(
             reverse("dashboard:internal-api:maps:mvt-min-max-per-hexagon"),
             data={"zoom": 8, "speciesIds[]": self.second_species.pk},
@@ -313,6 +329,8 @@ def test_min_max_in_hexagon_with_status_filter_anonymous(self):
             location=Point(4.36229, 50.64628, srid=4326),  # Lillois, bakkerij
         )
 
+        create_or_refresh_materialized_views(zoom_levels=[8])
+
         # At a zoom level that only shows Lillois or Andenne, it's 1-1
         response = self.client.get(
             reverse("dashboard:internal-api:maps:mvt-min-max-per-hexagon"),
@@ -337,6 +355,8 @@ def test_min_max_per_hexagon_with_area_filter(self):
             location=Point(4.36229, 50.64628, srid=4326),  # Lillois, bakkerij
         )
 
+        create_or_refresh_materialized_views(zoom_levels=[8])
+
         # We restrict ourselves to Andenne: only one observation
         response = self.client.get(
             reverse("dashboard:internal-api:maps:mvt-min-max-per-hexagon"),
@@ -400,8 +420,8 @@ def test_status_and_content_type(self):
         )
 
     def test_zoom_levels(self):
-        """Zoom levels 0-21 are supported"""
-        for zoom_level in range(0, 21):
+        """Zoom levels 0-14 are supported"""
+        for zoom_level in range(0, 14):
             response = self.client.get(self._build_valid_tile_url(zoom=zoom_level))
             self.assertEqual(response.status_code, 200)
             mapbox_vector_tile.decode(response.content)
@@ -756,7 +776,7 @@ def test_tiles_area_filter(self):
         # Case 4: A zoomed time on Lillois, should be empty because of the filtering
         base_url = reverse(
             self.server_url_name,
-            kwargs={"zoom": 17, "x": 67123, "y": 44083},
+            kwargs={"zoom": 14, "x": 8390, "y": 5510},
         )
         url_with_params = f"{base_url}?areaIds[]={self.public_area_andenne.pk}"
         response = self.client.get(url_with_params)
@@ -854,7 +874,7 @@ def test_tiles_status_filter_case2(self):
         # Case 2.2: there's nothing when zoomed on Lillois
         base_url = reverse(
             self.server_url_name,
-            kwargs={"zoom": 17, "x": 67123, "y": 44083},
+            kwargs={"zoom": 14, "x": 8390, "y": 5510},
         )
         url_with_params = f"{base_url}?status=unseen"
         response = self.client.get(url_with_params)
@@ -916,7 +936,7 @@ def test_tiles_species_filter(self):
         # Case 4: A zoomed time on Lillois, should be empty because of the filtering
         base_url = reverse(
             self.server_url_name,
-            kwargs={"zoom": 17, "x": 67123, "y": 44083},
+            kwargs={"zoom": 14, "x": 8390, "y": 5510},
         )
         url_with_params = f"{base_url}?speciesIds[]={self.first_species.pk}"
         response = self.client.get(url_with_params)
@@ -994,7 +1014,7 @@ def test_tiles_dataset_filter(self):
         # Case 4: A zoomed time on Lillois, should be empty because of the filtering
         base_url = reverse(
             self.server_url_name,
-            kwargs={"zoom": 17, "x": 67123, "y": 44083},
+            kwargs={"zoom": 14, "x": 8390, "y": 5510},
         )
         url_with_params = f"{base_url}?speciesIds[]={self.first_species.pk}"
         response = self.client.get(url_with_params)
@@ -1088,7 +1108,7 @@ def test_tiles_species_multiple_species_filters(self):
         # Case 4: A zoomed time on Lillois, we expect the three tetraodon observations
         base_url = reverse(
             self.server_url_name,
-            kwargs={"zoom": 17, "x": 67123, "y": 44083},
+            kwargs={"zoom": 14, "x": 8390, "y": 5510},
         )
         url_with_params = f"{base_url}?speciesIds[]={self.first_species.pk}&speciesIds[]={species_tetraodon.pk}"
         response = self.client.get(url_with_params)
@@ -1195,7 +1215,7 @@ def test_tiles_basic_data_in_hexagons(self):
         response = self.client.get(
             reverse(
                 self.server_url_name,
-                kwargs={"zoom": 17, "x": 67123, "y": 44083},
+                kwargs={"zoom": 14, "x": 8390, "y": 5510},
             )
         )
         decoded_tile = mapbox_vector_tile.decode(response.content)
@@ -1206,11 +1226,11 @@ def test_tiles_basic_data_in_hexagons(self):
             decoded_tile["default"]["features"][0]["properties"]["count"], 1
         )
 
-        # The next one is empty
+        # The next tile is empty
         response = self.client.get(
             reverse(
                 self.server_url_name,
-                kwargs={"zoom": 17, "x": 67124, "y": 44083},
+                kwargs={"zoom": 14, "x": 8391, "y": 5510},
             )
         )
         decoded_tile = mapbox_vector_tile.decode(response.content)
diff --git a/dashboard/views/helpers.py b/dashboard/views/helpers.py
index 911aed1..0c92169 100644
--- a/dashboard/views/helpers.py
+++ b/dashboard/views/helpers.py
@@ -2,12 +2,20 @@
 
 import ast
 import datetime
+import logging
+from string import Template
 from urllib.parse import unquote
 
+from django.db import connection
 from django.db.models import QuerySet
 from django.http import HttpRequest, JsonResponse, QueryDict
+from jinjasql import JinjaSql
 
 from dashboard.models import Observation, User
+from dashboard.utils import readable_string
+from django.conf import settings
+
+logger = logging.getLogger(__name__)
 
 
 # This class is only defined to make Mypy happy
@@ -152,3 +160,42 @@ def model_to_json_list(Model) -> JsonResponse:
     Model instances should have an as_dict property
     """
     return JsonResponse([entry.as_dict for entry in Model.objects.all()], safe=False)
+
+
+def create_or_refresh_all_materialized_views():
+    for hex_size in set(settings.ZOOM_TO_HEX_SIZE.values()):  # set to remove duplicates
+        create_or_refresh_materialized_view(hex_size)
+
+
+def create_or_refresh_materialized_views(zoom_levels: list[int]):
+    """Create or refresh a bunch of materialized views for a list of zoom levels"""
+    for zoom_level in zoom_levels:
+        create_or_refresh_materialized_view(settings.ZOOM_TO_HEX_SIZE[zoom_level])
+
+
+def create_or_refresh_materialized_view(hex_size_meters: int):
+    """Create or refresh a single materialized view for a specific hex size in meters"""
+    logger.info(
+        f"Creating or refreshing materialized view for hex size {hex_size_meters}"
+    )
+
+    sql_template = readable_string(
+        Template(
+            """
+        CREATE MATERIALIZED VIEW IF NOT EXISTS hexa_$hex_size_meters AS (
+         SELECT *
+         FROM dashboard_observation AS obs
+         JOIN ST_HexagonGrid($hex_size_meters,
+        ST_SetSRID(ST_EstimatedExtent('dashboard_observation', 'location'),
+        3857)) AS hexes ON ST_Intersects(obs.location, hexes.geom)
+        ) WITH NO DATA;
+        
+        REFRESH MATERIALIZED VIEW hexa_$hex_size_meters;
+        """
+        ).substitute(hex_size_meters=hex_size_meters)
+    )
+
+    j = JinjaSql()
+    query, bind_params = j.prepare_query(sql_template, {})
+    with connection.cursor() as cursor:
+        cursor.execute(query, bind_params)
diff --git a/dashboard/views/maps.py b/dashboard/views/maps.py
index 4103007..8b8519c 100644
--- a/dashboard/views/maps.py
+++ b/dashboard/views/maps.py
@@ -9,6 +9,7 @@
 from dashboard.models import Observation, Area, ObservationView, Species
 from dashboard.utils import readable_string
 from dashboard.views.helpers import filters_from_request, extract_int_request
+from django.conf import settings
 
 AREAS_TABLE_NAME = Area.objects.model._meta.db_table
 OBSERVATIONS_TABLE_NAME = Observation.objects.model._meta.db_table
@@ -21,54 +22,13 @@
 DB_DATE_EXCHANGE_FORMAT_PYTHON = "%Y-%m-%d"  # To be passed to strftime()
 DB_DATE_EXCHANGE_FORMAT_POSTGRES = "YYYY-MM-DD"  # To be used in SQL queries
 
-# Hexagon size (in meters) according to the zoom level. Adjust ZOOM_TO_HEX_SIZE_MULTIPLIER to simultaneously configure
-# all zoom levels
-ZOOM_TO_HEX_SIZE_MULTIPLIER = 2
-ZOOM_TO_HEX_SIZE_BASELINE = {
-    0: 640000,
-    1: 320000,
-    2: 160000,
-    3: 80000,
-    4: 40000,
-    5: 20000,
-    6: 10000,
-    7: 5000,
-    8: 2500,
-    9: 1250,
-    10: 675,
-    11: 335,
-    12: 160,
-    13: 80,
-    14: 40,
-    15: 20,
-    16: 10,
-    17: 5,
-    18: 5,
-    19: 5,
-    20: 5,
-}
-ZOOM_TO_HEX_SIZE = {
-    key: value * ZOOM_TO_HEX_SIZE_MULTIPLIER
-    for key, value in ZOOM_TO_HEX_SIZE_BASELINE.items()
-}
 
 # !! IMPORTANT !! Make sure the observation filtering here is equivalent to what's done in
 # other places (views.helpers.filtered_observations_from_request). Otherwise, observations returned on the map and on
 # other components (table, ...) will be inconsistent.
-JINJASQL_FRAGMENT_FILTER_OBSERVATIONS = Template(
-    """
-    SELECT * FROM $observations_table_name as obs
-    LEFT JOIN $species_table_name as species
-    ON obs.species_id = species.id
-    {% if status == 'seen' %}
-        INNER JOIN $observationview_table_name
-        ON obs.id = $observationview_table_name.observation_id
-    {% endif %}
-    
-    {% if area_ids %}
-    , (SELECT mpoly FROM $areas_table_name WHERE $areas_table_name.id IN {{ area_ids | inclause }}) AS areas
-    {% endif %}
-    WHERE (
+WHERE_CLAUSE = readable_string(
+    Template(
+        """ 
         1 = 1
         {% if species_ids %}
             AND obs.species_id IN {{ species_ids | inclause }}
@@ -100,33 +60,48 @@
                 SELECT ov1.id FROM $observationview_table_name ov1 WHERE ov1.user_id = {{ user_id }}
             )
         {% endif %}
+"""
+    ).substitute(
+        observationview_table_name=OBSERVATIONVIEWS_TABLE_NAME,
+        date_format=DB_DATE_EXCHANGE_FORMAT_POSTGRES,
+    )
+)
+
+JINJASQL_FRAGMENT_FILTER_OBSERVATIONS = Template(
+    """
+    SELECT * FROM $observations_table_name as obs
+    LEFT JOIN $species_table_name as species
+    ON obs.species_id = species.id
+    {% if status == 'seen' %}
+        INNER JOIN $observationview_table_name
+        ON obs.id = $observationview_table_name.observation_id
+    {% endif %}
+    
+    {% if area_ids %}
+    , (SELECT mpoly FROM $areas_table_name WHERE $areas_table_name.id IN {{ area_ids | inclause }}) AS areas
+    {% endif %}
+    WHERE (
+        $where_clause
     )
 """
 ).substitute(
+    observationview_table_name=OBSERVATIONVIEWS_TABLE_NAME,
     areas_table_name=AREAS_TABLE_NAME,
     species_table_name=SPECIES_TABLE_NAME,
     observations_table_name=OBSERVATIONS_TABLE_NAME,
-    observationview_table_name=OBSERVATIONVIEWS_TABLE_NAME,
-    date_format=DB_DATE_EXCHANGE_FORMAT_POSTGRES,
+    where_clause=WHERE_CLAUSE,
 )
 
 JINJASQL_FRAGMENT_AGGREGATED_GRID = Template(
     """
     SELECT COUNT(*), hexes.geom
-                    FROM
-                        ST_HexagonGrid(
-                            {{ hex_size_meters }},
-                            {% if grid_extent_viewport %}
-                                ST_TileEnvelope({{ zoom }}, {{ x }}, {{ y }})
-                            {% else %}
-                                ST_SetSRID(ST_EstimatedExtent('$observations_table_name', '$observations_field_name_point'), 3857)
-                            {% endif %} 
-                        ) AS hexes
-                    INNER JOIN ($jinjasql_fragment_filter_observations)
-                    AS dashboard_filtered_occ
-
-                    ON ST_Intersects(dashboard_filtered_occ.$observations_field_name_point, hexes.geom)
-                    GROUP BY hexes.geom
+    FROM
+        ST_HexagonGrid({{ hex_size_meters }}, ST_TileEnvelope({{ zoom }}, {{ x }}, {{ y }})) AS hexes
+        INNER JOIN ($jinjasql_fragment_filter_observations)
+    AS dashboard_filtered_occ
+
+    ON ST_Intersects(dashboard_filtered_occ.$observations_field_name_point, hexes.geom)
+    GROUP BY hexes.geom
 """
 ).substitute(
     observations_table_name=OBSERVATIONS_TABLE_NAME,
@@ -219,8 +194,7 @@ def mvt_tiles_observations_hexagon_grid_aggregated(
 
     sql_params = {
         # Map technicalities
-        "hex_size_meters": ZOOM_TO_HEX_SIZE[zoom],
-        "grid_extent_viewport": True,
+        "hex_size_meters": settings.ZOOM_TO_HEX_SIZE[zoom],
         "zoom": zoom,
         "x": x,
         "y": y,
@@ -267,17 +241,30 @@ def observation_min_max_in_hex_grid_json(request: HttpRequest):
         sql_template = readable_string(
             Template(
                 """
-        WITH grid AS ($jinjasql_fragment_aggregated_grid)
-        SELECT MIN(count), MAX(count) FROM grid;
-        """
+                WITH grid AS (
+                    SELECT COUNT(*)
+                    FROM (SELECT * FROM hexa_$hex_size_meters) AS obs
+                        LEFT JOIN dashboard_species as species ON obs.species_id = species.id
+                        
+                        {% if area_ids %}
+                        ,(SELECT mpoly FROM $areas_table_name WHERE $areas_table_name.id IN {{ area_ids | inclause }}) AS areas
+                        {% endif %}
+                    WHERE (
+                        $where_clause
+                    )      
+                GROUP BY obs.geom
+                )
+                
+                SELECT MIN(count), MAX(count) FROM grid;
+                """
             ).substitute(
-                jinjasql_fragment_aggregated_grid=JINJASQL_FRAGMENT_AGGREGATED_GRID
+                hex_size_meters=settings.ZOOM_TO_HEX_SIZE[zoom],
+                areas_table_name=AREAS_TABLE_NAME,
+                where_clause=WHERE_CLAUSE,
             )
         )
 
         sql_params = {
-            "hex_size_meters": ZOOM_TO_HEX_SIZE[zoom],
-            "grid_extent_viewport": False,
             "species_ids": species_ids,
             "datasets_ids": datasets_ids,
             "area_ids": area_ids,
diff --git a/djangoproject/settings.py b/djangoproject/settings.py
index 08f1cca..6e350d9 100644
--- a/djangoproject/settings.py
+++ b/djangoproject/settings.py
@@ -271,3 +271,30 @@
 TAGGIT_CASE_INSENSITIVE = True
 
 SELENIUM_HEADLESS_MODE = True
+
+# Hexagon size (in meters) according to the zoom level. Adjust ZOOM_TO_HEX_SIZE_MULTIPLIER to simultaneously configure
+# all zoom levels
+
+ZOOM_TO_HEX_SIZE_MULTIPLIER = 2
+ZOOM_TO_HEX_SIZE_BASELINE = {
+    0: 640000,
+    1: 320000,
+    2: 160000,
+    3: 80000,
+    4: 40000,
+    5: 20000,
+    6: 10000,
+    7: 5000,
+    8: 2500,
+    9: 1250,
+    10: 675,
+    11: 335,
+    12: 160,
+    13: 80,
+    14: 80,
+    # At higher zoom levels, the map shows the individual points
+}
+ZOOM_TO_HEX_SIZE = {
+    key: value * ZOOM_TO_HEX_SIZE_MULTIPLIER
+    for key, value in ZOOM_TO_HEX_SIZE_BASELINE.items()
+}

From b1be1c01bb7c274fa7e0f5733a8973fdf752c232 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20No=C3=A9?= <nicolas@niconoe.eu>
Date: Tue, 23 Jul 2024 12:15:04 +0200
Subject: [PATCH 03/12] To save disk space and import time, we only
 generate/update materialized views when necessary

---
 assets/ts/components/ObservationsMap.vue             |  8 ++++++--
 assets/ts/components/OuterObservationsMap.vue        |  1 +
 assets/ts/interfaces.ts                              |  1 +
 dashboard/management/commands/import_observations.py | 10 +++++++---
 dashboard/templatetags/gbif-alert_extras.py          |  1 +
 dashboard/views/helpers.py                           |  8 +++++---
 djangoproject/settings.py                            | 10 ++++++++++
 7 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/assets/ts/components/ObservationsMap.vue b/assets/ts/components/ObservationsMap.vue
index fa7b044..bbba445 100644
--- a/assets/ts/components/ObservationsMap.vue
+++ b/assets/ts/components/ObservationsMap.vue
@@ -63,7 +63,7 @@ export default defineComponent({
     baseLayerName: String,
     dataLayerOpacity: Number,
     areasToShow: {
-      type: Array as PropType<Array<number>>, // Array of area ids
+      type: Array as PropType<Array<Number>>, // Array of area ids
       default: [],
     },
     layerSwitchZoomLevel: {
@@ -71,6 +71,10 @@ export default defineComponent({
       type: Number,
       default: 13,
     },
+    zoomLevelMinMaxQuery: {
+      type: Number,
+      required: true,
+    },
   },
   data: function () {
     return {
@@ -242,7 +246,7 @@ export default defineComponent({
         if (this.aggregatedDataLayer) {
           this.map.removeLayer(this.aggregatedDataLayer as VectorTileLayer<Feature>);
         }
-        this.loadOccMinMax(this.initialPosition.initialZoom, this.filters);
+        this.loadOccMinMax(this.zoomLevelMinMaxQuery, this.filters);
         this.aggregatedDataLayer = this.createAggregatedDataLayer();
         this.map.addLayer(this.aggregatedDataLayer as VectorTileLayer<Feature>);
       }
diff --git a/assets/ts/components/OuterObservationsMap.vue b/assets/ts/components/OuterObservationsMap.vue
index bf902a8..aabac5f 100644
--- a/assets/ts/components/OuterObservationsMap.vue
+++ b/assets/ts/components/OuterObservationsMap.vue
@@ -44,6 +44,7 @@
       :base-layer-name="mapBaseLayer"
       :data-layer-opacity="dataLayerOpacity"
       :areas-to-show="filters.areaIds"
+      :zoom-level-min-max-query="frontendConfig.zoomLevelMinMaxQuery"
     ></Observations-Map>
   </div>
 </template>
diff --git a/assets/ts/interfaces.ts b/assets/ts/interfaces.ts
index 48c4e93..6c52948 100644
--- a/assets/ts/interfaces.ts
+++ b/assets/ts/interfaces.ts
@@ -101,6 +101,7 @@ export interface FrontEndConfig {
   authenticatedUser: boolean;
   userId?: number; // Only set if authenticatedUser is true
   mainMapConfig: MapConfig;
+  zoomLevelMinMaxQuery: number;
 }
 
 // Keep in sync with Models.Observation.as_dict()
diff --git a/dashboard/management/commands/import_observations.py b/dashboard/management/commands/import_observations.py
index 87eeee1..af740f3 100644
--- a/dashboard/management/commands/import_observations.py
+++ b/dashboard/management/commands/import_observations.py
@@ -19,7 +19,9 @@
 
 from dashboard.management.commands.helpers import get_dataset_name_from_gbif_api
 from dashboard.models import Species, Observation, DataImport, Dataset
-from dashboard.views.helpers import create_or_refresh_all_materialized_views
+from dashboard.views.helpers import (
+    create_or_refresh_materialized_views,
+)
 
 BULK_CREATE_CHUNK_SIZE = 10000
 
@@ -377,8 +379,10 @@ def handle(self, *args, **options) -> None:
                 "We'll now create or refresh the materialized views. This can take a while."
             )
 
-            # 7. Create or refresh the materialized views (for the map)
-            create_or_refresh_all_materialized_views()
+            # 7. Create or refresh the materialized view (for the map)
+            create_or_refresh_materialized_views(
+                zoom_levels=[settings.ZOOM_LEVEL_FOR_MIN_MAX_QUERY]
+            )
 
             # 8. Remove unused Dataset entries (and edit related alerts)
             for dataset in Dataset.objects.all():
diff --git a/dashboard/templatetags/gbif-alert_extras.py b/dashboard/templatetags/gbif-alert_extras.py
index 3a74626..c7a1181 100644
--- a/dashboard/templatetags/gbif-alert_extras.py
+++ b/dashboard/templatetags/gbif-alert_extras.py
@@ -108,6 +108,7 @@ def js_config_object(context):
             ).replace("1", "{id}"),
         },
         "mainMapConfig": settings.GBIF_ALERT["MAIN_MAP_CONFIG"],
+        "zoomLevelMinMaxQuery": settings.ZOOM_LEVEL_FOR_MIN_MAX_QUERY,
     }
     if context.request.user.is_authenticated:
         conf["userId"] = context.request.user.pk
diff --git a/dashboard/views/helpers.py b/dashboard/views/helpers.py
index 0c92169..c8d7a68 100644
--- a/dashboard/views/helpers.py
+++ b/dashboard/views/helpers.py
@@ -164,16 +164,18 @@ def model_to_json_list(Model) -> JsonResponse:
 
 def create_or_refresh_all_materialized_views():
     for hex_size in set(settings.ZOOM_TO_HEX_SIZE.values()):  # set to remove duplicates
-        create_or_refresh_materialized_view(hex_size)
+        create_or_refresh_single_materialized_view(hex_size)
 
 
 def create_or_refresh_materialized_views(zoom_levels: list[int]):
     """Create or refresh a bunch of materialized views for a list of zoom levels"""
     for zoom_level in zoom_levels:
-        create_or_refresh_materialized_view(settings.ZOOM_TO_HEX_SIZE[zoom_level])
+        create_or_refresh_single_materialized_view(
+            settings.ZOOM_TO_HEX_SIZE[zoom_level]
+        )
 
 
-def create_or_refresh_materialized_view(hex_size_meters: int):
+def create_or_refresh_single_materialized_view(hex_size_meters: int):
     """Create or refresh a single materialized view for a specific hex size in meters"""
     logger.info(
         f"Creating or refreshing materialized view for hex size {hex_size_meters}"
diff --git a/djangoproject/settings.py b/djangoproject/settings.py
index 6e350d9..4a919b5 100644
--- a/djangoproject/settings.py
+++ b/djangoproject/settings.py
@@ -298,3 +298,13 @@
     key: value * ZOOM_TO_HEX_SIZE_MULTIPLIER
     for key, value in ZOOM_TO_HEX_SIZE_BASELINE.items()
 }
+
+# The zoom level at which the minimum and maximum values are queried
+# That's the only zoom level where this calculation is done
+# By consequence, we generate a materialized view for this zoom level, so
+# the endpoint has good performances. We initially generated those views at each zoom
+# level, but it's lengthy (during import), eats disk space and is unnecessary.
+# The test suite also make sure the endpoint works at different zoom levels, so it
+# will need to generate more materialized views (so the endpoint works), but it's only
+# when running tests and with a tiny amount of data.
+ZOOM_LEVEL_FOR_MIN_MAX_QUERY = 8

From b1b182f515502973358aaba80c44fc81f99204a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20No=C3=A9?= <nicolas@niconoe.eu>
Date: Tue, 23 Jul 2024 12:52:32 +0200
Subject: [PATCH 04/12] Fixed soe backend errors while running selenium tests

---
 dashboard/views/maps.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/dashboard/views/maps.py b/dashboard/views/maps.py
index 8b8519c..0b0b36e 100644
--- a/dashboard/views/maps.py
+++ b/dashboard/views/maps.py
@@ -249,6 +249,10 @@ def observation_min_max_in_hex_grid_json(request: HttpRequest):
                         {% if area_ids %}
                         ,(SELECT mpoly FROM $areas_table_name WHERE $areas_table_name.id IN {{ area_ids | inclause }}) AS areas
                         {% endif %}
+                        {% if status == 'seen' %}
+                            INNER JOIN $observationview_table_name
+                            ON obs.id = $observationview_table_name.observation_id
+                        {% endif %}
                     WHERE (
                         $where_clause
                     )      
@@ -260,6 +264,7 @@ def observation_min_max_in_hex_grid_json(request: HttpRequest):
             ).substitute(
                 hex_size_meters=settings.ZOOM_TO_HEX_SIZE[zoom],
                 areas_table_name=AREAS_TABLE_NAME,
+                observationview_table_name=OBSERVATIONVIEWS_TABLE_NAME,
                 where_clause=WHERE_CLAUSE,
             )
         )

From 4aeead736af5495fd209e52ab669bb50510c883a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20No=C3=A9?= <nicolas@niconoe.eu>
Date: Tue, 23 Jul 2024 13:00:28 +0200
Subject: [PATCH 05/12] Make test resilient ot explicit ordering

---
 dashboard/tests/views/test_public_api.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/dashboard/tests/views/test_public_api.py b/dashboard/tests/views/test_public_api.py
index 7817580..8a4a04c 100644
--- a/dashboard/tests/views/test_public_api.py
+++ b/dashboard/tests/views/test_public_api.py
@@ -196,7 +196,7 @@ def test_observation_json_short_results(self):
         response = self.client.get(f"{base_url}?limit=10&page_number=1&mode=short")
         self.assertEqual(response.status_code, 200)
         json_data = response.json()
-        expected = [
+        expected_data = [
             {
                 "id": self.obs1.pk,
                 "lat": 50.48940999999999,
@@ -222,7 +222,8 @@ def test_observation_json_short_results(self):
                 "date": "2021-10-08",
             },
         ]
-        self.assertEqual(json_data["results"], expected)
+        for expected in expected_data:
+            self.assertIn(expected, json_data["results"])
 
     def test_observations_json_default_mode_normal(self):
         """Explicitly asking the normal mode brings the same result as not specifying a mode"""

From 8f0c87e7494c21e791388a850a312ae8947bd8c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20No=C3=A9?= <nicolas@niconoe.eu>
Date: Tue, 23 Jul 2024 13:07:45 +0200
Subject: [PATCH 06/12] Another attempt to fix tests on CI

---
 dashboard/tests/views/test_public_api.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/dashboard/tests/views/test_public_api.py b/dashboard/tests/views/test_public_api.py
index 8a4a04c..920a432 100644
--- a/dashboard/tests/views/test_public_api.py
+++ b/dashboard/tests/views/test_public_api.py
@@ -199,24 +199,24 @@ def test_observation_json_short_results(self):
         expected_data = [
             {
                 "id": self.obs1.pk,
-                "lat": 50.48940999999999,
-                "lon": 5.095129999999999,
+                "lat": self.obs1.lat,
+                "lon": self.obs1.lon,
                 "scientificName": "Procambarus fallax",
                 "speciesId": self.first_species.pk,
                 "date": "2021-09-13",
             },
             {
                 "id": self.obs2.pk,
-                "lat": 50.647279999999995,
-                "lon": 4.35978,
+                "lat": self.obs2.lat,
+                "lon": self.obs2.lon,
                 "scientificName": "Orconectes virilis",
                 "speciesId": self.second_species.pk,
                 "date": "2021-09-13",
             },
             {
                 "id": self.obs3.pk,
-                "lat": 50.647279999999995,
-                "lon": 4.35978,
+                "lat": self.obs3.lat,
+                "lon": self.obs3.lon,
                 "scientificName": "Orconectes virilis",
                 "speciesId": self.second_species.pk,
                 "date": "2021-10-08",

From 3bb29bb942b3d387c045e660281d968e0601d678 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20No=C3=A9?= <nicolas@niconoe.eu>
Date: Tue, 23 Jul 2024 14:10:47 +0200
Subject: [PATCH 07/12] Another attempt to fix tests on CI

---
 dashboard/tests/views/test_public_api.py | 45 +++++++++---------------
 1 file changed, 17 insertions(+), 28 deletions(-)

diff --git a/dashboard/tests/views/test_public_api.py b/dashboard/tests/views/test_public_api.py
index 920a432..bd8aea7 100644
--- a/dashboard/tests/views/test_public_api.py
+++ b/dashboard/tests/views/test_public_api.py
@@ -196,34 +196,23 @@ def test_observation_json_short_results(self):
         response = self.client.get(f"{base_url}?limit=10&page_number=1&mode=short")
         self.assertEqual(response.status_code, 200)
         json_data = response.json()
-        expected_data = [
-            {
-                "id": self.obs1.pk,
-                "lat": self.obs1.lat,
-                "lon": self.obs1.lon,
-                "scientificName": "Procambarus fallax",
-                "speciesId": self.first_species.pk,
-                "date": "2021-09-13",
-            },
-            {
-                "id": self.obs2.pk,
-                "lat": self.obs2.lat,
-                "lon": self.obs2.lon,
-                "scientificName": "Orconectes virilis",
-                "speciesId": self.second_species.pk,
-                "date": "2021-09-13",
-            },
-            {
-                "id": self.obs3.pk,
-                "lat": self.obs3.lat,
-                "lon": self.obs3.lon,
-                "scientificName": "Orconectes virilis",
-                "speciesId": self.second_species.pk,
-                "date": "2021-10-08",
-            },
-        ]
-        for expected in expected_data:
-            self.assertIn(expected, json_data["results"])
+
+        results = json_data["results"]
+
+        # Check the correct records are present
+        ids_in_results = [result["id"] for result in results]
+        self.assertEqual(ids_in_results, [self.obs1.pk, self.obs2.pk, self.obs3.pk])
+
+        # check the fields that are present in the short mode
+        for result in results:
+            self.assertIn("id", result)
+            self.assertIn("lat", result)
+            self.assertIn("lon", result)
+            self.assertIn("scientificName", result)
+            self.assertIn("speciesId", result)
+            self.assertIn("date", result)
+            # Check fields that should not be present in the short mode
+            self.assertNotIn("stableId", result)
 
     def test_observations_json_default_mode_normal(self):
         """Explicitly asking the normal mode brings the same result as not specifying a mode"""

From 248a70f2333cde614801b858f1c3cd3c66f43039 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20No=C3=A9?= <nicolas@niconoe.eu>
Date: Tue, 23 Jul 2024 14:19:41 +0200
Subject: [PATCH 08/12] Ignored some mypy errors

---
 dashboard/translation.py | 2 +-
 dashboard/views/maps.py  | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/dashboard/translation.py b/dashboard/translation.py
index 85bfb18..92b0f52 100644
--- a/dashboard/translation.py
+++ b/dashboard/translation.py
@@ -5,4 +5,4 @@
 
 @register(Species)
 class SpeciesTranslationOptions(TranslationOptions):
-    fields = ("vernacular_name",)
+    fields = ("vernacular_name",)  # type: ignore
diff --git a/dashboard/views/maps.py b/dashboard/views/maps.py
index 0b0b36e..5fca84b 100644
--- a/dashboard/views/maps.py
+++ b/dashboard/views/maps.py
@@ -277,15 +277,15 @@ def observation_min_max_in_hex_grid_json(request: HttpRequest):
         }
 
         if status_for_user and request.user.is_authenticated:
-            sql_params["status"] = status_for_user
-            sql_params["user_id"] = request.user.pk
+            sql_params["status"] = status_for_user  # type: ignore
+            sql_params["user_id"] = request.user.pk  # type: ignore
 
         if start_date:
-            sql_params["start_date"] = start_date.strftime(
+            sql_params["start_date"] = start_date.strftime(  # type: ignore
                 DB_DATE_EXCHANGE_FORMAT_PYTHON
             )
         if end_date:
-            sql_params["end_date"] = end_date.strftime(DB_DATE_EXCHANGE_FORMAT_PYTHON)
+            sql_params["end_date"] = end_date.strftime(DB_DATE_EXCHANGE_FORMAT_PYTHON)  # type: ignore
 
         j = JinjaSql()
         query, bind_params = j.prepare_query(sql_template, sql_params)

From 3fea31465f4071444b3dc89f2f0a28825cd7b48c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20No=C3=A9?= <nicolas@niconoe.eu>
Date: Thu, 25 Jul 2024 09:50:46 +0200
Subject: [PATCH 09/12] Another smaller map optimization

---
 README.md               | 16 +++++++++++++++-
 dashboard/views/maps.py |  4 ++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3094332..1f7cbeb 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,16 @@
+TODO:
+- to continue:
+  - new table in DB for the not seen, and populated. so we can now:
+    - adjust all the seen/not seen code to make sure it uses the new model
+    - add the user preference to choose which observations should be marked as seen automatically (delay)
+    - add code to mark as seen all the observations that are older than the delay (run each day after the data import?)
+    - test if data migration performance is enough for prod purposes !!
+    - (at the end): drop the old observationView table
+    - (clone the existing site/db to test everything before real deployment)?
+- confirm with Damiano that all observations are considered as seen whe a user create an account
+- New NL translations?
+
+
 # GBIF Alert
 
 <!-- badges: start -->
@@ -27,4 +40,5 @@ See [INSTALL.md](INSTALL.md) for more information.
 ## GBIF Alert instances in the wild
 
 - LIFE RIPARIAS Early Alert: [production](https://alert.riparias.be) / [development](https://dev-alert.riparias.be) (Targets riparian invasive species in Belgium)
-- [GBIF Alert demo instance](https://gbif-alert-demo.thebinaryforest.net/) (Always in sync with the `devel` branch of this repository)
\ No newline at end of file
+- [GBIF Alert demo instance](https://gbif-alert-demo.thebinaryforest.net/) (Always in sync with the `devel` branch of this repository)
+- The Belgian Biodiversity Platform uses GBIF alert under the hood as an API for the ManaIAS project. 
\ No newline at end of file
diff --git a/dashboard/views/maps.py b/dashboard/views/maps.py
index 5fca84b..f1134e8 100644
--- a/dashboard/views/maps.py
+++ b/dashboard/views/maps.py
@@ -60,6 +60,9 @@
                 SELECT ov1.id FROM $observationview_table_name ov1 WHERE ov1.user_id = {{ user_id }}
             )
         {% endif %}
+        {% if limit_to_tile %}
+            AND ST_Within(obs.location,  ST_TileEnvelope({{ zoom }}, {{ x }}, {{ y }}))
+        {% endif %}
 """
     ).substitute(
         observationview_table_name=OBSERVATIONVIEWS_TABLE_NAME,
@@ -140,6 +143,7 @@ def mvt_tiles_observations(
 
     sql_params = {
         # Map technicalities
+        "limit_to_tile": False,
         "zoom": zoom,
         "x": x,
         "y": y,

From 893b58bae427c8549e6bb04cc2e51fc6b947a006 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20No=C3=A9?= <nicolas@niconoe.eu>
Date: Thu, 25 Jul 2024 09:54:41 +0200
Subject: [PATCH 10/12] Prepare for release

---
 CHANGELOG.md       | 2 +-
 INSTALL.md         | 6 +++---
 docker-compose.yml | 6 +++---
 package.json       | 2 +-
 pyproject.toml     | 2 +-
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 754b103..d8fa2c3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,4 @@
-# Current (unreleased)
+# v1.7.5 (2024-07-25)
 
 - Fix a compatibility issue with Windows platform (data import script). Thanks, @sronveaux!
 - Major improvements under the hood to map performances (Thanks for the suggestion, @sronveaux and @silenius!)
diff --git a/INSTALL.md b/INSTALL.md
index 45757fa..7360ebf 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -44,14 +44,14 @@ properly configured.
 ### Prerequisites
 
 - Make sure [Docker](https://docs.docker.com/get-docker/) is installed on your system
-- Identify the latest release of GBIF Alert on GitHub at https://github.com/riparias/gbif-alert/tags (currently [v1.7.4](https://github.com/riparias/gbif-alert/releases/tag/v1.7.4))
+- Identify the latest release of GBIF Alert on GitHub at https://github.com/riparias/gbif-alert/tags (currently [v1.7.5](https://github.com/riparias/gbif-alert/releases/tag/v1.7.5))
 
 ### Installation steps
 
 - Create a new directory on your system, e.g. `invasive-fishes-nz` following the example above.
-- Go to the `docker-compose.yml` file from the latest release of GBIF Alert on GitHub: at the moment https://github.com/riparias/gbif-alert/blob/v1.7.4/docker-compose.yml (note that the URL contains the version number).
+- Go to the `docker-compose.yml` file from the latest release of GBIF Alert on GitHub: at the moment https://github.com/riparias/gbif-alert/blob/v1.7.5/docker-compose.yml (note that the URL contains the version number).
 - Save the file in the directory you have just created.
-- Go to the `local_settings_docker.template.py` file from the latest release of GBIF Alert on GitHub: at the moment https://github.com/riparias/gbif-alert/blob/v1.7.4/djangoproject/local_settings_docker.template.py.
+- Go to the `local_settings_docker.template.py` file from the latest release of GBIF Alert on GitHub: at the moment https://github.com/riparias/gbif-alert/blob/v1.7.5/djangoproject/local_settings_docker.template.py.
 - Save the file in the directory you have just created. 
 - Rename this file to `local_settings_docker.py`.
 - Open a terminal, navigate to the `invasive-fishes-nz` directory and run the following command: `docker-compose up`. 
diff --git a/docker-compose.yml b/docker-compose.yml
index 17e6bf0..2632ec5 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -2,7 +2,7 @@ version: '3'
 
 services:
   nginx:
-    image: niconoe/gbif-alert-nginx:1.7.4
+    image: niconoe/gbif-alert-nginx:1.7.5
     ports:
       - "1337:80"
     depends_on:
@@ -28,7 +28,7 @@ services:
     expose:
       - 6379
   gbif-alert:
-    image : niconoe/gbif-alert:1.7.4
+    image : niconoe/gbif-alert:1.7.5
     expose:
       - 8000
     depends_on:
@@ -44,7 +44,7 @@ services:
         target: /app/djangoproject/local_settings_docker.py
         read_only: true
   rqworker:
-    image: niconoe/gbif-alert:1.7.4
+    image: niconoe/gbif-alert:1.7.5
     entrypoint: poetry run python manage.py rqworker default
     depends_on:
       db:
diff --git a/package.json b/package.json
index 4cc2ea0..c490028 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "gbif-alert",
-  "version": "1.7.4",
+  "version": "1.7.5",
   "description": "[![Django CI](https://github.com/riparias/early-alert-webapp/actions/workflows/django_tests.yml/badge.svg)](https://github.com/riparias/early-warning-webapp/actions/workflows/django_tests.yml)",
   "main": "index.js",
   "scripts": {
diff --git a/pyproject.toml b/pyproject.toml
index aa2712e..3027453 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gbif-alert"
-version = "1.7.4"
+version = "1.7.5"
 description = "A GBIF-based early alert system"
 authors = ["Nicolas Noé <nicolas@niconoe.eu>"]
 package-mode = false

From 5ee69e6e15465ed7bd51cae5e6de6093fcf3f166 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20No=C3=A9?= <nicolas@niconoe.eu>
Date: Thu, 25 Jul 2024 09:56:24 +0200
Subject: [PATCH 11/12] Poetry (self) update

---
 poetry.lock | 92 ++---------------------------------------------------
 1 file changed, 3 insertions(+), 89 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 69e7bdc..ca829ac 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,10 +1,9 @@
-# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
 
 [[package]]
 name = "asgiref"
 version = "3.8.1"
 description = "ASGI specs, helper code, and adapters"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -19,7 +18,6 @@ tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"]
 name = "async-timeout"
 version = "4.0.3"
 description = "Timeout context manager for asyncio programs"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -31,7 +29,6 @@ files = [
 name = "atomicwrites"
 version = "1.4.1"
 description = "Atomic file writes."
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -42,7 +39,6 @@ files = [
 name = "attrs"
 version = "23.2.0"
 description = "Classes Without Boilerplate"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -62,7 +58,6 @@ tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "p
 name = "black"
 version = "23.12.1"
 description = "The uncompromising code formatter."
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -107,7 +102,6 @@ uvloop = ["uvloop (>=0.15.2)"]
 name = "certifi"
 version = "2024.2.2"
 description = "Python package for providing Mozilla's CA Bundle."
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -119,7 +113,6 @@ files = [
 name = "cffi"
 version = "1.16.0"
 description = "Foreign Function Interface for Python calling C code."
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -184,7 +177,6 @@ pycparser = "*"
 name = "charset-normalizer"
 version = "3.3.2"
 description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
-category = "main"
 optional = false
 python-versions = ">=3.7.0"
 files = [
@@ -284,7 +276,6 @@ files = [
 name = "click"
 version = "8.1.7"
 description = "Composable command line interface toolkit"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -299,7 +290,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""}
 name = "colorama"
 version = "0.4.6"
 description = "Cross-platform colored terminal text."
-category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
 files = [
@@ -311,7 +301,6 @@ files = [
 name = "crispy-bootstrap5"
 version = "2024.2"
 description = "Bootstrap5 template pack for django-crispy-forms"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -330,7 +319,6 @@ test = ["pytest", "pytest-django"]
 name = "defusedxml"
 version = "0.7.1"
 description = "XML bomb protection for Python stdlib modules"
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 files = [
@@ -342,7 +330,6 @@ files = [
 name = "diff-match-patch"
 version = "20230430"
 description = "Diff Match and Patch"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -357,7 +344,6 @@ dev = ["attribution (==1.6.2)", "black (==23.3.0)", "flit (==3.8.0)", "mypy (==1
 name = "django"
 version = "4.2.13"
 description = "A high-level Python web framework that encourages rapid development and clean, pragmatic design."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -378,7 +364,6 @@ bcrypt = ["bcrypt"]
 name = "django-cors-headers"
 version = "4.3.1"
 description = "django-cors-headers is a Django application for handling the server headers required for Cross-Origin Resource Sharing (CORS)."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -394,7 +379,6 @@ Django = ">=3.2"
 name = "django-crispy-forms"
 version = "2.1"
 description = "Best way to have Django DRY forms"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -409,7 +393,6 @@ django = ">=4.2"
 name = "django-gisserver"
 version = "1.3.0"
 description = "Django speaking WFS 2.0 (exposing GeoDjango model fields)"
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -430,7 +413,6 @@ tests = ["django-environ (>=0.4.5)", "lxml (>=4.5.0)", "psycopg2-binary (>=2.8.4
 name = "django-import-export"
 version = "3.3.9"
 description = "Django application and library for importing and exporting data with included admin integration."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -447,7 +429,6 @@ tablib = {version = "3.5.0", extras = ["html", "ods", "xls", "xlsx", "yaml"]}
 name = "django-maintenance-mode"
 version = "0.21.1"
 description = "shows a 503 error page when maintenance-mode is on."
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -462,7 +443,6 @@ python-fsutil = ">=0.13.1,<1.0.0"
 name = "django-markdownx"
 version = "4.0.7"
 description = "A comprehensive Markdown editor built for Django."
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -479,7 +459,6 @@ Pillow = "*"
 name = "django-modeltranslation"
 version = "0.18.13"
 description = "Translates Django models using a registration approach."
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -494,7 +473,6 @@ Django = ">=4.2"
 name = "django-rq"
 version = "2.10.2"
 description = "An app that provides django integration for RQ (Redis Queue)"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -515,7 +493,6 @@ testing = ["mock (>=2.0.0)"]
 name = "django-stubs"
 version = "4.2.3"
 description = "Mypy stubs for Django"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -532,13 +509,12 @@ types-PyYAML = "*"
 typing-extensions = "*"
 
 [package.extras]
-compatible-mypy = ["mypy (>=1.4.0,<1.5.0)"]
+compatible-mypy = ["mypy (==1.4.*)"]
 
 [[package]]
 name = "django-stubs-ext"
 version = "5.0.0"
 description = "Monkey-patching and extensions for django-stubs"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -554,7 +530,6 @@ typing-extensions = "*"
 name = "django-taggit"
 version = "5.0.1"
 description = "django-taggit is a reusable Django application for simple tagging."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -569,7 +544,6 @@ Django = ">=4.1"
 name = "et-xmlfile"
 version = "1.1.0"
 description = "An implementation of lxml.xmlfile for the standard library"
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -581,7 +555,6 @@ files = [
 name = "gbif-blocking-occurrence-download"
 version = "0.1.1"
 description = ""
-category = "main"
 optional = false
 python-versions = ">=3.8,<4.0"
 files = [
@@ -597,7 +570,6 @@ requests = ">=2.26.0,<3.0.0"
 name = "gunicorn"
 version = "20.1.0"
 description = "WSGI HTTP Server for UNIX"
-category = "dev"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -618,7 +590,6 @@ tornado = ["tornado (>=0.2)"]
 name = "h11"
 version = "0.14.0"
 description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -630,7 +601,6 @@ files = [
 name = "html2text"
 version = "2024.2.25"
 description = "Turn HTML into equivalent Markdown-structured text."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -641,7 +611,6 @@ files = [
 name = "idna"
 version = "3.7"
 description = "Internationalized Domain Names in Applications (IDNA)"
-category = "main"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -653,7 +622,6 @@ files = [
 name = "iniconfig"
 version = "2.0.0"
 description = "brain-dead simple config-ini parsing"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -665,7 +633,6 @@ files = [
 name = "jinja2"
 version = "3.0.3"
 description = "A very fast and expressive template engine."
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -683,7 +650,6 @@ i18n = ["Babel (>=2.7)"]
 name = "jinjasql"
 version = "0.1.8"
 description = "Generate SQL Queries and Corresponding Bind Parameters using a Jinja2 Template"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -698,7 +664,6 @@ Jinja2 = ">=2.5"
 name = "lru-dict"
 version = "1.3.0"
 description = "An Dict like LRU container."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -792,7 +757,6 @@ test = ["pytest"]
 name = "mapbox-vector-tile"
 version = "2.0.1"
 description = "Mapbox Vector Tile encoding and decoding."
-category = "dev"
 optional = false
 python-versions = ">=3.8,<4.0"
 files = [
@@ -812,7 +776,6 @@ proj = ["pyproj (>=3.4.1,<4.0.0)"]
 name = "markdown"
 version = "3.6"
 description = "Python implementation of John Gruber's Markdown."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -828,7 +791,6 @@ testing = ["coverage", "pyyaml"]
 name = "markuppy"
 version = "1.14"
 description = "An HTML/XML generator"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -839,7 +801,6 @@ files = [
 name = "markupsafe"
 version = "2.1.5"
 description = "Safely add untrusted strings to HTML/XML markup."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -909,7 +870,6 @@ files = [
 name = "mypy"
 version = "1.4.0"
 description = "Optional static typing for Python"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -955,7 +915,6 @@ reports = ["lxml"]
 name = "mypy-extensions"
 version = "1.0.0"
 description = "Type system extensions for programs checked with the mypy type checker."
-category = "dev"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -967,7 +926,6 @@ files = [
 name = "numpy"
 version = "1.26.4"
 description = "Fundamental package for array computing in Python"
-category = "dev"
 optional = false
 python-versions = ">=3.9"
 files = [
@@ -1013,7 +971,6 @@ files = [
 name = "odfpy"
 version = "1.4.1"
 description = "Python API and tools to manipulate OpenDocument files"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -1027,7 +984,6 @@ defusedxml = "*"
 name = "openpyxl"
 version = "3.1.2"
 description = "A Python library to read/write Excel 2010 xlsx/xlsm files"
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1042,7 +998,6 @@ et-xmlfile = "*"
 name = "orjson"
 version = "3.10.3"
 description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1098,7 +1053,6 @@ files = [
 name = "outcome"
 version = "1.3.0.post0"
 description = "Capture the outcome of Python function calls."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1113,7 +1067,6 @@ attrs = ">=19.2.0"
 name = "packaging"
 version = "24.0"
 description = "Core utilities for Python packages"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1125,7 +1078,6 @@ files = [
 name = "pathspec"
 version = "0.12.1"
 description = "Utility library for gitignore style pattern matching of file paths."
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1137,7 +1089,6 @@ files = [
 name = "pillow"
 version = "10.3.0"
 description = "Python Imaging Library (Fork)"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1224,7 +1175,6 @@ xmp = ["defusedxml"]
 name = "platformdirs"
 version = "4.2.2"
 description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`."
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1241,7 +1191,6 @@ type = ["mypy (>=1.8)"]
 name = "pluggy"
 version = "1.5.0"
 description = "plugin and hook calling mechanisms for python"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1257,7 +1206,6 @@ testing = ["pytest", "pytest-benchmark"]
 name = "protobuf"
 version = "4.25.3"
 description = ""
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1278,7 +1226,6 @@ files = [
 name = "psycopg2-binary"
 version = "2.9.9"
 description = "psycopg2 - Python-PostgreSQL Database Adapter"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1360,7 +1307,6 @@ files = [
 name = "py"
 version = "1.11.0"
 description = "library with cross-python path, ini-parsing, io, code, log facilities"
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 files = [
@@ -1372,7 +1318,6 @@ files = [
 name = "pyclipper"
 version = "1.3.0.post5"
 description = "Cython wrapper for the C++ translation of the Angus Johnson's Clipper library (ver. 6.4.2)"
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -1426,7 +1371,6 @@ files = [
 name = "pycparser"
 version = "2.22"
 description = "C parser in Python"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1438,7 +1382,6 @@ files = [
 name = "pysocks"
 version = "1.7.1"
 description = "A Python SOCKS client module. See https://github.com/Anorov/PySocks for more information."
-category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -1451,7 +1394,6 @@ files = [
 name = "pytest"
 version = "6.2.5"
 description = "pytest: simple powerful testing with Python"
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1476,7 +1418,6 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xm
 name = "python-dotenv"
 version = "1.0.1"
 description = "Read key-value pairs from a .env file and set them as environment variables"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1491,7 +1432,6 @@ cli = ["click (>=5.0)"]
 name = "python-dwca-reader"
 version = "0.16.0"
 description = "A simple Python package to read Darwin Core Archive (DwC-A) files."
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -1503,7 +1443,6 @@ files = [
 name = "python-fsutil"
 version = "0.14.1"
 description = "high-level file-system operations for lazy devs."
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -1515,7 +1454,6 @@ files = [
 name = "pyyaml"
 version = "6.0.1"
 description = "YAML parser and emitter for Python"
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1565,7 +1503,6 @@ files = [
 name = "redis"
 version = "5.0.4"
 description = "Python client for Redis database and key-value store"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1584,7 +1521,6 @@ ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"
 name = "requests"
 version = "2.32.2"
 description = "Python HTTP for Humans."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1606,7 +1542,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
 name = "requests-mock"
 version = "1.12.1"
 description = "Mock out responses from the requests package"
-category = "dev"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -1624,7 +1559,6 @@ fixture = ["fixtures"]
 name = "rq"
 version = "1.16.2"
 description = "RQ is a simple, lightweight, library for creating background jobs, and processing them."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1640,7 +1574,6 @@ redis = ">=3.5"
 name = "selenium"
 version = "4.21.0"
 description = ""
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1659,7 +1592,6 @@ urllib3 = {version = ">=1.26,<3", extras = ["socks"]}
 name = "setuptools"
 version = "70.0.0"
 description = "Easily download, build, install, upgrade, and uninstall Python packages"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1675,7 +1607,6 @@ testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metad
 name = "shapely"
 version = "2.0.4"
 description = "Manipulation and analysis of geometric objects"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1726,14 +1657,13 @@ files = [
 numpy = ">=1.14,<3"
 
 [package.extras]
-docs = ["matplotlib", "numpydoc (>=1.1.0,<1.2.0)", "sphinx", "sphinx-book-theme", "sphinx-remove-toctrees"]
+docs = ["matplotlib", "numpydoc (==1.1.*)", "sphinx", "sphinx-book-theme", "sphinx-remove-toctrees"]
 test = ["pytest", "pytest-cov"]
 
 [[package]]
 name = "sniffio"
 version = "1.3.1"
 description = "Sniff out which async library your code is running under"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1745,7 +1675,6 @@ files = [
 name = "sortedcontainers"
 version = "2.4.0"
 description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set"
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -1757,7 +1686,6 @@ files = [
 name = "sqlparse"
 version = "0.5.0"
 description = "A non-validating SQL parser."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1773,7 +1701,6 @@ doc = ["sphinx"]
 name = "tablib"
 version = "3.5.0"
 description = "Format agnostic tabular data library (XLS, JSON, YAML, CSV, etc.)"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1803,7 +1730,6 @@ yaml = ["pyyaml"]
 name = "tblib"
 version = "1.7.0"
 description = "Traceback serialization library."
-category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 files = [
@@ -1815,7 +1741,6 @@ files = [
 name = "toml"
 version = "0.10.2"
 description = "Python Library for Tom's Obvious, Minimal Language"
-category = "main"
 optional = false
 python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
 files = [
@@ -1827,7 +1752,6 @@ files = [
 name = "trio"
 version = "0.25.1"
 description = "A friendly Python library for async concurrency and I/O"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1847,7 +1771,6 @@ sortedcontainers = "*"
 name = "trio-websocket"
 version = "0.11.1"
 description = "WebSocket library for Trio"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1863,7 +1786,6 @@ wsproto = ">=0.14"
 name = "types-pytz"
 version = "2024.1.0.20240417"
 description = "Typing stubs for pytz"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1875,7 +1797,6 @@ files = [
 name = "types-pyyaml"
 version = "6.0.12.20240311"
 description = "Typing stubs for PyYAML"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1887,7 +1808,6 @@ files = [
 name = "types-requests"
 version = "2.32.0.20240523"
 description = "Typing stubs for requests"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1902,7 +1822,6 @@ urllib3 = ">=2"
 name = "typing-extensions"
 version = "4.12.0"
 description = "Backported and Experimental Type Hints for Python 3.8+"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1914,7 +1833,6 @@ files = [
 name = "tzdata"
 version = "2024.1"
 description = "Provider of IANA time zone data"
-category = "main"
 optional = false
 python-versions = ">=2"
 files = [
@@ -1926,7 +1844,6 @@ files = [
 name = "urllib3"
 version = "2.2.1"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1947,7 +1864,6 @@ zstd = ["zstandard (>=0.18.0)"]
 name = "wsproto"
 version = "1.2.0"
 description = "WebSockets state-machine based protocol implementation"
-category = "dev"
 optional = false
 python-versions = ">=3.7.0"
 files = [
@@ -1962,7 +1878,6 @@ h11 = ">=0.9.0,<1"
 name = "xlrd"
 version = "2.0.1"
 description = "Library for developers to extract data from Microsoft Excel (tm) .xls spreadsheet files"
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
 files = [
@@ -1979,7 +1894,6 @@ test = ["pytest", "pytest-cov"]
 name = "xlwt"
 version = "1.3.0"
 description = "Library to create spreadsheet files compatible with MS Excel 97/2000/XP/2003 XLS files, on any platform, with Python 2.6, 2.7, 3.3+"
-category = "main"
 optional = false
 python-versions = "*"
 files = [

From 1150342a0d8dad4aa1080e499bbfdfbe1ae36923 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20No=C3=A9?= <nicolas@niconoe.eu>
Date: Thu, 25 Jul 2024 10:00:38 +0200
Subject: [PATCH 12/12] Poetry (self) update

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 4caac03..b8881c3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -8,7 +8,7 @@ ENV PYTHONUNBUFFERED=1 \
     PIP_DEFAULT_TIMEOUT=100 \
     PIP_DISABLE_PIP_VERSION_CHECK=1 \
     PIP_NO_CACHE_DIR=1 \
-    POETRY_VERSION=1.4.0 \
+    POETRY_VERSION=1.8.3 \
     NVM_VERSION=0.39.1 \
     NVM_DIR=/root/.nvm \
     NODE_VERSION=19.7.0 \