fix grid generation

back/iarbre_data/management/commands/c02_init_grid.py

@@ -18,13 +18,13 @@
 import numpy as np
 import gc
 import itertools
-from shapely.geometry import Point
+from shapely.geometry import box
 from tqdm import tqdm
 from django.db import transaction
 
 
 def create_tiles_for_city(
-    city, grid_size, tile_shape_cls, logger, batch_size=int(1e6), unit=None, a=None
+    city, grid_size, tile_shape_cls, logger, batch_size=int(1e6), unit=None, a=1
 ) -> None:
     """
     Create tiles (square or hexagonal) for a specific city.
@@ -46,21 +46,21 @@ def create_tiles_for_city(
     xmin, ymin, xmax, ymax = city_geom.bounds
     # Bounds for the generation
     xmin, ymin, xmax, ymax = tile_shape_cls.adjust_bounds(
-        xmin, ymin, xmax, ymax, grid_size, a
+        xmin, ymin, xmax, ymax, grid_size, unit, a
     )
 
     tiles = []
     tile_count = 0
 
-    for x, y in tqdm(
+    for x, (i, y) in tqdm(
         tile_shape_cls.tile_positions(xmin, ymin, xmax, ymax, grid_size, unit, a)
     ):
-        point = Point([x, y])
-        if not city_geom.covers(point):
+        tile = box(x, y * a, x - 2 * grid_size, y * a - 2 * grid_size)
+        if not city_geom.intersects(tile):
             continue
 
         tile_count += 1
-        polygon = tile_shape_cls.create_tile_polygon(x, y, grid_size, unit, a)
+        polygon = tile_shape_cls.create_tile_polygon(x, y, grid_size, unit, a, i)
 
         iris_id = tile_shape_cls.get_iris_id(polygon)
 
@@ -100,7 +100,7 @@ class SquareTileShape(TileShape):
     """Generate square tile."""
 
     @staticmethod
-    def adjust_bounds(xmin, ymin, xmax, ymax, grid_size, a=None):
+    def adjust_bounds(xmin, ymin, xmax, ymax, grid_size, unit=None, a=None):
         """Snap bounds to the nearest grid alignment."""
         xmin = np.floor(xmin / grid_size) * grid_size
         ymin = np.floor(ymin / grid_size) * grid_size
@@ -113,11 +113,11 @@ def tile_positions(xmin, ymin, xmax, ymax, grid_size, unit=None, a=None):
         """Generate an iterator for all the position where to create a tile."""
         return itertools.product(
             np.arange(xmin, xmax + grid_size, grid_size),
-            np.arange(ymin, ymax + grid_size, grid_size),
+            enumerate(np.arange(ymin, ymax + grid_size, grid_size)),
         )
 
     @staticmethod
-    def create_tile_polygon(x, y, grid_size, unit=None, a=None):
+    def create_tile_polygon(x, y, grid_size, unit=None, a=None, i=None):
         """Create a single square and round geometries to optimize storage."""
         x1 = x - grid_size
         y1 = y + grid_size
@@ -128,10 +128,10 @@ class HexTileShape(TileShape):
     """Generate hexagonal tile."""
 
     @staticmethod
-    def adjust_bounds(xmin, ymin, xmax, ymax, grid_size, a):
+    def adjust_bounds(xmin, ymin, xmax, ymax, grid_size, unit, a):
         """Snap bounds to the nearest grid alignment."""
-        hex_width = 3 * grid_size
-        hex_height = 2 * grid_size * a
+        hex_width = 3 * unit
+        hex_height = 2 * unit * a
         xmin = hex_width * (np.floor(xmin / hex_width) - 1)
         ymin = hex_height * (np.floor(ymin / hex_height) - 1)
         xmax = hex_width * (np.ceil(xmax / hex_width) + 1)
@@ -142,22 +142,22 @@ def adjust_bounds(xmin, ymin, xmax, ymax, grid_size, a):
     def tile_positions(xmin, ymin, xmax, ymax, grid_size, unit, a):
         """Generate an iterator for all the position where to create a tile."""
         cols = np.arange(xmin, xmax, 3 * unit)
-        rows = np.arange(ymin, ymax, unit * a)
-        return itertools.product(cols, rows)
+        rows = np.arange(ymin / a, ymax / a, unit)
+        return itertools.product(cols, enumerate(rows))
 
     @staticmethod
-    def create_tile_polygon(x, y, grid_size, unit, a):
+    def create_tile_polygon(x, y, grid_size, unit, a, i):
         """Create a single hexagon and round geometries to optimize storage."""
-        offset = 1.5 * unit if y / a % 2 != 0 else 0
-        x0 = x + offset
+        offset = 1.5 * unit if i % 2 != 0 else 0
+        x0 = x - offset
         dim = [
-            (x0, y),
-            (x0 + unit, y),
-            (x0 + (1.5 * unit), y + unit * a),
-            (x0 + unit, y + 2 * unit * a),
-            (x0, y + 2 * unit * a),
-            (x0 - (0.5 * unit), y + unit * a),
-            (x0, y),
+            (x0, y * a),
+            (x0 + unit, y * a),
+            (x0 + (1.5 * unit), (y + unit) * a),
+            (x0 + unit, (y + (2 * unit)) * a),
+            (x0, (y + (2 * unit)) * a),
+            (x0 - (0.5 * unit), (y + unit) * a),
+            (x0, y * a),
         ]
         return Polygon([(round(x, 2), round(y, 2)) for (x, y) in dim], srid=TARGET_PROJ)
 
@@ -187,7 +187,7 @@ def clean_outside(selected_city, batch_size) -> None:
         with transaction.atomic():
             deleted_count, _ = (
                 Tile.objects.filter(id__in=batch_ids)
-                .exclude(geometry__within=city_union_geom.wkt)
+                .exclude(geometry__intersects=city_union_geom.wkt)
                 .delete()
             )
             total_deleted += deleted_count
@@ -311,6 +311,6 @@ def handle(self, *args, **options):
                 )
                 gc.collect()  # just to be sure gc is called...
         print("Removing duplicates...")
-        remove_duplicates()
+        remove_duplicates(Tile)
         if keep_outside is False:
             clean_outside(selected_city, batch_size)

back/iarbre_data/tests/test_c02_grid.py

@@ -27,37 +27,31 @@ def setUp(self):
     def test_create_square_tile(self):
         code = 69381
         selected_city = select_city(str(code))
-
         create_tiles_for_city(
-            selected_city.iloc[0],
-            self.grid_size,
-            SquareTileShape,
+            city=selected_city.iloc[0],
+            grid_size=self.grid_size,
+            tile_shape_cls=SquareTileShape,
             logger=logging.getLogger(__name__),
             batch_size=int(1e6),
-            a=None,
         )
         qs = City.objects.filter(code=code)
         df = load_geodataframe_from_db(qs, ["tiles_generated"])
         self.assertTrue(df.tiles_generated.values)
-        self.assertEqual(Tile.objects.count(), 37)
+        self.assertEqual(Tile.objects.count(), 61)
         tile = Tile.objects.first()
         self.assertEqual(tile.geometry.area, self.grid_size**2)
         coords = tile.geometry.coords[0]
         self.assertEqual(len(coords), 5)  # it's a square
         self.assertEqual(coords[0][0] - coords[2][0], self.grid_size)
         self.assertEqual(coords[1][1] - coords[0][1], self.grid_size)
-        self.assertTrue(
-            City.objects.filter(code=code)[0].geometry.intersects(tile.geometry)
-        )
 
     def test_create_hex_tile(self):
         code = 69381
         selected_city = select_city(str(code))
-
         create_tiles_for_city(
-            selected_city.iloc[0],
-            self.grid_size,
-            HexTileShape,
+            city=selected_city.iloc[0],
+            grid_size=self.grid_size,
+            tile_shape_cls=HexTileShape,
             logger=logging.getLogger(__name__),
             batch_size=int(1e6),
             unit=self.unit,
@@ -66,16 +60,13 @@ def test_create_hex_tile(self):
         qs = City.objects.filter(code=code)
         df = load_geodataframe_from_db(qs, ["tiles_generated"])
         self.assertTrue(df.tiles_generated.values)
-        self.assertEqual(Tile.objects.count(), 36)
+        self.assertEqual(Tile.objects.count(), 71)
         tile = Tile.objects.first()
         self.assertEqual(int(tile.geometry.area), self.grid_size**2 - 1)
         coords = tile.geometry.coords[0]
         self.assertEqual(len(coords), 7)  # it's a hex
         self.assertEqual(int(coords[1][0] - coords[0][0]), int(self.unit))
         self.assertEqual(int(coords[2][1] - coords[0][1]), int(self.a * self.unit))
-        self.assertTrue(
-            City.objects.filter(code=code)[0].geometry.intersects(tile.geometry)
-        )
 
     def test_clean_outside(self):
         codes = [69381, 69382]

back/iarbre_data/tests/test_c03_data.py

@@ -1,10 +1,17 @@
 from django.test import TestCase
 from iarbre_data.data_config import DATA_FILES, URL_FILES
 from iarbre_data.models import Data
-from iarbre_data.management.commands.c03_import_data import download_from_url, read_data, apply_actions, process_data, save_geometries
+from iarbre_data.management.commands.c03_import_data import (
+    download_from_url,
+    read_data,
+    apply_actions,
+    process_data,
+    save_geometries,
+)
 from iarbre_data.settings import TARGET_PROJ
 import geopandas as gpd
 
+
 class c03_dataTestCase(TestCase):
     def setUp(self):
         self.data_config = DATA_FILES[0]
@@ -15,13 +22,13 @@ def test_download_from_url(self):
         data_config = URL_FILES[1]
         df_url = download_from_url(data_config["url"], data_config["layer_name"])
         self.assertTrue(isinstance(df_url, gpd.GeoDataFrame))
-        self.assertTrue(hasattr(df_url, 'geometry'))
-        self.assertTrue(df_url.geometry.dtype == 'geometry')
+        self.assertTrue(hasattr(df_url, "geometry"))
+        self.assertTrue(df_url.geometry.dtype == "geometry")
 
     def test_read_data(self):
         self.assertTrue(isinstance(self.df, gpd.GeoDataFrame))
-        self.assertTrue(hasattr(self.df, 'geometry'))
-        self.assertTrue(self.df.geometry.dtype == 'geometry')
+        self.assertTrue(hasattr(self.df, "geometry"))
+        self.assertTrue(self.df.geometry.dtype == "geometry")
         self.assertTrue(self.df.geometry.crs == TARGET_PROJ)
         valid_geometries = self.df.geometry.notnull() & self.df.geometry.is_valid
         self.assertTrue(valid_geometries.all())
@@ -30,34 +37,31 @@ def test_apply_actions(self):
         actions = {"buffer_size": 1, "explode": True, "union": True, "simplify": 3}
         df = self.df.copy()[:5]
         df = apply_actions(df, actions)
-        self.assertTrue((df.geom_type == 'Polygon').all())
-
-        actions =  {
-                "filters": [
-                    {
-                        "name": "typeespacepublic",
-                        "value": "Aire de jeux",
-                    },
-                    {
-                        "name": "typeespacepublic",
-                        "value": "Espace piétonnier",
-                    },
-                ]
-            }
+        self.assertTrue((df.geom_type == "Polygon").all())
+
+        actions = {
+            "filters": [
+                {
+                    "name": "typeespacepublic",
+                    "value": "Aire de jeux",
+                },
+                {
+                    "name": "typeespacepublic",
+                    "value": "Espace piétonnier",
+                },
+            ]
+        }
         data_config = DATA_FILES[4]
         df = read_data(data_config)
         df = apply_actions(df, actions)
-        self.assertTrue((df.geom_type == 'Polygon').all())
+        self.assertTrue((df.geom_type == "Polygon").all())
         # Empty actions
         df = apply_actions(self.df, {})
-        self.assertTrue((df.geom_type == 'Polygon').all())
+        self.assertTrue((df.geom_type == "Polygon").all())
 
     def test_process_data(self):
         self.assertTrue(isinstance(self.datas, list))
 
     def test_save_geometries(self):
         save_geometries(self.datas, self.data_config)
         self.assertNotEquals(Data.objects.count(), 0)
-
-
-

back/iarbre_data/tests/test_c04_compute_factors.py

@@ -1,5 +1,4 @@
 import logging
-import time
 
 from django.test import TestCase