From 42bc1bbb16d4be23e7fb05cf3b082b650521e28c Mon Sep 17 00:00:00 2001 From: weiqi-tori Date: Tue, 26 Nov 2024 16:47:04 +0800 Subject: [PATCH 1/8] add recreational space per capita indicator --- city_metrix/layers/layer.py | 5 +++++ city_metrix/metrics/__init__.py | 1 + .../metrics/recreational_space_per_capita.py | 16 ++++++++++++++++ tests/test_metrics.py | 7 +++++++ 4 files changed, 29 insertions(+) create mode 100644 city_metrix/metrics/recreational_space_per_capita.py diff --git a/city_metrix/layers/layer.py b/city_metrix/layers/layer.py index 83fa9eb2..cad46bf5 100644 --- a/city_metrix/layers/layer.py +++ b/city_metrix/layers/layer.py @@ -135,6 +135,9 @@ def mean(self): def count(self): return self._zonal_stats("count") + def sum(self): + return self._zonal_stats("sum") + def _zonal_stats(self, stats_func): if box(*self.zones.total_bounds).area <= MAX_TILE_SIZE_DEGREES**2: stats = self._zonal_stats_tile(self.zones, [stats_func]) @@ -314,6 +317,8 @@ def _aggregate_stats(df, stats_func): elif stats_func == "mean": # mean must weight by number of pixels used for each tile return (df["mean"] * df["count"]).sum() / df["count"].sum() + elif stats_func == "sum": + return df["sum"].sum() def get_stats_funcs(stats_func): diff --git a/city_metrix/metrics/__init__.py b/city_metrix/metrics/__init__.py index d95cfa5c..6efe2ae4 100644 --- a/city_metrix/metrics/__init__.py +++ b/city_metrix/metrics/__init__.py @@ -5,3 +5,4 @@ from .urban_open_space import urban_open_space from .natural_areas import natural_areas from .era_5_met_preprocessing import era_5_met_preprocessing +from .recreational_space_per_capita import recreational_space_per_capita diff --git a/city_metrix/metrics/recreational_space_per_capita.py b/city_metrix/metrics/recreational_space_per_capita.py new file mode 100644 index 00000000..2e066d76 --- /dev/null +++ b/city_metrix/metrics/recreational_space_per_capita.py @@ -0,0 +1,16 @@ +from geopandas import GeoDataFrame, GeoSeries + +from city_metrix.layers import WorldPop, OpenStreetMap, OpenStreetMapClass + + +def recreational_space_per_capita(zones: GeoDataFrame, spatial_resolution=100) -> GeoSeries: + world_pop = WorldPop(spatial_resolution=spatial_resolution) + open_space = OpenStreetMap(osm_class=OpenStreetMapClass.OPEN_SPACE) + + # per 1000 people + world_pop_sum = world_pop.groupby(zones).sum() / 1000 + # convert square meter to hectare + open_space_counts = open_space.mask(world_pop).groupby(zones).count() + open_space_area = open_space_counts.fillna(0) * spatial_resolution ** 2 / 10000 + + return open_space_area / world_pop_sum diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 8fd42cc3..a68b8ea8 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -44,6 +44,13 @@ def test_natural_areas(): assert expected_zone_size == actual_indicator_size +def test_recreational_space_per_capita(): + indicator = recreational_space_per_capita(ZONES) + expected_zone_size = ZONES.geometry.size + actual_indicator_size = indicator.size + assert expected_zone_size == actual_indicator_size + + def test_urban_open_space(): indicator = urban_open_space(ZONES) expected_zone_size = ZONES.geometry.size From eb2f90ea870d83603cc3c2ec488c83042b69b960 Mon Sep 17 00:00:00 2001 From: weiqi-tori Date: Wed, 18 Dec 2024 17:57:59 +0800 Subject: [PATCH 2/8] reduce max cell size --- city_metrix/layers/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/city_metrix/layers/layer.py b/city_metrix/layers/layer.py index 9beae31a..c4a007ea 100644 --- a/city_metrix/layers/layer.py +++ b/city_metrix/layers/layer.py @@ -18,7 +18,7 @@ import shapely.geometry as geometry import pandas as pd -MAX_TILE_SIZE_DEGREES = 0.5 # TODO Why was this value selected? +MAX_TILE_SIZE_DEGREES = 0.2 # TODO Why was this value selected? class Layer: def __init__(self, aggregate=None, masks=[]): From 1c0979b163546da11aa7c54d76235973745ae85f Mon Sep 17 00:00:00 2001 From: weiqi-tori Date: Fri, 20 Dec 2024 18:04:57 +0800 Subject: [PATCH 3/8] update osmnx --- city_metrix/layers/layer.py | 2 +- city_metrix/layers/open_street_map.py | 4 ++-- environment.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/city_metrix/layers/layer.py b/city_metrix/layers/layer.py index 7407c9ef..bb52e6c4 100644 --- a/city_metrix/layers/layer.py +++ b/city_metrix/layers/layer.py @@ -20,7 +20,7 @@ import shapely.geometry as geometry import pandas as pd -MAX_TILE_SIZE_DEGREES = 0.2 # TODO Why was this value selected? +MAX_TILE_SIZE_DEGREES = 0.5 # TODO Why was this value selected? class Layer: def __init__(self, aggregate=None, masks=[]): diff --git a/city_metrix/layers/open_street_map.py b/city_metrix/layers/open_street_map.py index a4e54892..6a8c99db 100644 --- a/city_metrix/layers/open_street_map.py +++ b/city_metrix/layers/open_street_map.py @@ -40,11 +40,11 @@ def __init__(self, osm_class=None, **kwargs): self.osm_class = osm_class def get_data(self, bbox): - north, south, east, west = bbox[3], bbox[1], bbox[0], bbox[2] + left, top, right, bottom = bbox # Set the OSMnx configuration to disable caching ox.settings.use_cache = False try: - osm_feature = ox.features_from_bbox(bbox=(north, south, east, west), tags=self.osm_class.value) + osm_feature = ox.features_from_bbox(bbox=(left, bottom, right, top), tags=self.osm_class.value) # When no feature in bbox, return an empty gdf except ox._errors.InsufficientResponseError as e: osm_feature = gpd.GeoDataFrame(pd.DataFrame(columns=['osmid', 'geometry']+list(self.osm_class.value.keys())), geometry='geometry') diff --git a/environment.yml b/environment.yml index 39537969..3538e330 100644 --- a/environment.yml +++ b/environment.yml @@ -5,7 +5,7 @@ dependencies: - python=3.10 - earthengine-api=0.1.411 - geocube=0.4.2 - - geopandas=0.14.4 + - geopandas=1.0.1 - xarray=2024.7.0 - rioxarray=0.15.0 - odc-stac=0.3.8 @@ -14,7 +14,7 @@ dependencies: - xarray-spatial=0.3.7 - xee=0.0.15 - utm=0.7.0 - - osmnx=1.9.0 + - osmnx=2.0.0 - dask[complete]=2023.11.0 - matplotlib=3.8.2 - jupyterlab=4.0.10 From 13c78fb45562d329e5c01f94c4d0b0deec76a5c2 Mon Sep 17 00:00:00 2001 From: weiqi-tori Date: Mon, 23 Dec 2024 10:51:49 +0800 Subject: [PATCH 4/8] add gdal in env file --- environment.yml | 1 + setup.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 3538e330..40ce45a4 100644 --- a/environment.yml +++ b/environment.yml @@ -6,6 +6,7 @@ dependencies: - earthengine-api=0.1.411 - geocube=0.4.2 - geopandas=1.0.1 + - gdal=3.10.0 - xarray=2024.7.0 - rioxarray=0.15.0 - odc-stac=0.3.8 diff --git a/setup.py b/setup.py index a46fcde2..8d1d773a 100644 --- a/setup.py +++ b/setup.py @@ -17,8 +17,9 @@ "xee", "rioxarray", "utm", - "osmnx", + "osmnx>=2.0.0", "geopandas", + "gdal", "s3fs", "dask>=2023.11.0", "boto3", From 12e9c2119ccfef7243a3cefe78b66b1cdbdbbfd3 Mon Sep 17 00:00:00 2001 From: weiqi-tori Date: Mon, 23 Dec 2024 11:45:26 +0800 Subject: [PATCH 5/8] fix osm column name and tags issue --- city_metrix/layers/open_street_map.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/city_metrix/layers/open_street_map.py b/city_metrix/layers/open_street_map.py index 6a8c99db..af0455db 100644 --- a/city_metrix/layers/open_street_map.py +++ b/city_metrix/layers/open_street_map.py @@ -7,6 +7,13 @@ class OpenStreetMapClass(Enum): + # ALL includes all 29 primary features https://wiki.openstreetmap.org/wiki/Map_features + ALL = {'aerialway': True, 'aeroway': True, 'amenity': True, 'barrier': True, 'boundary': True, + 'building': True, 'craft': True, 'emergency': True, 'geological': True, 'healthcare': True, + 'highway': True, 'historic': True, 'landuse': True, 'leisure': True, 'man_made': True, + 'military': True, 'natural': True, 'office': True, 'place': True, 'power': True, + 'public_transport': True, 'railway': True, 'route': True, 'shop': True, 'sport': True, + 'telecom': True, 'tourism': True, 'water': True, 'waterway': True} OPEN_SPACE = {'leisure': ['park', 'nature_reserve', 'common', 'playground', 'pitch', 'track'], 'boundary': ['protected_area', 'national_park']} OPEN_SPACE_HEAT = {'leisure': ['park', 'nature_reserve', 'common', 'playground', 'pitch', 'garden', 'golf_course', 'dog_park', 'recreation_ground', 'disc_golf_course'], @@ -23,7 +30,7 @@ class OpenStreetMapClass(Enum): 'building': ['office', 'commercial', 'industrial', 'retail', 'supermarket'], 'shop': True} SCHOOLS = {'building': ['school',], - 'amenity': ['school', 'kindergarten']} + 'amenity': ['school', 'kindergarten']} HIGHER_EDUCATION = {'amenity': ['college', 'university'], 'building': ['college', 'university']} TRANSIT_STOP = {'amenity':['ferry_terminal'], @@ -35,7 +42,7 @@ class OpenStreetMapClass(Enum): class OpenStreetMap(Layer): - def __init__(self, osm_class=None, **kwargs): + def __init__(self, osm_class=OpenStreetMapClass.ALL, **kwargs): super().__init__(**kwargs) self.osm_class = osm_class @@ -47,7 +54,7 @@ def get_data(self, bbox): osm_feature = ox.features_from_bbox(bbox=(left, bottom, right, top), tags=self.osm_class.value) # When no feature in bbox, return an empty gdf except ox._errors.InsufficientResponseError as e: - osm_feature = gpd.GeoDataFrame(pd.DataFrame(columns=['osmid', 'geometry']+list(self.osm_class.value.keys())), geometry='geometry') + osm_feature = gpd.GeoDataFrame(pd.DataFrame(columns=['id', 'geometry']+list(self.osm_class.value.keys())), geometry='geometry') osm_feature.crs = "EPSG:4326" # Filter by geo_type @@ -62,7 +69,7 @@ def get_data(self, bbox): osm_feature = osm_feature[osm_feature.geom_type.isin(['Polygon', 'MultiPolygon'])] # keep only columns desired to reduce file size - keep_col = ['osmid', 'geometry'] + keep_col = ['id', 'geometry'] for key in self.osm_class.value: if key in osm_feature.columns: keep_col.append(key) From 9b373d2effdfd200289dc261fb261836c0ef46e8 Mon Sep 17 00:00:00 2001 From: weiqi-tori Date: Mon, 23 Dec 2024 14:16:26 +0800 Subject: [PATCH 6/8] drop zones index for ValueError: cannot insert level_0, already exists. --- city_metrix/layers/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/city_metrix/layers/layer.py b/city_metrix/layers/layer.py index 62f88330..afe18a38 100644 --- a/city_metrix/layers/layer.py +++ b/city_metrix/layers/layer.py @@ -127,7 +127,7 @@ class LayerGroupBy: def __init__(self, aggregate, zones, layer=None, masks=[]): self.aggregate = aggregate self.masks = masks - self.zones = zones.reset_index() + self.zones = zones.reset_index(drop=True) self.layer = layer def mean(self): From 4ec491e1f1ba0a38c3bdf4940af795a181c7a4f9 Mon Sep 17 00:00:00 2001 From: weiqi-tori Date: Mon, 23 Dec 2024 17:19:02 +0800 Subject: [PATCH 7/8] remove gdal and add xarray --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8d1d773a..8138282b 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ "utm", "osmnx>=2.0.0", "geopandas", - "gdal", + "xarray", "s3fs", "dask>=2023.11.0", "boto3", From 2e6a7b95b7cfe2100a12000fc7c5bf73927d76a7 Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 2 Jan 2025 14:38:31 -0800 Subject: [PATCH 8/8] Adding a master control for ignored tests --- tests/conftest.py | 7 +++++++ tests/test_layers.py | 5 +++-- tests/test_metrics.py | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 992894d4..e140ab7b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,13 @@ from city_metrix.layers.layer import create_fishnet_grid from geocube.api.core import make_geocube +# EXECUTE_IGNORED_TESTS is the master control for whether to execute tests decorated with +# pytest.mark.skipif. These tests are temporarily ignored due to some unresolved issue. +# Setting EXECUTE_IGNORED_TESTS to True turns on code execution. This should be done for local testing. +# The value must be set to False when pushing to GitHub since the ignored tests would otherwise fail +# in GitHub Actions. +EXECUTE_IGNORED_TESTS = False + def test_create_fishnet_grid(min_x, min_y, max_x, max_y, cell_size): # Slightly reduce aoi to avoid partial cells reduction = 0.000001 diff --git a/tests/test_layers.py b/tests/test_layers.py index 7875ab97..85c37086 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -31,6 +31,7 @@ UrbanLandUse, WorldPop ) +from tests.conftest import EXECUTE_IGNORED_TESTS from tests.resources.bbox_constants import BBOX_BRA_LAURO_DE_FREITAS_1 # Tests are implemented for the same bounding box in Brazil. @@ -53,12 +54,12 @@ def test_built_up_height(): data = BuiltUpHeight().get_data(BBOX) assert np.size(data) > 0 -@pytest.mark.skip(reason="CDS API needs personal access token file to run") +@pytest.mark.skipif(EXECUTE_IGNORED_TESTS == False, reason="CDS API needs personal access token file to run") def test_cams(): data = Cams().get_data(BBOX) assert np.size(data) > 0 -@pytest.mark.skip(reason="CDS API needs personal access token file to run") +@pytest.mark.skipif(EXECUTE_IGNORED_TESTS == False, reason="CDS API needs personal access token file to run") def test_era_5_hottest_day(): data = Era5HottestDay().get_data(BBOX) assert np.size(data) > 0 diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 8fd42cc3..841424e9 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -1,5 +1,5 @@ from city_metrix import * -from .conftest import ZONES +from .conftest import ZONES, EXECUTE_IGNORED_TESTS import pytest @@ -24,7 +24,7 @@ def test_built_land_without_tree_cover(): assert expected_zone_size == actual_indicator_size -@pytest.mark.skip(reason="CDS API needs personal access token file to run") +@pytest.mark.skipif(EXECUTE_IGNORED_TESTS == False, reason="CDS API needs personal access token file to run") def test_era_5_met_preprocess(): indicator = era_5_met_preprocessing(ZONES) assert len(indicator) == 24