From 4becd5298ce0cec834f7450f3cc70f320cf54977 Mon Sep 17 00:00:00 2001 From: donaldte Date: Thu, 7 Mar 2024 05:01:55 +0100 Subject: [PATCH 1/2] feat: Add support for BytesIO input in boundary processing --- osm_fieldwork/basemapper.py | 80 +++++++++++++++++++++++++------------ outreachy.py | 15 +++++++ tests/test_basemap.py | 27 +++++++++++++ 3 files changed, 96 insertions(+), 26 deletions(-) create mode 100644 outreachy.py diff --git a/osm_fieldwork/basemapper.py b/osm_fieldwork/basemapper.py index 84d988d3..b46b957a 100755 --- a/osm_fieldwork/basemapper.py +++ b/osm_fieldwork/basemapper.py @@ -28,6 +28,7 @@ import threading from pathlib import Path from typing import Union +from io import BytesIO import geojson import mercantile @@ -44,6 +45,7 @@ from pmtiles.writer import Writer as PMTileWriter from pySmartDL import SmartDL from shapely.geometry import shape +from shapely.geometry.base import BaseGeometry from shapely.ops import unary_union from osm_fieldwork.sqlite import DataFile, MapTile @@ -125,7 +127,7 @@ class BaseMapper(object): def __init__( self, - boundary: str, + boundary: Union[str, BytesIO], # Updated type hint to accept BytesIO base: str, source: str, xy: bool, @@ -133,7 +135,8 @@ def __init__( """Create an tile basemap for ODK Collect. Args: - boundary (str): A BBOX string or GeoJSON file of the AOI. + boundary (Union[str, BytesIO]): The boundary for the area you want. + Can be a BBOX string, or GeoJSON file of the AOI or BytesIO object containing GeoJSON data. The GeoJSON can contain multiple geometries. base (str): The base directory to cache map tile in source (str): The upstream data source for map tiles @@ -272,41 +275,66 @@ def tileExists( def makeBbox( self, - boundary: str, + boundary: Union[str, BytesIO], ) -> tuple[float, float, float, float]: """Make a bounding box from a shapely geometry. Args: - boundary (str): A BBOX string or GeoJSON file of the AOI. + boundary (str): A BBOX string or GeoJSON file of the AOI or BytesIO object containing GeoJSON data. The GeoJSON can contain multiple geometries. Returns: (list): The bounding box coordinates """ - if not boundary.lower().endswith((".json", ".geojson")): - # Is BBOX string + # verify that the boundary is a string or BytesIO object + if isinstance(boundary, str): + # If the boundary is a string, check if it's a BBOX string or a file path + if not boundary.lower().endswith((".json", ".geojson")): + # Parse BBOX string + try: + if "," in boundary: + bbox_parts = boundary.split(",") + else: + bbox_parts = boundary.split(" ") + bbox = tuple(float(x) for x in bbox_parts) + if len(bbox) == 4: + return bbox + else: + msg = f"BBOX string malformed: {bbox}" + log.error(msg) + raise ValueError(msg) from None + except Exception as e: + log.error(e) + msg = f"Failed to parse BBOX string: {boundary}" + raise ValueError(msg) from e + else: + # Load GeoJSON from file + with open(boundary, "r") as f: + try: + poly = geojson.load(f) + return self.extract_bbox(poly) + except Exception as e: + log.error(e) + msg = f"Failed to load GeoJSON file: {boundary}" + raise ValueError(msg) from e + elif isinstance(boundary, BytesIO): + # Process BytesIO object try: - if "," in boundary: - bbox_parts = boundary.split(",") - else: - bbox_parts = boundary.split(" ") - bbox = tuple(float(x) for x in bbox_parts) - if len(bbox) == 4: - # BBOX valid - return bbox - else: - msg = f"BBOX string malformed: {bbox}" - log.error(msg) - raise ValueError(msg) from None + boundary.seek(0) + geojson_data = boundary.read().decode('utf-8') + poly = geojson.loads(geojson_data) + return self.extract_bbox(poly) except Exception as e: log.error(e) - msg = f"Failed to parse BBOX string: {boundary}" - log.error(msg) - raise ValueError(msg) from None + raise ValueError("Failed to decode GeoJSON data from BytesIO object") from e + else: + raise ValueError(f"Invalid boundary type: {type(boundary)}. It must be a BBOX string or (.json, .geojson) flie or BytesIO object") + - log.debug(f"Reading geojson file: {boundary}") - with open(boundary, "r") as f: - poly = geojson.load(f) + def extract_bbox( + self, poly: Union[BaseGeometry, None] + ) -> tuple[float, float, float, float]: + """Extract bounding box from GeoJSON polygon.""" if "features" in poly: geometry = shape(poly["features"][0]["geometry"]) elif "geometry" in poly: @@ -327,7 +355,7 @@ def makeBbox( bbox = geometry.bounds # left, bottom, right, top # minX, minY, maxX, maxY - return bbox + return bbox def tileid_from_y_tile(filepath: Union[Path | str]): @@ -420,7 +448,7 @@ def create_basemap_file( Args: verbose (bool, optional): Enable verbose output if True. - boundary (str, optional): The boundary for the area you want. + boundary (Union[str, BytesIO], optional): The boundary for the area you want. tms (str, optional): Custom TMS URL. xy (bool, optional): Swap the X & Y coordinates when using a custom TMS if True. diff --git a/outreachy.py b/outreachy.py new file mode 100644 index 00000000..c6054161 --- /dev/null +++ b/outreachy.py @@ -0,0 +1,15 @@ +from io import BytesIO +from osm_fieldwork.basemapper import create_basemap_file + +with open("D:\\customers\\osm-fieldwork\\tests\\testdata\\Rollinsville.geojson", "rb") as f: + boundary = f.read() # Read the file into memory + boundary_bytesio = BytesIO(boundary) # Convert the file into a BytesIO object + + +create_basemap_file( + verbose=True, + boundary=boundary_bytesio, + outfile="outreachy.mbtiles", + zooms="12-15", + source="esri", +) diff --git a/tests/test_basemap.py b/tests/test_basemap.py index 7ee678b7..54774343 100755 --- a/tests/test_basemap.py +++ b/tests/test_basemap.py @@ -19,6 +19,7 @@ # """Test functionalty of basemapper.py.""" +import io import logging import os import shutil @@ -66,5 +67,31 @@ def test_create(): assert hits == 2 +def test_create_with_byteio(): + """Test loading with a BytesIO boundary""" + hits = 0 + with open(boundary, "rb") as f: + boundary_bytes = io.BytesIO(f.read()) + basemap = BaseMapper(boundary_bytes, base, "topo", False) + tiles = list() + for level in [8, 9, 10, 11, 12]: + basemap.getTiles(level) + tiles += basemap.tiles + + if len(tiles) == 5: + hits += 1 + + if tiles[0].x == 52 and tiles[1].y == 193 and tiles[2].x == 211: + hits += 1 + + outf = DataFile(outfile, basemap.getFormat()) + outf.writeTiles(tiles, base) + + os.remove(outfile) + shutil.rmtree(base) + + assert hits == 2 + if __name__ == "__main__": test_create() + test_create_with_byteio() From 1ad2e4976ae508314413c98c092c9505cbf50556 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Mar 2024 04:22:54 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- osm_fieldwork/basemapper.py | 23 +++++++++++------------ outreachy.py | 5 +++-- tests/test_basemap.py | 3 ++- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/osm_fieldwork/basemapper.py b/osm_fieldwork/basemapper.py index b46b957a..13446664 100755 --- a/osm_fieldwork/basemapper.py +++ b/osm_fieldwork/basemapper.py @@ -26,9 +26,9 @@ import re import sys import threading +from io import BytesIO from pathlib import Path from typing import Union -from io import BytesIO import geojson import mercantile @@ -127,7 +127,7 @@ class BaseMapper(object): def __init__( self, - boundary: Union[str, BytesIO], # Updated type hint to accept BytesIO + boundary: Union[str, BytesIO], # Updated type hint to accept BytesIO base: str, source: str, xy: bool, @@ -135,7 +135,7 @@ def __init__( """Create an tile basemap for ODK Collect. Args: - boundary (Union[str, BytesIO]): The boundary for the area you want. + boundary (Union[str, BytesIO]): The boundary for the area you want. Can be a BBOX string, or GeoJSON file of the AOI or BytesIO object containing GeoJSON data. The GeoJSON can contain multiple geometries. base (str): The base directory to cache map tile in @@ -275,7 +275,7 @@ def tileExists( def makeBbox( self, - boundary: Union[str, BytesIO], + boundary: Union[str, BytesIO], ) -> tuple[float, float, float, float]: """Make a bounding box from a shapely geometry. @@ -321,19 +321,18 @@ def makeBbox( # Process BytesIO object try: boundary.seek(0) - geojson_data = boundary.read().decode('utf-8') + geojson_data = boundary.read().decode("utf-8") poly = geojson.loads(geojson_data) return self.extract_bbox(poly) except Exception as e: log.error(e) raise ValueError("Failed to decode GeoJSON data from BytesIO object") from e else: - raise ValueError(f"Invalid boundary type: {type(boundary)}. It must be a BBOX string or (.json, .geojson) flie or BytesIO object") + raise ValueError( + f"Invalid boundary type: {type(boundary)}. It must be a BBOX string or (.json, .geojson) flie or BytesIO object" + ) - - def extract_bbox( - self, poly: Union[BaseGeometry, None] - ) -> tuple[float, float, float, float]: + def extract_bbox(self, poly: Union[BaseGeometry, None]) -> tuple[float, float, float, float]: """Extract bounding box from GeoJSON polygon.""" if "features" in poly: geometry = shape(poly["features"][0]["geometry"]) @@ -355,7 +354,7 @@ def extract_bbox( bbox = geometry.bounds # left, bottom, right, top # minX, minY, maxX, maxY - return bbox + return bbox def tileid_from_y_tile(filepath: Union[Path | str]): @@ -448,7 +447,7 @@ def create_basemap_file( Args: verbose (bool, optional): Enable verbose output if True. - boundary (Union[str, BytesIO], optional): The boundary for the area you want. + boundary (Union[str, BytesIO], optional): The boundary for the area you want. tms (str, optional): Custom TMS URL. xy (bool, optional): Swap the X & Y coordinates when using a custom TMS if True. diff --git a/outreachy.py b/outreachy.py index c6054161..01f9722e 100644 --- a/outreachy.py +++ b/outreachy.py @@ -1,9 +1,10 @@ from io import BytesIO + from osm_fieldwork.basemapper import create_basemap_file with open("D:\\customers\\osm-fieldwork\\tests\\testdata\\Rollinsville.geojson", "rb") as f: - boundary = f.read() # Read the file into memory - boundary_bytesio = BytesIO(boundary) # Convert the file into a BytesIO object + boundary = f.read() # Read the file into memory + boundary_bytesio = BytesIO(boundary) # Convert the file into a BytesIO object create_basemap_file( diff --git a/tests/test_basemap.py b/tests/test_basemap.py index 54774343..490d557e 100755 --- a/tests/test_basemap.py +++ b/tests/test_basemap.py @@ -90,7 +90,8 @@ def test_create_with_byteio(): os.remove(outfile) shutil.rmtree(base) - assert hits == 2 + assert hits == 2 + if __name__ == "__main__": test_create()