From 78b5d7e7e5d4991afc61e676b14f1a2da96989a2 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Thu, 22 Sep 2022 15:34:42 +0545 Subject: [PATCH] remove fiona since its no longer used --- requirements.docker.txt | 1 - requirements.txt | 1 - src/galaxy/app.py | 152 ++++++++++++++++++++-------------------- src/galaxy/config.py | 2 +- 4 files changed, 77 insertions(+), 79 deletions(-) diff --git a/requirements.docker.txt b/requirements.docker.txt index 9d0a5f1b..1bf94ede 100644 --- a/requirements.docker.txt +++ b/requirements.docker.txt @@ -26,7 +26,6 @@ sphinx==4.2.0 area==1.1.1 orjson==3.6.7 boto3==1.24.38 -Fiona==1.8.21 fastapi-versioning==0.10.0 redis==4.3.4 celery==5.2.7 diff --git a/requirements.txt b/requirements.txt index 9d0a5f1b..1bf94ede 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,6 @@ sphinx==4.2.0 area==1.1.1 orjson==3.6.7 boto3==1.24.38 -Fiona==1.8.21 fastapi-versioning==0.10.0 redis==4.3.4 celery==5.2.7 diff --git a/src/galaxy/app.py b/src/galaxy/app.py index 760ae580..fb09751e 100644 --- a/src/galaxy/app.py +++ b/src/galaxy/app.py @@ -37,8 +37,8 @@ from area import area import subprocess from json import dumps -import fiona -from fiona.crs import from_epsg +# import fiona +# from fiona.crs import from_epsg import time import shutil import boto3 @@ -1029,80 +1029,80 @@ def query2geojson(con, extraction_query, dump_temp_file_path): f.write(post_geojson) logging.debug("Server side Query Result Post Processing Done") - @staticmethod - def query2shapefile(con, point_query, line_query, poly_query, point_schema, line_schema, poly_schema, dump_temp_file_path): - """Function that transfer db query to shp""" - # schema: it is a simple dictionary with geometry and properties as keys - # schema = {'geometry': 'LineString','properties': {'test': 'int'}} - file_paths = [] - if point_query: - logging.debug("Writing Point Shapefile") - - schema = {'geometry': 'Point', 'properties': point_schema, } - point_file_path = f"""{dump_temp_file_path}_point.shp""" - # open a fiona object - pointShp = fiona.open(point_file_path, mode='w', driver='ESRI Shapefile', encoding='UTF-8', - schema=schema, crs="EPSG:4326") - - with con.cursor(name='fetch_raw') as cursor: # using server side cursor - cursor.itersize = 1000 # chunk size to get 1000 row at a time in client side - cursor.execute(point_query) - for row in cursor: - pointShp.write(orjson.loads(row[0])) - - cursor.close() # closing connection to avoid memory issues - # close fiona object - pointShp.close() - file_paths.append(point_file_path) - file_paths.append(f"""{dump_temp_file_path}_point.shx""") - file_paths.append(f"""{dump_temp_file_path}_point.cpg""") - file_paths.append(f"""{dump_temp_file_path}_point.dbf""") - file_paths.append(f"""{dump_temp_file_path}_point.prj""") - - if line_query: - logging.debug("Writing Line Shapefile") - - schema = {'geometry': 'LineString', 'properties': line_schema, } - # print(schema) - line_file_path = f"""{dump_temp_file_path}_line.shp""" - with fiona.open(line_file_path, 'w', encoding='UTF-8', crs=from_epsg(4326), driver='ESRI Shapefile', schema=schema) as layer: - with con.cursor(name='fetch_raw') as cursor: # using server side cursor - cursor.itersize = 1000 # chunk size to get 1000 row at a time in client side - cursor.execute(line_query) - for row in cursor: - layer.write(orjson.loads(row[0])) - - cursor.close() # closing connection to avoid memory issues - # close fiona object - layer.close() - file_paths.append(line_file_path) - file_paths.append(f"""{dump_temp_file_path}_line.shx""") - file_paths.append(f"""{dump_temp_file_path}_line.cpg""") - file_paths.append(f"""{dump_temp_file_path}_line.dbf""") - file_paths.append(f"""{dump_temp_file_path}_line.prj""") - - if poly_query: - logging.debug("Writing Poly Shapefile") - - poly_file_path = f"""{dump_temp_file_path}_poly.shp""" - schema = {'geometry': 'Polygon', 'properties': poly_schema, } - - with fiona.open(poly_file_path, 'w', encoding='UTF-8', crs=from_epsg(4326), driver='ESRI Shapefile', schema=schema) as layer: - with con.cursor(name='fetch_raw') as cursor: # using server side cursor - cursor.itersize = 1000 # chunk size to get 1000 row at a time in client side - cursor.execute(poly_query) - for row in cursor: - layer.write(orjson.loads(row[0])) - - cursor.close() # closing connection to avoid memory issues - # close fiona object - layer.close() - file_paths.append(poly_file_path) - file_paths.append(f"""{dump_temp_file_path}_poly.shx""") - file_paths.append(f"""{dump_temp_file_path}_poly.cpg""") - file_paths.append(f"""{dump_temp_file_path}_poly.dbf""") - file_paths.append(f"""{dump_temp_file_path}_poly.prj""") - return file_paths + # @staticmethod + # def query2shapefile(con, point_query, line_query, poly_query, point_schema, line_schema, poly_schema, dump_temp_file_path): + # """Function that transfer db query to shp""" + # # schema: it is a simple dictionary with geometry and properties as keys + # # schema = {'geometry': 'LineString','properties': {'test': 'int'}} + # file_paths = [] + # if point_query: + # logging.debug("Writing Point Shapefile") + + # schema = {'geometry': 'Point', 'properties': point_schema, } + # point_file_path = f"""{dump_temp_file_path}_point.shp""" + # # open a fiona object + # pointShp = fiona.open(point_file_path, mode='w', driver='ESRI Shapefile', encoding='UTF-8', + # schema=schema, crs="EPSG:4326") + + # with con.cursor(name='fetch_raw') as cursor: # using server side cursor + # cursor.itersize = 1000 # chunk size to get 1000 row at a time in client side + # cursor.execute(point_query) + # for row in cursor: + # pointShp.write(orjson.loads(row[0])) + + # cursor.close() # closing connection to avoid memory issues + # # close fiona object + # pointShp.close() + # file_paths.append(point_file_path) + # file_paths.append(f"""{dump_temp_file_path}_point.shx""") + # file_paths.append(f"""{dump_temp_file_path}_point.cpg""") + # file_paths.append(f"""{dump_temp_file_path}_point.dbf""") + # file_paths.append(f"""{dump_temp_file_path}_point.prj""") + + # if line_query: + # logging.debug("Writing Line Shapefile") + + # schema = {'geometry': 'LineString', 'properties': line_schema, } + # # print(schema) + # line_file_path = f"""{dump_temp_file_path}_line.shp""" + # with fiona.open(line_file_path, 'w', encoding='UTF-8', crs=from_epsg(4326), driver='ESRI Shapefile', schema=schema) as layer: + # with con.cursor(name='fetch_raw') as cursor: # using server side cursor + # cursor.itersize = 1000 # chunk size to get 1000 row at a time in client side + # cursor.execute(line_query) + # for row in cursor: + # layer.write(orjson.loads(row[0])) + + # cursor.close() # closing connection to avoid memory issues + # # close fiona object + # layer.close() + # file_paths.append(line_file_path) + # file_paths.append(f"""{dump_temp_file_path}_line.shx""") + # file_paths.append(f"""{dump_temp_file_path}_line.cpg""") + # file_paths.append(f"""{dump_temp_file_path}_line.dbf""") + # file_paths.append(f"""{dump_temp_file_path}_line.prj""") + + # if poly_query: + # logging.debug("Writing Poly Shapefile") + + # poly_file_path = f"""{dump_temp_file_path}_poly.shp""" + # schema = {'geometry': 'Polygon', 'properties': poly_schema, } + + # with fiona.open(poly_file_path, 'w', encoding='UTF-8', crs=from_epsg(4326), driver='ESRI Shapefile', schema=schema) as layer: + # with con.cursor(name='fetch_raw') as cursor: # using server side cursor + # cursor.itersize = 1000 # chunk size to get 1000 row at a time in client side + # cursor.execute(poly_query) + # for row in cursor: + # layer.write(orjson.loads(row[0])) + + # cursor.close() # closing connection to avoid memory issues + # # close fiona object + # layer.close() + # file_paths.append(poly_file_path) + # file_paths.append(f"""{dump_temp_file_path}_poly.shx""") + # file_paths.append(f"""{dump_temp_file_path}_poly.cpg""") + # file_paths.append(f"""{dump_temp_file_path}_poly.dbf""") + # file_paths.append(f"""{dump_temp_file_path}_poly.prj""") + # return file_paths @staticmethod def get_grid_id(geom, cur): diff --git a/src/galaxy/config.py b/src/galaxy/config.py index d9cda9bb..356e1f27 100644 --- a/src/galaxy/config.py +++ b/src/galaxy/config.py @@ -44,7 +44,7 @@ "logging config is not supported , Supported fields are : debug,error,warning,info , Logging to default :debug") level = logging.DEBUG -logging.getLogger("fiona").propagate = False # disable fiona logging +# logging.getLogger("fiona").propagate = False # disable fiona logging logging.basicConfig(format='%(asctime)s - %(message)s', level=level) logging.getLogger('boto3').propagate = False # disable boto3 logging