Skip to content

Commit

Permalink
GCBM Refactor idea Draft
Browse files Browse the repository at this point in the history 
Signed-off-by: Freeman <[email protected]>
  • Loading branch information
@Freeman-kuch
Freeman-kuch committed Oct 28, 2022
1 parent 674b919 commit 3354a5c
Show file tree
Hide file tree
Showing 3 changed files with 109 additions and 61 deletions.
47 changes: 34 additions & 13 deletions local/rest_api_skeleton/Helpers/for_requests.py
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,47 @@
from flask_restful import reqparse
from werkzeug.datastructures import FileStorage
from werkzeug.datastructures import FileStorage


def title_query():
query = reqparse.RequestParser()
query.add_argument("title", required=True, location="form" )
return query.parse_args()
query.add_argument("title", required=True, location="form")
return query.parse_args()


def classifier_query():
query = reqparse.RequestParser()
query.add_argument("classifiers",type=FileStorage, required=True, action="append", location="files" )
query.add_argument("attributes", location= "form")
return query.parse_args()
query.add_argument(
"classifiers",
type=FileStorage,
required=True,
action="append",
location="files",
)
query.add_argument("attributes", location="form")
return query.parse_args()


def disturbance_query():
query = reqparse.RequestParser()
query.add_argument("disturbances",type=FileStorage, required=True, action="append", location="files" )
query.add_argument("attributes", location= "form")
return query.parse_args()
query.add_argument(
"disturbances",
type=FileStorage,
required=True,
action="append",
location="files",
)
query.add_argument("attributes", location="form")
return query.parse_args()


def miscellaneous_query():
query = reqparse.RequestParser()
query.add_argument("miscellaneous",type=FileStorage, required=True, action="append", location="files" )
query.add_argument("attributes", location= "form")
return query.parse_args()

query.add_argument(
"miscellaneous",
type=FileStorage,
required=True,
action="append",
location="files",
)
query.add_argument("attributes", location="form")
return query.parse_args()
117 changes: 72 additions & 45 deletions local/rest_api_skeleton/Helpers/preprocess.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
import json
import rasterio as rst

def launch_run(title, input_dir):

def launch_run(title, input_dir):
s = time.time()
with open(f"{input_dir}/gcbm_logs.csv", "w+") as f:
res = subprocess.Popen(
Expand Down Expand Up @@ -36,32 +37,43 @@ def launch_run(title, input_dir):
"response": "Operation executed successfully. Downloadable links for input and output are attached in the response. Alternatively, you may also download this simulation input and output results by making a request at gcbm/download with the title in the body.",
}

class Configs:

class Configs:
def __init__(self, input_dir) -> None:
self.input_dir = input_dir
self.Rasters = []
self.Rastersm = []
self.Rastersm = []
self.nodatam = []
self.nodata = []
self.cellLatSize = []
self.cellLonSize = []
self.paths = []
self.lst = []
self.provider_config = open(f"{os.getcwd()}/templates/provider_config.json", "r+")
self.provider_config = open(
f"{os.getcwd()}/templates/provider_config.json", "r+"
)

def get_config_templates(self):
if not os.path.exists(f"{self.input_dir}/templates"):
shutil.copytree(
f"{os.getcwd()}/templates", f"{self.input_dir}/templates", dirs_exist_ok=False
f"{os.getcwd()}/templates",
f"{self.input_dir}/templates",
dirs_exist_ok=False,
)
self.provider_config = open(
f"{self.input_dir}/templates/provider_config.json", "r+"
)
self.provider_config = open(f"{self.input_dir}/templates/provider_config.json", "r+")

def get_modules_cbm_config(self):
with open(f"{self.input_dir}/templates/modules_cbm.json", "r+") as modules_cbm_config:
with open(
f"{self.input_dir}/templates/modules_cbm.json", "r+"
) as modules_cbm_config:
data = json.load(modules_cbm_config)
disturbances = [file.split(".")[0][:-5] for file in os.listdir(f"{self.input_dir}/disturbances/")] # drop `_moja` to match modules_cbm.json template
modules_cbm_config.seek(0)
disturbances = [
file.split(".")[0][:-5]
for file in os.listdir(f"{self.input_dir}/disturbances/")
] # drop `_moja` to match modules_cbm.json template
modules_cbm_config.seek(0)
data["Modules"]["CBMDisturbanceListener"]["settings"]["vars"] = disturbances
json.dump(data, modules_cbm_config, indent=4)
modules_cbm_config.truncate()
Expand All @@ -70,19 +82,18 @@ def get_modules_cbm_config(self):
def database_writes(self):
data = json.load(self.provider_config)
for file in os.listdir(f"{self.input_dir}/db/"):
data["Providers"]["SQLite"] = {"type": "SQLite", "path": file }
data["Providers"]["SQLite"] = {"type": "SQLite", "path": file}
self.provider_config.seek(0)


def write_configs(self, config_type : str):
def write_configs(self, config_type: str):
data = json.load(self.provider_config)
for file in os.listdir(f"{self.input_dir}/{config_type}/"):
d = dict()
d["name"] = file[:-10]
d["layer_path"] = file
d["layer_prefix"] = file[:-5]
self.lst.append(d)
if config_type == "disturbances" or config_type == "classifiers":
if config_type == "disturbances" or config_type == "classifiers":
for root, _, files in os.walk(
os.path.abspath(f"{self.input_dir}/{config_type}/")
):
Expand All @@ -91,15 +102,15 @@ def write_configs(self, config_type : str):
self.Rasters.append(fp)
self.paths.append(fp)
for self.nd in self.Rasters:
img = rst.open(self.nd)
t = img.transform
img = rst.open(self.nd)
t = img.transform
x = t[0]
y = -t[4]
n = img.nodata
n = img.nodata
self.cellLatSize.append(x)
self.cellLonSize.append(y)
self.nodata.append(n)
result = all(element == self.cellLatSize[0] for element in self.cellLatSize)
result = all(element == self.cellLatSize[0] for element in self.cellLatSize)
if result:
cellLat = x
cellLon = y
Expand All @@ -112,8 +123,15 @@ def write_configs(self, config_type : str):
print("Corrupt files")

self.provider_config.seek(0)
new_values = {"cellLonSize": cellLon,"cellLatSize": cellLat, "blockLonSize": blockLon, "blockLatSize": blockLat, "tileLatSize": tileLat, "tileLonSize": tileLon}
data["Providers"]["RasterTiled"]["layers"] = self.lst
new_values = {
"cellLonSize": cellLon,
"cellLatSize": cellLat,
"blockLonSize": blockLon,
"blockLatSize": blockLat,
"tileLatSize": tileLat,
"tileLonSize": tileLon,
}
data["Providers"]["RasterTiled"]["layers"] = self.lst
data["Providers"]["RasterTiled"].update(new_values)

json.dump(data, self.provider_config, indent=4)
Expand All @@ -132,21 +150,23 @@ def write_configs(self, config_type : str):
}

self.study_area = {
"tile_size": tileLat,
"block_size": blockLat,
"tiles": [
{
"x": int(t[2]),
"y": int(t[5]),
"index": 12674,
}
],
"pixel_size": cellLat,
"layers": [],
}
"tile_size": tileLat,
"block_size": blockLat,
"tiles": [
{
"x": int(t[2]),
"y": int(t[5]),
"index": 12674,
}
],
"pixel_size": cellLat,
"layers": [],
}

def add_file_to_path(self, config_type):
for root, _, files in os.walk(os.path.abspath(f"{self.input_dir}/{config_type}/")):
for root, _, files in os.walk(
os.path.abspath(f"{self.input_dir}/{config_type}/")
):
for file in files:
fp = os.path.join(root, file)
self.paths.append(fp)
Expand All @@ -157,15 +177,18 @@ def copy_directory(self):

def flatten_directory(self, config_type):
shutil.rmtree((f"{self.input_dir}/{config_type}/"))



class DisturbanceConfig(Configs):
def __init__(self, input_dir, config_file : str, attribute : dict = None ) -> None:
def __init__(self, input_dir, config_file: str, attribute: dict = None) -> None:
super().__init__(input_dir)
self.config_file = config_file
self.attribute = attribute

def disturbances_special(self):
with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file:
with open(
f"{self.input_dir}/{self.config_file}", "w", encoding="utf8"
) as json_file:
self.dictionary["attributes"] = self.attribute
json.dump(self.dictionary, json_file, indent=4)
with open(
Expand Down Expand Up @@ -194,14 +217,17 @@ def __call__(self):
self.add_file_to_path("disturbances")
self.copy_directory()


class ClassifierConfig(Configs):
def __init__(self, input_dir, config_file : str, attribute : dict ) -> None:
def __init__(self, input_dir, config_file: str, attribute: dict) -> None:
super().__init__(input_dir)
self.config_file = config_file
self.attribute = attribute

def classifier_special(self):
with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file:
with open(
f"{self.input_dir}/{self.config_file}", "w", encoding="utf8"
) as json_file:
self.dictionary["attributes"] = self.attribute
json.dump(self.dictionary, json_file, indent=4)

Expand All @@ -225,16 +251,17 @@ def __call__(self):
self.add_file_to_path("classifiers")
self.copy_directory()


class MiscellaneousConfig(Configs):
def __init__(self, input_dir, config_file : str, attribute : dict ) -> None:
def __init__(self, input_dir, config_file: str, attribute: dict) -> None:
super().__init__(input_dir)
self.config_file = config_file
self.attribute = attribute
def miscellaneous_special(self):

def miscellaneous_special(self):
for root, _, files in os.walk(
os.path.abspath(f"{self.input_dir}/miscellaneous/")
):
):
for file in files:
fp2 = os.path.join(root, file)
self.Rastersm.append(fp2)
Expand All @@ -244,9 +271,11 @@ def miscellaneous_special(self):
d = img.nodata
self.nodatam.append(d)
"""this is an experimental thing"""
with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file:
with open(
f"{self.input_dir}/{self.config_file}", "w", encoding="utf8"
) as json_file:
self.dictionary["layer_type"] = "GridLayer"
if self.config_file == "mean_annual_temperature_moja.json":
if self.config_file == "mean_annual_temperature_moja.json":
self.dictionary["layer_data"] = "Float32"
else:
self.dictionary["layer_data"] = "Int16"
Expand All @@ -273,5 +302,3 @@ def __call__(self):
self.add_file_to_path("miscellaneous")
self.copy_directory()
self.flatten_directory("miscellaneous")


6 changes: 3 additions & 3 deletions local/rest_api_skeleton/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
from flask_restful import Api
from Endpoints.gcbm_endpoints import disturbance, Run, title, classifier, miscellaneous

app = Flask(__name__)
api = Api()
app = Flask(__name__)
api = Api()

api.add_resource(disturbance, "/gcbm/upload/disturbances")
api.add_resource(title, "/gcbm/create")
Expand All @@ -12,5 +12,5 @@
api.add_resource(Run, "/gcbm/run")


if __name__ == "__main__":
if __name__ == "__main__":
app.run(debug=True)

0 comments on commit 3354a5c

Please sign in to comment.