diff --git a/local/rest_api_gcbm/app.py b/local/rest_api_gcbm/app.py index 1feed739..469ac0fa 100644 --- a/local/rest_api_gcbm/app.py +++ b/local/rest_api_gcbm/app.py @@ -1,752 +1,13 @@ -from crypt import methods -from threading import Thread - -from numpy import append -from run_distributed import * -from flask_autoindex import AutoIndex -from flask_swagger_ui import get_swaggerui_blueprint -from flask_swagger import swagger -from datetime import timedelta -from datetime import datetime -from google.cloud import storage, pubsub_v1 -from google.api_core.exceptions import AlreadyExists -import logging -import shutil -import json -import time -import subprocess +from flask_restful import Api +from gcbm_simulation.routes import initialize_routes +from flask import Flask import os -import flask.scaffold -import rasterio as rst -from flask import jsonify -from config_table import rename_columns -import sqlite3 -from preprocess import get_config_templates, get_modules_cbm_config, get_provider_config - -flask.helpers._endpoint_from_view_func = flask.scaffold._endpoint_from_view_func -from flask_restful import Resource, Api, reqparse -from flask import Flask, send_from_directory, request, jsonify, redirect, send_file -from flask_cors import CORS app = Flask(__name__) -CORS( - app, - origins=[ - "http://127.0.0.1:8080/", - "http://127.0.0.1:8000", - "http://localhost:5000", - "http://localhost:8000", - r"^https://.+example.com$", - ], -) -# ppath = "/" -# AutoIndex(app, browse_root=ppath) api = Api(app) - -# logger config -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -c_handler = logging.StreamHandler() -c_handler.setLevel(logging.DEBUG) -c_format = logging.Formatter("%(name)s - %(levelname)s - %(message)s") -c_handler.setFormatter(c_format) -logger.addHandler(c_handler) - - -### swagger specific ### -SWAGGER_URL = "/swagger" -API_URL = "/static/swagger.json" -SWAGGERUI_BLUEPRINT = get_swaggerui_blueprint( - SWAGGER_URL, API_URL, config={"app_name": "FLINT-GCBM REST API"} -) -app.register_blueprint(SWAGGERUI_BLUEPRINT, url_prefix=SWAGGER_URL) -### end swagger specific ### - - -@app.route("/spec") -def spec(): - swag = swagger(app) - swag["info"]["version"] = "1.0" - swag["info"]["title"] = "FLINT-GCBM Rest Api" - f = open("./static/swagger.json", "w+") - json.dump(swag, f) - return jsonify(swag) - - -@app.route("/help/", methods=["GET"]) -def help(arg): - """ - Get Help Section - --- - tags: - - help - parameters: - - name: arg - in: path - description: Help info about named section. Pass all to get all info - required: true - type: string - responses: - 200: - description: Help - """ - s = time.time() - if arg == "all": - res = subprocess.run(["/opt/gcbm/moja.cli", "--help"], stdout=subprocess.PIPE) - else: - res = subprocess.run( - ["/opt/gcbm/moja.cli", "--help-section", arg], stdout=subprocess.PIPE - ) - e = time.time() - - response = { - "exitCode": res.returncode, - "execTime": e - s, - "response": res.stdout.decode("utf-8"), - } - return {"data": response}, 200 - - -@app.route("/version", methods=["GET"]) -def version(): - """ - Get Version of FLINT - --- - tags: - - version - responses: - 200: - description: Version - """ - s = time.time() - res = subprocess.run(["/opt/gcbm/moja.cli", "--version"], stdout=subprocess.PIPE) - e = time.time() - - response = { - "exitCode": res.returncode, - "execTime": e - s, - "response": res.stdout.decode("utf-8"), - } - return {"data": response}, 200 - - -@app.route("/gcbm/new", methods=["POST"]) -def gcbm_new(): - """ - Create a new GCBM simulation with a title. - --- - tags: - - gcbm - responses: - 200: - parameters: - - in: body - name: title - required: true - schema: - type: string - description: Create a new simulation for GCBM Implementation of FLINT - """ - # Default title = simulation - title = request.form.get("title") or "simulation" - # Sanitize title - title = "".join(c for c in title if c.isalnum()) - # input_dir = f"{title}" - input_dir = f"{os.getcwd()}/input/{title}" - if not os.path.exists(f"{input_dir}"): - os.makedirs(f"{input_dir}") - message = "New simulation started. Please move on to the next stage for uploading files at /gcbm/upload." - else: - message = "Simulation already exists. Please check the list of simulations present before proceeding with a new simulation at gcbm/list. You may also download the input and output files for this simulation at gcbm/download sending parameter title in the body." - - return {"data": message}, 200 - - -@app.route("/gcbm/upload", methods=["POST"]) -def gcbm_upload(): - """ - Upload files for GCBM Dynamic implementation of FLINT - --- - tags: - - gcbm - responses: - 200: - parameters: - - in: body - name: title - required: true - schema: - type: string - description: File upload for GCBM Implementation FLINT - """ - - # Default title = simulation - title = request.form.get("title") or "simulation" - # Sanitize title - title = "".join(c for c in title if c.isalnum()) - - # Create project directory - input_dir = f"{os.getcwd()}/input/{title}" - if not os.path.exists(f"{input_dir}"): - os.makedirs(f"{input_dir}") - logging.debug(os.getcwd()) - - # input files follow a strict structure - if not os.path.exists(f"{input_dir}/disturbances"): - os.makedirs(f"{input_dir}/disturbances") - if not os.path.exists(f"{input_dir}/classifiers"): - os.makedirs(f"{input_dir}/classifiers") - if not os.path.exists(f"{input_dir}/db"): - os.makedirs(f"{input_dir}/db") - if not os.path.exists(f"{input_dir}/miscellaneous"): - os.makedirs(f"{input_dir}/miscellaneous") - - # store files following structure defined in curl.md - if "disturbances" in request.files: - for file in request.files.getlist("disturbances"): - file.save(f"{input_dir}/disturbances/{file.filename}") - else: - return {"error": "Missing configuration file"}, 400 - - if "classifiers" in request.files: - for file in request.files.getlist("classifiers"): - file.save(f"{input_dir}/classifiers/{file.filename}") - else: - return {"error": "Missing configuration file"}, 400 - - if "db" in request.files: - for file in request.files.getlist("db"): - file.save(f"{input_dir}/db/{file.filename}") - else: - return {"error": "Missing configuration file"}, 400 - - if "miscellaneous" in request.files: - for file in request.files.getlist("miscellaneous"): - file.save(f"{input_dir}/miscellaneous/{file.filename}") - else: - return {"error": "Missing configuration file"}, 400 - - return { - "data": "All files uploaded succesfully. Proceed to the next step of the API at gcbm/dynamic." - } - - -@app.route("/config", methods=["POST"]) -def config_table(): - obj = request.get_json() - print(obj) - input_dir = f"{os.getcwd()}/input/{obj['simulation_name']}" - response = dict() - try: - return { - "status": 1, - "response": rename_columns(obj["tables"], obj["simulation_name"]), - } - except Exception: - return {"status": 0, "error": Exception} - - -@app.route("/upload/db/tables", methods=["POST"]) -def send_table(): - """ - Get GCBM table names from database file - --- - tags: - - gcbm - responses: - 200: - parameters: - - in: Params - name: title - type: string - description: GCBM table output - """ - # Default title = simulation - title = request.form.get("title") or "simulation" - input_dir = f"{os.getcwd()}/input/{title}/db/" - conn = sqlite3.connect(input_dir + "gcbm_input.db") - sql_query = """SELECT name FROM sqlite_master - WHERE type='table';""" - - tables = conn.execute(sql_query).fetchall() - resp = dict() - # iterate over all the table name - for table_name in tables: - schema = [] - ins = "PRAGMA table_info('" + table_name[0] + "')" - # key as the table name and values as the column names - for row in conn.execute(ins).fetchall(): - schema.append(row[1]) - resp[table_name[0]] = schema - return resp, 200 - - -@app.route("/gcbm/table/rename", methods=["POST"]) -def rename_table(): - """ - Rename a table - --- - tags: - - gcbm - responses: - 200: - parameters: - - in: Params - name: title - type: string - name: previous - type: string - nsme: new - type: string - description: Status indicating success/failure in performing the rename - """ - title = request.form.get("title") or "simulation" - input_dir = f"{os.getcwd()}/input/{title}/db/" - try: - connection = sqlite3.connect(input_dir + "gcbm_input.db") - cursor = connection.cursor() - previous_name = request.form.get("previous") - new_name = request.form.get("new") - cursor.execute(f"ALTER TABLE {previous_name} rename to {new_name}") - return {"status": 1} - except Exception as exception: - return {"status": 0, "error": str(exception)} - - -@app.route("/gcbm/dynamic", methods=["POST"]) -def gcbm_dynamic(): - """ - Get GCBM Dynamic implementation of FLINT - --- - tags: - - gcbm - responses: - 200: - parameters: - - in: body - name: title - required: true - schema: - type: string - description: GCBM Implementation FLINT - """ - # Default title = simulation - title = request.form.get("title") or "simulation" - - # Sanitize title - title = "".join(c for c in title if c.isalnum()) - input_dir = f"{os.getcwd()}/input/{title}" - - get_config_templates(input_dir) - get_modules_cbm_config(input_dir) - get_provider_config(input_dir) - - if not os.path.exists(f"{input_dir}"): - os.makedirs(f"{input_dir}") - - thread = Thread(target=launch_run, kwargs={"title": title, "input_dir": input_dir}) - thread.start() - # subscriber_path = create_topic_and_sub(title) - return {"status": "Run started"}, 200 - - -@app.route("/gcbm/getConfig", methods=["POST"]) -def getConfig(): - """ - Return .json for the input .tiff files - --- - tags: - - gcbm - responses: - 200: - parameters: - - in: body - name: title - required: true - schema: - type: string - description: Name of the Simulation - name: file_name - required: true - schema: - type: string - description: Name of the File - """ - # Default title = Simulation - title = request.form.get("title").strip() - file_name = request.form.get("file_name").strip() - input_dir = f"{os.getcwd()}/input/{title}" - - # check if title is empty - if title == "": - return {"error": "No Simulation name specified"}, 400 - - # check if file_name is empty - if file_name == "": - return {"error": "No file name specified"}, 400 - - # Check if simulation exists or not - if not os.path.exists(f"{input_dir}"): - return {"error": "Simulation with the name " + title + " doesn't exists"}, 400 - - input_dir_file = f"{input_dir}/{file_name}.json" - # Check if file exists or not - if not os.path.exists(f"{input_dir_file}"): - return {"error": "File with name " + file_name + " doesn't exists"}, 400 - - # Return the json for the corresponding file name - file_obj = open(f"{input_dir_file}") - obj = json.load(file_obj) - return {"data": obj}, 200 - - -def launch_run(title, input_dir): - s = time.time() - logging.debug("Starting run") - with open(f"{input_dir}/gcbm_logs.csv", "w+") as f: - res = subprocess.Popen( - [ - "/opt/gcbm/moja.cli", - "--config_file", - "gcbm_config.cfg", - "--config_provider", - "provider_config.json", - ], - stdout=f, - cwd=f"{input_dir}", - ) - logging.debug("Communicating") - (output, err) = res.communicate() - logging.debug("Communicated") - - if not os.path.exists(f"{input_dir}/output"): - logging.error(err) - return "OK" - logging.debug("Output exists") - - # cut and paste output folder to app/output/simulation_name - shutil.copytree(f"{input_dir}/output", (f"{os.getcwd()}/output/{title}")) - shutil.make_archive( - f"{os.getcwd()}/output/{title}", "zip", f"{os.getcwd()}/output/{title}" - ) - shutil.rmtree((f"{input_dir}/output")) - logging.debug("Made archive") - e = time.time() - - logging.debug("Generated URL") - response = { - "exitCode": res.returncode, - "execTime": e - s, - "response": "Operation executed successfully. Downloadable links for input and output are attached in the response. Alternatively, you may also download this simulation input and output results by making a request at gcbm/download with the title in the body.", - } - - -@app.route("/gcbm/download", methods=["POST"]) -def gcbm_download(): - """ - Download GCBM Input and Output - --- - tags: - - gcbm - responses: - 200: - parameters: - - in: body - name: title - required: true - schema: - type: string - description: GCBM Download FLINT - """ - # Default title = simulation - title = request.form.get("title") or "simulation" - # Sanitize title - title = "".join(c for c in title if c.isalnum()) - - output_dir = f"{os.getcwd()}/output/{title}.zip" - input_dir = f"{os.getcwd()}/input/{title}" - - # if the title has an input simulation and there is no output simulation then they should check the status. - if not os.path.exists(f"{output_dir}") and os.path.exists(f"{input_dir}"): - return { - "message": "You simulation is currently running, check the status via /gcbm/status" - } - - # if there is no input simulation and no output simulation then the simulation does not exist. - elif not os.path.exists(f"{output_dir}") and not os.path.exists(f"{input_dir}"): - return { - "message": "You don't have a simulation with this title kindly check the title and try again" - } - - return send_file( - f"{os.getcwd()}/output/{title}.zip", attachment_filename="{title}.zip", - ) - - -@app.route("/gcbm/list", methods=["GET"]) -def gcbm_list_simulations(): - """ - Get GCBM Simulations List - --- - tags: - - gcbm - responses: - 200: - description: GCBM Simulations List - """ - - list = [] - for file in os.listdir(f"{os.getcwd()}/input"): - list.append(file) - - return ( - { - "data": list, - "message": "To create a new simulation, create a request at gcbm/new. To access the results of the existing simulations, create a request at gcbm/download.", - }, - 200, - ) - - -@app.route("/gcbm/status", methods=["POST"]) -def status(): - """ - Get status of a simulation - --- - tags: - - gcbm - responses: - 200: - parameters: - - in: body - name: title - required: true - schema: - type: string - description: Get status of simulation - """ - # Default title = simulation - title = request.form.get("title") or "simulation" - # Sanitize title - title = "".join(c for c in title if c.isalnum()) - - if os.path.isfile(f"{os.getcwd()}/input/{title}/output.zip"): - message = "Output is ready to download at gcbm/download" - else: - message = "In Progress" - - return {"finished": message} - - -@app.route("/check", methods=["GET", "POST"]) -def check(): - return "Checks OK", 200 - - -@app.route("/", methods=["GET"]) -def home(): - return "FLINT.Cloud API" - - -@app.route("/upload/title", methods=["POST"]) -def getTitle(): - """ - Get simulation title for GCBM implementation of FLINT - --- - tags: - - gcbm-upload - responses: - 200: - parameters: - - in: body - name: title - required: true - schema: - type: string - description: Get simulation title for GCBM - """ - # Default sim_title = simulation - sim_title = request.form.get("title") or "simulation" - - # Sanitize title - sim_title = "".join(c for c in sim_title if c.isalnum()) - - # input_dir = f"{title}" - input_dir = f"{os.getcwd()}/input/{sim_title}" - if not os.path.exists(f"{input_dir}"): - os.makedirs(f"{input_dir}") - message = "New {sim_title} simulation started. Please move on to the next stage for uploading files at /gcbm/upload." - else: - message = "Simulation already exists with name {sim_title}. Please check the list of simulations present before proceeding with a new simulation at gcbm/list. You may also download the input and output files for this simulation at gcbm/download sending parameter title in the body." - - return {"data": message}, 200 - - -@app.route("/gcbm/upload/disturbances", methods=["POST"]) -def gcbm_disturbances(): - """ - Disturbances file for GCBM Dynamic implementation of FLINT - --- - tags: - - gcbm-upload - responses: - 200: - parameters: - - in: body - name: string - required: true - schema: - type: string - description: Disturbances File upload for GCBM Implementation FLINT - """ - - # Get the title from the payload - title = request.form.get("title") or "simulation" - - # Check for project directory else create one - input_dir = f"{os.getcwd()}/input/{title}" - if not os.path.exists(f"{input_dir}"): - os.makedirs(f"{input_dir}") - logging.debug(os.getcwd()) - - # input files follow a strict structure - if not os.path.exists(f"{input_dir}/disturbances"): - os.makedirs(f"{input_dir}/disturbances") - - # store disturbances file in a new folder - if "disturbances" in request.files: - for file in request.files.getlist("disturbances"): - file.save(f"{input_dir}/disturbances/{file.filename}") - else: - return {"error": "Missing disturbances file"}, 400 - - return {"data": "Disturbances file uploaded succesfully. Proceed to the next step."} - - -@app.route("/gcbm/upload/classifiers", methods=["POST"]) -def gcbm_classifiers(): - """ - Classifiers file for GCBM Dynamic implementation of FLINT - --- - tags: - - gcbm-upload - responses: - 200: - parameters: - - in: body - name: string - required: true - schema: - type: string - description: Classifiers File upload for GCBM Implementation FLINT - """ - - # Get the title from the payload - title = request.form.get("title") or "simulation" - - # Check for project directory else create one - input_dir = f"{os.getcwd()}/input/{title}" - if not os.path.exists(f"{input_dir}"): - os.makedirs(f"{input_dir}") - logging.debug(os.getcwd()) - - # input files follow a strict structure - if not os.path.exists(f"{input_dir}/classifiers"): - os.makedirs(f"{input_dir}/classifiers") - - # store disturbances file in a new folder - if "classifiers" in request.files: - for file in request.files.getlist("classifiers"): - file.save(f"{input_dir}/classifiers/{file.filename}") - else: - return {"error": "Missing classifiers file"}, 400 - - return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} - - -@app.route("/gcbm/upload/miscellaneous", methods=["POST"]) -def gcbm_miscellaneous(): - """ - Miscellaneous file for GCBM Dynamic implementation of FLINT - --- - tags: - - gcbm-upload - responses: - 200: - parameters: - - in: body - name: string - required: true - schema: - type: string - description: Miscellaneous File upload for GCBM Implementation FLINT - """ - - # Get the title from the payload - title = request.form.get("title") or "simulation" - - # Check for project directory else create one - input_dir = f"{os.getcwd()}/input/{title}" - if not os.path.exists(f"{input_dir}"): - os.makedirs(f"{input_dir}") - logging.debug(os.getcwd()) - - # input files follow a strict structure - if not os.path.exists(f"{input_dir}/miscellaneous"): - os.makedirs(f"{input_dir}/miscellaneous") - - # store miscellaneous file in a new folder - if "miscellaneous" in request.files: - for file in request.files.getlist("miscellaneous"): - file.save(f"{input_dir}/miscellaneous/{file.filename}") - else: - return {"error": "Missing miscellaneous file"}, 400 - - return { - "data": "Miscellaneous file uploaded succesfully. Proceed to the next step." - } - - -@app.route("/gcbm/upload/db", methods=["POST"]) -def gcbm_db(): - """ - db file for GCBM Dynamic implementation of FLINT - --- - tags: - - gcbm-upload - responses: - 200: - parameters: - - in: body - name: string - required: true - schema: - type: string - description: db File upload for GCBM Implementation FLINT - """ - - # Get the title from the payload - title = request.form.get("title") or "simulation" - - # Check for project directory else create one - input_dir = f"{os.getcwd()}/input/{title}" - if not os.path.exists(f"{input_dir}"): - os.makedirs(f"{input_dir}") - logging.debug(os.getcwd()) - - # input files follow a strict structure - if not os.path.exists(f"{input_dir}/db"): - os.makedirs(f"{input_dir}/db") - - # store miscellaneous file in a new folder - if "db" in request.files: - for file in request.files.getlist("db"): - file.save(f"{input_dir}/db/{file.filename}") - else: - return {"error": "Missing db file"}, 400 - - return {"data": "db file uploaded succesfully. Proceed to the next step."} +initialize_routes(api) if __name__ == "__main__": app.run(debug=True, host="0.0.0.0", port=int(os.environ.get("PORT", 8080))) + diff --git a/local/rest_api_gcbm/gcbm_simulation/preprocess.py b/local/rest_api_gcbm/gcbm_simulation/preprocess.py new file mode 100644 index 00000000..f221f70a --- /dev/null +++ b/local/rest_api_gcbm/gcbm_simulation/preprocess.py @@ -0,0 +1,168 @@ +import logging +import shutil +import json +import time +import subprocess +import os + + +class GCBMSimulation: + def __init__(self, input_dir): + self.input_dir = input_dir + self.Rastersm = [] + self.rasters = [] + self.paths = [] + self.nodatam = [] + self.dictionary = {} + self.study_area = {} + self.provider_config = open( + f"{self.input_dir}/templates/provider_config.json", "r+" + ) + + def get_uploaded_files_details(self): + with open( + f"{self.input_dir}/upload_details.json", "w+", encoding="utf8" + ) as upload_details: + upload_details_dictionary = json.load(upload_details) + return upload_details_dictionary + + def set_config_templates(self): + if not os.path.exists(f"{self.input_dir}/templates"): + shutil.copytree( + f"{os.getcwd()}/templates", + f"{self.input_dir}/templates", + dirs_exist_ok=False, + ) + + def add_database_to_provider_config(self): + data = json.load(self.provider_config) + upload_details_dictionary = self.get_uploaded_files_details() + for file in upload_details_dictionary: + if file["category"] == "db": + data["Providers"]["SQLite"] = {"path": file["path"], "type": "SQLite"} + self.provider_config.seek(0) + + # disturbances, #classifiers, #miscellaneous e.t.c, + def add_files_to_provider_config_layers(self): + data = json.load(self.provider_config) + layer = [] + upload_details_dictionary = self.get_uploaded_files_details() + for file in upload_details_dictionary: + dic = { + "name": file, + "layer_path": file["path"], + "layer_prefix": file["path"][:5], + } + layer.append(dic) + data["Providers"]["RasterTiled"]["layers"] += layer + + def generate_provider_config(self): + data = json.load(self.provider_config) + nodata = [] + cellLatSize = [] + cellLonSize = [] + + # if there is disturbances. + upload_details_dictionary = self.get_uploaded_files_details() + for file in upload_details_dictionary: + if file["category"] == "disturbances": + self.rasters.append(file["path"]) + + for nd in self.rasters: + img = rst.open(nd) + t = img.transform + x = t[0] + y = -t[4] + n = img.nodata + cellLatSize.append(x) + cellLonSize.append(y) + nodata.append(n) + + result = all(element == cellLatSize[0] for element in cellLatSize) + print(result) + if result: + cellLat = x + cellLon = y + nd = n + blockLat = x * 400 + blockLon = y * 400 + tileLat = x * 4000 + tileLon = y * 4000 + else: + print("Corrupt files") + + self.provider_config.seek(0) + + data["Providers"]["RasterTiled"]["cellLonSize"] = cellLon + data["Providers"]["RasterTiled"]["cellLatSize"] = cellLat + data["Providers"]["RasterTiled"]["blockLonSize"] = blockLon + data["Providers"]["RasterTiled"]["blockLatSize"] = blockLat + data["Providers"]["RasterTiled"]["tileLatSize"] = tileLat + data["Providers"]["RasterTiled"]["tileLonSize"] = tileLon + + json.dump(data, self.provider_config, indent=4) + self.provider_config.truncate() + + self.dictionary = { + "layer_type": "GridLayer", + "layer_data": "Byte", + "nodata": nd, + "tileLatSize": tileLat, + "tileLonSize": tileLon, + "blockLatSize": blockLat, + "blockLonSize": blockLon, + "cellLatSize": cellLat, + "cellLonSize": cellLon, + } + + self.study_area = { + "tile_size": tileLat, + "block_size": blockLat, + "tiles": [{"x": int(t[2]), "y": int(t[5]), "index": 12674}], + "pixel_size": cellLat, + "layers": [], + } + + def set_attributes(self, file_name: str, payload: dict): + with open( + f"{self.input_dir}/{file_name}.json", "w", encoding="utf8" + ) as json_file: + self.dictionary["attributes"] = payload + json.dump(self.dictionary, json_file, indent=4) + + def set_study_area(self): + with open( + f"{self.input_dir}/study_area.json", "w", encoding="utf" + ) as json_file: + study_area = [] + + upload_details_dictionary = self.get_uploaded_files_details() + for file in upload_details_dictionary: + if file["category"] == "miscellaneous": + study_area.append({"name": file[:10], "type": "VectorLayer"}) + elif file["category"] == "classifiers": + study_area.append( + { + "name": file[:10], + "type": "VectorLayer", + "tags": ["classifier"], + } + ) + elif file["category"] == "disturbances": + study_area.append( + { + "name": file[:10], + "type": "VectorLayer", + "tags": ["classifier"], + } + ) + study_area.append( + { + "name": file[:10], + "type": "DisturbanceLayer", + "tags": ["disturbance"], + } + ) + + self.study_area["layers"] = study_area + json.dump(self.study_area, json_file, indent=4) diff --git a/local/rest_api_gcbm/gcbm_simulation/routes.py b/local/rest_api_gcbm/gcbm_simulation/routes.py new file mode 100644 index 00000000..d9e869b9 --- /dev/null +++ b/local/rest_api_gcbm/gcbm_simulation/routes.py @@ -0,0 +1,9 @@ +from .views import Gcbm, GbcmUpload, Config, GcbmRun, DownloadGcbmResult + +def initialize_routes(api): + api.add_resource(Gcbm, '/gcbm/new') + api.add_resource(GbcmUpload, '/gcbm/upload/?category=category') + api.add_resource(GcbmRun, '/dynamic') + api.add_resource(Config, '/config') + api.add_resource(DownloadGcbmResult, '/gcbm/download') + diff --git a/local/rest_api_gcbm/gcbm_simulation/utils.py b/local/rest_api_gcbm/gcbm_simulation/utils.py new file mode 100644 index 00000000..c6fdcee5 --- /dev/null +++ b/local/rest_api_gcbm/gcbm_simulation/utils.py @@ -0,0 +1,46 @@ +import logging +import shutil +import json +import time +import subprocess +import os + +def launch_run(title, input_dir): + s = time.time() + logging.debug("Starting run") + with open(f"{input_dir}/gcbm_logs.csv", "w+") as f: + res = subprocess.Popen( + [ + "/opt/gcbm/moja.cli", + "--config_file", + "gcbm_config.cfg", + "--config_provider", + "provider_config.json", + ], + stdout=f, + cwd=f"{input_dir}", + ) + logging.debug("Communicating") + (output, err) = res.communicate() + logging.debug("Communicated") + + if not os.path.exists(f"{input_dir}/output"): + logging.error(err) + return "OK" + logging.debug("Output exists") + + # cut and paste output folder to app/output/simulation_name + shutil.copytree(f"{input_dir}/output", (f"{os.getcwd()}/output/{title}")) + shutil.make_archive( + f"{os.getcwd()}/output/{title}", "zip", f"{os.getcwd()}/output/{title}" + ) + shutil.rmtree((f"{input_dir}/output")) + logging.debug("Made archive") + e = time.time() + + logging.debug("Generated URL") + response = { + "exitCode": res.returncode, + "execTime": e - s, + "response": "Operation executed successfully. Downloadable links for input and output are attached in the response. Alternatively, you may also download this simulation input and output results by making a request at gcbm/download with the title in the body.", + } \ No newline at end of file diff --git a/local/rest_api_gcbm/gcbm_simulation/views.py b/local/rest_api_gcbm/gcbm_simulation/views.py new file mode 100644 index 00000000..8e5120b0 --- /dev/null +++ b/local/rest_api_gcbm/gcbm_simulation/views.py @@ -0,0 +1,145 @@ +from nis import cat +import os +import shutil +import json +from unicodedata import category +from .preprocess import GCBMSimulation +import rasterio as rst +from flask import jsonify, request +from flask_restful import Resource +from threading import Thread +from .utils import launch_run + + + + +class Gcbm(Resource): + + #create new gcbm + def post(self): + title = request.form.get("title") or "simulation" + title = "".join(c for c in title if c.isalnum()) + # input_dir = f"{title}" + input_dir = f"{os.getcwd()}/input/{title}" + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + message = "New {title} simulation started. Please move on to the next stage for uploading files at /gcbm/upload." + else: + message = "Simulation already exists with name {title}. Please check the list of simulations present before proceeding with a new simulation at gcbm/list. You may also download the input and output files for this simulation at gcbm/download sending parameter title in the body." + return {"data": message}, 200 + + #get a list of simulations + def get(self): + pass + + + +""" + +Upload details stores the information about the files upload in a json file. + +sample_db = { + "filename":{ + "path": "", + "category":"disturbance" + }, + "filename": { + "path": "", + "category": "classifier" + } +} """ + + +class GbcmUpload(Resource): + + # this is used to fetch all files uploaded in regards to that simulation + def get(self): + title = request.form.get("title") or "simulation" + input_dir = f"{os.getcwd()}/input/{title}" + if os.path.exists(f"{input_dir}/upload_details.json"): + with open( + f"{input_dir}/upload_details.json", "w+", encoding="utf8" + ) as upload_details: + upload_details_dictionary = json.load(upload_details) + return jsonify( + { + "message": "File retrived successfully", + "data": upload_details_dictionary, + } + ) + + # upload files to a paticular category. + def post(self): + """ sample - payload ={ + "title": "run4", + "file": "file-uploaded", + "category": "classifier" + } """ + + title = request.form.get("title") or "simulation" + category = request.args.get("category") + files = request.files.getlist("file") + + input_dir = f"{os.getcwd()}/input/{title}" + + if os.path.exists(f"{input_dir}/upload_details.json"): + with open( + f"{input_dir}/upload_details.json", "w+", encoding="utf8" + ) as upload_details: + upload_details_dictionary = json.load(upload_details) + else: + with open( + f"{input_dir}/upload_details.json", "w", encoding="utf8" + ) as upload_details: + upload_details_dictionary = {} + # uploadDetailsDictionary = json.load(upload_details) + # print(uploadDetailsDictionary) + + for file in files: + file.save(f"{input_dir}/{file.filename}") + upload_details_dictionary[file.filename] = { + "path": f"{input_dir}/{file.filename}", + "category": category, + } + json.dump(upload_details_dictionary, upload_details, indent=4) + + return { + "data": "All files uploaded succesfully. Proceed to the next step of the API at gcbm/dynamic." + } + + +class Config(Resource): + + # this get request will fetch the config templates. E.g modules.json, provider.json , and return them so that user can see them as seen on flint UI here -- https://flint-ui.vercel.app/gcbm/configurations/local-domain + def get(self): + pass + + def post(self): + pass + + +# Running the simulation +class GcbmRun(Resource): + def post(self): + title = request.form.get("title") or "simulation" + input_dir = f"{os.getcwd()}/input/{title}" + + gcbm_simulation = GCBMSimulation(input_dir=input_dir) + gcbm_simulation.set_config_templates() + gcbm_simulation.add_database_to_provider_config() + gcbm_simulation.add_files_to_provider_config_layers() + gcbm_simulation.generate_provider_config() + + thread = Thread( + target=launch_run, kwargs={"title": title, "input_dir": input_dir} + ) + thread.start() + return {"status": "Run started"}, 200 + + +class DownloadGcbmResult(Resource): + + #download result /or check status of list is not avaliable + def get(self): + pass + diff --git a/local/rest_api_gcbm/preprocess.py b/local/rest_api_gcbm/preprocess.py deleted file mode 100644 index 261d2ed8..00000000 --- a/local/rest_api_gcbm/preprocess.py +++ /dev/null @@ -1,354 +0,0 @@ -import os -import shutil -import json -import rasterio as rst - - -def get_config_templates(input_dir): - if not os.path.exists(f"{input_dir}/templates"): - shutil.copytree( - f"{os.getcwd()}/templates", f"{input_dir}/templates", dirs_exist_ok=False - ) - - -# TODO: there needs to be a link between the files configured here append -# the ["vars"] attribute of modules_cbm.json -> CBMDisturbanceListener -# current hack is to drop the last five characters, but thats very fragile -def get_modules_cbm_config(input_dir): - with open(f"{input_dir}/templates/modules_cbm.json", "r+") as modules_cbm_config: - disturbances = [] - data = json.load(modules_cbm_config) - for file in os.listdir(f"{input_dir}/disturbances/"): - disturbances.append( - file.split(".")[0][:-5] - ) # drop `_moja` to match modules_cbm.json template - modules_cbm_config.seek(0) - data["Modules"]["CBMDisturbanceListener"]["settings"]["vars"] = disturbances - json.dump(data, modules_cbm_config, indent=4) - modules_cbm_config.truncate() - - -def get_provider_config(input_dir): - with open(f"{input_dir}/templates/provider_config.json", "r+") as provider_config: - lst = [] - data = json.load(provider_config) - - for file in os.listdir(f"{input_dir}/db/"): - d = dict() - d["path"] = file - d["type"] = "SQLite" - data["Providers"]["SQLite"] = d - provider_config.seek(0) - - for file in os.listdir(f"{input_dir}/disturbances/"): - d = dict() - d["name"] = file[:-10] - d["layer_path"] = file - d["layer_prefix"] = file[:-5] - lst.append(d) - provider_config.seek(0) - data["Providers"]["RasterTiled"]["layers"] = lst - - for file in os.listdir(f"{input_dir}/classifiers/"): - d = dict() - d["name"] = file[:-10] - d["layer_path"] = file - d["layer_prefix"] = file[:-5] - lst.append(d) - provider_config.seek(0) - data["Providers"]["RasterTiled"]["layers"] = lst - - for file in os.listdir(f"{input_dir}/miscellaneous/"): - d = dict() - d["name"] = file[:-10] - d["layer_path"] = file - d["layer_prefix"] = file[:-5] - lst.append(d) - provider_config.seek(0) - data["Providers"]["RasterTiled"]["layers"] = lst - - Rasters = [] - Rastersm = [] - nodatam = [] - nodata = [] - cellLatSize = [] - cellLonSize = [] - paths = [] - - for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/disturbances/")): - for file in files: - fp = os.path.join(root, file) - Rasters.append(fp) - paths.append(fp) - - for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/classifiers/")): - for file in files: - fp1 = os.path.join(root, file) - Rasters.append(fp1) - paths.append(fp1) - - for nd in Rasters: - img = rst.open(nd) - t = img.transform - x = t[0] - y = -t[4] - n = img.nodata - cellLatSize.append(x) - cellLonSize.append(y) - nodata.append(n) - - result = all(element == cellLatSize[0] for element in cellLatSize) - if result: - cellLat = x - cellLon = y - nd = n - blockLat = x * 400 - blockLon = y * 400 - tileLat = x * 4000 - tileLon = y * 4000 - else: - print("Corrupt files") - - provider_config.seek(0) - - data["Providers"]["RasterTiled"]["cellLonSize"] = cellLon - data["Providers"]["RasterTiled"]["cellLatSize"] = cellLat - data["Providers"]["RasterTiled"]["blockLonSize"] = blockLon - data["Providers"]["RasterTiled"]["blockLatSize"] = blockLat - data["Providers"]["RasterTiled"]["tileLatSize"] = tileLat - data["Providers"]["RasterTiled"]["tileLonSize"] = tileLon - - json.dump(data, provider_config, indent=4) - provider_config.truncate() - - dictionary = { - "layer_type": "GridLayer", - "layer_data": "Byte", - "nodata": nd, - "tileLatSize": tileLat, - "tileLonSize": tileLon, - "blockLatSize": blockLat, - "blockLonSize": blockLon, - "cellLatSize": cellLat, - "cellLonSize": cellLon, - } - - # should be able to accept variable number of inputs, but requires - # means for user to specify/verify correct ["attributes"] - def get_input_layers(): - for root, dirs, files in os.walk( - os.path.abspath(f"{input_dir}/miscellaneous/") - ): - for file in files: - fp2 = os.path.join(root, file) - Rastersm.append(fp2) - - for i in Rastersm: - img = rst.open(i) - d = img.nodata - nodatam.append(d) - - with open( - f"{input_dir}/initial_age_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["layer_type"] = "GridLayer" - dictionary["layer_data"] = "Int16" - dictionary["nodata"] = nodatam[1] - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/mean_annual_temperature_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["layer_type"] = "GridLayer" - dictionary["layer_data"] = "Float32" - dictionary["nodata"] = nodatam[0] - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/Classifier1_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["layer_type"] = "GridLayer" - dictionary["layer_data"] = "Byte" - dictionary["nodata"] = nd - dictionary["attributes"] = { - "1": "TA", - "2": "BP", - "3": "BS", - "4": "JP", - "5": "WS", - "6": "WB", - "7": "BF", - "8": "GA", - } - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/Classifier2_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["layer_type"] = "GridLayer" - dictionary["layer_data"] = "Byte" - dictionary["nodata"] = nd - dictionary["attributes"] = {"1": "5", "2": "6", "3": "7", "4": "8"} - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/disturbances_2011_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["layer_data"] = "Byte" - dictionary["nodata"] = nd - dictionary["attributes"] = { - "1": {"year": 2011, "disturbance_type": "Wildfire", "transition": 1} - } - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/disturbances_2012_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["attributes"] = { - "1": {"year": 2012, "disturbance_type": "Wildfire", "transition": 1} - } - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/disturbances_2013_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["attributes"] = { - "1": { - "year": 2013, - "disturbance_type": "Mountain pine beetle — Very severe impact", - "transition": 1, - }, - "2": { - "year": 2013, - "disturbance_type": "Wildfire", - "transition": 1, - }, - } - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/disturbances_2014_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["attributes"] = { - "1": { - "year": 2014, - "disturbance_type": "Mountain pine beetle — Very severe impact", - "transition": 1, - } - } - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/disturbances_2015_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["attributes"] = { - "1": {"year": 2016, "disturbance_type": "Wildfire", "transition": 1} - } - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/disturbances_2016_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["attributes"] = { - "1": {"year": 2016, "disturbance_type": "Wildfire", "transition": 1} - } - json.dump(dictionary, json_file, indent=4) - - with open( - f"{input_dir}/disturbances_2018_moja.json", "w", encoding="utf8" - ) as json_file: - dictionary["attributes"] = { - "1": { - "year": 2018, - "disturbance_type": "Mountain pine beetle — Low impact", - "transition": 1, - } - } - json.dump(dictionary, json_file, indent=4) - - get_input_layers() - - def get_study_area(): - study_area = { - "tile_size": tileLat, - "block_size": blockLat, - "tiles": [ - { - "x": int(t[2]), - "y": int(t[5]), - "index": 12674, - } - ], - "pixel_size": cellLat, - "layers": [], - } - - with open(f"{input_dir}/study_area.json", "w", encoding="utf") as json_file: - list = [] - - for file in os.listdir(f"{input_dir}/miscellaneous/"): - d1 = dict() - d1["name"] = file[:-10] - d1["type"] = "VectorLayer" - list.append(d1) - study_area["layers"] = list - - for file in os.listdir(f"{input_dir}/classifiers/"): - d1 = dict() - d1["name"] = file[:-10] - d1["type"] = "VectorLayer" - d1["tags"] = ["classifier"] - list.append(d1) - study_area["layers"] = list - - for file in os.listdir(f"{input_dir}/disturbances/"): - d1 = dict() - d1["name"] = file[:-10] - d1["type"] = "DisturbanceLayer" - d1["tags"] = ["disturbance"] - list.append(d1) - study_area["layers"] = list - - json.dump(study_area, json_file, indent=4) - - get_study_area() - - for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/disturbances/")): - for file in files: - fp = os.path.join(root, file) - Rasters.append(fp) - paths.append(fp) - - for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/classifiers/")): - for file in files: - fp1 = os.path.join(root, file) - Rasters.append(fp1) - paths.append(fp1) - - for root, dirs, files in os.walk( - os.path.abspath(f"{input_dir}/miscellaneous/") - ): - for file in files: - fp2 = os.path.join(root, file) - paths.append(fp2) - - for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/templates/")): - for file in files: - fp3 = os.path.join(root, file) - paths.append(fp3) - - for root, dirs, files in os.walk(os.path.abspath(f"{input_dir}/db/")): - for file in files: - fp4 = os.path.join(root, file) - paths.append(fp4) - - # copy files to input directory - for i in paths: - shutil.copy2(i, (f"{input_dir}")) - - # delete folders from input directory - shutil.rmtree((f"{input_dir}/disturbances/")) - shutil.rmtree((f"{input_dir}/templates/")) - shutil.rmtree((f"{input_dir}/classifiers/")) - shutil.rmtree((f"{input_dir}/miscellaneous/")) - shutil.rmtree((f"{input_dir}/db/"))