Refactor GCBM API

local/rest_api_gcbm/templates/modules_cbm.json

@@ -13,9 +13,7 @@
             "order": 3,
             "library": "moja.modules.cbm",
             "settings": {
-                "vars": [
-
-                ]
+                "vars": []
             }
         },
         "CBMDisturbanceEventModule": {
@@ -56,4 +54,4 @@
             "library": "internal.flint"
         }
     }
-}
+}

local/rest_api_gcbm/templates/provider_config.json

@@ -1,7 +1,7 @@
 {
     "Providers": {
         "SQLite": {
-            "path": "../input_database/gcbm_input.db",
+            "path": "",
             "type": "SQLite"
         },
         "RasterTiled": {

local/rest_api_gcbm/templates/variables.json

@@ -107,16 +107,6 @@
                 "provider": "SQLite"
             }
         },
-        "initial_classifier_set": {
-            "transform": {
-                "type": "CompositeTransform",
-                "library": "internal.flint",
-                "vars": [
-                    "Classifier2",
-                    "Classifier1"
-                ]
-            }
-        },
         "disturbance_matrix_associations": {
             "transform": {
                 "queryString": "SELECT dt.name AS disturbance_type, dma.spatial_unit_id, dma.disturbance_matrix_id FROM disturbance_matrix_association dma INNER JOIN disturbance_type dt ON dma.disturbance_type_id = dt.id",
@@ -165,93 +155,13 @@
                 "provider": "SQLite"
             }
         },
-        "mean_annual_temperature": {
-            "transform": {
-                "library": "internal.flint",
-                "type": "LocationIdxFromFlintDataTransform",
-                "provider": "RasterTiled",
-                "data_id": "mean_annual_temperature"
-            }
-        },
         "decay_parameters": {
             "transform": {
                 "queryString": "SELECT p.name AS pool, dp.base_decay_rate AS organic_matter_decay_rate, dp.prop_to_atmosphere AS prop_to_atmosphere, dp.q10 AS q10, dp.reference_temp AS reference_temp, dp.max_rate AS max_decay_rate_soft FROM decay_parameter dp INNER JOIN dom_pool dom ON dp.dom_pool_id = dom.id INNER JOIN pool p ON p.id = dom.pool_id",
                 "type": "SQLQueryTransform",
                 "library": "internal.flint",
                 "provider": "SQLite"
             }
-        },
-        "disturbances_2012": {
-            "transform": {
-                "library": "internal.flint",
-                "type": "LocationIdxFromFlintDataTransform",
-                "provider": "RasterTiled",
-                "data_id": "disturbances_2012"
-            }
-        },
-        "disturbances_2011": {
-            "transform": {
-                "library": "internal.flint",
-                "type": "LocationIdxFromFlintDataTransform",
-                "provider": "RasterTiled",
-                "data_id": "disturbances_2011"
-            }
-        },
-        "disturbances_2015": {
-            "transform": {
-                "library": "internal.flint",
-                "type": "LocationIdxFromFlintDataTransform",
-                "provider": "RasterTiled",
-                "data_id": "disturbances_2015"
-            }
-        },
-        "disturbances_2014": {
-            "transform": {
-                "library": "internal.flint",
-                "type": "LocationIdxFromFlintDataTransform",
-                "provider": "RasterTiled",
-                "data_id": "disturbances_2014"
-            }
-        },
-        "disturbances_2018": {
-            "transform": {
-                "library": "internal.flint",
-                "type": "LocationIdxFromFlintDataTransform",
-                "provider": "RasterTiled",
-                "data_id": "disturbances_2018"
-            }
-        },
-        "disturbances_2016": {
-            "transform": {
-                "library": "internal.flint",
-                "type": "LocationIdxFromFlintDataTransform",
-                "provider": "RasterTiled",
-                "data_id": "disturbances_2016"
-            }
-        },
-        "Classifier2": {
-            "transform": {
-                "library": "internal.flint",
-                "type": "LocationIdxFromFlintDataTransform",
-                "provider": "RasterTiled",
-                "data_id": "Classifier2"
-            }
-        },
-        "Classifier1": {
-            "transform": {
-                "library": "internal.flint",
-                "type": "LocationIdxFromFlintDataTransform",
-                "provider": "RasterTiled",
-                "data_id": "Classifier1"
-            }
-        },
-        "disturbances_2013": {
-            "transform": {
-                "library": "internal.flint",
-                "type": "LocationIdxFromFlintDataTransform",
-                "provider": "RasterTiled",
-                "data_id": "disturbances_2013"
-            }
         }
     }
-}
+}

local/rest_api_gcbm/tests/compare_json.py

@@ -0,0 +1,52 @@
+import os
+import json
+import sys
+
+def compare_json(json_1, json_2):
+    try:
+        assert sorted(json_1.items()) == sorted(json_2.items())
+        return True
+    except AssertionError:
+        return False
+
+def read_json(path: str):
+    with open(path, "r") as _file:
+        return json.load(_file)
+
+if __name__ == "__main__":
+    if len(sys.argv) >= 3:
+        original_folder = sys.argv[1]
+        new_folder = sys.argv[2]
+        print(f"Note: using {original_folder} for original JSONs and {new_folder} for new JSONs folders.")
+    else:
+        original_folder = "tests/reference/"
+        new_folder = "input/test-run/"
+    original_paths = sorted(os.listdir(original_folder))
+
+    #TODO: how do we check non-json config
+    new_paths = [file for file in sorted(os.listdir(new_folder)) if ".json" in file]
+
+    didnt_match = []
+    matched = []
+    for path, new_path in zip(original_paths, new_paths):
+
+        path = original_folder + path
+        new_path = new_folder + new_path
+
+        print(f"Now comparing: {path} vs {new_path}")
+        if ".json" in path:
+            obj_original, obj_new = read_json(path), read_json(new_path)
+            did_match = compare_json(obj_original, obj_new)
+
+            if not did_match:
+                didnt_match.append(new_path)
+            else:
+                matched.append(path)
+
+
+    print(f"Failures: {len(didnt_match)}/{len(new_paths)}")
+    print(f"Success: {len(matched)}/{len(new_paths)}")
+
+    if len(didnt_match) != 0:
+        print("Files that didn't match are: ")
+        print(didnt_match)

local/rest_api_gcbm/tests/readme.md

@@ -0,0 +1,6 @@
+## Compare auto-generated config against the GCBM Demo run
+
+Steps:
+1. Extract `GCBM_New_Demo_Run.zip` into `local/rest_api_gcbm/tests/`
+2. Run `python3 gcbm.py`
+3. Run `python3 tests/compare_json.py`

local/rest_api_gcbm/tests/reference/Classifier1.json

@@ -0,0 +1,21 @@
+{
+    "layer_type": "GridLayer",
+    "layer_data": "Byte",
+    "nodata": 255,
+    "tileLatSize": 1.0,
+    "tileLonSize": 1.0,
+    "blockLatSize": 0.1,
+    "blockLonSize": 0.1,
+    "cellLatSize": 0.00025,
+    "cellLonSize": 0.00025,
+    "attributes": {
+        "1": "TA",
+        "2": "BP",
+        "3": "BS",
+        "4": "JP",
+        "5": "WS",
+        "6": "WB",
+        "7": "BF",
+        "8": "GA"
+    }
+}

local/rest_api_gcbm/tests/reference/Classifier2.json

@@ -0,0 +1,17 @@
+{
+    "layer_type": "GridLayer",
+    "layer_data": "Byte",
+    "nodata": 255,
+    "tileLatSize": 1.0,
+    "tileLonSize": 1.0,
+    "blockLatSize": 0.1,
+    "blockLonSize": 0.1,
+    "cellLatSize": 0.00025,
+    "cellLonSize": 0.00025,
+    "attributes": {
+        "1": "5",
+        "2": "6",
+        "3": "7",
+        "4": "8"
+    }
+}

local/rest_api_gcbm/tests/reference/disturbances_2011.json

@@ -0,0 +1,18 @@
+{
+    "layer_type": "GridLayer",
+    "layer_data": "Byte",
+    "nodata": 255,
+    "tileLatSize": 1.0,
+    "tileLonSize": 1.0,
+    "blockLatSize": 0.1,
+    "blockLonSize": 0.1,
+    "cellLatSize": 0.00025,
+    "cellLonSize": 0.00025,
+    "attributes": {
+        "1": {
+            "year": 2011,
+            "disturbance_type": "Wildfire",
+            "transition": 1
+        }
+    }
+}

local/rest_api_gcbm/tests/reference/disturbances_2012.json

@@ -0,0 +1,18 @@
+{
+    "layer_type": "GridLayer",
+    "layer_data": "Byte",
+    "nodata": 255,
+    "tileLatSize": 1.0,
+    "tileLonSize": 1.0,
+    "blockLatSize": 0.1,
+    "blockLonSize": 0.1,
+    "cellLatSize": 0.00025,
+    "cellLonSize": 0.00025,
+    "attributes": {
+        "1": {
+            "year": 2012,
+            "disturbance_type": "Wildfire",
+            "transition": 1
+        }
+    }
+}

local/rest_api_gcbm/tests/reference/disturbances_2013.json

@@ -0,0 +1,23 @@
+{
+    "layer_type": "GridLayer",
+    "layer_data": "Byte",
+    "nodata": 255,
+    "tileLatSize": 1.0,
+    "tileLonSize": 1.0,
+    "blockLatSize": 0.1,
+    "blockLonSize": 0.1,
+    "cellLatSize": 0.00025,
+    "cellLonSize": 0.00025,
+    "attributes": {
+        "1": {
+            "year": 2013,
+            "disturbance_type": "Mountain pine beetle — Very severe impact",
+            "transition": 1
+        },
+        "2": {
+            "year": 2013,
+            "disturbance_type": "Wildfire",
+            "transition": 1
+        }
+    }
+}

local/rest_api_gcbm/tests/reference/disturbances_2014.json

@@ -0,0 +1,18 @@
+{
+    "layer_type": "GridLayer",
+    "layer_data": "Byte",
+    "nodata": 255,
+    "tileLatSize": 1.0,
+    "tileLonSize": 1.0,
+    "blockLatSize": 0.1,
+    "blockLonSize": 0.1,
+    "cellLatSize": 0.00025,
+    "cellLonSize": 0.00025,
+    "attributes": {
+        "1": {
+            "year": 2014,
+            "disturbance_type": "Mountain pine beetle — Very severe impact",
+            "transition": 1
+        }
+    }
+}

local/rest_api_gcbm/tests/reference/disturbances_2015.json

@@ -0,0 +1,18 @@
+{
+    "layer_type": "GridLayer",
+    "layer_data": "Byte",
+    "nodata": 255,
+    "tileLatSize": 1.0,
+    "tileLonSize": 1.0,
+    "blockLatSize": 0.1,
+    "blockLonSize": 0.1,
+    "cellLatSize": 0.00025,
+    "cellLonSize": 0.00025,
+    "attributes": {
+        "1": {
+            "year": 2015,
+            "disturbance_type": "Wildfire",
+            "transition": 1
+        }
+    }
+}

local/rest_api_gcbm/tests/reference/disturbances_2016.json

@@ -0,0 +1,18 @@
+{
+    "layer_type": "GridLayer",
+    "layer_data": "Byte",
+    "nodata": 255,
+    "tileLatSize": 1.0,
+    "tileLonSize": 1.0,
+    "blockLatSize": 0.1,
+    "blockLonSize": 0.1,
+    "cellLatSize": 0.00025,
+    "cellLonSize": 0.00025,
+    "attributes": {
+        "1": {
+            "year": 2016,
+            "disturbance_type": "Wildfire",
+            "transition": 1
+        }
+    }
+}

local/rest_api_gcbm/tests/reference/disturbances_2018.json

@@ -0,0 +1,18 @@
+{
+    "layer_type": "GridLayer",
+    "layer_data": "Byte",
+    "nodata": 255,
+    "tileLatSize": 1.0,
+    "tileLonSize": 1.0,
+    "blockLatSize": 0.1,
+    "blockLonSize": 0.1,
+    "cellLatSize": 0.00025,
+    "cellLonSize": 0.00025,
+    "attributes": {
+        "1": {
+            "year": 2018,
+            "disturbance_type": "Mountain pine beetle — Low impact",
+            "transition": 1
+        }
+    }
+}

local/rest_api_gcbm/tests/reference/gcbm_config.cfg

@@ -0,0 +1,7 @@
+config=localdomain.json
+config=pools_cbm.json
+config=modules_cbm.json
+config=modules_output.json
+config=spinup.json
+config=variables.json
+config=internal_variables.json