Feature/update schema

.github/workflows/test_database.yaml

@@ -35,22 +35,6 @@ jobs:
           python -m pip install --upgrade pip
           pip install . -r requirements.txt
 
-      - name: Create Stored Procedures
-        env:
-          DB_HOST: ${{secrets.DB_HOST}}
-          DB_NAME: ${{secrets.DB_NAME}}
-          DB_USER: ${{secrets.DB_USER}}
-          DB_PASSWORD: ${{secrets.DB_PASSWORD}}
-        run: |
-          export PGPASSWORD=$DB_PASSWORD
-          psql -h $DB_HOST -U $DB_USER -d $DB_NAME -a -f ./src/evagram/database/sql/proc_create_owners.sql
-          psql -h $DB_HOST -U $DB_USER -d $DB_NAME -a -f ./src/evagram/database/sql/proc_create_experiments.sql
-          psql -h $DB_HOST -U $DB_USER -d $DB_NAME -a -f ./src/evagram/database/sql/proc_create_groups.sql
-          psql -h $DB_HOST -U $DB_USER -d $DB_NAME -a -f ./src/evagram/database/sql/proc_create_plots.sql
-          psql -h $DB_HOST -U $DB_USER -d $DB_NAME -a -f ./src/evagram/database/sql/proc_create_variables.sql
-          psql -h $DB_HOST -U $DB_USER -d $DB_NAME -a -f ./src/evagram/database/sql/proc_create_observations.sql
-          psql -h $DB_HOST -U $DB_USER -d $DB_NAME -a -f ./src/evagram/database/sql/proc_create_observation_variable.sql
-
       - name: Run Input Tool
         env:
           DB_HOST: ${{secrets.DB_HOST}}

src/evagram/database/dataset.json

@@ -66,7 +66,7 @@
         {
             "plot_id": 322,
             "plot_file": "brightnessTemperature_10_hofx-vs-gsihofxbc.pkl",
-            "experiment_id": 3
+            "experiment_id": 12
         },
         {
             "plot_id": 323,
@@ -76,7 +76,7 @@
         {
             "plot_id": 114,
             "plot_file": "brightnessTemperature_8_effectiveerror-vs-gsifinalerror.pkl",
-            "experiment_id": 1
+            "experiment_id": 3
         },
         {
             "plot_id": 165,
@@ -85,7 +85,7 @@
         },
         {
             "plot_id": 966,
-            "plot_file": "windEastward_effectiveerrordiff-vs-gsifinalerror.pkl",
+            "plot_file": "windEastward__effectiveerrordiff-vs-gsifinalerror.pkl",
             "experiment_id": 96
         }
     ],

src/evagram/database/input_tool.py

@@ -2,6 +2,8 @@
 import json
 import os
 import sys
+import argparse
+from pathlib import Path
 import psycopg2
 from dotenv import load_dotenv
 
@@ -14,9 +16,47 @@
 db_user = os.environ.get('DB_USER')
 db_password = os.environ.get('DB_PASSWORD')
 
-# can be modified to the file path of experiment data
-EXPERIMENT_DATA_PATH = './tests/eva/'
+# default path configurations
+EXPERIMENT_DATA_PATH = './tests/eva'
 DATASET_PATH = './src/evagram/database/'
+PROCEDURES_PATH = './src/evagram/database/sql'
+
+
+def main(args):
+    global EXPERIMENT_DATA_PATH
+    parser = argparse.ArgumentParser()
+    parser.add_argument("experiment_path")
+    args = parser.parse_args(args)
+    experiment_path = Path(args.experiment_path)
+    if experiment_path.exists():
+        EXPERIMENT_DATA_PATH = args.experiment_path
+
+    conn = psycopg2.connect(
+        host=db_host,
+        port=db_port,
+        dbname=db_name,
+        user=db_user,
+        password=db_password
+    )
+    cur = conn.cursor()
+
+    create_procedures(cur)
+    drop_tables(cur)
+    create_tables(cur)
+    load_dataset_to_db(cur)
+
+    conn.commit()
+    cur.close()
+    conn.close()
+
+
+def create_procedures(cur):
+    for proc in os.listdir(PROCEDURES_PATH):
+        proc_file = os.path.join(PROCEDURES_PATH, proc)
+        if os.path.isfile(proc_file) and proc.startswith("proc_") and proc.endswith(".sql"):
+            contents = open(proc_file, 'r')
+            cur.execute(contents.read())
+            contents.close()
 
 
 def create_tables(cur):
@@ -32,8 +72,6 @@ def create_tables(cur):
     cur.execute("CALL public.create_observations();")
     # Plots table
     cur.execute("CALL public.create_plots();")
-    # Observation Variable join table
-    cur.execute("CALL public.create_observation_variable();")
 
 
 def drop_tables(cur):
@@ -43,7 +81,6 @@ def drop_tables(cur):
     cur.execute("DROP TABLE IF EXISTS groups CASCADE")
     cur.execute("DROP TABLE IF EXISTS observations CASCADE")
     cur.execute("DROP TABLE IF EXISTS variables CASCADE")
-    cur.execute("DROP TABLE IF EXISTS observation_variable CASCADE")
 
 
 def load_dataset_to_db(cur):
@@ -138,13 +175,8 @@ def add_plot(cur, plot_obj, observation_dirs):
     plot_components = filename_no_extension.split("_")
 
     var_name = plot_components[0]
-    # parse channel fields for brightnessTemperature observations
-    if var_name == "brightnessTemperature":
-        channel = plot_components[1]
-        group_name = plot_components[2]
-    else:
-        channel = None
-        group_name = plot_components[1]
+    channel = plot_components[1] if plot_components[1] != '' else None
+    group_name = plot_components[2]
 
     # insert observation, variable, group dynamically if not exist in database
     cur.execute("SELECT observation_id FROM observations WHERE observation_name=%s",
@@ -194,39 +226,13 @@ def add_plot(cur, plot_obj, observation_dirs):
     plot_obj["script"] = script
     plot_obj["observation_id"] = observation_id
     plot_obj["group_id"] = group_id
+    plot_obj["variable_id"] = variable_id
 
     # insert plot to database
     insert_table_record(cur, plot_obj, "plots")
 
-    # create relationship between observation and variable in join table
-    # only if at least one of them was just inserted
-    if new_observation or new_variable:
-        observation_variable_obj = {
-            "observation_id": observation_id,
-            "variable_id": variable_id
-        }
-        insert_table_record(cur, observation_variable_obj, "observation_variable")
-
     return 0
 
 
 if __name__ == "__main__":
-    if len(sys.argv) > 1:
-        EXPERIMENT_DATA_PATH = sys.argv[1]
-
-    conn = psycopg2.connect(
-        host=db_host,
-        port=db_port,
-        dbname=db_name,
-        user=db_user,
-        password=db_password
-    )
-    cur = conn.cursor()
-
-    drop_tables(cur)
-    create_tables(cur)
-    load_dataset_to_db(cur)
-
-    conn.commit()
-    cur.close()
-    conn.close()
+    main(sys.argv[1:])

src/evagram/database/sql/proc_create_plots.sql

@@ -6,27 +6,59 @@ CREATE OR REPLACE PROCEDURE public.create_plots(
 	)
 LANGUAGE 'sql'
 AS $BODY$
-CREATE TABLE IF NOT EXISTS plots (
-	plot_id serial PRIMARY KEY,
-	div VARCHAR,
-	script VARCHAR,
-	experiment_id INTEGER NOT NULL,
-	group_id INTEGER NOT NULL,
-	observation_id INTEGER NOT NULL,
-	CONSTRAINT fk_experiment
-		FOREIGN KEY (experiment_id)
-			REFERENCES experiments(experiment_id)
-			ON DELETE CASCADE,
-	CONSTRAINT fk_group
-		FOREIGN KEY (group_id)
-			REFERENCES groups(group_id)
-			ON DELETE CASCADE,
-	CONSTRAINT fk_observation
-		FOREIGN KEY (observation_id)
-			REFERENCES observations(observation_id)
-			ON DELETE CASCADE,
-	UNIQUE(experiment_id, group_id, observation_id)
-);
+CREATE TABLE IF NOT EXISTS public.plots
+(
+    plot_id integer NOT NULL GENERATED BY DEFAULT AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 2147483647 CACHE 1 ),
+    div character varying COLLATE pg_catalog."default",
+    script character varying COLLATE pg_catalog."default",
+    experiment_id integer NOT NULL,
+    group_id integer NOT NULL,
+    observation_id integer NOT NULL,
+    variable_id integer NOT NULL,
+    CONSTRAINT plots_pkey PRIMARY KEY (plot_id),
+    CONSTRAINT plots_experiment_id_group_id_o_78d8b74e_uniq UNIQUE (experiment_id, group_id, observation_id, variable_id),
+    CONSTRAINT plots_experiment_id_79cb5574_fk_experiments_experiment_id FOREIGN KEY (experiment_id)
+        REFERENCES public.experiments (experiment_id) 
+        ON DELETE CASCADE,
+    CONSTRAINT plots_group_id_85eaf2d2_fk_groups_group_id FOREIGN KEY (group_id)
+        REFERENCES public.groups (group_id) 
+        ON DELETE CASCADE,
+    CONSTRAINT plots_observation_id_26813e98_fk_observations_observation_id FOREIGN KEY (observation_id)
+        REFERENCES public.observations (observation_id) 
+        ON DELETE CASCADE,
+    CONSTRAINT plots_variable_id_394e0899_fk_variables_variable_id FOREIGN KEY (variable_id)
+        REFERENCES public.variables (variable_id) 
+        ON DELETE CASCADE
+)
+
+
+
+-- CREATE TABLE IF NOT EXISTS plots (
+-- 	plot_id serial PRIMARY KEY,
+-- 	div VARCHAR,
+-- 	script VARCHAR,
+-- 	experiment_id INTEGER NOT NULL,
+-- 	group_id INTEGER NOT NULL,
+-- 	observation_id INTEGER NOT NULL,
+-- 	variable_id INTEGER NOT NULL,
+-- 	CONSTRAINT fk_experiment
+-- 		FOREIGN KEY (experiment_id)
+-- 			REFERENCES experiments(experiment_id)
+-- 			ON DELETE CASCADE,
+-- 	CONSTRAINT fk_group
+-- 		FOREIGN KEY (group_id)
+-- 			REFERENCES groups(group_id)
+-- 			ON DELETE CASCADE,
+-- 	CONSTRAINT fk_observation
+-- 		FOREIGN KEY (observation_id)
+-- 			REFERENCES observation_variable(observation_id)
+-- 			ON DELETE CASCADE,
+-- 	CONSTRAINT fk_variable
+-- 		FOREIGN KEY (variable_id)
+-- 			REFERENCES variables(variable_id)
+-- 			ON DELETE CASCADE,
+-- 	UNIQUE(experiment_id, group_id, observation_id, variable_id)
+
 $BODY$;
 ALTER PROCEDURE public.create_plots()
     OWNER TO postgres;