fixing the DB dump all cybercastor riverscapes script

lib/cybercastor/cybercastor/add_job.py

@@ -383,12 +383,9 @@ def change_task_status(cc_api: CybercastorAPI, job_id, tasks, op):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument(
-        'job_json', help='The job specification JSON file', type=str)
-    parser.add_argument('stage', help='Cybercastor API stage',
-                        type=str, default='production')
-    parser.add_argument('--verbose', help='(optional) a little extra logging ',
-                        action='store_true', default=False)
+    parser.add_argument('job_json', help='The job specification JSON file', type=str)
+    parser.add_argument('stage', help='Cybercastor API stage', type=str, default='production')
+    parser.add_argument('--verbose', help='(optional) a little extra logging ', action='store_true', default=False)
 
     args = dotenv.parse_args_env(parser)
 

lib/cybercastor/cybercastor/classes/RiverscapesAPI.py

@@ -1,6 +1,6 @@
 import os
 import webbrowser
-from http.server import BaseHTTPRequestHandler, HTTPServer
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from typing import Dict
 from urllib.parse import urlencode, urlparse, urlunparse
 import requests

lib/cybercastor/cybercastor/dump_all.py

@@ -12,7 +12,7 @@
 from cybercastor.lib.dump.dump_views import dump_views
 
 
-def dump_all(sqlite_db_dir, cybercastor_api_url, username, password, template_geom, stage):
+def dump_all(sqlite_db_dir, cc_stage, template_geom, rs_stage):
     """_summary_
 
     Args:
@@ -26,12 +26,10 @@ def dump_all(sqlite_db_dir, cybercastor_api_url, username, password, template_ge
     log = Logger('Dump all Riverscapes and Cybercastor data to sqlite')
 
     if not os.path.exists(template_geom):
-        log.error(
-            f'The GeoPackge with HUC geoemtry does not exist: {template_geom}')
-        raise Exception(
-            f'The GeoPackge with HUC geoemtry does not exist: {template_geom}')
+        log.error(f'The GeoPackge with HUC geoemtry does not exist: {template_geom}')
+        raise Exception(f'The GeoPackge with HUC geoemtry does not exist: {template_geom}')
 
-    sqlite_db_path = os.path.join(sqlite_db_dir, f'DataExchange_{stage}.gpkg')
+    sqlite_db_path = os.path.join(sqlite_db_dir, f'DataExchange_{rs_stage}.gpkg')
 
     # TODO: TEMPORARY Cleanup DB File
     # if os.path.exists(sqlite_db_path):
@@ -47,9 +45,9 @@ def dump_all(sqlite_db_dir, cybercastor_api_url, username, password, template_ge
     create_database('cybercastor/lib/dump/schema.sql', sqlite_db_path)
 
     # Then add the cybercastor data
-    # dump_cybercastor(sqlite_db_path, cybercastor_api_url, username, password, stage)
+    # dump_cybercastor(sqlite_db_path, cc_stage, stage)
     # Then add the riverscapes data (authentication will be a browser popup)
-    dump_riverscapes(template_geom, stage)
+    dump_riverscapes(sqlite_db_path, rs_stage)
     # # Then write any additional views
     # dump_views(sqlite_db_path)
 
@@ -86,32 +84,20 @@ def create_database(schema_file_path: str, db_path: str):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument(
-        'output_db_path', help='The final resting place of the SQLITE DB', type=str)
-    parser.add_argument('cybercastor_api_url',
-                        help='URL to the cybercastor API', type=str)
-    parser.add_argument(
-        'username', help='Cybercastor API URL Username', type=str)
-    parser.add_argument(
-        'password', help='Cybercastor API URL Password', type=str)
-    parser.add_argument(
-        'template_geom', help='the template gpkg of huc10 geometry', type=str)
-    parser.add_argument(
-        'stage', help='Riverscapes stage', type=str, default='production')
-    parser.add_argument('--verbose', help='(optional) a little extra logging ',
-                        action='store_true', default=False)
+    parser.add_argument('output_db_path', help='The final resting place of the SQLITE DB', type=str)
+    parser.add_argument('cc_stage', help='Cybercastor API stage', type=str, default='production')
+    parser.add_argument('template_geom', help='the template gpkg of huc10 geometry', type=str)
+    parser.add_argument('rs_stage', help='Riverscapes stage', type=str, default='production')
+    parser.add_argument('--verbose', help='(optional) a little extra logging ', action='store_true', default=False)
     args = dotenv.parse_args_env(parser)
 
     # Initiate the log file
     log = Logger("SQLite Riverscapes Dump")
     log.setup(logPath=os.path.join(args.output_db_path,
               "dump_sqlite.log"), verbose=args.verbose)
 
-    fixedurl = args.cybercastor_api_url.replace(':/', '://')
-
     try:
-        dump_all(args.output_db_path, fixedurl, args.username,
-                 args.password, args.template_geom, args.stage)
+        dump_all(args.output_db_path, args.cc_stage, args.template_geom, args.rs_stage)
 
     except Exception as e:
         log.error(e)

lib/cybercastor/cybercastor/lib/dump/dump_riverscapes.py

@@ -45,7 +45,7 @@ def dump_riverscapes(sqlite_db_path, stage):
     current_date = datetime.now()
     grand_total = 0
 
-    # Determine last created date projects in the database. 
+    # Determine last created date projects in the database.
     # Delete all projects that were in that same day and then start the download
     # for that day over again. This will ensure we don't have duplicates.
     curs.execute("SELECT MAX(created_on) FROM rs_projects")