From e180a5597abe56a7faa2d097d8340c818b923e6e Mon Sep 17 00:00:00 2001 From: Matt Reimer Date: Thu, 14 Nov 2024 16:12:04 -0800 Subject: [PATCH] updating demo scripts with new param --- .../riverscapes/lib/dump/dump_riverscapes.py | 2 +- lib/riverscapes/riverscapes/merge-projects.py | 2 +- .../riverscapes/merge_rme_scrapes.py | 2 +- lib/riverscapes/riverscapes/scrape-rme.py | 2 +- .../riverscapes/scripts/addTags.py | 2 +- .../riverscapes/scripts/byteSize.py | 2 +- .../riverscapes/scripts/changeOwnership.py | 2 +- .../riverscapes/scripts/changeVisibility.py | 2 +- .../riverscapes/scripts/deleteProjects.py | 2 +- .../riverscapes/scripts/demo_search.py | 19 +++++++++---------- .../riverscapes/scripts/file_download.py | 2 +- .../riverscapes/scripts/rebuild_tiles.py | 2 +- .../riverscapes/scripts/searchTest.py | 2 +- 13 files changed, 21 insertions(+), 22 deletions(-) diff --git a/lib/riverscapes/riverscapes/lib/dump/dump_riverscapes.py b/lib/riverscapes/riverscapes/lib/dump/dump_riverscapes.py index 0b9905e8..7447a38d 100644 --- a/lib/riverscapes/riverscapes/lib/dump/dump_riverscapes.py +++ b/lib/riverscapes/riverscapes/lib/dump/dump_riverscapes.py @@ -49,7 +49,7 @@ def dump_riverscapes(rs_api: RiverscapesAPI, db_path: str): searchParams.createdOnFrom = last_inserted # Create a timedelta object with a difference of 1 day - for project, _stats, _searchtotal in rs_api.search(searchParams, progress_bar=True, page_size=100): + for project, _stats, _searchtotal, _prg in rs_api.search(searchParams, progress_bar=True, page_size=100): # Attempt to retrieve the huc10 from the project metadata if it exists huc10 = None diff --git a/lib/riverscapes/riverscapes/merge-projects.py b/lib/riverscapes/riverscapes/merge-projects.py index 510792d9..d486b8d7 100644 --- a/lib/riverscapes/riverscapes/merge-projects.py +++ b/lib/riverscapes/riverscapes/merge-projects.py @@ -420,7 +420,7 @@ def main(): }) projects_lookup: Dict[str, RiverscapesProject] = {} - for project, _stats, search_total in api.search(search_params, progress_bar=True): + for project, _stats, search_total, _prg in api.search(search_params, progress_bar=True): if search_total < 2: log.error(f'Insufficient number of projects ({search_total}) found with type {args.project_type} and tags {args.project_tags}. 2 or more needed.') sys.exit(1) diff --git a/lib/riverscapes/riverscapes/merge_rme_scrapes.py b/lib/riverscapes/riverscapes/merge_rme_scrapes.py index 51c12b45..6c7a9a9e 100644 --- a/lib/riverscapes/riverscapes/merge_rme_scrapes.py +++ b/lib/riverscapes/riverscapes/merge_rme_scrapes.py @@ -28,7 +28,7 @@ def merge_rme_scrapes(rs_api: RiverscapesAPI, search_params: RiverscapesSearchPa log = Logger('Merge RME Scrapes') # Create a timedelta object with a difference of 1 day - for project, _stats, _searchtotal in rs_api.search(search_params, progress_bar=True, page_size=100): + for project, _stats, _searchtotal, _prg in rs_api.search(search_params, progress_bar=True, page_size=100): # Attempt to retrieve the huc10 from the project metadata if it exists huc10 = None diff --git a/lib/riverscapes/riverscapes/scrape-rme.py b/lib/riverscapes/riverscapes/scrape-rme.py index 73b75c2c..fd645c8f 100644 --- a/lib/riverscapes/riverscapes/scrape-rme.py +++ b/lib/riverscapes/riverscapes/scrape-rme.py @@ -37,7 +37,7 @@ def scrape_rme(rs_api: RiverscapesAPI, search_params: RiverscapesSearchParams, d metric_ids = {} # Loop over all projects yielded by the search - for project, _stats, _searchtotal in rs_api.search(search_params, progress_bar=True, page_size=100): + for project, _stats, _searchtotal, _prg in rs_api.search(search_params, progress_bar=True, page_size=100): try: # Attempt to retrieve the huc10 from the project metadata if it exists huc10 = None diff --git a/lib/riverscapes/riverscapes/scripts/addTags.py b/lib/riverscapes/riverscapes/scripts/addTags.py index 92d95e3f..2604c8c1 100644 --- a/lib/riverscapes/riverscapes/scripts/addTags.py +++ b/lib/riverscapes/riverscapes/scripts/addTags.py @@ -53,7 +53,7 @@ def add_tag(riverscapes_api: RiverscapesAPI): # Create a timedelta object with a difference of 1 day total = 0 - for project, _stats, search_total in riverscapes_api.search(search_params, progress_bar=True): + for project, _stats, search_total, _prg in riverscapes_api.search(search_params, progress_bar=True): total = search_total if any(tag not in project.tags for tag in tags): changeable_projects.append(project) diff --git a/lib/riverscapes/riverscapes/scripts/byteSize.py b/lib/riverscapes/riverscapes/scripts/byteSize.py index 240369e5..12f5997d 100644 --- a/lib/riverscapes/riverscapes/scripts/byteSize.py +++ b/lib/riverscapes/riverscapes/scripts/byteSize.py @@ -33,7 +33,7 @@ def total_bytes_calc(api: RiverscapesAPI): log.title("Loop over each project and \"DO\" somewthing with each one") # Note how we keep the page size low here because byte size can be a little more expensive to calculate - for project, _stats, _total in api.search(RiverscapesSearchParams({"projectTypeId": "vbet"}), progress_bar=True, page_size=100): + for project, _stats, _total, _prg in api.search(RiverscapesSearchParams({"projectTypeId": "vbet"}), progress_bar=True, page_size=100): size = project.json['totalSize'] total_bytes += size diff --git a/lib/riverscapes/riverscapes/scripts/changeOwnership.py b/lib/riverscapes/riverscapes/scripts/changeOwnership.py index d675f5ff..68f9eb6e 100644 --- a/lib/riverscapes/riverscapes/scripts/changeOwnership.py +++ b/lib/riverscapes/riverscapes/scripts/changeOwnership.py @@ -38,7 +38,7 @@ def change_owner(riverscapes_api: RiverscapesAPI): changeable_projects: List[RiverscapesProject] = [] total = 0 - for project, _stats, search_total in riverscapes_api.search(search_params, progress_bar=True): + for project, _stats, search_total, _prg in riverscapes_api.search(search_params, progress_bar=True): total = search_total if project.json['ownedBy']['id'] != new_org_id: changeable_projects.append(project) diff --git a/lib/riverscapes/riverscapes/scripts/changeVisibility.py b/lib/riverscapes/riverscapes/scripts/changeVisibility.py index 56cd19bd..df0f9359 100644 --- a/lib/riverscapes/riverscapes/scripts/changeVisibility.py +++ b/lib/riverscapes/riverscapes/scripts/changeVisibility.py @@ -42,7 +42,7 @@ def changeVis(riverscapes_api: RiverscapesAPI): changeable_projects: List[RiverscapesProject] = [] total = 0 - for project, _stats, search_total in riverscapes_api.search(search_params, progress_bar=True): + for project, _stats, search_total, _prg in riverscapes_api.search(search_params, progress_bar=True): total = search_total if project.visibility != new_visibility: changeable_projects.append(project) diff --git a/lib/riverscapes/riverscapes/scripts/deleteProjects.py b/lib/riverscapes/riverscapes/scripts/deleteProjects.py index 3bac8c51..e780129b 100644 --- a/lib/riverscapes/riverscapes/scripts/deleteProjects.py +++ b/lib/riverscapes/riverscapes/scripts/deleteProjects.py @@ -43,7 +43,7 @@ def delete_by_tags(riverscapes_api: RiverscapesAPI): deletable_projects: List[RiverscapesProject] = [] total = 0 - for project, _stats, _search_total in riverscapes_api.search(search_params, progress_bar=True): + for project, _stats, _search_total, _prg in riverscapes_api.search(search_params, progress_bar=True): deletable_projects.append(project) # Now write all projects to a log file as json diff --git a/lib/riverscapes/riverscapes/scripts/demo_search.py b/lib/riverscapes/riverscapes/scripts/demo_search.py index 84c5d6d1..b5fb15a6 100644 --- a/lib/riverscapes/riverscapes/scripts/demo_search.py +++ b/lib/riverscapes/riverscapes/scripts/demo_search.py @@ -40,7 +40,7 @@ def simple_search(api: RiverscapesAPI): # NOTE: RiverscapesSearchParams will throw errors for common mistakes and anything it doesn't recognize as valid # ==================================================================================================== log.title("All possible search parameters") - search_count = [x for x, _stats, _total in api.search(RiverscapesSearchParams({ + search_count = [x for x, _stats, _total, _prg in api.search(RiverscapesSearchParams({ "projectTypeId": "vbet", # Only return projects of this type "keywords": "my search terms", # This will give a warning since keyword searches are not that useful in programmatic searches "name": "my project name", @@ -77,7 +77,7 @@ def simple_search(api: RiverscapesAPI): # NB: We set max_results here for demo purposes so this doesn't take 20 minutes but you probably don't want to do that in production # ==================================================================================================== log.title("Loop over each project and \"DO\" somewthing with each one") - for project, stats, _total in api.search(RiverscapesSearchParams({"projectTypeId": "vbet"}), progress_bar=True, max_results=1234): + for project, stats, _total, _prg in api.search(RiverscapesSearchParams({"projectTypeId": "vbet"}), progress_bar=True, max_results=1234): # Do a thing (like tag the project, delete it etc.) # INSERT THING DOING HERE log.debug(f"Project {project.id} has {len(project.json['tags'])} tags") @@ -92,7 +92,7 @@ def simple_search(api: RiverscapesAPI): "Runner": "Cybercastor", } }) - searched_project_ids = [p.id for p, _stats, _total in api.search(search_params, progress_bar=True, max_results=1234)] + searched_project_ids = [p.id for p, _stats, _total, _prg in api.search(search_params, progress_bar=True, max_results=1234)] log.debug(f"Found {len(searched_project_ids)} projects") # EXAMPLE Collect all the metadata for each project by id @@ -103,7 +103,7 @@ def simple_search(api: RiverscapesAPI): # Used https://geojson.io to get a rough bbox to limit this query roughing to washington state (which makes the query cheaper) "bbox": [-125.40936477595693, 45.38966396117303, -116.21237724715607, 49.470853578429626] }) - searched_project_meta = {p.id: p.project_meta for p, _stats, _total in api.search(search_params, progress_bar=True, max_results=1234)} + searched_project_meta = {p.id: p.project_meta for p, _stats, _total, _prg in api.search(search_params, progress_bar=True, max_results=1234)} log.info(f"Found {len(searched_project_meta)} projects") @@ -165,7 +165,7 @@ def simple_search_with_cache(api: RiverscapesAPI): data = json.load(f) else: log.info("Querying the server for fresh data") - data = {p.id: p.project_meta for p, _stats, _total in api.search(search_params, progress_bar=True, max_results=1234)} + data = {p.id: p.project_meta for p, _stats, _total, _prg in api.search(search_params, progress_bar=True, max_results=1234)} # Save the data to a file for later with open(cache_filename, 'w', encoding='utf8') as f: json.dump(data, f, indent=2) @@ -198,7 +198,7 @@ def stream_to_file(api: RiverscapesAPI): f_json.write('[\n') f_csv.write("id, project_type, huc, model_version, created_date\n") counter = 0 - for proj, _stats, _total in api.search(search_params, progress_bar=True, max_results=1234): + for proj, _stats, _total, _prg in api.search(search_params, progress_bar=True, max_results=1234): if counter > 0: f_json.write(',\n') json.dump(proj.json, f_json, indent=2) @@ -241,7 +241,7 @@ def find_duplicates(api: RiverscapesAPI): # ... # } huc_lookup = {} - for project, _stats, _total in api.search(search_params, progress_bar=True): + for project, _stats, _total, _prg in api.search(search_params, progress_bar=True): if project.project_type is None: raise Exception(f"Project {project.id} has no project type. This is likely a query error") @@ -280,7 +280,7 @@ def proj_print_str(proj: RiverscapesProject): # You can instantiate the API like this but you need to call refresh_token # Note how it will ask y ou interactively for the stage. Specify the stage like this: # riverscapes_api = RiverscapesAPI(stage='dev') to not get the prompt - # + # # Also yout need to call riverscapes_api.shutdown() when you're done with it. riverscapes_api = RiverscapesAPI() riverscapes_api.refresh_token() @@ -296,11 +296,10 @@ def proj_print_str(proj: RiverscapesProject): # If you put it inside a finally block it will always run (even if there's an error or a keyboard interrupt like ctrl+c) riverscapes_api.shutdown() - # But there's a better way! (Method 2) # ==================================================================================================== # OR you can instantiate it with a "with" statement like this - # This might be slightly preferred because it handles the refresh token AND automatically shuts down the + # This might be slightly preferred because it handles the refresh token AND automatically shuts down the # polling process that refreshes the token when you're done with it with RiverscapesAPI() as riverscapes_api: simple_search(riverscapes_api) diff --git a/lib/riverscapes/riverscapes/scripts/file_download.py b/lib/riverscapes/riverscapes/scripts/file_download.py index 5cf4a2ba..f6c41667 100644 --- a/lib/riverscapes/riverscapes/scripts/file_download.py +++ b/lib/riverscapes/riverscapes/scripts/file_download.py @@ -33,7 +33,7 @@ def download_files(riverscapes_api: RiverscapesAPI): # Make the search and download all necessary files # ================================================================================================================ - for project, _stats, _total in riverscapes_api.search(search_params): + for project, _stats, _total, _prg in riverscapes_api.search(search_params): # Since we're searching for a huc we can pretty reliably assume that we're only going to get one project dlhuc = project.project_meta['HUC'] diff --git a/lib/riverscapes/riverscapes/scripts/rebuild_tiles.py b/lib/riverscapes/riverscapes/scripts/rebuild_tiles.py index 5918baf1..3f393797 100644 --- a/lib/riverscapes/riverscapes/scripts/rebuild_tiles.py +++ b/lib/riverscapes/riverscapes/scripts/rebuild_tiles.py @@ -99,7 +99,7 @@ def rebuild_web_tiles(riverscapes_api: RiverscapesAPI): changeable_projects: List[RiverscapesProject] = [] total = 0 - for project, _stats, search_total in riverscapes_api.search(search_params, progress_bar=True): + for project, _stats, search_total, _prg in riverscapes_api.search(search_params, progress_bar=True): total = search_total changeable_projects.append(project['item']) diff --git a/lib/riverscapes/riverscapes/scripts/searchTest.py b/lib/riverscapes/riverscapes/scripts/searchTest.py index 94918880..e69f8f70 100644 --- a/lib/riverscapes/riverscapes/scripts/searchTest.py +++ b/lib/riverscapes/riverscapes/scripts/searchTest.py @@ -22,7 +22,7 @@ def change_owner(riverscapes_api: RiverscapesAPI): # Make the search and collect all the data # ================================================================================================================ - for project, _stats, search_total in riverscapes_api.search(search_params, progress_bar=True): + for project, _stats, search_total, _prg in riverscapes_api.search(search_params, progress_bar=True): print(search_total)