From 6bf1e02dc9ae3b958883e3f4692230ef73b55977 Mon Sep 17 00:00:00 2001 From: James Mathews Date: Tue, 28 May 2024 13:46:06 -0400 Subject: [PATCH] Reduce memory footprint with an optimization in cache checking (#321) --- spatialprofilingtoolbox/ondemand/fast_cache_assessor.py | 2 ++ .../workflow/common/structure_centroids.py | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/spatialprofilingtoolbox/ondemand/fast_cache_assessor.py b/spatialprofilingtoolbox/ondemand/fast_cache_assessor.py index a6a755ae..682fcc24 100644 --- a/spatialprofilingtoolbox/ondemand/fast_cache_assessor.py +++ b/spatialprofilingtoolbox/ondemand/fast_cache_assessor.py @@ -45,6 +45,8 @@ def block_until_available(self): else: verbose=False up_to_date = self._cache_is_up_to_date(verbose=verbose) + if up_to_date: + break if verbose: logger.debug('Waiting for cache to be available.') check_count += 1 diff --git a/spatialprofilingtoolbox/workflow/common/structure_centroids.py b/spatialprofilingtoolbox/workflow/common/structure_centroids.py index 78acab18..f91091c3 100644 --- a/spatialprofilingtoolbox/workflow/common/structure_centroids.py +++ b/spatialprofilingtoolbox/workflow/common/structure_centroids.py @@ -93,10 +93,12 @@ def load_from_db(self, study: str | None = None) -> None: SELECT specimen, blob_contents FROM ondemand_studies_index osi WHERE osi.blob_type='centroids'; ''', (study, )) - specimens_to_blobs = tuple(cursor.fetchall()) self._studies[study] = {} - for _, blob in specimens_to_blobs: - obj = pickle.loads(blob) + while True: + row = cursor.fetchone() + if row is None: + break + obj = pickle.loads(row[1]) for key, value in obj.items(): if not key in self._studies: self._studies[study][key] = {}