Merge branch 'main' into parallel-cggnn

CHANGELOG.md

@@ -1,3 +1,12 @@
+# v0.16.2
+- Deprecates heavy index on large tables:
+  - Adds a new table for tracking scope ranges.
+  - Converts the former `source_specimen` column on `expression_quantification` to a `SERIAL`` integer.
+  - Makes tabular import keep track of ranges per-specimen in the new range_definitions table.
+  - Updates the "optimized" sparse matrix query to use the ranges rather than the former huge index.
+  - Deprecates the modify-constraints CLI entrypoint (only used internally now).
+  - Deprecates the expression indexing module, CLI entrypoint, etc.
+
 # v0.16.0
 - Separates datasets into own databases:
   - `DBCursor` and `DBConnection` usage streamlined, typically requires study-scoping (dataset-scoping).

pyproject.toml.unversioned

@@ -174,12 +174,10 @@ packages = [
 ]
 "spatialprofilingtoolbox.db.scripts" = [
     "create_schema.py",
-    "modify_constraints.py",
     "guess_channels_from_object_files.py",
     "status.py",
     "retrieve_feature_matrices.py",
     "drop.py",
-    "index_expressions_table.py",
     "drop_ondemand_computations.py"
 ]
 "spatialprofilingtoolbox.db.data_model" = [

spatialprofilingtoolbox/db/data_model/performance_tweaks.sql

@@ -14,6 +14,16 @@ ADD UNIQUE (feature_specification, specifier, ordinality) ;
 ALTER TABLE two_cohort_feature_association_test
 ADD UNIQUE (selection_criterion_1, selection_criterion_2, test, p_value, feature_tested) ;
 
+ALTER TABLE expression_quantification
+ADD range_identifier_integer SERIAL ;
+
+CREATE TABLE range_definitions (
+    scope_identifier VARCHAR(512),
+    tablename VARCHAR(512),
+    lowest_value INT,
+    highest_value INT
+) ;
+
 CREATE EXTENSION IF NOT EXISTS tablefunc;
 
 CREATE TABLE sample_strata (

spatialprofilingtoolbox/db/scripts/drop_ondemand_computations.py

@@ -16,6 +16,7 @@ def main():
         description='Drop ondemand-computed features, including values, specifiers, etc.'
     )
     add_argument(parser, 'database config')
+    add_argument(parser, 'study name')
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument(
         '--pending-only',
@@ -30,7 +31,7 @@ def main():
     args = parser.parse_args()
 
     config_file = get_and_validate_database_config(args)
-    with DBCursor(database_config_file=config_file) as cursor:
+    with DBCursor(database_config_file=config_file, study=args.study_name) as cursor:
         if args.pending_only:
             OnDemandComputationsDropper.drop(cursor, pending_only=True)
         if args.all:

spatialprofilingtoolbox/ondemand/scripts/cache_expressions_data_array.py

@@ -8,7 +8,6 @@
 from os import getcwd
 import sys
 
-from spatialprofilingtoolbox.db.expressions_table_indexer import ExpressionsTableIndexer
 from spatialprofilingtoolbox.workflow.common.structure_centroids import StructureCentroids
 from spatialprofilingtoolbox.ondemand.defaults import EXPRESSIONS_INDEX_FILENAME
 from spatialprofilingtoolbox.workflow.common.cli_arguments import add_argument
@@ -48,7 +47,6 @@ def main():
         message = '%s was not found, will do feature matrix pull after all.'
         logger.info(message, EXPRESSIONS_INDEX_FILENAME)
 
-    # ExpressionsTableIndexer.ensure_indexed_expressions_tables(database_config_file)
     puller = SparseMatrixPuller(database_config_file)
     puller.pull_and_write_to_files()