Update dbapi usage for psycopg v3

build/apiserver/Dockerfile

@@ -2,7 +2,7 @@ FROM python:3.11-slim
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update && apt install -y gcc libpq-dev curl && apt-get clean
 ARG PIP_NO_CACHE_DIR=1
-RUN python -m pip install psycopg2==2.9.9
+RUN python -m pip install psycopg==3.1.19
 RUN python -m pip install adiscstudies==0.11.0
 RUN python -m pip install numba==0.59.1
 RUN python -m pip install attrs==23.2.0

build/db/Dockerfile

@@ -9,7 +9,7 @@ RUN apt install python3-venv -y && apt-get clean
 RUN apt install python3-pip -y && apt-get clean
 RUN apt install -y libpq-dev && apt-get clean
 RUN apt install -y libgdal-dev && apt-get clean
-RUN python3 -m pip install --break-system-packages psycopg2==2.9.6
+RUN python3 -m pip install --break-system-packages psycopg==3.1.19
 RUN python3 -m pip install --break-system-packages adiscstudies==0.11.0
 RUN python3 -m pip install --break-system-packages numba==0.59.1
 RUN python3 -m pip install --break-system-packages attrs==23.2.0

build/ondemand/Dockerfile

@@ -4,7 +4,7 @@ ARG PIP_NO_CACHE_DIR=1
 RUN apt update && apt install -y gcc g++ libpq-dev && apt-get clean
 WORKDIR /usr/src/app
 RUN apt install -y libgdal-dev && apt-get clean
-RUN python -m pip install psycopg2==2.9.9
+RUN python -m pip install psycopg==3.1.19
 RUN python -m pip install adiscstudies==0.11.0
 RUN python -m pip install numba==0.59.0
 RUN python -m pip install attrs==23.2.0

pyproject.toml.unversioned

@@ -19,7 +19,7 @@ classifiers = [
 ]
 requires-python = ">=3.9"
 dependencies = [
-    "psycopg2==2.9.9",
+    "psycopg==3.1.19",
     "adiscstudies==0.11.0",
     "numba==0.59.1",
     "attrs==23.2.0",

spatialprofilingtoolbox/db/accessors/cells.py

@@ -6,7 +6,7 @@
 from itertools import islice
 from itertools import product
 
-from psycopg2.extensions import cursor as Psycopg2Cursor
+from psycopg import cursor as PsycopgCursor
 
 from spatialprofilingtoolbox.db.exchange_data_formats.cells import CellsData
 from spatialprofilingtoolbox.db.exchange_data_formats.cells import BitMaskFeatureNames
@@ -149,7 +149,7 @@ def _format_cell_bytes(cls, args: tuple[int, tuple[float, float], bytes]) -> byt
     def fetch_one_or_else(
         query: str,
         args: tuple,
-        cursor: Psycopg2Cursor,
+        cursor: PsycopgCursor,
         error_message: str,
     ) -> Any:
         cursor.execute(query, args)

spatialprofilingtoolbox/db/accessors/study.py

@@ -3,7 +3,7 @@
 from typing import cast
 import re
 
-from psycopg2.errors import UndefinedTable
+from psycopg.errors import UndefinedTable
 
 from spatialprofilingtoolbox.db.simple_method_cache import simple_instance_method_cache
 from spatialprofilingtoolbox.workflow.common.export_features import ADIFeatureSpecificationUploader

spatialprofilingtoolbox/db/create_data_analysis_study.py

@@ -3,7 +3,7 @@
 import datetime
 from typing import cast
 
-from psycopg2.extensions import connection as Connection
+from psycopg import connection as Connection
 
 from spatialprofilingtoolbox.standalone_utilities.log_formats import colorized_logger
 
@@ -54,9 +54,11 @@ def create(self) -> str:
         cursor.execute('''
             INSERT INTO data_analysis_study(name)
             VALUES (%s) ;
+        ''', (name,))
+        cursor.execute('''
             INSERT INTO study_component(primary_study, component_study)
             VALUES (%s, %s) ;
-        ''', (name, self.study, name))
+        ''', (self.study, name))
         cursor.close()
         self.connection.commit()
         logger.info('Inserted data analysis study: "%s"', name)

spatialprofilingtoolbox/db/database_connection.py

@@ -10,12 +10,12 @@
 from typing import Callable
 from inspect import getfullargspec
 
-from psycopg2 import connect
-from psycopg2.extensions import connection as Connection
-from psycopg2.extensions import cursor as Psycopg2Cursor
-from psycopg2 import Error as Psycopg2Error
-from psycopg2 import OperationalError
-from psycopg2.errors import DuplicateDatabase
+from psycopg import connect
+from psycopg import connection as Connection
+from psycopg import cursor as PsycopgCursor
+from psycopg import Error as PsycopgError
+from psycopg import OperationalError
+from psycopg.errors import DuplicateDatabase
 from attr import define
 
 from spatialprofilingtoolbox.db.credentials import DBCredentials
@@ -56,7 +56,7 @@ def is_connected(self):
 
 
 class DBConnection(ConnectionProvider):
-    """Provides a psycopg2 Postgres database connection. Takes care of connecting and disconnecting.
+    """Provides a psycopg Postgres database connection. Takes care of connecting and disconnecting.
     """
     autocommit: bool
 
@@ -74,7 +74,7 @@ def __init__(self,
                 study_database = self._retrieve_study_database(credentials, study)
                 credentials.update_database(study_database)
             super().__init__(self.make_connection(credentials))
-        except Psycopg2Error as exception:
+        except PsycopgError as exception:
             message = 'Failed to connect to database: %s, %s'
             logger.error(message, credentials.endpoint, credentials.database)
             raise exception
@@ -118,15 +118,15 @@ def __exit__(self, exception_type, exception_value, traceback):
 
 class DBCursor(DBConnection):
     """Context manager for shortcutting right to provision of a cursor."""
-    cursor: Psycopg2Cursor
+    cursor: PsycopgCursor
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-    def get_cursor(self) -> Psycopg2Cursor:
+    def get_cursor(self) -> PsycopgCursor:
         return self.cursor
 
-    def set_cursor(self, cursor: Psycopg2Cursor) -> None:
+    def set_cursor(self, cursor: PsycopgCursor) -> None:
         self.cursor = cursor
 
     def __enter__(self):
@@ -166,7 +166,7 @@ def _check_database_is_ready() -> bool:
             cursor.execute('SELECT * FROM study_lookup;')
             _ = cursor.fetchall()
             return True
-    except Psycopg2Error as _:
+    except PsycopgError as _:
         return False
 
 
@@ -239,7 +239,7 @@ def create_database(database_config_file: str | None, database_name: str) -> Non
 @define
 class SimpleReadOnlyProvider:
     """State-holder for basic read-only one-time database data provider classes."""
-    cursor: Psycopg2Cursor
+    cursor: PsycopgCursor
 
 
 class QueryCursor:

spatialprofilingtoolbox/db/importance_score_transcriber.py

@@ -8,7 +8,7 @@
     to_datetime,
     Timestamp,
 )
-from psycopg2.extensions import connection as Connection
+from psycopg import connection as Connection
 
 from spatialprofilingtoolbox.util import STRFTIME_FORMAT
 from spatialprofilingtoolbox.graphs.plugin_constants import PLUGIN_ALIASES

spatialprofilingtoolbox/db/ondemand_dropper.py

@@ -3,7 +3,7 @@
 from typing import cast
 import re
 
-from psycopg2.extensions import cursor as Psycopg2Cursor
+from psycopg import cursor as PsycopgCursor
 
 from spatialprofilingtoolbox.standalone_utilities.log_formats import colorized_logger
 
@@ -14,7 +14,7 @@ class OnDemandComputationsDropper:
     """Drop ondemand-computed feature values, specifications, etc."""
 
     @staticmethod
-    def drop(cursor: Psycopg2Cursor, pending_only: bool = False, drop_all: bool = False):
+    def drop(cursor: PsycopgCursor, pending_only: bool = False, drop_all: bool = False):
         specifications = cast(list[str], OnDemandComputationsDropper.get_droppable(
             cursor,
             pending_only=pending_only,
@@ -24,7 +24,7 @@ def drop(cursor: Psycopg2Cursor, pending_only: bool = False, drop_all: bool = Fa
 
     @staticmethod
     def get_droppable(
-        cursor: Psycopg2Cursor,
+        cursor: PsycopgCursor,
         pending_only: bool = False,
         drop_all: bool = False,
     ) -> list[str] | None:
@@ -45,7 +45,7 @@ def get_droppable(
         return None
 
     @staticmethod
-    def drop_features(cursor: Psycopg2Cursor, specifications: list[str]):
+    def drop_features(cursor: PsycopgCursor, specifications: list[str]):
         for specification in specifications:
             queries = [
                 'DELETE FROM pending_feature_computation WHERE feature_specification=%s ;',

spatialprofilingtoolbox/db/schema_infuser.py

@@ -6,9 +6,9 @@
 import re
 
 import pandas as pd
-from psycopg2 import Error as Psycopg2Error
-from psycopg2.errors import UndefinedTable
-from psycopg2.errors import DuplicateTable
+from psycopg import Error as PsycopgError
+from psycopg.errors import UndefinedTable
+from psycopg.errors import DuplicateTable
 
 from spatialprofilingtoolbox.db.database_connection import create_database
 from spatialprofilingtoolbox.db.credentials import metaschema_database
@@ -43,7 +43,7 @@ def setup_lightweight_metaschema(self, force=False):
 
         try:
             self._verbose_sql_execute(('grant_on_tables.sql', 'grant appropriate access to users'))
-        except Psycopg2Error as exception:
+        except PsycopgError as exception:
             logger.warning('Could not run grant privileges script. Possibly users are not set up.')
             logger.warning(exception)
 
@@ -64,7 +64,7 @@ def setup_schema(self, force=False):
         self._verbose_sql_execute(('performance_tweaks.sql', 'tweak main schema'))
         try:
             self._verbose_sql_execute(('grant_on_tables.sql', 'grant appropriate access to users'))
-        except Psycopg2Error as exception:
+        except PsycopgError as exception:
             logger.warning('Could not run grant privileges script. Possibly users are not set up.')
             logger.warning(exception)
 

spatialprofilingtoolbox/db/scripts/create_cells_bundles.py

@@ -1,7 +1,7 @@
 """Utility to pre-create cell data JSON payloads for application/client-use and save them to the database."""
 import argparse
 
-from psycopg2 import cursor as Psycopg2Cursor
+from psycopg import cursor as PsycopgCursor
 
 from spatialprofilingtoolbox.db.database_connection import get_and_validate_database_config
 from spatialprofilingtoolbox.workflow.common.cli_arguments import add_argument

spatialprofilingtoolbox/db/source_file_parser_interface.py

@@ -5,7 +5,7 @@
 """
 import re
 
-import psycopg2
+import psycopg
 
 from spatialprofilingtoolbox.standalone_utilities.log_formats import colorized_logger
 
@@ -75,7 +75,7 @@ def get_next_integer_identifier(tablename, cursor, key_name='identifier'):
         cursor.execute(f'SELECT {key_name} FROM {tablename};')
         try:
             identifiers = cursor.fetchall()
-        except psycopg2.ProgrammingError:
+        except psycopg.ProgrammingError:
             return 0
         known_integer_identifiers = [
             int(i[0]) for i in identifiers if SourceToADIParser.is_integer(i[0])]

spatialprofilingtoolbox/db/squidpy_metrics.py

@@ -1,7 +1,7 @@
 """Make squidpy metrics that don't require specific phenotype selection available."""
 
 from pandas import DataFrame
-from psycopg2.extensions import cursor as Psycopg2Cursor
+from psycopg import cursor as PsycopgCursor
 
 from spatialprofilingtoolbox.db.database_connection import DBConnection
 from spatialprofilingtoolbox.db.feature_matrix_extractor import FeatureMatrixExtractor

spatialprofilingtoolbox/db/stratification_puller.py

@@ -2,7 +2,7 @@
 from typing import cast
 
 from pandas import DataFrame
-from psycopg2.extensions import cursor as Psycopg2Cursor
+from psycopg import cursor as PsycopgCursor
 
 from spatialprofilingtoolbox.db.database_connection import DBCursor
 from spatialprofilingtoolbox.db.database_connection import retrieve_study_names

spatialprofilingtoolbox/db/study_dropper.py

@@ -1,6 +1,6 @@
 """Drop a single study."""
 
-from psycopg2.errors import InvalidCatalogName
+from psycopg.errors import InvalidCatalogName
 
 from spatialprofilingtoolbox.db.database_connection import DBCursor
 from spatialprofilingtoolbox.db.database_connection import DBConnection
@@ -33,8 +33,8 @@ def matches_base_study(_study: str) -> bool:
                 return
             database_name = matches[0][0]
 
-        with DBConnection(database_config_file=database_config_file, autocommit = False) as connection:
-            connection.set_session(autocommit = True)
+        with DBConnection(database_config_file=database_config_file, autocommit = True) as connection:
+            connection.autocommit = True
             with connection.cursor() as cursor:
                 try:
                     cursor.execute('DROP DATABASE %s ;' % database_name)

spatialprofilingtoolbox/workflow/common/export_features.py

@@ -9,7 +9,7 @@
 import re
 
 import pandas as pd
-from psycopg2.extensions import connection as Psycopg2Connection
+from psycopg import connection as PsycopgConnection
 
 from spatialprofilingtoolbox.db.source_file_parser_interface import SourceToADIParser
 from spatialprofilingtoolbox.db.database_connection import ConnectionProvider
@@ -30,7 +30,7 @@ class ADIFeaturesUploader(SourceToADIParser):
     quiet: bool
 
     def __init__(self,
-        connection: Psycopg2Connection,
+        connection: PsycopgConnection,
         data_analysis_study,
         derivation_and_number_specifiers,
         impute_zeros=False,
@@ -275,15 +275,15 @@ def ondemand_descriptor():
     def get_data_analysis_study(measurement_study, cursor):
         cursor.execute('''
         SELECT sc.primary_study FROM study_component sc
-        WHERE sc.component_study=%s
+        WHERE sc.component_study=%s ;
         ''', (measurement_study,))
         study = cursor.fetchall()[0][0]
 
         cursor.execute('''
         SELECT das.name
         FROM data_analysis_study das
         JOIN study_component sc ON sc.component_study=das.name
-        WHERE sc.primary_study=%s
+        WHERE sc.primary_study=%s ;
         ''', (study,))
         rows = cursor.fetchall()
         ondemand = ADIFeatureSpecificationUploader.ondemand_descriptor()
@@ -293,9 +293,10 @@ def get_data_analysis_study(measurement_study, cursor):
         data_analysis_study = ADIFeatureSpecificationUploader.form_ondemand_study_name(study)
         cursor.execute('''
         INSERT INTO data_analysis_study (name) VALUES (%s) ;
+        ''', (data_analysis_study,))
+        cursor.execute('''
         INSERT INTO study_component (primary_study, component_study) VALUES (%s , %s) ;
-        ''', (data_analysis_study, study, data_analysis_study))
-        # cursor.commit()
+        ''', (study, data_analysis_study))
         return data_analysis_study
 
     @staticmethod

spatialprofilingtoolbox/workflow/common/sparse_matrix_puller.py

@@ -371,12 +371,12 @@ def _get_sparse_entries(self,
         sparse_entries: list = []
         number_log_messages = 0
         with DBCursor(database_config_file=self.database_config_file, study=study_name) as cursor:
-            query, parameters = self._get_sparse_matrix_query_specimen_specific(
+            query = self._get_sparse_matrix_query_specimen_specific(
                 cursor,
                 specimen,
                 histological_structures,
             )
-            cursor.execute(query, parameters)
+            cursor.execute(query)
             total = cursor.rowcount
             while cursor.rownumber < total - 1:
                 current_number_stored = len(sparse_entries)
@@ -392,15 +392,16 @@ def _get_sparse_matrix_query_specimen_specific(self,
         cursor,
         specimen: str,
         histological_structures: set[int] | None,
-    ) -> tuple[str, tuple]:
+    ) -> str:
         structures_present = histological_structures is not None
         parameters: list[str | tuple[str, ...] | int] = []
         range_definition = SparseMatrixPuller._retrieve_expressions_range(cursor, specimen)
         query = self._sparse_entries_query(structures_present)
-        parameters = [range_definition[0], range_definition[1]]
         if histological_structures is not None:
-            parameters.append(tuple(str(hs_id) for hs_id in histological_structures))
-        return (query, tuple(parameters))
+            query = query % (range_definition[0], range_definition[1], f'{tuple(str(hs_id) for hs_id in histological_structures)}')
+        else:
+            query = query % (range_definition[0], range_definition[1])
+        return query
 
     @staticmethod
     def _retrieve_expressions_range(cursor, scope: str) -> tuple[int, int]:

spatialprofilingtoolbox/workflow/common/structure_centroids.py

@@ -92,7 +92,7 @@ def load_from_db(self, study: str | None = None) -> None:
                 cursor.execute('''
                     SELECT specimen, blob_contents FROM ondemand_studies_index osi
                     WHERE osi.blob_type='centroids';
-                ''', (study, ))
+                ''')
                 self._studies[study] = {}
                 while True:
                     row = cursor.fetchone()