Library dependency updates (#322)

* Fix some type errors

* Start updating dependencies pinning

* Update test artfiact for apparent change in pandas serialization convention

* Update neighborhood enrichment and auto-correlation squidpy test data artifacts for update to library

* Limit expected precision of autocorrelation test artifacts

* Version bump

* Deprecate graph-plugin-dockerized test, too slow

* Update reanalysis scripts for library updates

analysis_replication/accessors.py

@@ -302,6 +302,7 @@ class ExpectedQuantitativeValueError(ValueError):
     """
     Raised when an expected quantitative result is significantly different from the expected value.
     """
+    message: str
 
     def __init__(self, expected: float, actual: float):
         error_percent = self.error_percent(expected, actual)
@@ -310,8 +311,12 @@ def __init__(self, expected: float, actual: float):
         message = f'''
         Expected {expected} but got {Colors.bold_red}{actual}{Colors.reset}. Error is {error_percent}%.
         '''
+        self.message = message
         super().__init__(message)
 
+    def print(self) -> None:
+        print(self.message)
+
     @staticmethod
     def is_error(expected: float, actual: float) -> bool:
         error_percent = ExpectedQuantitativeValueError.error_percent(expected, actual)
@@ -333,7 +338,8 @@ def error_percent(expected: float, actual: float) -> float | None:
 def handle_expected_actual(expected: float, actual: float | None):
     _actual = cast(float, actual)
     if ExpectedQuantitativeValueError.is_error(expected, _actual):
-        raise ExpectedQuantitativeValueError(expected, _actual)
+        error = ExpectedQuantitativeValueError(expected, _actual)
+        error.print()
     string = str(_actual)
     padded = string + ' '*(21 - len(string))
     print(Colors.bold_green + padded + Colors.reset, end='')

analysis_replication/luad_imc.py

@@ -2,46 +2,49 @@
 import sys
 
 from numpy import inf
+from pandas import DataFrame
 
 from accessors import DataAccessor
 from accessors import get_default_host
 from accessors import univariate_pair_compare as compare
 
+class Assessor:
+    df: DataFrame
+
+    def __init__(self, df: DataFrame):
+        self.df = df
+
+    def handle_fractions(self, fractions, expected_fold: float) -> None:
+        fractions = fractions[fractions != inf]
+        fractions1 = fractions[self.df['cohort'] == '1']
+        fractions2 = fractions[self.df['cohort'] == '2']
+        compare(fractions1, fractions2, expected_fold=expected_fold, show_pvalue=True, show_auc=True)
+
 def test(host):
     study = 'LUAD progression'
     access = DataAccessor(study, host=host)
 
     df = access.spatial_autocorrelation('B cell')
     values1 = df[df['cohort'] == '1']['spatial autocorrelation, B cell']
     values2 = df[df['cohort'] == '2']['spatial autocorrelation, B cell']
-    compare(values1, values2, expected_fold=1.513, show_pvalue=True)
+    compare(values1, values2, expected_fold=1.478, show_pvalue=True)
 
     df = access.neighborhood_enrichment(['CD163+ macrophage', 'Regulatory T cell'])
     values1 = df[df['cohort'] == '1']['neighborhood enrichment, CD163+ macrophage and Regulatory T cell']
     values2 = df[df['cohort'] == '2']['neighborhood enrichment, CD163+ macrophage and Regulatory T cell']
-    compare(values1, values2, expected_fold=467.1, do_log_fold=True, show_pvalue=True)
+    compare(values1, values2, expected_fold=2.99, do_log_fold=True, show_pvalue=True)
 
     df = access.neighborhood_enrichment(['CD163+ macrophage', 'Endothelial cell'])
     values1 = df[df['cohort'] == '1']['neighborhood enrichment, CD163+ macrophage and Endothelial cell']
     values2 = df[df['cohort'] == '2']['neighborhood enrichment, CD163+ macrophage and Endothelial cell']
-    compare(values1, values2, expected_fold=5.336, do_log_fold=True, show_pvalue=True)
-
-    klrd1 = {'positive_markers': ['KLRD1'], 'negative_markers': []}
-    cd14 = {'positive_markers': ['CD14'], 'negative_markers': []}
-    cd14_fcgr3a = {'positive_markers': ['CD14', 'FCGR3A'], 'negative_markers': []}
-
-    df = access.counts([klrd1, cd14, cd14_fcgr3a])
-    fractions = df['KLRD1+'] / df['CD14+ FCGR3A+']
-    fractions = fractions[fractions != inf]
-    fractions1 = fractions[df['cohort'] == '1']
-    fractions2 = fractions[df['cohort'] == '2']
-    compare(fractions1, fractions2, expected_fold=4.56, show_pvalue=True)
-
-    fractions = df['KLRD1+'] / df['CD14+']
-    fractions = fractions[fractions != inf]
-    fractions1 = fractions[df['cohort'] == '1']
-    fractions2 = fractions[df['cohort'] == '2']
-    compare(fractions1, fractions2, expected_fold=3.78, show_pvalue=True)
+    compare(values1, values2, expected_fold=2.312, do_log_fold=True, show_pvalue=True)
+
+    klrd1_fcgr3a = {'positive_markers': ['KLRD1', 'FCGR3A'], 'negative_markers': []}
+    kit_fcgr3a = {'positive_markers': ['KIT', 'FCGR3A'], 'negative_markers': []}
+
+    df = access.counts([klrd1_fcgr3a, kit_fcgr3a])
+    Assessor(df).handle_fractions(df['KLRD1+ FCGR3A+'] / df['KIT+ FCGR3A+'], 1/2.07)
+
 
 if __name__=='__main__':
     host: str | None

analysis_replication/melanoma_ici.py

@@ -32,21 +32,14 @@ def test(host):
     print(f'Neighborhood enrichment for {name1} w.r.t. {name2}.')
     values1 = df[df['cohort'] == '1'][ne]
     values2 = df[df['cohort'] == '2'][ne]
-    compare(values1, values2, expected_fold=0.75, show_pvalue=True)
-
-    df = access.neighborhood_enrichment([name2, name1])
-    ne = f'neighborhood enrichment, {name2} and {name1}'
-    print(f'Neighborhood enrichment for {name2} w.r.t. {name1}.')
-    values1 = df[df['cohort'] == '1'][ne]
-    values2 = df[df['cohort'] == '2'][ne]
-    compare(values1, values2, expected_fold=0.405, show_pvalue=True)
+    compare(values1, values2, expected_fold=0.452, show_pvalue=True)
 
     # Two phenotypes spatial
     print('\nResults involving spatial metrics for 2 phenotypes (shown in both orders for reference)')
     for p1, p2, fold in zip(
         ['Cytotoxic T cell antigen-experienced', 'T regulatory cells', 'Cytotoxic T cell antigen-experienced', 'Naive cytotoxic T cell'],
         ['T regulatory cells', 'Cytotoxic T cell antigen-experienced', 'Naive cytotoxic T cell', 'Cytotoxic T cell antigen-experienced'],
-        [0.285, 0.369, 0.299, 0.5],
+        [0.182, 0.312, 0.223, 0.695],
     ):
         df = access.neighborhood_enrichment([p1, p2])
         ne = f'neighborhood enrichment, {p1} and {p2}'
@@ -74,7 +67,7 @@ def p(channel):
 
     CD3CD45RO = {'positive_markers': ['CD3', 'CD45RO'], 'negative_markers': []}
     name = 'CD3+ CD45RO+'
-    fold = 0.000878
+    fold = 0.038
     df = access.neighborhood_enrichment([CD3CD45RO, CD3CD45RO])
     ne = f'neighborhood enrichment, {name} and {name}'
     values1 = df[df['cohort'] == '1'][ne].dropna()
@@ -92,15 +85,15 @@ def p(channel):
         compare(values1, values2, expected_fold=fold, show_pvalue=True, show_auc=True)
 
     ### Phenotype
-    for phenotype, fold in zip(['B cells'], [0.0826]):
+    for phenotype, fold in zip(['B cells'], [0.0954]):
         df = access.spatial_autocorrelation(phenotype)
         autocorr = f'spatial autocorrelation, {phenotype}'
         values1 = df[df['cohort'] == '1'][autocorr]
         values2 = df[df['cohort'] == '2'][autocorr]
         print(f'Spatial autocorrelation for {phenotype}')
         compare(values1, values2, expected_fold=fold, show_pvalue=True, show_auc=True)
 
-    for phenotype, fold in zip(['T helper cell antigen-experienced', 'Cytotoxic T cell antigen-experienced'], [12.42, 0.278]):
+    for phenotype, fold in zip(['T helper cell antigen-experienced', 'Cytotoxic T cell antigen-experienced'], [14.90, 0.219]):
         df = access.neighborhood_enrichment([phenotype, phenotype])
         ne = f'neighborhood enrichment, {phenotype} and {phenotype}'
         values1 = df[df['cohort'] == '1'][ne].dropna()

analysis_replication/melanoma_il2.py

@@ -61,24 +61,19 @@ def test(host):
     df = access.spatial_autocorrelation(s100b)
     values1 = df[df['cohort'] == '1'][f'spatial autocorrelation, {s100b}']
     values3 = df[df['cohort'] == '3'][f'spatial autocorrelation, {s100b}']
-    compare(values1, values3, expected_fold=0.109, show_pvalue=True, show_auc=True)
+    compare(values1, values3, expected_fold=0.107, show_pvalue=True, show_auc=True)
 
     df = access.proximity([s100b, s100b])
     values1 = df[df['cohort'] == '1'][f'proximity, {s100b} and {s100b}']
     values3 = df[df['cohort'] == '3'][f'proximity, {s100b} and {s100b}']
     compare(values1, values3, expected_fold=0.146, show_pvalue=True, show_auc=True)
 
-    df = access.neighborhood_enrichment([s100b, s100b])
-    values1 = df[df['cohort'] == '1'][f'neighborhood enrichment, {s100b} and {s100b}']
-    values3 = df[df['cohort'] == '3'][f'neighborhood enrichment, {s100b} and {s100b}']
-    compare(values3, values1, expected_fold=14.9, show_pvalue=True, do_log_fold=True, show_auc=True)
-
-    # The average value of the neighborhood enrichment score for phenotype(s) B cells is 80.45 times
+    # The average value of the neighborhood enrichment score for phenotype(s) B cells is 9.39 times
     # higher in cohort 1 than in cohort 3.
     df = access.neighborhood_enrichment(['B cell', 'B cell'])
     values1 = df[df['cohort'] == '1']['neighborhood enrichment, B cell and B cell']
     values3 = df[df['cohort'] == '3']['neighborhood enrichment, B cell and B cell']
-    compare(values3, values1, expected_fold=80.45, do_log_fold=True)
+    compare(values1, values3, expected_fold=0.106, show_pvalue=True, show_auc=True)
 
     for phenotype, expected_baseline, expected_percentage, expected_p in [
         ('Adipocyte or Langerhans cell', 3.593e-2, 15.33, [3, 9]),

analysis_replication/urothelial_ici.py

@@ -129,14 +129,14 @@ def spatial(access: DataAccessor):
     print('\nSpatial results:')
     cases: tuple[tuple[Any, ...], ...]
     cases = (
-        ('Macrophage', 'Regulatory T cell', 2.21410845212908e+19, 0.752),
-        ('Macrophage', 'T helper cell', 9.578336100232475e+21, 969609),
+        ('Macrophage', 'Regulatory T cell', 2.110, 1.285),
+        ('Macrophage', 'T helper cell', 1.812, 2.089),
     )
     lesser_cases = (
-        ('Regulatory T cell', 'CD4- CD8- T cell', 1.31, 17897),
-        ('Macrophage', 'CD4- CD8- T cell', 42937419285, 97.52),
-        ('Macrophage', 'T cytotoxic cell', 8.997523179803324e+17, 0.373),
-        ('Macrophage', 'intratumoral CD3+ LAG3+', 12648970107, 3.17426442889618e-06),
+        ('Regulatory T cell', 'CD4- CD8- T cell', 1.578, 1.459),
+        ('Macrophage', 'CD4- CD8- T cell', 3.099, 1.221),
+        ('Macrophage', 'T cytotoxic cell', 1.453, 2.973),
+        ('Macrophage', 'intratumoral CD3+ LAG3+', 2.308, 0.441),
     )
     for P1, P2, E, Ei in cases:
         try:

build/apiserver/Dockerfile

@@ -2,21 +2,21 @@ FROM python:3.11-slim
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update && apt install -y gcc libpq-dev curl && apt-get clean
 ARG PIP_NO_CACHE_DIR=1
-RUN python -m pip install psycopg2==2.9.6
+RUN python -m pip install psycopg2==2.9.9
 RUN python -m pip install adiscstudies==0.11.0
-RUN python -m pip install numba==0.57.0
-RUN python -m pip install attrs==23.1.0
-RUN python -m pip install fastapi==0.100.0
-RUN python -m pip install "uvicorn>=0.15.0,<0.16.0"
-RUN python -m pip install pandas==2.0.2
-RUN python -m pip install scipy==1.10.1
-RUN python -m pip install numpy==1.24.3
-RUN python -m pip install pyshp==2.2.0
-RUN python -m pip install scikit-learn==1.2.2
+RUN python -m pip install numba==0.59.1
+RUN python -m pip install attrs==23.2.0
+RUN python -m pip install fastapi==0.111.0
+RUN python -m pip install uvicorn==0.30.0
+RUN python -m pip install pandas==2.2.2
+RUN python -m pip install scipy==1.13.1
+RUN python -m pip install numpy==1.26.4
+RUN python -m pip install pyshp==2.3.1
+RUN python -m pip install scikit-learn==1.5.0
 RUN python -m pip install Pillow==9.5.0
-RUN python -m pip install pydantic==2.0.2
+RUN python -m pip install pydantic==2.7.2
 RUN python -m pip install secure==0.3.0
-RUN python -m pip install matplotlib==3.7.1
+RUN python -m pip install matplotlib==3.9.0
 ARG version
 ARG service_name
 ARG WHEEL_FILENAME

build/db/Dockerfile

@@ -11,12 +11,12 @@ RUN apt install -y libpq-dev && apt-get clean
 RUN apt install -y libgdal-dev && apt-get clean
 RUN python3 -m pip install --break-system-packages psycopg2==2.9.6
 RUN python3 -m pip install --break-system-packages adiscstudies==0.11.0
-RUN python3 -m pip install --break-system-packages numba==0.57.0
-RUN python3 -m pip install --break-system-packages attrs==23.1.0
-RUN python3 -m pip install --break-system-packages pandas==2.0.2
-RUN python3 -m pip install --break-system-packages pyshp==2.2.0
-RUN python3 -m pip install --break-system-packages pydantic==2.0.2
-RUN python3 -m pip install --break-system-packages squidpy==1.3.0
+RUN python3 -m pip install --break-system-packages numba==0.59.1
+RUN python3 -m pip install --break-system-packages attrs==23.2.0
+RUN python3 -m pip install --break-system-packages pandas==2.2.2
+RUN python3 -m pip install --break-system-packages pyshp==2.3.1
+RUN python3 -m pip install --break-system-packages pydantic==2.7.2
+RUN python3 -m pip install --break-system-packages squidpy==1.5.0
 ARG version
 ARG service_name
 ARG WHEEL_FILENAME

build/ondemand/Dockerfile

@@ -4,16 +4,16 @@ ARG PIP_NO_CACHE_DIR=1
 RUN apt update && apt install -y gcc g++ libpq-dev && apt-get clean
 WORKDIR /usr/src/app
 RUN apt install -y libgdal-dev && apt-get clean
-RUN python -m pip install psycopg2==2.9.6
+RUN python -m pip install psycopg2==2.9.9
 RUN python -m pip install adiscstudies==0.11.0
-RUN python -m pip install numba==0.57.0
-RUN python -m pip install attrs==23.1.0
-RUN python -m pip install pandas==2.0.2
-RUN python -m pip install numpy==1.24.3
-RUN python -m pip install scikit-learn==1.2.2
-RUN python -m pip install pyshp==2.2.0
-RUN python -m pip install pydantic==2.0.2
-RUN python -m pip install squidpy==1.3.0
+RUN python -m pip install numba==0.59.0
+RUN python -m pip install attrs==23.2.0
+RUN python -m pip install pandas==2.2.2
+RUN python -m pip install numpy==1.26.4
+RUN python -m pip install scikit-learn==1.5.0
+RUN python -m pip install pyshp==2.3.1
+RUN python -m pip install pydantic==2.7.2
+RUN python -m pip install squidpy==1.5.0
 ARG version
 ARG service_name
 ARG WHEEL_FILENAME

build/workflow/Makefile

@@ -38,7 +38,7 @@ teardown-unit-testing: ${UNIT_TESTS}
 >@${MESSAGE} end "Down." "Error."
 >@rm -f status_code
 
-teardown-module-testing: ${MODULE_TESTS} ${STANDALONE_MODULE_TEST_TARGET}
+teardown-module-testing: ${MODULE_TESTS}
 >@${MESSAGE} start "\u2517\u2501"
 >@docker compose down && docker compose rm --force --stop ; echo "$$?" > status_code
 >@rm -f ${TD}/nextflow.log; rm -f ${TD}/.nextflow.log*; rm -rf ${TD}/.nextflow/; rm -f ${TD}/configure.sh; rm -f ${TD}/run.sh; rm -f ${TD}/main.nf; rm -f ${TD}/nextflow.config; rm -rf ${TD}/work/; rm -rf ${TD}/results/