feat!: Initial support for apiv2 (#1085)

This is a checkpoint: Basic relationships and filters work, but I haven't had a chance to test all of the filter operators and update test infra yet. --------- Co-authored-by: uermel <[email protected]>
chanzuckerberg · Oct 30, 2024 · 1efdb33 · 1efdb33
1 parent 596e2a3
commit 1efdb33
Show file tree

Hide file tree

Showing 57 changed files with 6,580 additions and 14,004 deletions.
diff --git a/client/python/cryoet_data_portal/Makefile b/client/python/cryoet_data_portal/Makefile
@@ -1,17 +1,38 @@
+SHELL := /bin/bash
+
+export BACKEND_DIR=/tmp/cryoet-data-portal-backend-test-infra
+.PHONY: set-api-version
+set-api-version:
+	if [ ! -e $(BACKEND_DIR) ]; then git clone https://github.com/chanzuckerberg/cryoet-data-portal-backend.git $(BACKEND_DIR); fi
+	export STAGING_SHA=$$(./tests/test_infra/get_deployment.sh staging); \
+	if [ -n "$${STAGING_SHA}" ]; then \
+	    echo "Setting backend to SHA: $${STAGING_SHA}"; \
+	    cd $(BACKEND_DIR); \
+	    git fetch; \
+	    git reset --hard $$STAGING_SHA; \
+	else \
+	    echo "Could not get staging SHA!"; \
+	    exit 1; \
+	fi
+	cd $(BACKEND_DIR) && make apiv2-init
+
 .PHONY: test-infra
-test-infra:
-	# Use force-recreate to handle re-initializing the database schema and data
-	# rather than maintaining incremental migrations.
-	# See the test_infra/hasura/README.md for more details.
-	cd tests && docker compose up -d --wait --wait-timeout 300 --force-recreate
+test-infra: set-api-version
+	cp tests/test_infra/seed_db.py $(BACKEND_DIR)/apiv2/scripts
+	cd $(BACKEND_DIR) && docker compose run graphql-api python3 scripts/seed_db.py
 	cd ./tests/test_infra/; ./seed_moto.sh
 
+.PHONY: clean
+clean:
+	if [ -e $(BACKEND_DIR) ]; then cd $(BACKEND_DIR); make clean; fi
+	rm -rf $(BACKEND_DIR)
+
 .PHONY: coverage
 coverage:
 	export AWS_REGION=us-west-2; \
 		export AWS_ACCESS_KEY_ID=test; \
 		export AWS_SECRET_ACCESS_KEY=test; \
-		export BOTO_ENDPOINT_URL=http://localhost:4000; \
+		export BOTO_ENDPOINT_URL=http://localhost:5566; \
 		export BOTO_SIGNATURE_VERSION=s3v4; \
 		coverage run --parallel-mode -m pytest -v -rP --durations=20 ./tests/
 
@@ -20,13 +41,13 @@ test:
 	export AWS_REGION=us-west-2; \
 		export AWS_ACCESS_KEY_ID=test; \
 		export AWS_SECRET_ACCESS_KEY=test; \
-		export BOTO_ENDPOINT_URL=http://localhost:4000; \
+		export BOTO_ENDPOINT_URL=http://localhost:5566; \
 		export BOTO_SIGNATURE_VERSION=s3v4; \
 		pytest -vvv -s . $(TEST)
 
 .PHONY: codegen
 codegen:
-	python -m cryoet_data_portal._codegen
+	cd src && python3 -m cryoet_data_portal._codegen
 	# Need to run pre-commit twice because black and ruff fight with each other.
 	# Ignore the return code because that is non-zero when pre-commit applies a fix.
 	-pre-commit run --files src/cryoet_data_portal/_models.py src/cryoet_data_portal/data/schema.graphql

diff --git a/client/python/cryoet_data_portal/pyproject.toml b/client/python/cryoet_data_portal/pyproject.toml
@@ -1,38 +1,39 @@
 [build-system]
-requires = ["setuptools>=64"]
+requires = ["setuptools>=64","setuptools-scm"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "cryoet_data_portal"
 version = "3.1.1"
 description = "API Client to facilitate the use of the CryoET Portal. For more information about the API and the project visit https://github.com/chanzuckerberg/cryoet-data-portal/"
 authors = [
-    { name = "Chan Zuckerberg Initiative", email = "[email protected]" }
+  { name = "Chan Zuckerberg Initiative", email = "[email protected]" },
 ]
 license = { text = "MIT" }
 readme = "README.md"
 requires-python = ">= 3.7"
 classifiers = [
-    "Development Status :: 4 - Beta",
-    "Intended Audience :: Developers",
-    "Intended Audience :: Information Technology",
-    "Intended Audience :: Science/Research",
-    "License :: OSI Approved :: MIT License",
-    "Programming Language :: Python",
-    "Topic :: Scientific/Engineering :: Bio-Informatics",
-    "Operating System :: POSIX :: Linux",
-    "Operating System :: MacOS :: MacOS X",
-    "Programming Language :: Python :: 3.7",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
-    "Programming Language :: Python :: 3.10",
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "Intended Audience :: Information Technology",
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python",
+  "Topic :: Scientific/Engineering :: Bio-Informatics",
+  "Operating System :: POSIX :: Linux",
+  "Operating System :: MacOS :: MacOS X",
+  "Programming Language :: Python :: 3.7",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
 ]
-dependencies= [
-    "requests",
-    "boto3",
-    "deepmerge",
-    "gql[requests]",
-    "tqdm",
+dependencies = [
+  "requests",
+  "boto3",
+  "deepmerge",
+  "gql[requests]",
+  "tqdm",
+  "strcase",
 ]
 
 [project.urls]
@@ -42,8 +43,15 @@ documentation = "https://chanzuckerberg.github.io/cryoet-data-portal/python-api.
 
 [tool.setuptools.packages.find]
 where = ["src"]
-include = ["cryoet_data_portal*"]  # package names should match these glob patterns (["*"] by default)
-exclude = ["tests*"]  # exclude packages matching these glob patterns (empty by default)
+include = [
+  "cryoet_data_portal*",
+] # package names should match these glob patterns (["*"] by default)
+exclude = [
+  "tests*",
+] # exclude packages matching these glob patterns (empty by default)
+
+[tool.setuptools]
+include-package-data = true
 
 [tool.setuptools_scm]
 root = "../../.."
@@ -55,6 +63,4 @@ warn_unreachable = true
 strict = true
 
 [tool.pytest.ini_options]
-markers = [
-    "expensive: too expensive to run regularly or in CI",
-]
+markers = ["expensive: too expensive to run regularly or in CI"]
diff --git a/client/python/cryoet_data_portal/src/cryoet_data_portal/__init__.py b/client/python/cryoet_data_portal/src/cryoet_data_portal/__init__.py
@@ -6,14 +6,22 @@
 
 from ._client import Client
 from ._models import (
+    Alignment,
     Annotation,
     AnnotationAuthor,
     AnnotationFile,
+    AnnotationMethodLink,
+    AnnotationShape,
     Dataset,
     DatasetAuthor,
     DatasetFunding,
     Deposition,
     DepositionAuthor,
+    DepositionType,
+    Frame,
+    FrameAcquisitionFile,
+    GainFile,
+    PerSectionAlignmentParameters,
     Run,
     TiltSeries,
     Tomogram,
@@ -25,17 +33,25 @@
 
 __all__ = [
     "Client",
+    "Alignment",
     "Annotation",
-    "AnnotationFile",
     "AnnotationAuthor",
+    "AnnotationFile",
+    "AnnotationMethodLink",
+    "AnnotationShape",
     "Dataset",
     "DatasetAuthor",
     "DatasetFunding",
     "Deposition",
     "DepositionAuthor",
+    "DepositionType",
+    "Frame",
+    "FrameAcquisitionFile",
+    "GainFile",
+    "PerSectionAlignmentParameters",
     "Run",
     "TiltSeries",
-    "Tomogram",
     "TomogramAuthor",
     "TomogramVoxelSpacing",
+    "Tomogram",
 ]
diff --git a/client/python/cryoet_data_portal/src/cryoet_data_portal/_client.py b/client/python/cryoet_data_portal/src/cryoet_data_portal/_client.py
@@ -8,7 +8,7 @@
 
 from cryoet_data_portal._constants import USER_AGENT
 
-DEFAULT_URL = "https://graphql.cryoetdataportal.cziscience.com/v1/graphql"
+DEFAULT_URL = "https://graphql.cryoetdataportal.czscience.com/graphql"
 
 
 class Client:
@@ -46,16 +46,22 @@ def __init__(self, url: Optional[str] = None):
         self.client = GQLClient(transport=transport, schema=schema_str)
         self.ds = DSLSchema(self.client.schema)
 
-    def build_query(self, cls, gql_class_name: str, query_filters=None):
+    def build_query(
+        self,
+        cls,
+        root_field: str,
+        gql_class_name: str,
+        query_filters=None,
+    ):
         ds = self.ds
         query_filters = {} if not query_filters else {"where": query_filters}
         gql_type = getattr(ds, gql_class_name)
         scalar_fields = [
-            getattr(gql_type, fieldname) for fieldname in cls._get_scalar_fields()
+            getattr(gql_type, fieldname) for fieldname in cls._get_gql_fields()
         ]
         query = dsl_gql(
             DSLQuery(
-                getattr(ds.query_root, gql_class_name)(**query_filters).select(
+                getattr(ds.Query, root_field)(**query_filters).select(
                     *scalar_fields,
                 ),
             ),
@@ -64,8 +70,11 @@ def build_query(self, cls, gql_class_name: str, query_filters=None):
 
     def find(self, cls, query_filters=None):
         gql_type = cls._get_gql_type()
-        response = self.client.execute(self.build_query(cls, gql_type, query_filters))
-        return [cls(self, **item) for item in response[gql_type]]
+        gql_root = cls._get_gql_root_field()
+        response = self.client.execute(
+            self.build_query(cls, gql_root, gql_type, query_filters),
+        )
+        return [cls(self, **item) for item in response[gql_root]]
 
     def find_one(self, *args, **kwargs):
         for result in self.find(*args, **kwargs):