From cf2f5303b59df796292af973a6c25dd495202b27 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 5 Apr 2024 00:10:34 -0700 Subject: [PATCH 1/9] WIP --- caveclient/endpoints.py | 4 +++ caveclient/jsonservice.py | 67 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/caveclient/endpoints.py b/caveclient/endpoints.py index 39d98b69..192849d1 100644 --- a/caveclient/endpoints.py +++ b/caveclient/endpoints.py @@ -219,6 +219,10 @@ "upload_state_w_id": json_v1 + "/post/{state_id}", "get_state": json_v1 + "/{state_id}", "get_state_raw": json_v1 + "/raw/{state_id}", + "get_properties": json_v1 + "/property/{state_id}/info", + "upload_properties": json_v1 + "/property/post", + "get_properties_raw": json_v1 + "/property/raw/{state_id}", + "upload_properties_w_id": json_v1 + "/property/post/{state_id}", } json_legacy = "{json_server_address}/nglstate" diff --git a/caveclient/jsonservice.py b/caveclient/jsonservice.py index 6f904be3..f1253110 100644 --- a/caveclient/jsonservice.py +++ b/caveclient/jsonservice.py @@ -186,6 +186,26 @@ def get_state_json(self, state_id): handle_response(response, as_json=False) return json.loads(response.content) + def get_property_json(self, state_id): + """Download a Neuroglancer JSON state + + Parameters + ---------- + state_id : int + ID of a JSON state uploaded to the state service. + + Returns + ------- + dict + JSON specifying a Neuroglancer state. + """ + url_mapping = self.default_url_mapping + url_mapping["state_id"] = state_id + url = self._endpoints["get_property"].format_map(url_mapping) + response = self.session.get(url) + handle_response(response, as_json=False) + return json.loads(response.content) + def upload_state_json(self, json_state, state_id=None, timestamp=None): """Upload a Neuroglancer JSON state @@ -224,6 +244,44 @@ def upload_state_json(self, json_state, state_id=None, timestamp=None): response_re = re.search(".*\/(\d+)", str(response.content)) return int(response_re.groups()[0]) + def upload_property_json(self, property_json, state_id=None, timestamp=None): + """Upload a Neuroglancer JSON state + + Parameters + ---------- + propery_json : dict + Dict representation of a neuroglancer segment properties json + state_id : int + ID of a JSON state uploaded to the state service. + Using a state_id is an admin feature. + timestamp: time.time + Timestamp for json state date. Requires state_id. + + Returns + ------- + int + state_id of the uploaded JSON state + """ + url_mapping = self.default_url_mapping + + if state_id is None: + url = self._endpoints["upload_properties"].format_map(url_mapping) + else: + url_mapping = self.default_url_mapping + url_mapping["state_id"] = state_id + url = self._endpoints["upload_properties_w_id"].format_map(url_mapping) + + response = self.session.post( + url, + data=json.dumps( + property_json, + default=neuroglancer_json_encoder, + ), + ) + handle_response(response, as_json=False) + response_re = re.search(".*\/(\d+)", str(response.content)) + return int(response_re.groups()[0]) + def save_state_json_local(self, json_state, filename, overwrite=False): """Save a Neuroglancer JSON state to a JSON file locally. @@ -251,6 +309,7 @@ def build_neuroglancer_url( ngl_url=None, target_site=None, static_url=False, + format_propeties=False, ): """Build a URL for a Neuroglancer deployment that will automatically retrieve specified state. If the datastack is specified, this is prepopulated from the info file field "viewer_site". @@ -269,7 +328,8 @@ def build_neuroglancer_url( Default is None. static_url : bool If True, treats "state_id" as a static URL directly to the JSON and does not use the state service. - + format_propeties : bool + If True, formats the url as a segment_properties info file Returns ------- str @@ -304,6 +364,11 @@ def build_neuroglancer_url( target_site_error = "A specified target_site must be one of 'seunglab', 'cave-explorer' or 'mainline'" raise ValueError(target_site_error) + if format_propeties: + url_mapping = self.default_url_mapping + url_mapping["state_id"] = state_id + get_state_url = self._endpoints["get_property"][:-5].format_map(url_mapping) + url = "precomputed://" + auth_text + get_state_url if static_url: url = ngl_url + parameter_text + state_id else: From 23169dd85042b1f198ef1fc4865a5019c43b6567 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 5 Apr 2024 12:43:37 -0700 Subject: [PATCH 2/9] fix formatting --- caveclient/jsonservice.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/caveclient/jsonservice.py b/caveclient/jsonservice.py index f1253110..40baafeb 100644 --- a/caveclient/jsonservice.py +++ b/caveclient/jsonservice.py @@ -367,8 +367,11 @@ def build_neuroglancer_url( if format_propeties: url_mapping = self.default_url_mapping url_mapping["state_id"] = state_id - get_state_url = self._endpoints["get_property"][:-5].format_map(url_mapping) + get_state_url = self._endpoints["get_properties"][:-5].format_map( + url_mapping + ) url = "precomputed://" + auth_text + get_state_url + return url if static_url: url = ngl_url + parameter_text + state_id else: From f2838eadcfacb5e7231a35877d03c9e38974858d Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 6 Apr 2024 00:06:11 -0700 Subject: [PATCH 3/9] add version reqs --- caveclient/jsonservice.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/caveclient/jsonservice.py b/caveclient/jsonservice.py index 40baafeb..530b7b25 100644 --- a/caveclient/jsonservice.py +++ b/caveclient/jsonservice.py @@ -10,6 +10,7 @@ ClientBase, _api_endpoints, handle_response, + _check_version_compatibility, ) from .endpoints import ( default_global_server_address, @@ -186,6 +187,7 @@ def get_state_json(self, state_id): handle_response(response, as_json=False) return json.loads(response.content) + @_check_version_compatibility(">=0.4.0") def get_property_json(self, state_id): """Download a Neuroglancer JSON state @@ -244,6 +246,7 @@ def upload_state_json(self, json_state, state_id=None, timestamp=None): response_re = re.search(".*\/(\d+)", str(response.content)) return int(response_re.groups()[0]) + @_check_version_compatibility(">=0.4.0") def upload_property_json(self, property_json, state_id=None, timestamp=None): """Upload a Neuroglancer JSON state From e925a84359c1d664cb0cf77a2c079f89bc3b0945 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 6 Apr 2024 00:17:46 -0700 Subject: [PATCH 4/9] improve server version handling for no server version --- caveclient/base.py | 14 +++++++++++--- caveclient/chunkedgraph.py | 1 - caveclient/jsonservice.py | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/caveclient/base.py b/caveclient/base.py index 054a1d13..74c858ad 100644 --- a/caveclient/base.py +++ b/caveclient/base.py @@ -215,6 +215,7 @@ def __init__( self._api_version = api_version self._endpoints = endpoints self._fc = over_client + self._server_version = self._get_version() @property def fc(self): @@ -234,7 +235,11 @@ def api_version(self): def _get_version(self) -> Optional[Version]: endpoint_mapping = self.default_url_mapping - url = self._endpoints.get("get_version", None).format_map(endpoint_mapping) + endpoint = self._endpoints.get("get_version", None) + if endpoint is None: + return None + + url = endpoint.format_map(endpoint_mapping) response = self.session.get(url) if response.status_code == 404: # server doesn't have this endpoint yet return None @@ -349,8 +354,11 @@ def _version_fails_constraint(version: Version, constraint: str = None): if constraint is None: return False else: - specifier = SpecifierSet(constraint) - return version not in specifier + if version is None: + return True + else: + specifier = SpecifierSet(constraint) + return version not in specifier @parametrized diff --git a/caveclient/chunkedgraph.py b/caveclient/chunkedgraph.py index 8fa7c8bf..a9d8d052 100644 --- a/caveclient/chunkedgraph.py +++ b/caveclient/chunkedgraph.py @@ -189,7 +189,6 @@ def __init__( self._default_timestamp = timestamp self._table_name = table_name self._segmentation_info = None - self._server_version = self._get_version() @property def default_url_mapping(self): diff --git a/caveclient/jsonservice.py b/caveclient/jsonservice.py index 530b7b25..7c909d14 100644 --- a/caveclient/jsonservice.py +++ b/caveclient/jsonservice.py @@ -187,7 +187,7 @@ def get_state_json(self, state_id): handle_response(response, as_json=False) return json.loads(response.content) - @_check_version_compatibility(">=0.4.0") + @_check_version_compatibility(method_constraint=">=0.4.0") def get_property_json(self, state_id): """Download a Neuroglancer JSON state From 7628a60053c5b9489e272ffb87d930a969b50ef3 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 6 Apr 2024 00:30:13 -0700 Subject: [PATCH 5/9] fix error --- caveclient/jsonservice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caveclient/jsonservice.py b/caveclient/jsonservice.py index 7c909d14..b9b38410 100644 --- a/caveclient/jsonservice.py +++ b/caveclient/jsonservice.py @@ -9,8 +9,8 @@ from .base import ( ClientBase, _api_endpoints, - handle_response, _check_version_compatibility, + handle_response, ) from .endpoints import ( default_global_server_address, From 90515a24b1cd02fd002fb28cd2dcf971e5bc9f5f Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 6 Apr 2024 00:30:22 -0700 Subject: [PATCH 6/9] =?UTF-8?q?Bump=20version:=205.18.0=20=E2=86=92=205.18?= =?UTF-8?q?.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- caveclient/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index dfecc1cc..1e49ec63 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.18.0 +current_version = 5.18.1 commit = True tag = True diff --git a/caveclient/__init__.py b/caveclient/__init__.py index f50517f8..39bd2abc 100644 --- a/caveclient/__init__.py +++ b/caveclient/__init__.py @@ -1,4 +1,4 @@ -__version__ = "5.18.0" +__version__ = "5.18.1" from .frameworkclient import CAVEclient From b69d04a9ba89f86a8401e62f07081d90dede620b Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sun, 7 Apr 2024 14:32:22 -0700 Subject: [PATCH 7/9] fix: non-kwarg error for nglstate feature --- caveclient/base.py | 22 +++++++++++----------- caveclient/endpoints.py | 1 + 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/caveclient/base.py b/caveclient/base.py index 74c858ad..5e21d613 100644 --- a/caveclient/base.py +++ b/caveclient/base.py @@ -407,17 +407,17 @@ def wrapper(*args, **kwargs): ) raise ServerIncompatibilityError(msg) - - for kwarg, kwarg_constraint in kwarg_use_constraints.items(): - if _version_fails_constraint(self.server_version, kwarg_constraint): - msg = ( - f"Use of keyword argument `{kwarg}` in `{method.__name__}` " - "is only permitted " - f"for server version {kwarg_constraint}, your server " - f"version is {self.server_version}. Contact your system " - "administrator to update the server version." - ) - raise ServerIncompatibilityError(msg) + if kwarg_use_constraints is not None: + for kwarg, kwarg_constraint in kwarg_use_constraints.items(): + if _version_fails_constraint(self.server_version, kwarg_constraint): + msg = ( + f"Use of keyword argument `{kwarg}` in `{method.__name__}` " + "is only permitted " + f"for server version {kwarg_constraint}, your server " + f"version is {self.server_version}. Contact your system " + "administrator to update the server version." + ) + raise ServerIncompatibilityError(msg) out = method(*args, **kwargs) return out diff --git a/caveclient/endpoints.py b/caveclient/endpoints.py index 192849d1..b871883f 100644 --- a/caveclient/endpoints.py +++ b/caveclient/endpoints.py @@ -223,6 +223,7 @@ "upload_properties": json_v1 + "/property/post", "get_properties_raw": json_v1 + "/property/raw/{state_id}", "upload_properties_w_id": json_v1 + "/property/post/{state_id}", + "get_version": json_v1 + "/version", } json_legacy = "{json_server_address}/nglstate" From 9420ebb4e1c87f4e9023a75c87c34101111a9f4e Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sun, 7 Apr 2024 20:08:51 -0700 Subject: [PATCH 8/9] =?UTF-8?q?Bump=20version:=205.18.1=20=E2=86=92=205.19?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- caveclient/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1e49ec63..ae0a4085 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.18.1 +current_version = 5.19.0 commit = True tag = True diff --git a/caveclient/__init__.py b/caveclient/__init__.py index 39bd2abc..bc3940cf 100644 --- a/caveclient/__init__.py +++ b/caveclient/__init__.py @@ -1,4 +1,4 @@ -__version__ = "5.18.1" +__version__ = "5.19.0" from .frameworkclient import CAVEclient From d8c528fea877faee31d2f84992c4d2d85a57c910 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 8 Apr 2024 08:50:26 -0700 Subject: [PATCH 9/9] adding to changelog --- docs/changelog.md | 94 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 5a68d24a..f94251ab 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,6 +1,100 @@ --- title: Changelog --- +## 5.19.0 (April 8, 2024) +- Added generalized support for detecting server versions to provide timely exceptions to users +- Used new support to check that chunkegraph has updated version before using spatial bounds kwarg +on client.chunkedgraph.level2_chunk_graph +- Added support for postign and getting segment properties files to client.state + +## 5.18.0 +- Added serialization support for pandas.index + +## 5.17.3 +- Minor documentation typo fix + +## 5.17.2 +- Bug fixes related to table_manager interface + +## 5.17.1 +- Bug fixes related to table_manager interface + +## 5.17.0 +- Fix attrs in dataframe attributes of client.materialize results to remove numpy arrays to allow concatenation of dataframes +- Added getting multiple schemas in one call to improve initialization of table_manager interface of materialization + +## 5.16.1 +- Bugfix on client.chunkedgrpah.level2_chunk_graph + +## 5.16.0 +- Added bounding box query to client.chunkedgraph.level2_chunk_graph +- Fix default materialization version client when server not advertising correctly + +## 5.15.1 (Jan 18, 2024) +- minor improvements to release process + +## 5.15.0 (Jan 18, 2024) +- Improved documentation with types +- Improved testing on more python versions +- Bugfixes for pyton 3.12 compatability + +## 5.14.0 (November 24, 2023) +- Made automatic detection of neuroglancer versioning when constructing link shortener links + +## 5.13.0 (October 26, 2023) +- Add option to get expired versions to client.materialize.get_versions + +## 5.12.1 (October 16, 2023) +- Bugfixes for client.chunkedgraph.get_latest_roots + +## 5.12.0 (October 16, 2023) +- Improved logic for client.chunkedgraph.get_latest_roots to work forward or backwards in time + +## 5.11.0 (September 19, 2023) +- Added filter_regex_dict options to client.materialize.query_table interface + +## 5.10.2 (August 16,2023) +- Fixed pyarrow support for live_live query + +## 5.10.1 (August 14,2023) +- Changed random_sample argument to be an integer number of annotations rather than a floating fraction of table +- Added option to live_query + +## 5.9.0 (August 14, 2023) +- Added support for native pyarrow deserialization, allowing upgrade to pyarrow version + +## 5.8.0 +- Allowed int64 root ids to serialize properly +- Added warning that client.materialize.tables interface is in beta + +## 5.7.0 +- Fix to ensure stop_layer is at least 1 +- Added client.chunkedgraph.suggest_latest_roots + +## 5.6.0 +- Added views to client.materialize.tables interface +- Added optional argument to allow invalid root ids when querying live live, versus creating an exception + + +## 5.5.1 +- documentation fixes on client.materialize.join_query + +## 5.5.0 +- added methods for different neuroglancer state formats to client.state. + +## 5.4.3 +- Added 'view' querying options to materialization +- Added client.materialize.tables interface +- Added client.materialize.get_tables_metadata to get all metadata in one call + +## 5.2.0 +- Added local caching of datastack names > server_address to simplify initialization of clients +with servers other than global.daf-apis.com. + +Cache is saved on a local file ~/.cloudvolume/secrets/cave_datastack_to_server_map.json + +Cache will populate the first time caveclient.CAVEclient('my_datastack', server_address="https://my_server.com") +is called. Subsequent calls can then just be caveclient.CAVEclient('my_datastack'). ## 5.1.0