Skeleton dev

.github/workflows/dev.yml

@@ -38,4 +38,9 @@ jobs:
 
       - name: Test with pytest
         run: |
-          uvx --from poethepoet poe test
+          uvx --from poethepoet poe test
+
+      - name: Upload to codecov
+        uses: codecov/codecov-action@v2
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}

README.md

@@ -2,6 +2,7 @@
 
 ![PyPI - Version](https://img.shields.io/pypi/v/CAVEclient)
 [![build status](https://github.com/CAVEconnectome/CAVEclient/actions/workflows/daily.yml/badge.svg)](https://github.com/CAVEconnectome/CAVEclient/actions/workflows/daily.yml) [![Downloads](https://static.pepy.tech/badge/caveclient)](https://pepy.tech/project/caveclient)
+[![codecov](https://codecov.io/gh/CAVEconnectome/CAVEclient/graph/badge.svg?token=KVI1AG6B8A)](https://codecov.io/gh/CAVEconnectome/CAVEclient)
 
 CAVE is short for Connectome Annotation Versioning Engine. CAVE is a set of microservices
 that provide a framework for storing and versioning connectomics data and large sets of
@@ -27,8 +28,16 @@ pip install caveclient[cv]
 
 ## Python version support
 
-Currently we are officially supporting and testing against Python 3.8, 3.9, 3.10, 3.11 and 3.12.
+Currently we are officially supporting and testing against Python 3.9, 3.10, 3.11 and 3.12.
 
 ## Documentation
 
 You can find full documentation at [caveconnectome.github.io/CAVEclient](https://caveconnectome.github.io/CAVEclient).
+
+## Issues
+
+We welcome bug reports and questions. Please post an informative issue on the [GitHub issue tracker](https://github.com/CAVEconnectome/CAVEclient/issues).
+
+## Development
+
+To view information about developing CAVEclient, see our [contributing guide](https://caveconnectome.github.io/CAVEclient/contributing).

caveclient/annotationengine.py

@@ -312,6 +312,10 @@
         if user_id is not None:
             metadata["user_id"] = user_id
         if notice_text is not None:
+            if notice_text.lower() == "none":
+                # This alteration is actually redundant with similar code in the server,
+                # but there's no harm in doubling up on it here.
+                notice_text = ""
             metadata["notice_text"] = notice_text
         if reference_table is not None:
             metadata["table_metadata"] = {
@@ -347,31 +351,32 @@
         ----------
         table_name (str): name of table to update
         description (str, optional): text description of the the table.
-            Defaults to None (will not update).
+            Defaults to the None type, in which case no change will be made to this metadata field.
         flat_segmentation_source (str, optional): cloudpath to a flat segmentation associated with this table.
-            Defaults to None (will not update).
+            Defaults to the None type, in which case no change will be made to this metadata field.
         read_permission: str, optional
             What permissions to give the table for reading. One of
             PRIVATE: only you can read this table. Intended to be used for sorting out bugs.
             GROUP: only members that share a group with you can read (intended for within group vetting)
             PUBLIC: anyone with permissions to read this datastack can read this data
-            Defaults to None (will not update).
+            Defaults to the None type, in which case no change will be made to this metadata field.
         write_permission: str, optional
             What permissions to give the table for writing.  One of
             PRIVATE: only you can write to this table
             GROUP: only members that share a group with you can write (excluding some groups)
             PUBLIC: Anyone can write to this table. Note all data is logged, and deletes are done
             by marking rows as deleted, so all data is always recoverable
-            Defaults to None (will not update).
+            Defaults to the None type, in which case no change will be made to this metadata field.
         user_id (int, optional): change ownership of this table to this user_id.
             Note, if you use this you will not be able to update the metadata on this table any longer
             and depending on permissions may not be able to read or write to it
-            Defaults to None. (will not update)
+            Defaults to the None type, in which case no change will be made to this metadata field.
         notice_text: str, optional
             Text the user will see when querying this table. Can be used to warn users of flaws,
             and uncertainty in the data, or to advertise citations that should be used with this table.
-            If you wish to remove the notice_text pass an empty string.
-            Defaults to None. (will not update)
+            If you wish to remove the notice_text pass an empty string, or "none", "None", "NONE",
+            or any other capitaliztion of the word "none".
+            Defaults to the None type, in which case no change will be made to this metadata field.
         aligned_volume_name : str or None, optional
             Name of the aligned_volume. If None, uses the one specified in the client.
         """
@@ -406,10 +411,11 @@
         if user_id is not None:
             metadata["user_id"] = user_id
         if notice_text is not None:
-            if notice_text == "None":
-                metadata["notice_text"] = ""
-            else:
-                metadata["notice_text"] = notice_text
+            if notice_text.lower() == "none":
+                # This alteration is actually redundant with similar code in the server,
+                # but there's no harm in doubling up on it here.
+                notice_text = ""
+            metadata["notice_text"] = notice_text
 
         data = {"table_name": table_name, "metadata": metadata}
         response = self.session.put(url, json=data)

caveclient/skeletonservice.py

@@ -8,6 +8,7 @@
 from io import BytesIO, StringIO
 from typing import List, Literal, Optional, Union
 
+import numpy as np
 import pandas as pd
 from cachetools import TTLCache, cached
 from packaging.version import Version
@@ -22,6 +23,9 @@
 
 SERVER_KEY = "skeleton_server_address"
 
+MAX_BULK_ASYNCHRONOUS_SKELETONS = 10000
+BULK_ASYNC_SKELETONS_BATCH_SIZE = 100
+
 
 class NoL2CacheException(Exception):
     def __init__(self, value=""):
@@ -375,6 +379,9 @@
                 f"Unknown skeleton version: {skeleton_version}. Valid options: {valid_skeleton_versions}"
             )
 
+        if isinstance(root_ids, np.ndarray):
+            root_ids = root_ids.tolist()
+
         if isinstance(root_ids, int):
             root_ids = str(root_ids)
         elif isinstance(root_ids, List):
@@ -669,20 +676,47 @@
             )
             skeleton_version = -1
 
-        url = self._build_bulk_async_endpoint(
-            root_ids, datastack_name, skeleton_version
-        )
-        response = self.session.get(url)
-        self.raise_for_status(response, log_warning=log_warning)
+        if isinstance(root_ids, np.ndarray):
+            root_ids = root_ids.tolist()
+        if not isinstance(root_ids, list):
+            raise ValueError(
+                f"root_ids must be a list or numpy array of root_ids, not a {type(root_ids)}"
+            )
 
-        estimated_async_time_secs_upper_bound = float(response.text)
+        if len(root_ids) > MAX_BULK_ASYNCHRONOUS_SKELETONS:
+            logging.warning(
+                f"The number of root_ids exceeds the current limit of {MAX_BULK_ASYNCHRONOUS_SKELETONS}. Only the first {MAX_BULK_ASYNCHRONOUS_SKELETONS} will be processed."
+            )
+            root_ids = root_ids[:MAX_BULK_ASYNCHRONOUS_SKELETONS]
 
-        if verbose_level >= 1:
-            logging.info(
-                f"Queued asynchronous skeleton generation for root_ids: {root_ids}"
+        estimated_async_time_secs_upper_bound_sum = 0
+        for batch in range(0, len(root_ids), BULK_ASYNC_SKELETONS_BATCH_SIZE):
+            rids_one_batch = root_ids[batch : batch + BULK_ASYNC_SKELETONS_BATCH_SIZE]
+
+            url = self._build_bulk_async_endpoint(
+                rids_one_batch, datastack_name, skeleton_version
             )
-            logging.info(
-                f"Upper estimate to generate {len(root_ids)} skeletons: {estimated_async_time_secs_upper_bound} seconds"
+            response = self.session.get(url)
+            self.raise_for_status(response, log_warning=log_warning)
+
+            estimated_async_time_secs_upper_bound = float(response.text)
+            estimated_async_time_secs_upper_bound_sum += (
+                estimated_async_time_secs_upper_bound
             )
 
-        return f"Upper estimate to generate {len(root_ids)} skeletons: {estimated_async_time_secs_upper_bound} seconds"
+            if verbose_level >= 1:
+                logging.info(
+                    f"Queued asynchronous skeleton generation for one batch of root_ids: {rids_one_batch}"
+                )
+                logging.info(
+                    f"Upper estimate to generate one batch of {len(rids_one_batch)} skeletons: {estimated_async_time_secs_upper_bound} seconds"
+                )
+
+        if estimated_async_time_secs_upper_bound_sum < 60:
+            return f"Upper estimate to generate all {len(root_ids)} skeletons: {estimated_async_time_secs_upper_bound_sum:.0f} seconds"
+        if estimated_async_time_secs_upper_bound_sum < 3600:
+            return f"Upper estimate to generate all {len(root_ids)} skeletons: {(estimated_async_time_secs_upper_bound_sum / 60):.1f} minutes"
+        # With a 10000 skeleton limit, the maximum time about 12 hours, so we don't need to check for more than that.
+        if True:  # estimated_async_time_secs_upper_bound_sum < 86400:
+            return f"Upper estimate to generate all {len(root_ids)} skeletons: {(estimated_async_time_secs_upper_bound_sum / 3600):.1f} hours"
+        # return f"Upper estimate to generate all {len(root_ids)} skeletons: {(estimated_async_time_secs_upper_bound_sum / 86400):.2f} days"

docs/changelog.md

@@ -1,6 +1,10 @@
 ---
 title: Changelog
 ---
+## 7.4.0 (December 12, 2024)
+annotation module: Added support for "None", "none", etc. to remove notice_text and 
+improved documentation on method
+
 ## 7.3.2 (December 10, 2024)
 Dropped oldest supported numpy requirements and declared python 3.8 support
 

docs/contributing.md

@@ -45,7 +45,7 @@ If you are proposing a feature:
 - Keep the scope as narrow as possible, to make it easier to implement.
 - Remember that while contributions are welcome, developer/maintainer time is limited.
 
-## Get Started
+## Modifying Code
 
 Ready to contribute? Here's how to set up `{{ names.package }}` for local development.
 
@@ -56,17 +56,22 @@ Ready to contribute? Here's how to set up `{{ names.package }}` for local develo
 git clone [email protected]:your_name_here/{{ names.repo_title }}.git
 ```
 
-- Ensure [pip](https://pip.pypa.io/en/stable/installation/) is installed.
-- Create a virtual environment (here we use venv):
+- We use [`uv`](https://docs.astral.sh/uv/) for various developer tasks. Ensure you have `uv` installed according to the [installation instructions](https://docs.astral.sh/uv/getting-started/installation/).
+
+!!! note
+
+    While we recommend using `uv` as described here, these tasks could also be achieved using `pip` to install and run the various required tools. You can view the development requirements and build/check commands in our [`pyproject.toml`](https://github.com/CAVEconnectome/CAVEclient/blob/master/pyproject.toml), so we avoid duplicating them here.
+
+- Navigate to the newly cloned directory, e.g.:
 
   ```console
-  python3 -m venv .venv
+  cd {{ names.repo_title }}
   ```
 
-- Start your virtualenv:
+- Create a synced virtual environment, optionally specifying a Python version:
 
   ```console
-  source .venv/bin/activate
+  uv sync --python 3.12
   ```
 
 - Create a branch for local development:
@@ -76,37 +81,57 @@ git clone [email protected]:your_name_here/{{ names.repo_title }}.git
   ```
 
 - Make your changes locally
-- Install development requirements:
+
+- If you have added code that should be tested, add [tests](https://github.com/{{ config.repo_name }}/tree/master/tests).
+
+- If you have modified dependencies in any way, make sure to run
 
   ```console
-  pip install -r test_requirements.txt
-  pip install -e .
+  uv sync
   ```
 
-- When you're done making changes, check that your changes pass the
-  tests by running [pytest](https://docs.pytest.org/en/):
+- Make sure you have added documentation for any additions or modifications to public functions or classes. You can build the documentation locally to make sure it renders correctly with:
 
   ```console
-  pytest tests
+  uvx --from poethepoet poe doc-build
   ```
 
-  Note that once you submit your pull request, GitHub Actions will run the tests also,
-  including on multiple operating systems and Python versions. Your pull request will
-  have to pass on all of these before it can be merged.
+## Automated Checks
 
-- Ensure your contribution meets style guidelines. First, install [ruff](https://docs.astral.sh/ruff/):
+- Run the autoformatter:
 
   ```console
-  pip install ruff
+  uvx --from poethepoet poe lint-fix
   ```
 
-- Fix linting and formatting. From the root of the repository, run the following commands:
+- Ensure that your changes pass the checks that will be run on Github Actions, including building the documentation, checking the formatting of the code, and running the tests. To run all at once, do:
 
   ```console
-  ruff check . --extend-select I --fix
-  ruff format .
+  uvx --from poethepoet poe checks
   ```
 
+- You may be interested in running some of these checks individually, such as:
+    - To run the tests:
+
+      ```console
+      uvx --from poethepoet poe test
+      ```
+
+    - To build the documentation:
+
+      ```console
+      uvx --from poethepoet poe doc-build
+      ```
+
+    - To run the linter
+
+      ```console
+      uvx --from poethepoet poe lint
+      ```
+
+## Submitting a Pull Request
+
+- Ensure your code has passed all of the tests described above.
 - Commit your changes and push your branch to GitHub:
 
   ```console
@@ -117,22 +142,11 @@ git clone [email protected]:your_name_here/{{ names.repo_title }}.git
 
 - [Submit a pull request](https://github.com/{{ config.repo_name }}/compare) through the GitHub website.
 
-## Pull Request Guidelines
-
 Before you submit a pull request, check that it meets these guidelines:
 
 - The pull request should include tests if adding a new feature.
 - The docs should be updated with whatever changes you have made. Put
   your new functionality into a function with a docstring, and make sure the new
-  functionality is documented after building the documentation.
-
-## Documentation style
-
-We use [mkdocs](https://www.mkdocs.org/) to build the documentation. In particular, we
-use the [mkdocs-material](https://squidfunk.github.io/mkdocs-material/) theme, and a
-variety of other extensions.
-
-!!! note
+  functionality is documented after building the documentation (described above).
 
-    More information codifying our documentation style and principles coming soon. For
-    now, just try to follow the style of the existing documentation.
+- Once you submit a pull request, automated checks will run. You may require administrator approval before running these checks if this is your first time contributing to the repo.

pyproject.toml

@@ -20,7 +20,7 @@ maintainers = [{ name = "CAVE Developers" }]
 name = "caveclient"
 readme = "README.md"
 requires-python = ">=3.9"
-version = "7.3.2"
+version = "7.4.0"
 
 [project.urls]
 Documentation = "https://caveconnectome.github.io/CAVEclient/"
@@ -62,7 +62,7 @@ test = ["pytest", "pytest-cov", "pytest-env", "pytest-mock", "responses"]
 allow_dirty = false
 commit = true
 commit_args = ""
-current_version = "7.3.2"
+current_version = "7.4.0"
 ignore_missing_version = false
 message = "Bump version: {current_version} → {new_version}"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
@@ -90,7 +90,9 @@ dry-bump = "uv run bump-my-version bump --dry-run --verbose"
 lint = ['lint-check', 'lint-format']
 lint-check = "uv run ruff check . --extend-select I"
 lint-format = "uv run ruff format . --check"
-test = "uv run pytest tests"
+test = "uv run pytest --cov --cov-report xml tests"
+lint-fix = "uv run ruff format caveclient tests"
+
 
 [build-system]
 build-backend = "hatchling.build"