diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index e16214c8..06170adf 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 5.11.0
+current_version = 5.14.0
commit = True
tag = True
diff --git a/.github/workflows/annotationframeworkclient.yml b/.github/workflows/annotationframeworkclient.yml
deleted file mode 100644
index aab164ff..00000000
--- a/.github/workflows/annotationframeworkclient.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
-name: CAVE Client
-
-on:
- push:
- branches:
- - master
- paths-ignore:
- - "README.rst"
- pull_request:
- branches: master
-
-jobs:
- test:
- name: Test against different Python versions
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- - name: Set up Python 3.7
- uses: actions/setup-python@v2
- with:
- python-version: 3.7
-
- - uses: actions/cache@v2
- with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-pip-${{ hashFiles('**/test_requirements.txt') }}
- restore-keys: |
- ${{ runner.os }}-pip-
-
- - name: Install dependencies
- run: |
- python -m pip install --upgrade pip
- pip install flake8 pytest
- pip install -r requirements.txt
- if [ -f test_requirements.txt ]; then pip install -r test_requirements.txt; fi
-
- - name: Lint with flake8
- run: |
- # stop the build if there are Python syntax errors or undefined names
- # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
- # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-
- - name: Test with pytest
- run: |
- pytest
diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
new file mode 100644
index 00000000..477775e7
--- /dev/null
+++ b/.github/workflows/daily.yml
@@ -0,0 +1,8 @@
+name: build status
+on:
+ schedule:
+ - cron: "8 15 * * *" # 7:08am PST
+ workflow_dispatch:
+jobs:
+ build:
+ uses: ./.github/workflows/dev.yml
diff --git a/.github/annotationframeworkclient.yml b/.github/workflows/dev.yml
similarity index 60%
rename from .github/annotationframeworkclient.yml
rename to .github/workflows/dev.yml
index aab164ff..667fa271 100644
--- a/.github/annotationframeworkclient.yml
+++ b/.github/workflows/dev.yml
@@ -1,7 +1,7 @@
# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-name: CAVE Client
+name: Development tests
on:
push:
@@ -11,31 +11,41 @@ on:
- "README.rst"
pull_request:
branches: master
+ # Allows you to run this workflow manually from the Actions tab
+ workflow_dispatch:
+ # Allows other workflows to trigger this workflow
+ workflow_call:
jobs:
test:
name: Test against different Python versions
- runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python-version: [3.7, 3.8, 3.9, 3.11]
+ os: [ubuntu-latest, windows-latest, macos-latest]
+
+ runs-on: ${{ matrix.os }}
steps:
- - uses: actions/checkout@v2
- - name: Set up Python 3.7
- uses: actions/setup-python@v2
+ - uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
with:
- python-version: 3.7
+ python-version: ${{ matrix.python-version }}
- - uses: actions/cache@v2
+ - uses: actions/cache@v4
with:
path: ~/.cache/pip
- key: ${{ runner.os }}-pip-${{ hashFiles('**/test_requirements.txt') }}
+ key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('**/test_requirements.txt') }}
restore-keys: |
- ${{ runner.os }}-pip-
+ ${{ runner.os }}-${{ matrix.python-version }}-pip-
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install flake8 pytest
pip install -r requirements.txt
- if [ -f test_requirements.txt ]; then pip install -r test_requirements.txt; fi
+ pip install -r test_requirements.txt
- name: Lint with flake8
run: |
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
new file mode 100644
index 00000000..4eae4dce
--- /dev/null
+++ b/.github/workflows/docs.yml
@@ -0,0 +1,60 @@
+name: dev docs
+
+on:
+ # run this workflow when dev checks pass
+ push:
+ branches:
+ - master
+
+ # Allows you to run this workflow manually from the Actions tab
+ workflow_dispatch:
+
+jobs:
+ # This workflow contains a single job called "docs"
+ docs:
+ name: Build and deploy (dev) docs
+
+ strategy:
+ matrix:
+ os: [ubuntu-latest]
+ python-version: ["3.11"]
+
+ runs-on: ${{ matrix.os }}
+
+ steps:
+ - name: Get repo
+ uses: actions/checkout@v4
+
+ - name: Get gh-pages branch
+ run: git fetch origin gh-pages --depth=1
+
+ - name: Setup Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Update pip
+ run: python -m pip install --upgrade pip
+
+ - name: Cache for virtual environment
+ uses: actions/cache@v3
+ with:
+ path: ./.venv
+ key: venv-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('doc_requirements.txt') }}
+
+ - name: Install dependencies
+ run: pip install -r doc_requirements.txt
+
+ - name: Configure git for github-actions[bot]
+ run: |
+ git config --global user.name "github-actions[bot]"
+ git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+ # - name: Build and push versioned docs with mike
+ # run: |
+ # poetry run mike deploy --push --update-aliases dev
+ # poetry run mike set-default stable --push
+
+ - name: Build and push docs
+ run: |
+ mkdocs gh-deploy --force --remote-branch gh-pages --remote-name origin
diff --git a/.readthedocs.yml b/.readthedocs.yml
deleted file mode 100644
index 83426bb2..00000000
--- a/.readthedocs.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-# .readthedocs.yaml
-# Read the Docs configuration file
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-# Required
-version: 2
-
-# Set the version of Python and other tools you might need
-build:
- os: ubuntu-22.04
- tools:
- python: "3.11"
-
-# Build documentation in the docs/ directory with Sphinx
-sphinx:
- configuration: docs/conf.py
-
-# We recommend specifying your dependencies to enable reproducible builds:
-# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-python:
- install:
- - requirements: docs/requirements.txt
diff --git a/CAVEclientExamples.ipynb b/CAVEclientExamples.ipynb
deleted file mode 100644
index 3eb7e240..00000000
--- a/CAVEclientExamples.ipynb
+++ /dev/null
@@ -1,541 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "source": [
- "### 1. One client to rule them all\n",
- "\n",
- "The Connectome Annotation Versioning Engine framework consists of a number of different services, each with a specific set of tasks that it can perform through REST endpoints. This module is designed to ease programmatic interaction with all of the various endpoints. Going forward, we also will be increasingly using authentication tokens for programmatic access to most if not all of the services. In order to collect a given server, datastack name, and user token together into a coherent package that can be used on multiple endpoints, we will use a CAVEclient that can build appropriately configured clients for each of the specific services.\n",
- "\n",
- "The following examples cover each of the client subtypes that are associated with a single service. The ImageryClient, which is a more complex collection of tools, will be covered elsewhere."
- ],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "#### Setting up your credentials\n",
- "\n",
- "Most services require you to authenticate yourself. In order to use authenticated services in the CAVEclient, you need to get a token for yourself and save it to your hard drive. This token should be treated as a personal secret, like a password.\n",
- "\n",
- "The easiest way to get your token is to use the client in its so-called \"global\" mode, without reference to any particular dataset. Once your client is initialized, you can get instructions for how to get a new token by running `client.auth.get_new_token()`. If your server is not for the MICrONs project, you may need to set a server address when initializing the CAVEclient.\n",
- "\n",
- "Note: If you have already set up Graphene authetication in CloudVolume, the same token applies. The CAVEclient will read from and write to the same default file as CloudVolume, so you probably need to do nothing to use it. If you need multiple tokens for different projects, please read the documentation.\n"
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "from caveclient import CAVEclient\n",
- "\n",
- "client = CAVEclient()\n",
- "client.auth.get_new_token()"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "Follow the instructions that are printed out. If you cannot log in in step 2, contact the project administrators. Copy the token (only the long alphanumeric part!) and save it to your drive by running the command specified in step 3a after replacing the PASTE_YOUR_TOKEN_HERE bit with the value you got after login. Once this is done, you shouldn't need to do it again for a long while. Note that every time you do the refresh_token step, you invalidate any previous token. That means that if you want to use the CAVEclient on multiple computers, you need to copy the same token to each device instead of going through these steps multiple times."
- ],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "#### Initializing a CAVEclient\n",
- "\n",
- "Most services require the use of a specific datastack. Once you have set up credentials on your computer. we can specify a datastack."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "from caveclient import CAVEclient\n",
- "\n",
- "datastack_name = 'minnie65_phase3_v0'\n",
- "client = CAVEclient(datastack_name)"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "Just to confirm that this works, let's see if we can get the EM image source from the InfoService. If you get a reasonable looking path, everything is okay."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "print(f\"The image source is: {client.info.image_source()}\")"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "If you don't know what datastacks exist, you can start a client with or without a datastack name (like we did when we set the auth token) and run `client.info.get_datastacks()`"
- ],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "### 2. Authentication Service\n",
- "\n",
- "The AuthClient handles storing and loading your token or tokens and inserting it into requests in other clients. We can access the auth client from `client.auth`. Once you have saved a token, you probably won't interact with this client very often, however it makes it convenient for saving new tokens the first time and has some other handy features. For example, to check what your token is (for example to set up a second computer)."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "auth = client.auth\n",
- "print(f\"My current token is: {auth.token}\")"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "#### Loading saved tokens\n",
- "The credentials save by default in `~/.cloudvolume/secrets/cave-secret.json`. You can try opening that file to see what we just created.\n",
- "\n",
- "If we had wanted to use a different file or a different json key, we could have specified that in `auth.save_token`.\n",
- "\n",
- "Because we used the default values, this token is used automatically when we intialize a new CAVEclient. If we wanted to use a different token file, token key, or even directly specify a token we could also do so here. Of course, a bad token will cause an unauthorized error."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "client = CAVEclient(datastack_name)\n",
- "print(f\"Now my basic token is: {client.auth.token}\")\n",
- "\n",
- "client_direct = CAVEclient(datastack_name, auth_token='another_fake_token_678')\n",
- "print(f\"A directly specified token is: {client_direct.auth.token}\")"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "If you use a CAVEclient, the AuthClient and its token will be automatically applied to any other services without further use. "
- ],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "### 3. Info Service\n",
- "A datastack has a number of complex paths to various data sources that together comprise a datastack. Rather than hardcode these paths, the InfoService allows one to query the location of each data source. This is also convenient in case data sources change.\n",
- "\n",
- "An InfoClient is accessed at `client.info`."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "client.info"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "client = CAVEclient(datastack_name)\n",
- "print(f\"This is an info client for {client.info.datastack_name} on {client.info.server_address}\")"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "#### Accessing datastack information\n",
- "All of the information accessible for the datastack can be seen as a dict using `get_datastack_info()`."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "client.info.get_datastack_info()"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "Individual entries can be found as well. Use tab autocomplete to see the various possibilities."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "client.info.local_server()"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "### 4. JSON Service\n",
- "\n",
- "We store the JSON description of a Neuroglancer state in a simple database at the JSON Service. This is a convenient way to build states to distribute to people, or pull states to parse work by individuals. The JSON Client is at `client.state`. Note that the state service will work with or without a datastack set."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "client.state"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "#### Retrieving a state\n",
- "\n",
- "JSON states are found simply by their ID, which you get when uploading a state. You can download a state with `get_state_json`."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "example_id = 5762925562167296\n",
- "example_state = client.state.get_state_json(example_id)\n",
- "example_state['layers'][0]"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "#### Uploading a state\n",
- "You can also upload states with `upload_state_json`. If you do this, the state id is returned by the function. Note that there is no easy way to query what you uploaded later, so be VERY CAREFUL with this state id if you wish to see it again.\n",
- "\n",
- "*Note: If you are working with a Neuroglancer Viewer object or similar, in order to upload, use viewer.state.to_json() to generate this representation.*"
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "example_state['layers'][0]['name'] = 'example_name'\n",
- "new_id = client.state.upload_state_json(example_state)"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "test_state = client.state.get_state_json(new_id)\n",
- "test_state['layers'][0]['name']"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "#### Generating Neuroglancer URLs\n",
- "\n",
- "Neuroglancer can automatically look up a JSON state based on its ID if you pass the URL to it correctly. The function `build_neuroglancer_url` helps format these correctly. Note that you need to specify the base URL for the Neuroglancer deployment you wish to use."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "url = client.state.build_neuroglancer_url(state_id=new_id, ngl_url='https://neuromancer-seung-import.appspot.com')\n",
- "print(url)"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "### 5. ChunkedGraph\n",
- "\n",
- "The ChunkedGraph client allows one to interact with the ChunkedGraph, which stores and updates the supervoxel agglomeration graph. This is most often useful for looking up an object root id of a supervoxel or looking up supervoxels belonging to a root id. The ChunkedGraph client is at `client.chunkedgraph`."
- ],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "#### Look up a supervoxel\n",
- "Usually in Neuroglancer, one never notices supervoxel ids, but they are important for programmatic work. In order to look up the root id for a location in space, one needs to use the supervoxel segmentation to get the associated supervoxel id. The ChunkedGraph client makes this easy using the `get_root_ids` method."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "from caveclient import CAVEclient\n",
- "\n",
- "datastack_name = 'minnie65_phase3_v0'\n",
- "client = CAVEclient(datastack_name)"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "sv_id = 109362238070465629\n",
- "client.chunkedgraph.get_root_id(supervoxel_id=sv_id)"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "However, as proofreading occurs, the root id that a supervoxel belongs to can change. By default, this function returns the current state, however one can also provide a UTC timestamp to get the root id at a particular moment in history. This can be useful for reproducible analysis."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "import datetime\n",
- "\n",
- "date_3_days_ago = datetime.datetime.now() - datetime.timedelta(days=3)\n",
- "\n",
- "# I looked up the UTC POSIX timestamp from a day in early 2019. \n",
- "#timestamp = datetime.datetime.utcfromtimestamp(1546595253)\n",
- "\n",
- "client.chunkedgraph.get_root_id(supervoxel_id=sv_id, timestamp=date_3_days_ago)"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "#### Getting supervoxels for a root id\n",
- "\n",
- "A root id is associated with a particular agglomeration of supervoxels, which can be found with the `get_leaves` method. A new root id is generated for every new change in the chunkedgraph, so time stamps do not apply."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "root_id = 864691134988869442\n",
- "client.chunkedgraph.get_leaves(root_id)"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "### 7. AnnotationEngine\n",
- "\n",
- "The AnnotationClient is used to interact with the AnnotationEngine service to create tables from existing schema, upload new data, and download existing annotations. Note that annotations in the AnnotationEngine are not linked to any particular segmentation, and thus do not include any root ids. An annotation client is accessed with `client.annotation`."
- ],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "#### Get existing tables\n",
- "\n",
- "A list of the existing tables for the datastack can be found at with `get_tables`."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "all_tables = client.annotation.get_tables()\n",
- "all_tables"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "Each table has three main properties that can be useful to know:\n",
- "* `table_name` : The table name, used to refer to it when uploading or downloading annotations. This is also passed through to the table in the Materialized database.\n",
- "* `schema_name` : The name of the table's schema from EMAnnotationSchemas (see below).\n",
- "* `max_annotation_id` : An upper limit on the number of annotations already contained in the table."
- ],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "#### Downloading annotations\n",
- "\n",
- "You can download the JSON representation of a data point through the `get_annotation` method. This can be useful if you need to look up information on unmaterialized data, or to see what a properly templated annotation looks like."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "table_name = all_tables[0]\n",
- "annotation_id = 1\n",
- "client.annotation.get_annotation(annotation_ids=1, table_name=table_name)"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "#### Create a new table\n",
- "\n",
- "One can create a new table with a specified schema with the `create_table` method:\n",
- "\n",
- "```\n",
- "client.annotation.create_table(table_name='test_table',\n",
- " schema_name='microns_func_coreg')\n",
- "```\n",
- "\n",
- "Now, new data can be generated as a dict or list of dicts following the schema and uploaded with `post_annotation`.\n",
- "For example, a `microns_func_coreg` point needs to have:\n",
- " * `type` set to `microns_func_coreg`\n",
- " * `pt` set to a dict with `position` as a key and the xyz location as a value.\n",
- " * `func_id` set to an integer.\n",
- " \n",
- "The following will create a new annotation and upload it to the service: \n",
- "```\n",
- "new_data = {'type': 'microns_func_coreg',\n",
- " 'pt': {'position': [1,2,3]},\n",
- " 'func_id': 0}\n",
- " \n",
- "client.annotation.post_annotation(table_name='test_table', data=[new_data])\n",
- "```"
- ],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "### 7. EMAnnotationSchemas\n",
- "\n",
- "The EMAnnotationSchemas client lets one look up the available schemas and how they are defined. This is mostly used for programmatic interactions between services, but can be useful when looking up schema definitions for new tables."
- ],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "#### Get the list of schema \n",
- "One can get the list of all available schema with the `schema` method. Currently, new schema have to be generated on the server side, although we aim to have a generic set available to use."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "client.schema.schema()"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "#### View a specific schema\n",
- "\n",
- "The details of each schema can be viewed with the `schema_definition` method, formatted as per JSONSchema. "
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "example_schema = client.schema.schema_definition('microns_func_coreg')\n",
- "example_schema"
- ],
- "outputs": [],
- "metadata": {}
- },
- {
- "cell_type": "markdown",
- "source": [
- "This is mostly useful for programmatic interaction between services at the moment, but can also be used to inspect the expected form of an annotation by digging into the format."
- ],
- "metadata": {}
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "source": [
- "example_schema['definitions']['FunctionalCoregistration']"
- ],
- "outputs": [],
- "metadata": {}
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.7.7"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}
\ No newline at end of file
diff --git a/FlyWireAnnotationTutorial.ipynb b/FlyWireAnnotationTutorial.ipynb
deleted file mode 100644
index 1f18659b..00000000
--- a/FlyWireAnnotationTutorial.ipynb
+++ /dev/null
@@ -1,323 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# The CAVEclient\n",
- "\n",
- "The CAVEclient is a client side library to allow easy interaction with the services within CAVE (connectome annotation versioning engine, also known as Dynamic Annotation Framework), eg. the annotations, stateserver. The github repository is public:\n",
- "https://github.com/seung-lab/CAVEclient\n",
- "\n",
- "The library can be installed directly from the github repository or from the prebuilt versions using pip:\n",
- "```\n",
- "pip install caveclient\n",
- "```\n",
- "\n",
- "\n",
- "## Tutorials\n",
- "\n",
- "This tutorial mainly covers the interactions with the materialized annotation tables. More information and better explanations of the other functionalities of the client can be found in the following tutorial. Please be advised that depending on your permission level you may not be able to execute all queries in this tutorial with the preset parameters as it was written with defaults for iarpa's microns project:\n",
- "https://github.com/seung-lab/CAVEclient/blob/master/CAVEclientExamples.ipynb\n",
- "\n",
- "\n",
- "## Authentication & Authorization\n",
- "\n",
- "If this is your first time to interact with any part of CAVE, chances are you need to setup your local credentials for your FlyWire account first. Please follow the section \"Setting up your credentials\" at the beginning of the tutorial above to do so.\n",
- "\n",
- "You will need to have access to the FlyWire's production dataset to retrieve annotations. Otherwise you will see\n",
- "\n",
- "```HTTPError: 403 Client Error: FORBIDDEN for url```\n",
- "\n",
- "errors upon querying the materialization server."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Initializing the CAVEclient\n",
- "\n",
- "The FrameworkClient is instantiated with a datastack name. A datastack is a set of segmentation, and annotation tables and lives within an aligned volume (the coordinate space). FlyWire's main datastack is `flywire_fafb_production`, the aligned volume is `fafb_seung_alignment_v0` (v14.1). For convenience, there are other defaults set on the datastack level."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [],
- "source": [
- "import numpy as np\n",
- "import datetime\n",
- "import pandas as pd\n",
- "from caveclient import CAVEclient"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [],
- "source": [
- "datastack_name = \"flywire_fafb_production\"\n",
- "client = CAVEclient(datastack_name)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Annotation tables\n",
- "\n",
- "Annotations are represented by points in space and parameters (such as size, type). At specific timepoints, annotations are combined with the (proofread) segmentation to create a materialized version of the annotation table. The AnnotationEngine (`client.annotation`) owns the raw annotations and the Materialization Service (`client.materialize`) owns the materialized versions of these tables. \n",
- "\n",
- "To check what annotation tables are visible to you run"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "https://prod.flywire-daf.com/annotation/api/v2/aligned_volume/fafb_seung_alignment_v0/table\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "['synapses_nt_v1', 'nuclei_v1']"
- ]
- },
- "execution_count": 3,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "client.annotation.get_tables()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Creating a table"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "All users with permissions to proofread in FlyWire can create annotation tables and upload annotations. Currently, only the user who created a table can upload annotations to it; all other users have read access to the table.\\\n",
- "\n",
- "Creating a table requires a unique `table_name`, a `schema`, a `description` and a `voxel_resolution`. The `voxel_resolution` defines the resolution with which annotations are uploaded. For instance, `[1, 1, 1]` would mean that annotations in this table will be uploaded in nanometer space. \n",
- "\n",
- "Schemas are managed in a separate [repository](https://github.com/seung-lab/EMAnnotationSchemas). Schemas can freely be chosen from [there](https://globalv1.daf-apis.com/schema/views/). If no applicable schema is available, we encourage users to create a new schema and submit a pull-request. \n",
- "\n",
- "In the example below, we are going to use the [`bound_tag`](https://globalv1.daf-apis.com/schema/views/type/bound_tag/view) schema which is an annotation with one coordinate and a text field. All annotations in a table follow the same schema."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "To avoid the creation of a large list of test tables, the following code is embedded in markdown. "
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "```\n",
- "\n",
- "table_name = \"my_table\"\n",
- "\n",
- "description = \"\"\"\n",
- "This is a test table to demonstrate table creation and annotation upload.\n",
- "The data in this table is random and should not be used for analysis.\n",
- "\n",
- "This table was create by ...\"\"\"\n",
- "\n",
- "client.annotation.create_table(table_name=random_table_name,\n",
- " schema_name=\"bound_tag\",\n",
- " description=description,\n",
- " voxel_resolution=[1, 1, 1])\n",
- " \n",
- "```"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "All tables are listed here: https://prod.flywire-daf.com/annotation/views/aligned_volume/fafb_seung_alignment_v0\n",
- "\n",
- "A specific table can be viewed here: https://prod.flywire-daf.com/annotation/views/aligned_volume/fafb_seung_alignment_v0/table/table_name (with table_name replaced)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Uploading and updating annotations"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Next, we generate some annotations to be uploaded to the new table. Tables can be uploaded from pandas DataFrames with columns according to the schema. One can include an `id` column to specify specific annotation, otherwise `id`s are assigned by the backend. "
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "```\n",
- "random_locations_nm = np.random.randint([448000, 185000, 87000], [588000, 292000, 90000], size=[100, 3], dtype=int)\n",
- "random_tags = [f\"tag {i}\" for i in range(100)]\n",
- "\n",
- "random_annotation_data = pd.DataFrame.from_dict({\"pt_position\": list(random_locations_nm), \n",
- " \"tag\": random_tags,\n",
- " \"id\": np.arange(100, 200)})\n",
- "```"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Annotations can be uploaded using `client.annotation.post_annotation` or `client.annotation.post_annotation_df`."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "```\n",
- "client.annotation.post_annotation_df(table_name=random_table_name,\n",
- " df=random_annotation_data,\n",
- " position_columns=[\"pt_position\"])\n",
- "```"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Updating annotations works similarly to uploading them in the first place. Updating requires `id`s to be defined."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "```\n",
- "random_locations_nm = np.random.randint([448000, 185000, 87000], [588000, 292000, 90000], size=[10, 3], dtype=int)\n",
- "random_tags = [f\"tag {i}\" for i in range(10)]\n",
- "\n",
- "random_annotation_data = pd.DataFrame.from_dict({\"pt_position\": list(random_locations_nm), \n",
- " \"tag\": random_tags,\n",
- " \"id\": np.arange(100, 110)})\n",
- " \n",
- "client.annotation.update_annotation_df(table_name=random_table_name,\n",
- " df=random_annotation_data,\n",
- " position_columns=[\"pt_position\"])\n",
- "```"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Reading annotations"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Annotations can be read directly from the annotation service. Annotations can be read by ID. Here, we use the nucleus table (`nuclei_v1`, see it online [here](https://prod.flywire-daf.com/annotation/views/aligned_volume/fafb_seung_alignment_v0/table/nuclei_v1)) as example."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "[{'volume': 6.65092096,\n",
- " 'bb_start_position': [718496, 261792, 114640],\n",
- " 'valid': True,\n",
- " 'bb_end_position': [722016, 265120, 116200],\n",
- " 'pt_position': [720000, 263360, 115520],\n",
- " 'superceded_id': None,\n",
- " 'created': '2021-06-23 19:55:35.166396',\n",
- " 'deleted': 'None',\n",
- " 'id': 7415718},\n",
- " {'volume': 11.52339968,\n",
- " 'bb_start_position': [708800, 262112, 128200],\n",
- " 'valid': True,\n",
- " 'bb_end_position': [712064, 264832, 131080],\n",
- " 'pt_position': [710592, 263392, 129800],\n",
- " 'superceded_id': None,\n",
- " 'created': '2021-06-23 19:55:35.160196',\n",
- " 'deleted': 'None',\n",
- " 'id': 7416439}]"
- ]
- },
- "execution_count": 5,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "client.annotation.get_annotation(table_name=\"nuclei_v1\",\n",
- " annotation_ids=[7416439, 7415718])"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "No segment IDs will be assigned to the annotations through this interface. To access annotations with segment IDs assigned, one must use the [materialization interface](https://github.com/seung-lab/CAVEclient/blob/master/FlyWireSynapseTutorial.ipynb).\n",
- "\n",
- "For this to work, at least one materialization run has to complete after the upload. Currently, materialization happen every to every second date."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.8.5"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}
diff --git a/FlyWireSynapseTutorial.ipynb b/FlyWireSynapseTutorial.ipynb
deleted file mode 100644
index a1dd977f..00000000
--- a/FlyWireSynapseTutorial.ipynb
+++ /dev/null
@@ -1,1715 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# The CAVEclient\n",
- "\n",
- "The CAVEclient is a client side library to allow easy interaction with the services within CAVE (connectome annotation versioning engine, also known as Dynamic Annotation Framework), eg. the annotations, stateserver. The github repository is public:\n",
- "https://github.com/seung-lab/CAVEclient\n",
- "\n",
- "The library can be installed directly from the github repository or from the prebuilt versions using pip:\n",
- "```\n",
- "pip install caveclient\n",
- "```\n",
- "\n",
- "\n",
- "## Tutorials\n",
- "\n",
- "This tutorial mainly covers the interactions with the materialized annotation tables. More information and better explanations of the other functionalities of the client can be found in the following tutorial. Please be advised that depending on your permission level you may not be able to execute all queries in this tutorial with the preset parameters as it was written with defaults for iarpa's microns project:\n",
- "https://github.com/seung-lab/CAVEclient/blob/master/CAVEclientExamples.ipynb\n",
- "\n",
- "\n",
- "## Authentication & Authorization\n",
- "\n",
- "If this is your first time to interact with any part of CAVE, chances are you need to setup your local credentials for your FlyWire account first. Please follow the section \"Setting up your credentials\" at the beginning of the tutorial above to do so.\n",
- "\n",
- "You will need to have access to the FlyWire's production dataset to retrieve annotations. Otherwise you will see\n",
- "\n",
- "```HTTPError: 403 Client Error: FORBIDDEN for url```\n",
- "\n",
- "errors upon querying the materialization server."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Initialize CAVEclient\n",
- "\n",
- "The CAVEclient is instantiated with a datastack name. A datastack is a set of segmentation, and annotation tables and lives within an aligned volume (the coordinate space). FlyWire's main datastack is `flywire_fafb_production`, the aligned volume is `fafb_seung_alignment_v0` (v14.1). For convenience, there are other defaults set on the datastack level."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [],
- "source": [
- "import numpy as np\n",
- "import datetime\n",
- "from caveclient import CAVEclient"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [],
- "source": [
- "datastack_name = \"flywire_fafb_production\"\n",
- "client = CAVEclient(datastack_name)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Annotation tables\n",
- "\n",
- "Annotations are represented by points in space and parameters (such as size, type). At specific timepoints, annotations are combined with the (proofread) segmentation to create a materialized version of the annotation table. The AnnotationEngine (`client.annotation`) owns the raw annotations and the Materialization Service (`client.materialize`) owns the materialized versions of these tables. \n",
- "\n",
- "To check what annotation tables are visible to you run"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "https://prod.flywire-daf.com/annotation/api/v2/aligned_volume/fafb_seung_alignment_v0/table\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "['synapses_nt_v1']"
- ]
- },
- "execution_count": 3,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "client.annotation.get_tables()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Every table has metadata associated with it which includes information about the owner/creator, a description and a schema that annotations in this table follow. Please review the metadata of any table you might use in the future before using it as it might contain instructions and restrictions for its usage and how to credit it's creators. For instance, the (v1) synapse table (`synapses_nt_v1`) includes an extensive description on all its columns, credits people that created it, contains instructions for citing this resource among others:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "FlyWire synapse description\r\n",
- "Synapse version: 20191211\r\n",
- "NT version: 20201223\r\n",
- "\r\n",
- "Synapses in this table consist of a pre- and a postsynaptic point (in nm), confidence scores, and neurotransmitter information. \r\n",
- "The synapses were predicted by Buhmann et al [1] for the v14 alignment of the FAFB dataset. The FlyWire team remapped these synapses into the v14.1 space used by FlyWire with an accuracy of <64nm (therefore, this is a potential source of error). This version of the Buhmann et al. synapses was trained on the initial training set from the calyx and performance varies across brain areas accordingly.\r\n",
- "Buhmann et al. assigned two scores to their synapses representing different measurements of confidence. The “connection_score” column contains the scores assigned by them during prediction (higher is more confident) and “cleft_score” contains the scores acquired by Buhmann et al. by using the synapse segmentation from Heinrich et al. [2] (higher is more confident). For the latter, Buhmann et al. found the highest value in Heinrich et al.’s segmentation along the line between the pre- and postsynaptic point for each synapse. This represents an individual assessment of the validity of each synapse because Buhmann et al. found synapses without using the segmentation by Heinrich et al.. Buhmann et al. suggested using the product of both scores, others found that the “cleft_score” alone with a threshold of 50-100 to be a good filter. We suggest validating analyses with various thresholds.\r\n",
- "Some synapses were (falsely) annotated multiple times. We found that most of these can be fixed by merging synapses between the same partners that are <150nm apart on the presynaptic side. The threshold might vary for different cell types and again we suggest validating analyses with various thresholds.\r\n",
- "Eckstein et al. [3] developed a classifier to predict probabilities for each of six neurotransmitters for each synapse (sum to 1, Dopamine (DA), Acetylcholine (ACh), Glutamate (Glut), Octopamine (OCT), Serotonin (SER), Gabaergic (GABA)). Davi Bock, Gregory Jefferis and Eric Perlman contributed infrastructure to run their classifier on all synapses from Buhmann et al. . For some synapses (<20k) no neurotransmitter information could be acquired. Probabilities for these have been set to all 0. Additionally, the “valid_nt” column can be used to filter these out (False=no neurotransmitter information available). Improving this classifier through the acquisition of additional ground truth is actively worked on. The classifications of ACh, GABA and Glu appear to be more robust than DA, OCT and SER in this version. Additionally, there was little training data from the optic lobe and results are more robust in the central brain. \r\n",
- "\r\n",
- "For any use of these synapses in a study, please cite Buhmann et al. [1]. For using the “cleft_score” please also cite Heinrich et al. [2]. When including the neurotransmitter information please also cite Eckstein et al. [3].\r\n",
- "\r\n",
- "The prediction and dissemination of the neurotransmitter information was supported by NIH BRAIN Initiative (grant 1RF1MH120679-01); additional work including assembling ground truth data was also supported by Wellcome trust (203261/Z/16/Z). \r\n",
- "\r\n",
- "[1] Julia Buhmann, Arlo Sheridan, Stephan Gerhard, Renate Krause, Tri Nguyen, Larissa Heinrich, Philipp Schlegel, Wei-Chung Allen Lee, Rachel Wilson, Stephan Saalfeld, Gregory Jefferis, Davi Bock, Srinivas Turaga, Matthew Cook, Jan Funke. 2019. “Automatic Detection of Synaptic Partners in a Whole-Brain Drosophila EM Dataset”. bioRxiv\r\n",
- "[2] Heinrich L., Funke J., Pape C., Nunez-Iglesias J., Saalfeld S. 2018. “Synaptic Cleft Segmentation in Non-isotropic Volume Electron Microscopy of the Complete Drosophila Brain”. MICCAI 2018. Lecture Notes in Computer Science, vol 11071. Springer\r\n",
- "[3] Nils Eckstein, Alexander S. Bates, Michelle Du, Volker Hartenstein, Gregory S.X.E. Jefferis, Jan Funke. 2020. “Neurotransmitter Classification from Electron Microscopy Images at Synaptic Sites in Drosophila”. bioRxiv\r\n",
- "\n"
- ]
- }
- ],
- "source": [
- "meta_data = client.annotation.get_table_metadata(\"synapses_nt_v1\")\n",
- "print(meta_data[\"description\"])"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "The meta data contains information about the schema which ultimately determines how annotations in a table are structured. All annotations in a table follow the same schema. The synapse table follows the `fly_nt_synapse` schema:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "'fly_nt_synapse'"
- ]
- },
- "execution_count": 5,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "meta_data[\"schema_type\"]"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Materialized annotation tables & Queries"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "```\n",
- "materialization = annotation + segmentation snapshot\n",
- "```\n",
- "\n",
- "As the segmentation and annotations change over time, we need to create snapshots of a combined view of them (materialized versions). Materialized versions of the annotation tables are (automatically) generated at a certain frequency. In addition to that, we are planning to include an option to retrieve any timestamp since the latest materialization (\"live\") but that is not available at the moment. \n",
- "\n",
- "There are usually a number of materialized versions available at the same time:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "[56, 64, 66, 67, 68, 70]"
- ]
- },
- "execution_count": 6,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "client.materialize.get_versions()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Each version comes with meta data about the time when it was created and when it will be deleted (expired). Different tables have different lifetimes and some may be LTS versions. The exact frequency and lifetime of tables will depend on how the community is using these tables. "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "{'valid': True,\n",
- " 'time_stamp': datetime.datetime(2021, 6, 10, 8, 10, 0, 286750, tzinfo=datetime.timezone.utc),\n",
- " 'id': 59,\n",
- " 'version': 70,\n",
- " 'datastack': 'flywire_fafb_production',\n",
- " 'expires_on': datetime.datetime(2021, 6, 12, 8, 10, 0, 286750, tzinfo=datetime.timezone.utc)}"
- ]
- },
- "execution_count": 7,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "latest_version = max(client.materialize.get_versions())\n",
- "client.materialize.get_version_metadata(latest_version)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Generally, specifying versions for the materialize service is optional. The latest version is used if no version is defined. \n",
- "\n",
- "Each materialization version contains a set of annotation tables. At the moment all tables are included in a materialization but in the future we might not include all tables in every materialization:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "['synapses_nt_v1']"
- ]
- },
- "execution_count": 8,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "client.materialize.get_tables()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Queries"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Here, we demonstrate some queries with the synapses from Buhmann et al.. For some essential annotation types, default tables are define in the centralized info service. This way, one automatically uses the latest synapse table after an update. "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "synapses_nt_v1\n"
- ]
- }
- ],
- "source": [
- "synapse_table = client.info.get_datastack_info()['synapse_table']\n",
- "print(synapse_table)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Each table in this list is stored as a SQL table on the backend. The client allows users to query these tables through the frontend of the Materialization Service conventiently without the need for SQL specific language. The client will format the results as pandas dataframes. Queries are restricted to a size of 200k rows to not overwhelm the server. Should a query result in a larger list of rows, only the first 200k are returned. For bulk downloads (eg. for data preservation before a publication) please contact us.\n",
- "\n",
- "To demonstrate this this query would pull the entire table but will only gather 200k rows (should take <2min). A warning will be raised if the query is cut short."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "WARNING:root:201 - \"Limited query to 200000 rows\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CPU times: user 764 ms, sys: 187 ms, total: 951 ms\n",
- "Wall time: 6.76 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "\n",
- "syn_df = client.materialize.query_table(synapse_table)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 11,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "200000\n"
- ]
- }
- ],
- "source": [
- "print(len(syn_df))"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Here, we set the materialization version specifically. If the materialization version is not specified, the query defaults to the most recent version.\n",
- "\n",
- "Let's take a brief look at the columns to illustrate how the materialization extends an annotation table:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 12,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " id | \n",
- " valid | \n",
- " pre_pt_supervoxel_id | \n",
- " pre_pt_root_id | \n",
- " post_pt_supervoxel_id | \n",
- " post_pt_root_id | \n",
- " connection_score | \n",
- " cleft_score | \n",
- " gaba | \n",
- " ach | \n",
- " glut | \n",
- " oct | \n",
- " ser | \n",
- " da | \n",
- " valid_nt | \n",
- " pre_pt_position | \n",
- " post_pt_position | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | 0 | \n",
- " 102406485 | \n",
- " t | \n",
- " 81559230397890019 | \n",
- " 720575940633908343 | \n",
- " 81559230397890088 | \n",
- " 720575940572044374 | \n",
- " 217.020920 | \n",
- " 122 | \n",
- " 0.001483 | \n",
- " 0.988252 | \n",
- " 1.837718e-05 | \n",
- " 0.000066 | \n",
- " 0.000337 | \n",
- " 0.009844 | \n",
- " t | \n",
- " [636900, 135040, 152840] | \n",
- " [636832, 135124, 152880] | \n",
- "
\n",
- " \n",
- " | 1 | \n",
- " 101781363 | \n",
- " t | \n",
- " 81139629272907063 | \n",
- " 720575940630969051 | \n",
- " 81139629272948248 | \n",
- " 720575940599614314 | \n",
- " 22.191496 | \n",
- " 0 | \n",
- " 0.105925 | \n",
- " 0.352661 | \n",
- " 3.122027e-01 | \n",
- " 0.000107 | \n",
- " 0.176082 | \n",
- " 0.053023 | \n",
- " t | \n",
- " [613864, 292888, 152600] | \n",
- " [613868, 292984, 152640] | \n",
- "
\n",
- " \n",
- " | 2 | \n",
- " 102336123 | \n",
- " t | \n",
- " 83251997268285653 | \n",
- " 720575940627576594 | \n",
- " 83181628524108478 | \n",
- " 720575940590270455 | \n",
- " 115.574806 | \n",
- " 57 | \n",
- " 0.005971 | \n",
- " 0.927559 | \n",
- " 7.128069e-04 | \n",
- " 0.063133 | \n",
- " 0.000020 | \n",
- " 0.002605 | \n",
- " t | \n",
- " [732880, 370040, 151320] | \n",
- " [732752, 370080, 151320] | \n",
- "
\n",
- " \n",
- " | 3 | \n",
- " 101781421 | \n",
- " t | \n",
- " 73469504877006214 | \n",
- " 720575940589372239 | \n",
- " 73469504876999566 | \n",
- " 720575940587527270 | \n",
- " 141.460388 | \n",
- " 144 | \n",
- " 0.000468 | \n",
- " 0.553068 | \n",
- " 1.522532e-08 | \n",
- " 0.000013 | \n",
- " 0.000003 | \n",
- " 0.446448 | \n",
- " t | \n",
- " [166996, 294144, 151280] | \n",
- " [166920, 294024, 151280] | \n",
- "
\n",
- " \n",
- " | 4 | \n",
- " 101781599 | \n",
- " t | \n",
- " 80998960504056992 | \n",
- " 720575940613732364 | \n",
- " 80998960504043951 | \n",
- " 720575940613732364 | \n",
- " 38.557209 | \n",
- " 37 | \n",
- " 0.674273 | \n",
- " 0.012040 | \n",
- " 2.504642e-01 | \n",
- " 0.001070 | \n",
- " 0.012926 | \n",
- " 0.049227 | \n",
- " t | \n",
- " [604412, 294744, 151520] | \n",
- " [604364, 294860, 151520] | \n",
- "
\n",
- " \n",
- "
\n",
- "
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " id | \n",
- " valid | \n",
- " pre_pt_supervoxel_id | \n",
- " pre_pt_root_id | \n",
- " post_pt_supervoxel_id | \n",
- " post_pt_root_id | \n",
- " connection_score | \n",
- " cleft_score | \n",
- " gaba | \n",
- " ach | \n",
- " glut | \n",
- " oct | \n",
- " ser | \n",
- " da | \n",
- " valid_nt | \n",
- " pre_pt_position | \n",
- " post_pt_position | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | 0 | \n",
- " 211365569 | \n",
- " t | \n",
- " 77832229442872849 | \n",
- " 720575940627197566 | \n",
- " 77832229442883229 | \n",
- " 720575940612001489 | \n",
- " 52.161354 | \n",
- " 100 | \n",
- " 0.050798 | \n",
- " 0.706145 | \n",
- " 0.103963 | \n",
- " 0.020856 | \n",
- " 0.007026 | \n",
- " 0.111212 | \n",
- " t | \n",
- " [418808, 286736, 110840] | \n",
- " [418760, 286616, 110880] | \n",
- "
\n",
- " \n",
- " | 1 | \n",
- " 217890780 | \n",
- " t | \n",
- " 77691423235424395 | \n",
- " 720575940627197566 | \n",
- " 77691423235425756 | \n",
- " 720575940612001489 | \n",
- " 102.692398 | \n",
- " 145 | \n",
- " 0.004396 | \n",
- " 0.970069 | \n",
- " 0.000883 | \n",
- " 0.000840 | \n",
- " 0.000148 | \n",
- " 0.023665 | \n",
- " t | \n",
- " [412236, 283056, 121640] | \n",
- " [412208, 283148, 121680] | \n",
- "
\n",
- " \n",
- " | 2 | \n",
- " 6180240 | \n",
- " t | \n",
- " 77761173637894262 | \n",
- " 720575940627197566 | \n",
- " 77761173637888123 | \n",
- " 720575940612001489 | \n",
- " 227.389908 | \n",
- " 158 | \n",
- " 0.011852 | \n",
- " 0.966436 | \n",
- " 0.005491 | \n",
- " 0.002103 | \n",
- " 0.000580 | \n",
- " 0.013538 | \n",
- " t | \n",
- " [414292, 245008, 144520] | \n",
- " [414160, 245016, 144520] | \n",
- "
\n",
- " \n",
- " | 3 | \n",
- " 51374154 | \n",
- " t | \n",
- " 77691285729116258 | \n",
- " 720575940627197566 | \n",
- " 77691285729100925 | \n",
- " 720575940612001489 | \n",
- " 111.818047 | \n",
- " 132 | \n",
- " 0.008370 | \n",
- " 0.705035 | \n",
- " 0.009202 | \n",
- " 0.000005 | \n",
- " 0.272142 | \n",
- " 0.005246 | \n",
- " t | \n",
- " [409856, 274840, 93040] | \n",
- " [409984, 274868, 93040] | \n",
- "
\n",
- " \n",
- " | 4 | \n",
- " 237434942 | \n",
- " t | \n",
- " 77832023284765928 | \n",
- " 720575940627197566 | \n",
- " 77832023284771485 | \n",
- " 720575940612001489 | \n",
- " 161.775467 | \n",
- " 141 | \n",
- " 0.023719 | \n",
- " 0.890757 | \n",
- " 0.013778 | \n",
- " 0.002228 | \n",
- " 0.019519 | \n",
- " 0.049999 | \n",
- " t | \n",
- " [417900, 272952, 119880] | \n",
- " [417844, 273044, 119880] | \n",
- "
\n",
- " \n",
- " | ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- "
\n",
- " \n",
- " | 121 | \n",
- " 236131087 | \n",
- " t | \n",
- " 77761585821121562 | \n",
- " 720575940627197566 | \n",
- " 77761585821120349 | \n",
- " 720575940612001489 | \n",
- " 88.254326 | \n",
- " 116 | \n",
- " 0.007150 | \n",
- " 0.958711 | \n",
- " 0.001355 | \n",
- " 0.016179 | \n",
- " 0.000408 | \n",
- " 0.016197 | \n",
- " t | \n",
- " [414620, 270192, 119680] | \n",
- " [414520, 270216, 119680] | \n",
- "
\n",
- " \n",
- " | 122 | \n",
- " 217890770 | \n",
- " t | \n",
- " 77691423235405859 | \n",
- " 720575940627197566 | \n",
- " 77691423235425756 | \n",
- " 720575940612001489 | \n",
- " 65.530502 | \n",
- " 145 | \n",
- " 0.003493 | \n",
- " 0.976093 | \n",
- " 0.000120 | \n",
- " 0.003406 | \n",
- " 0.000033 | \n",
- " 0.016854 | \n",
- " t | \n",
- " [412216, 283052, 121600] | \n",
- " [412128, 283092, 121640] | \n",
- "
\n",
- " \n",
- " | 123 | \n",
- " 147115465 | \n",
- " t | \n",
- " 77832023284730175 | \n",
- " 720575940627197566 | \n",
- " 77832023284716150 | \n",
- " 720575940612001489 | \n",
- " 13.454404 | \n",
- " 142 | \n",
- " 0.014081 | \n",
- " 0.748565 | \n",
- " 0.001552 | \n",
- " 0.000199 | \n",
- " 0.003485 | \n",
- " 0.232117 | \n",
- " t | \n",
- " [419512, 274800, 118480] | \n",
- " [419480, 274656, 118480] | \n",
- "
\n",
- " \n",
- " | 124 | \n",
- " 221946020 | \n",
- " t | \n",
- " 77832160723343355 | \n",
- " 720575940627197566 | \n",
- " 77832160723346346 | \n",
- " 720575940612001489 | \n",
- " 1038.665283 | \n",
- " 143 | \n",
- " 0.048305 | \n",
- " 0.895462 | \n",
- " 0.023994 | \n",
- " 0.006970 | \n",
- " 0.001213 | \n",
- " 0.024056 | \n",
- " t | \n",
- " [417700, 282552, 108800] | \n",
- " [417580, 282568, 108800] | \n",
- "
\n",
- " \n",
- " | 125 | \n",
- " 237435004 | \n",
- " t | \n",
- " 77832023284778536 | \n",
- " 720575940627197566 | \n",
- " 77832023284792016 | \n",
- " 720575940612001489 | \n",
- " 91.194901 | \n",
- " 114 | \n",
- " 0.056194 | \n",
- " 0.560103 | \n",
- " 0.066058 | \n",
- " 0.014366 | \n",
- " 0.059227 | \n",
- " 0.244052 | \n",
- " t | \n",
- " [419064, 274020, 120360] | \n",
- " [419164, 273968, 120400] | \n",
- "
\n",
- " \n",
- "
\n",
- "
96 rows × 17 columns
\n",
- "
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " id | \n",
- " valid | \n",
- " pre_pt_supervoxel_id | \n",
- " pre_pt_root_id | \n",
- " post_pt_supervoxel_id | \n",
- " post_pt_root_id | \n",
- " connection_score | \n",
- " cleft_score | \n",
- " gaba | \n",
- " ach | \n",
- " glut | \n",
- " oct | \n",
- " ser | \n",
- " da | \n",
- " valid_nt | \n",
- " pre_pt_position | \n",
- " post_pt_position | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | 0 | \n",
- " 43917406 | \n",
- " t | \n",
- " 84727748031271775 | \n",
- " 720575940637186126 | \n",
- " 84727748031264584 | \n",
- " 720575940626445100 | \n",
- " 63.022396 | \n",
- " 124 | \n",
- " 0.010390 | \n",
- " 0.950272 | \n",
- " 0.002132 | \n",
- " 0.008374 | \n",
- " 3.866737e-04 | \n",
- " 0.028445 | \n",
- " t | \n",
- " [820188, 250388, 153800] | \n",
- " [820212, 250440, 153840] | \n",
- "
\n",
- " \n",
- " | 1 | \n",
- " 61506994 | \n",
- " t | \n",
- " 83320304562319572 | \n",
- " 720575940637186126 | \n",
- " 83320304562328649 | \n",
- " 720575940619082317 | \n",
- " 54.087387 | \n",
- " 139 | \n",
- " 0.000389 | \n",
- " 0.997138 | \n",
- " 0.000002 | \n",
- " 0.001707 | \n",
- " 9.331072e-07 | \n",
- " 0.000762 | \n",
- " t | \n",
- " [740408, 246192, 190520] | \n",
- " [740380, 246032, 190560] | \n",
- "
\n",
- " \n",
- " | 2 | \n",
- " 232489725 | \n",
- " t | \n",
- " 84094429400951571 | \n",
- " 720575940637186126 | \n",
- " 84094429400956510 | \n",
- " 720575940590374582 | \n",
- " 270.478394 | \n",
- " 137 | \n",
- " 0.064027 | \n",
- " 0.356340 | \n",
- " 0.515614 | \n",
- " 0.004106 | \n",
- " 6.949618e-03 | \n",
- " 0.052963 | \n",
- " t | \n",
- " [782552, 248476, 180040] | \n",
- " [782508, 248392, 180080] | \n",
- "
\n",
- " \n",
- " | 3 | \n",
- " 232945196 | \n",
- " t | \n",
- " 84094429400971745 | \n",
- " 720575940637186126 | \n",
- " 84024060656788915 | \n",
- " 720575940583849998 | \n",
- " 100.080338 | \n",
- " 144 | \n",
- " 0.021885 | \n",
- " 0.813248 | \n",
- " 0.028637 | \n",
- " 0.006434 | \n",
- " 7.459946e-03 | \n",
- " 0.122336 | \n",
- " t | \n",
- " [782036, 250656, 180240] | \n",
- " [781968, 250780, 180200] | \n",
- "
\n",
- " \n",
- " | 4 | \n",
- " 239559768 | \n",
- " t | \n",
- " 84094429400828504 | \n",
- " 720575940637186126 | \n",
- " 84094429400827280 | \n",
- " 720575940626431532 | \n",
- " 31.991150 | \n",
- " 146 | \n",
- " 0.087519 | \n",
- " 0.123452 | \n",
- " 0.744683 | \n",
- " 0.002957 | \n",
- " 2.260582e-02 | \n",
- " 0.018783 | \n",
- " t | \n",
- " [783352, 250516, 176320] | \n",
- " [783288, 250484, 176280] | \n",
- "
\n",
- " \n",
- " | ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- "
\n",
- " \n",
- " | 1807 | \n",
- " 232945305 | \n",
- " t | \n",
- " 84094429400962617 | \n",
- " 720575940637186126 | \n",
- " 84094429400990219 | \n",
- " 720575940633432206 | \n",
- " 51.244499 | \n",
- " 111 | \n",
- " 0.006596 | \n",
- " 0.946592 | \n",
- " 0.003757 | \n",
- " 0.034190 | \n",
- " 8.990807e-05 | \n",
- " 0.008774 | \n",
- " t | \n",
- " [782376, 251864, 180800] | \n",
- " [782468, 251780, 180840] | \n",
- "
\n",
- " \n",
- " | 1808 | \n",
- " 232948742 | \n",
- " t | \n",
- " 84024060656803275 | \n",
- " 720575940637186126 | \n",
- " 84024060656799478 | \n",
- " 720575940627765099 | \n",
- " 11.450621 | \n",
- " 67 | \n",
- " 0.032924 | \n",
- " 0.898681 | \n",
- " 0.032187 | \n",
- " 0.001579 | \n",
- " 1.981975e-03 | \n",
- " 0.032648 | \n",
- " t | \n",
- " [780720, 250520, 180640] | \n",
- " [780840, 250488, 180640] | \n",
- "
\n",
- " \n",
- " | 1810 | \n",
- " 239559737 | \n",
- " t | \n",
- " 84094429400832117 | \n",
- " 720575940637186126 | \n",
- " 84094429400820474 | \n",
- " 720575940620162332 | \n",
- " 275.813080 | \n",
- " 142 | \n",
- " 0.010846 | \n",
- " 0.939684 | \n",
- " 0.026259 | \n",
- " 0.020127 | \n",
- " 7.136991e-05 | \n",
- " 0.003012 | \n",
- " t | \n",
- " [784380, 249888, 176080] | \n",
- " [784280, 249884, 176040] | \n",
- "
\n",
- " \n",
- " | 1811 | \n",
- " 239559595 | \n",
- " t | \n",
- " 84094429400800897 | \n",
- " 720575940637186126 | \n",
- " 84094429400805747 | \n",
- " 720575940627076396 | \n",
- " 241.319305 | \n",
- " 142 | \n",
- " 0.002015 | \n",
- " 0.988075 | \n",
- " 0.000435 | \n",
- " 0.007664 | \n",
- " 2.856799e-06 | \n",
- " 0.001808 | \n",
- " t | \n",
- " [784652, 251304, 175600] | \n",
- " [784708, 251216, 175600] | \n",
- "
\n",
- " \n",
- " | 1812 | \n",
- " 242611511 | \n",
- " t | \n",
- " 84587010543015811 | \n",
- " 720575940637186126 | \n",
- " 84587010543017084 | \n",
- " 720575940498620410 | \n",
- " 16.303911 | \n",
- " 58 | \n",
- " 0.190740 | \n",
- " 0.491727 | \n",
- " 0.150436 | \n",
- " 0.009635 | \n",
- " 2.533201e-02 | \n",
- " 0.132131 | \n",
- " t | \n",
- " [811568, 250932, 156880] | \n",
- " [811472, 250896, 156880] | \n",
- "
\n",
- " \n",
- "
\n",
- "
1422 rows × 17 columns
\n",
- "