From 34356c91581dd7763770e459c99c3b9573517fad Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Fri, 6 Jan 2023 16:04:31 -0500 Subject: [PATCH 01/24] added appspot and state share config bumped appengine runtime to python312 since "Runtime python27 is end of support and no longer allowed" removed theadsafe - "This field is not supported with runtime [python312] and can safely be removed." removed skip_files (no longer supported) "api_version" field is not allowed in runtime python312. forgot to add .gcloudignore (replacement of skip_files --- .github/workflows/build.yml | 319 +++++----------------------- .github/workflows/build_preview.yml | 29 --- appengine/frontend/.gcloudignore | 5 + appengine/frontend/app.yaml | 19 ++ config/state_servers.json | 6 + 5 files changed, 82 insertions(+), 296 deletions(-) delete mode 100644 .github/workflows/build_preview.yml create mode 100644 appengine/frontend/.gcloudignore create mode 100644 appengine/frontend/app.yaml create mode 100644 config/state_servers.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0c6d1ad621..bdb9662fe4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,19 +3,21 @@ name: Build on: push: branches: - - master + - spelunker tags: - v** pull_request: jobs: - client: + build-and-deploy: + permissions: + contents: "read" + id-token: "write" + deployments: "write" strategy: matrix: os: - "ubuntu-latest" - - "windows-latest" - - "macos-latest" runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -29,273 +31,56 @@ jobs: cache-dependency-path: | package-lock.json examples/**/package-lock.json + fetch-depth: 0 - run: npm install - - run: npm run format:fix - - name: Check for dirty working directory - run: git diff --exit-code - - run: npm run lint:check - name: Typecheck with TypeScript run: npm run typecheck - - name: Build client bundles - run: | - build_info="{'tag':'$(git describe --always --tags)', 'url':'https://github.com/google/neuroglancer/commit/$(git rev-parse HEAD)', 'timestamp':'$(date)'}" - npm run build -- --no-typecheck --no-lint --define NEUROGLANCER_BUILD_INFO="${build_info}" - echo $build_info > ./dist/client/version.json + - name: Get branch name (merge) + if: github.event_name != 'pull_request' shell: bash - - name: Build Python client bundles - run: npm run build-python -- --no-typecheck --no-lint - - run: npm run build-package - - run: npm publish --dry-run - working-directory: dist/package - - uses: ./.github/actions/setup-firefox - - name: Run JavaScript tests (including WebGL) - run: npm test - if: ${{ runner.os != 'macOS' }} - - name: Run JavaScript tests (excluding WebGL) - run: npm test -- --project node - if: ${{ runner.os == 'macOS' }} - - name: Run JavaScript benchmarks - run: npm run benchmark - - name: Upload NPM package as artifact - uses: actions/upload-artifact@v4 - with: - name: npm-package - path: dist/package - if: ${{ runner.os == 'Linux' }} - - name: Upload client as artifact - uses: actions/upload-artifact@v4 - with: - name: client - path: dist/client - if: ${{ runner.os == 'Linux' }} - - # Builds Python package and runs Python tests - # - # On ubuntu-latest, this also runs browser-based tests. On Mac OS and - # Windows, this only runs tests that do not require a browser, since a working - # headless WebGL2 implementation is not available on Github actions. - python-tests: - strategy: - matrix: - python-version: - - "3.9" - - "3.12" - os: - - "ubuntu-latest" - - "windows-latest" - - "macos-latest" - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - # Need full history to determine version number. - fetch-depth: 0 - - uses: actions/setup-node@v4 - with: - node-version: 20.x - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - # Uncomment the action below for an interactive shell - # - name: Setup tmate session - # uses: mxschmitt/action-tmate@v3 - - name: Install Python packaging/test tools - run: pip install tox nox wheel numpy -r python/requirements-test.txt - - uses: ./.github/actions/setup-firefox - - run: nox -s lint format mypy - - name: Check for dirty working directory - run: git diff --exit-code - - name: Test with tox - run: tox -e ${{ fromJSON('["skip-browser-tests","firefox-xvfb"]')[runner.os == 'Linux'] }} - # Verify that editable install works - - name: Install in editable form - run: pip install -e . --config-settings editable_mode=strict - - name: Run Python tests against editable install (excluding WebGL) - working-directory: python/tests - run: pytest -vv --skip-browser-tests - - python-build-package: - strategy: - matrix: - include: - - os: "ubuntu-latest" - cibw_build: "*" - wheel_identifier: "linux" - - os: "windows-latest" - cibw_build: "*" - wheel_identifier: "windows" - - os: "macos-14" - cibw_build: "*_x86_64" - wheel_identifier: "macos_x86_64" - - os: "macos-14" - cibw_build: "*_arm64" - wheel_identifier: "macos_arm64" - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - # Need full history to determine version number. - fetch-depth: 0 - - uses: actions/setup-node@v4 - with: - node-version: 20.x - cache: "npm" - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.x - - name: Get pip cache dir - id: pip-cache - run: | - echo "dir=$(pip cache dir)" >> "$GITHUB_OUTPUT" - - run: npm install - - run: | - build_info="{'tag':'$(git describe --always --tags)', 'url':'https://github.com/google/neuroglancer/commit/$(git rev-parse HEAD)', 'timestamp':'$(date)'}" - npm run build-python -- --no-typecheck --no-lint --define NEUROGLANCER_BUILD_INFO="${build_info}" + run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV + - name: Get branch name (pull request) + if: github.event_name == 'pull_request' + shell: bash + run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV + - run: echo "BRANCH_NAME_URL=$(echo ${{ env.BRANCH_NAME }} | tr / - | tr _ -)" >> $GITHUB_ENV + - name: Get build info + run: echo "BUILD_INFO={\"tag\":\"$(git describe --always --tags)\", \"url\":\"https://github.com/${{github.repository}}/commit/$(git rev-parse HEAD)\", \"timestamp\":\"$(date)\", \"branch\":\"${{github.repository}}/${{env.BRANCH_NAME}}\"}" >> $GITHUB_ENV shell: bash - name: Check for dirty working directory run: git diff --exit-code - - name: Install setuptools - run: pip install setuptools - - name: Build Python source distribution (sdist) - run: python setup.py sdist --format gztar - if: ${{ runner.os == 'Linux' }} - - name: Install cibuildwheel - run: pip install cibuildwheel - - name: Build Python wheels - run: bash -xve ./python/build_tools/cibuildwheel.sh - env: - # On Linux, share pip cache with manylinux docker containers - CIBW_ENVIRONMENT_LINUX: PIP_CACHE_DIR=/host${{ steps.pip-cache.outputs.dir }} - CIBW_BEFORE_ALL_LINUX: /project/python/build_tools/cibuildwheel_linux_cache_setup.sh /host${{ steps.pip-cache.outputs.dir }} - CIBW_BUILD: ${{ matrix.cibw_build }} - - name: Upload wheels as artifacts - uses: actions/upload-artifact@v4 - with: - name: python-wheels-${{ matrix.wheel_identifier }} - path: | - dist/*.whl - dist/*.tar.gz - - docs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Setup Graphviz - uses: ts-graphviz/setup-graphviz@b1de5da23ed0a6d14e0aeee8ed52fdd87af2363c # v2.0.2 - with: - macos-skip-brew-update: "true" - - name: Install nox - run: pip install nox - - name: Build docs - run: nox -s docs - - name: Upload docs as artifact - uses: actions/upload-artifact@v4 - with: - name: docs - path: | - dist/docs - - publish-package: - # Only publish package on push to tag or default branch. - if: ${{ github.event_name == 'push' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master') }} - runs-on: ubuntu-latest - needs: - - "client" - - "python-build-package" - - "docs" - steps: - - uses: actions/checkout@v4 - - name: Use Node.js - uses: actions/setup-node@v4 - with: - node-version: 20.x - registry-url: "https://registry.npmjs.org" - - uses: actions/download-artifact@v4 - with: - pattern: python-wheels-* - path: dist - merge-multiple: true - - uses: actions/download-artifact@v4 - with: - name: npm-package - path: npm-package - # - name: Publish to PyPI (test server) - # uses: pypa/gh-action-pypi-publish@54b39fb9371c0b3a6f9f14bb8a67394defc7a806 # 2020-09-25 - # with: - # user: __token__ - # password: ${{ secrets.pypi_test_token }} - - name: Publish to PyPI (main server) - uses: pypa/gh-action-pypi-publish@54b39fb9371c0b3a6f9f14bb8a67394defc7a806 # 2020-09-25 - with: - user: __token__ - password: ${{ secrets.pypi_token }} - if: ${{ startsWith(github.ref, 'refs/tags/v') }} - - name: Publish to NPM registry - if: ${{ startsWith(github.ref, 'refs/tags/v') }} - run: npm publish - working-directory: npm-package - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - # Download dist/client after publishing to PyPI, because PyPI publish - # action fails if dist/client directory is present. - - uses: actions/download-artifact@v4 - with: - name: client - path: dist/client - - name: Publish client to Firebase hosting - uses: FirebaseExtended/action-hosting-deploy@v0 - with: - firebaseServiceAccount: "${{ secrets.FIREBASE_HOSTING_SERVICE_ACCOUNT_KEY }}" - projectId: neuroglancer-demo - channelId: live - target: app - # Download dist/docs after publishing to PyPI, because PyPI publish - # action fails if dist/docs directory is present. - - uses: actions/download-artifact@v4 - with: - name: docs - path: dist/docs - - name: Publish docs to Firebase hosting - uses: FirebaseExtended/action-hosting-deploy@v0 - with: - firebaseServiceAccount: "${{ secrets.FIREBASE_HOSTING_SERVICE_ACCOUNT_KEY }}" - projectId: neuroglancer-demo - channelId: live - target: docs - - ngauth: - strategy: - matrix: - os: - - ubuntu-latest - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Setup go - uses: actions/setup-go@v5 - with: - go-version-file: ngauth_server/go.mod - cache-dependency-path: ngauth_server/go.sum - - run: go build . - working-directory: ngauth_server - wasm: - # Ensures that .wasm files are reproducible. - strategy: - matrix: - os: - - ubuntu-latest - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - run: ./src/mesh/draco/build.sh - - run: ./src/sliceview/compresso/build.sh - - run: ./src/sliceview/png/build.sh - - run: ./src/sliceview/jxl/build.sh - # Check that there are no differences. - - run: git diff --exit-code + - name: Build client bundles + run: npm run build -- --no-typecheck --no-lint --define STATE_SERVERS=$(cat config/state_servers.json | tr -d " \t\n\r") --define NEUROGLANCER_BUILD_INFO='${{ env.BUILD_INFO }}' --define CUSTOM_BINDINGS=$(cat config/custom-keybinds.json | tr -d " \t\n\r") + - name: Write build info + run: echo $BUILD_INFO > ./dist/client/version.json + shell: bash + - run: cp -r ./dist/client appengine/frontend/static/ + - name: start deployment + uses: bobheadxi/deployments@v1 + id: deployment + with: + step: start + token: ${{ secrets.GITHUB_TOKEN }} + env: ${{ env.BRANCH_NAME }} + desc: Setting up staging deployment for ${{ env.BRANCH_NAME }} + - id: "auth" + uses: "google-github-actions/auth@v1" + with: + workload_identity_provider: "projects/483670036293/locations/global/workloadIdentityPools/neuroglancer-github/providers/github" + service_account: "chris-apps-deploy@seung-lab.iam.gserviceaccount.com" + - id: deploy + uses: google-github-actions/deploy-appengine@main + with: + version: ${{ env.GITHUB_SHA }} + deliverables: appengine/frontend/app.yaml + promote: true + - name: update deployment status + uses: bobheadxi/deployments@v1 + if: always() + with: + step: finish + token: ${{ secrets.GITHUB_TOKEN }} + env: ${{ steps.deployment.outputs.env }} + env_url: ${{ steps.deploy.outputs.url }} + status: ${{ job.status }} + deployment_id: ${{ steps.deployment.outputs.deployment_id }} diff --git a/.github/workflows/build_preview.yml b/.github/workflows/build_preview.yml deleted file mode 100644 index 9dcac7d1e1..0000000000 --- a/.github/workflows/build_preview.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Build preview - -on: - pull_request: - -jobs: - upload: - strategy: - matrix: - node-version: - - "20.x" - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - run: npm install - - run: | - build_info="{'tag':'$(git describe --always --tags)', 'url':'https://github.com/google/neuroglancer/commit/$(git rev-parse HEAD)', 'timestamp':'$(date)'}" - npm run build -- --no-typecheck --no-lint --define NEUROGLANCER_BUILD_INFO="${build_info}" - - name: Upload client as artifact - uses: actions/upload-artifact@v4 - with: - name: client - path: | - dist/client/* diff --git a/appengine/frontend/.gcloudignore b/appengine/frontend/.gcloudignore new file mode 100644 index 0000000000..c2edb00e92 --- /dev/null +++ b/appengine/frontend/.gcloudignore @@ -0,0 +1,5 @@ +.gcloudignore +.envrc +.git +.gitignore +node_modules/ \ No newline at end of file diff --git a/appengine/frontend/app.yaml b/appengine/frontend/app.yaml new file mode 100644 index 0000000000..7d375d9d1b --- /dev/null +++ b/appengine/frontend/app.yaml @@ -0,0 +1,19 @@ +runtime: python312 + +service: neuroglancer + +handlers: + # Handle the main page by serving the index page. + # Note the $ to specify the end of the path, since app.yaml does prefix matching. + - url: /$ + static_files: static/index.html + upload: static/index.html + login: optional + secure: always + redirect_http_response_code: 301 + + - url: / + static_dir: static + login: optional + secure: always + redirect_http_response_code: 301 diff --git a/config/state_servers.json b/config/state_servers.json new file mode 100644 index 0000000000..c61efbd860 --- /dev/null +++ b/config/state_servers.json @@ -0,0 +1,6 @@ +{ + "cave": { + "url": "middleauth+https://global.daf-apis.com/nglstate/api/v1/post", + "default": true + } +} From b92dd9815f75497a4e43afcceb244f7efbf030c6 Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Fri, 5 May 2023 09:06:07 -0400 Subject: [PATCH 02/24] specific app config for cave-explorer --- appengine/frontend/app.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appengine/frontend/app.yaml b/appengine/frontend/app.yaml index 7d375d9d1b..9e6dfcfb07 100644 --- a/appengine/frontend/app.yaml +++ b/appengine/frontend/app.yaml @@ -1,6 +1,6 @@ runtime: python312 -service: neuroglancer +service: base-cave handlers: # Handle the main page by serving the index page. From 7ac51b33170f4641ba7e73482ba80f84bcaa43d0 Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Mon, 15 May 2023 12:52:41 -0400 Subject: [PATCH 03/24] specific app config for spelunker --- appengine/frontend/app.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appengine/frontend/app.yaml b/appengine/frontend/app.yaml index 9e6dfcfb07..9d82ed9370 100644 --- a/appengine/frontend/app.yaml +++ b/appengine/frontend/app.yaml @@ -1,6 +1,6 @@ runtime: python312 -service: base-cave +service: spelunker handlers: # Handle the main page by serving the index page. From cc0c00e6bfd658f0a3511eddc435080292c3011b Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Tue, 16 May 2023 11:13:46 -0400 Subject: [PATCH 04/24] trying to reduce flicker --- src/chunk_manager/frontend.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/chunk_manager/frontend.ts b/src/chunk_manager/frontend.ts index bc85aea127..96e76bfa1b 100644 --- a/src/chunk_manager/frontend.ts +++ b/src/chunk_manager/frontend.ts @@ -268,9 +268,10 @@ export class ChunkQueueManager extends SharedObject { if (newState !== oldState) { switch (newState) { case ChunkState.GPU_MEMORY: - // console.log("Copying to GPU", chunk); chunk.copyToGPU(this.gl); - visibleChunksChanged = true; + if (chunk.constructor.name !== "ManifestChunk") { + visibleChunksChanged = true; + } break; case ChunkState.SYSTEM_MEMORY: if (oldState === ChunkState.GPU_MEMORY) { From c07a868aa529b183dc50690ed67de9b59467b668 Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Tue, 6 Dec 2022 16:51:02 -0500 Subject: [PATCH 05/24] added graphene find path tool using l2cache when available for fast and more accurate find path check for l2cache existing is now cached and only checked when necessary --- src/datasource/graphene/frontend.ts | 1051 ++++++++++++++++++++------ src/datasource/graphene/graphene.css | 39 +- src/status.ts | 1 + src/ui/annotations.ts | 294 +++---- src/util/json.ts | 10 + 5 files changed, 1022 insertions(+), 373 deletions(-) diff --git a/src/datasource/graphene/frontend.ts b/src/datasource/graphene/frontend.ts index 04a1281b69..aab9fd20f3 100644 --- a/src/datasource/graphene/frontend.ts +++ b/src/datasource/graphene/frontend.ts @@ -20,8 +20,11 @@ import { AnnotationDisplayState, AnnotationLayerState, } from "#src/annotation/annotation_layer_state.js"; +import type { MultiscaleAnnotationSource } from "#src/annotation/frontend_source.js"; import type { + Annotation, AnnotationReference, + AnnotationSource, Line, Point, } from "#src/annotation/index.js"; @@ -142,9 +145,11 @@ import { } from "#src/trackable_value.js"; import { AnnotationLayerView, + makeAnnotationListElement, MergedAnnotationStates, PlaceLineTool, } from "#src/ui/annotations.js"; +import { getDefaultAnnotationListBindings } from "#src/ui/default_input_event_bindings.js"; import type { ToolActivation } from "#src/ui/tool.js"; import { LayerTool, @@ -153,10 +158,11 @@ import { registerLegacyTool, registerTool, } from "#src/ui/tool.js"; -import type { Uint64Set } from "#src/uint64_set.js"; +import { Uint64Set } from "#src/uint64_set.js"; import { packColor } from "#src/util/color.js"; import type { Owned } from "#src/util/disposable.js"; import { RefCounted } from "#src/util/disposable.js"; +import { removeChildren } from "#src/util/dom.js"; import type { ValueOrError } from "#src/util/error.js"; import { makeValueOrError, valueOrThrow } from "#src/util/error.js"; import { EventActionMap } from "#src/util/event_action_map.js"; @@ -174,7 +180,9 @@ import { verifyEnumString, verifyFiniteFloat, verifyFinitePositiveFloat, + verifyFloatArray, verifyInt, + verifyIntegerArray, verifyNonnegativeInt, verifyObject, verifyObjectProperty, @@ -182,7 +190,9 @@ import { verifyOptionalString, verifyPositiveInt, verifyString, + verifyStringArray, } from "#src/util/json.js"; +import { MouseEventBinder } from "#src/util/mouse_bindings.js"; import { getObjectId } from "#src/util/object_id.js"; import { NullarySignal } from "#src/util/signal.js"; import type { @@ -216,6 +226,7 @@ const RED_COLOR_SEGMENT_PACKED = new Uint64(packColor(RED_COLOR_SEGMENT)); const BLUE_COLOR_SEGMENT_PACKED = new Uint64(packColor(BLUE_COLOR_SEGMENT)); const TRANSPARENT_COLOR_PACKED = new Uint64(packColor(TRANSPARENT_COLOR)); const MULTICUT_OFF_COLOR = vec4.fromValues(0, 0, 0, 0.5); +const WHITE_COLOR = vec3.fromValues(1, 1, 1); class GrapheneMeshSource extends WithParameters( WithCredentialsProvider()(MeshSource), @@ -230,6 +241,8 @@ class GrapheneMeshSource extends WithParameters( class AppInfo { segmentationUrl: string; meshingUrl: string; + l2CacheUrl: string; + table: string; supported_api_versions: number[]; constructor(infoUrl: string, obj: any) { // .../1.0/... is the legacy link style @@ -240,8 +253,11 @@ class AppInfo { if (match === null) { throw Error(`Graph URL invalid: ${infoUrl}`); } - this.segmentationUrl = `${match[1]}/segmentation/api/v${PYCG_APP_VERSION}/table/${match[2]}`; - this.meshingUrl = `${match[1]}/meshing/api/v${PYCG_APP_VERSION}/table/${match[2]}`; + this.table = match[2]; + const { table } = this; + this.segmentationUrl = `${match[1]}/segmentation/api/v${PYCG_APP_VERSION}/table/${table}`; + this.meshingUrl = `${match[1]}/meshing/api/v${PYCG_APP_VERSION}/table/${table}`; + this.l2CacheUrl = `${match[1]}/l2cache/api/v${PYCG_APP_VERSION}`; try { verifyObject(obj); @@ -706,7 +722,7 @@ export class GrapheneDataSource extends PrecomputedDataSource { ); } catch (e) { if (isNotFoundError(e)) { - if (parameters.type === "mesh") { + if (parameters["type"] === "mesh") { console.log("does this happen?"); } } @@ -815,47 +831,81 @@ function restoreSegmentSelection(obj: any): SegmentSelection { }; } +const segmentSelectionToJSON = (x: SegmentSelection) => { + return { + [SEGMENT_ID_JSON_KEY]: x.segmentId.toJSON(), + [ROOT_ID_JSON_KEY]: x.rootId.toJSON(), + [POSITION_JSON_KEY]: [...x.position], + }; +}; + const ID_JSON_KEY = "id"; -const ERROR_JSON_KEY = "error"; +const SEGMENT_ID_JSON_KEY = "segmentId"; +const ROOT_ID_JSON_KEY = "rootId"; +const POSITION_JSON_KEY = "position"; +const SINK_JSON_KEY = "sink"; +const SOURCE_JSON_KEY = "source"; + const MULTICUT_JSON_KEY = "multicut"; const FOCUS_SEGMENT_JSON_KEY = "focusSegment"; const SINKS_JSON_KEY = "sinks"; const SOURCES_JSON_KEY = "sources"; -const SEGMENT_ID_JSON_KEY = "segmentId"; -const ROOT_ID_JSON_KEY = "rootId"; -const POSITION_JSON_KEY = "position"; + const MERGE_JSON_KEY = "merge"; const MERGES_JSON_KEY = "merges"; const AUTOSUBMIT_JSON_KEY = "autosubmit"; -const SINK_JSON_KEY = "sink"; -const SOURCE_JSON_KEY = "source"; -const MERGED_ROOT_JSON_KEY = "mergedRoot"; const LOCKED_JSON_KEY = "locked"; +const MERGED_ROOT_JSON_KEY = "mergedRoot"; +const ERROR_JSON_KEY = "error"; + +const FIND_PATH_JSON_KEY = "findPath"; +const TARGET_JSON_KEY = "target"; +const CENTROIDS_JSON_KEY = "centroids"; +const PRECISION_MODE_JSON_KEY = "precision"; -class GrapheneState implements Trackable { +class GrapheneState extends RefCounted implements Trackable { changed = new NullarySignal(); public multicutState = new MulticutState(); public mergeState = new MergeState(); + public findPathState = new FindPathState(); constructor() { - this.multicutState.changed.add(() => { - this.changed.dispatch(); - }); - this.mergeState.changed.add(() => { - this.changed.dispatch(); - }); + super(); + this.registerDisposer( + this.multicutState.changed.add(() => { + this.changed.dispatch(); + }), + ); + this.registerDisposer( + this.mergeState.changed.add(() => { + this.changed.dispatch(); + }), + ); + this.registerDisposer( + this.findPathState.changed.add(() => { + this.changed.dispatch(); + }), + ); + } + + replaceSegments(oldValues: Uint64Set, newValues: Uint64Set) { + this.multicutState.replaceSegments(oldValues, newValues); + this.mergeState.replaceSegments(oldValues, newValues); + this.findPathState.replaceSegments(oldValues, newValues); } reset() { this.multicutState.reset(); this.mergeState.reset(); + this.findPathState.reset(); } toJSON() { return { [MULTICUT_JSON_KEY]: this.multicutState.toJSON(), [MERGE_JSON_KEY]: this.mergeState.toJSON(), + [FIND_PATH_JSON_KEY]: this.findPathState.toJSON(), }; } @@ -866,6 +916,9 @@ class GrapheneState implements Trackable { verifyOptionalObjectProperty(x, MERGE_JSON_KEY, (value) => { this.mergeState.restoreState(value); }); + verifyOptionalObjectProperty(x, FIND_PATH_JSON_KEY, (value) => { + this.findPathState.restoreState(value); + }); } } @@ -887,6 +940,31 @@ class MergeState extends RefCounted implements Trackable { this.registerDisposer(this.merges.changed.add(this.changed.dispatch)); } + replaceSegments(oldValues: Uint64Set, newValues: Uint64Set) { + const { + merges: { value: merges }, + } = this; + const newValue = newValues.size === 1 ? [...newValues][0] : undefined; + for (const merge of merges) { + if (merge.source && oldValues.has(merge.source.rootId)) { + if (newValue) { + merge.source.rootId = newValue; + } else { + this.reset(); + return; + } + } + if (merge.sink && oldValues.has(merge.sink.rootId)) { + if (newValue) { + merge.sink.rootId = newValue; + } else { + this.reset(); + return; + } + } + } + } + reset() { this.merges.value = []; this.autoSubmit.reset(); @@ -894,15 +972,6 @@ class MergeState extends RefCounted implements Trackable { toJSON() { const { merges, autoSubmit } = this; - - const segmentSelectionToJSON = (x: SegmentSelection) => { - return { - [SEGMENT_ID_JSON_KEY]: x.segmentId.toJSON(), - [ROOT_ID_JSON_KEY]: x.rootId.toJSON(), - [POSITION_JSON_KEY]: [...x.position], - }; - }; - const mergeToJSON = (x: MergeSubmission) => { const res: any = { [ID_JSON_KEY]: x.id, @@ -910,17 +979,14 @@ class MergeState extends RefCounted implements Trackable { [SINK_JSON_KEY]: segmentSelectionToJSON(x.sink), [SOURCE_JSON_KEY]: segmentSelectionToJSON(x.source!), }; - if (x.mergedRoot) { res[MERGED_ROOT_JSON_KEY] = x.mergedRoot.toJSON(); } if (x.error) { res[ERROR_JSON_KEY] = x.error; } - return res; }; - return { [MERGES_JSON_KEY]: merges.value.filter((x) => x.source).map(mergeToJSON), [AUTOSUBMIT_JSON_KEY]: autoSubmit.toJSON(), @@ -966,6 +1032,147 @@ class MergeState extends RefCounted implements Trackable { } } +class FindPathState extends RefCounted implements Trackable { + changed = new NullarySignal(); + triggerPathUpdate = new NullarySignal(); + + source = new TrackableValue( + undefined, + (x) => x, + ); + target = new TrackableValue( + undefined, + (x) => x, + ); + centroids = new TrackableValue([], (x) => x); + precisionMode = new TrackableBoolean(true); + + get path() { + const path: Line[] = []; + const { + source: { value: source }, + target: { value: target }, + centroids: { value: centroids }, + } = this; + if (!source || !target || centroids.length === 0) { + return path; + } + for (let i = 0; i < centroids.length - 1; i++) { + const pointA = centroids[i]; + const pointB = centroids[i + 1]; + const line: Line = { + pointA: vec3.fromValues(pointA[0], pointA[1], pointA[2]), + pointB: vec3.fromValues(pointB[0], pointB[1], pointB[2]), + id: "", + type: AnnotationType.LINE, + properties: [], + }; + path.push(line); + } + const firstLine: Line = { + pointA: source.position, + pointB: path[0].pointA, + id: "", + type: AnnotationType.LINE, + properties: [], + }; + const lastLine: Line = { + pointA: path[path.length - 1].pointB, + pointB: target.position, + id: "", + type: AnnotationType.LINE, + properties: [], + }; + + return [firstLine, ...path, lastLine]; + } + + constructor() { + super(); + this.registerDisposer( + this.source.changed.add(() => { + this.centroids.reset(); + this.changed.dispatch(); + }), + ); + this.registerDisposer( + this.target.changed.add(() => { + this.centroids.reset(); + this.changed.dispatch(); + }), + ); + this.registerDisposer(this.centroids.changed.add(this.changed.dispatch)); + } + + replaceSegments(oldValues: Uint64Set, newValues: Uint64Set) { + const { + source: { value: source }, + target: { value: target }, + } = this; + const newValue = newValues.size === 1 ? [...newValues][0] : undefined; + const sourceChanged = !!source && oldValues.has(source.rootId); + const targetChanged = !!target && oldValues.has(target.rootId); + if (newValue) { + if (sourceChanged) { + source.rootId = newValue; + } + if (targetChanged) { + target.rootId = newValue; + } + // don't want to fire off multiple changed + if (sourceChanged || targetChanged) { + if (this.centroids.value.length) { + this.centroids.reset(); + this.triggerPathUpdate.dispatch(); + } else { + this.changed.dispatch(); + } + } + } else { + if (sourceChanged || targetChanged) { + this.reset(); + } + } + } + + reset() { + this.source.reset(); + this.target.reset(); + this.centroids.reset(); + this.precisionMode.reset(); + } + + toJSON() { + const { + source: { value: source }, + target: { value: target }, + centroids, + precisionMode, + } = this; + return { + [SOURCE_JSON_KEY]: source ? segmentSelectionToJSON(source) : undefined, + [TARGET_JSON_KEY]: target ? segmentSelectionToJSON(target) : undefined, + [CENTROIDS_JSON_KEY]: centroids.toJSON(), + [PRECISION_MODE_JSON_KEY]: precisionMode.toJSON(), + }; + } + + restoreState(x: any) { + verifyOptionalObjectProperty(x, SOURCE_JSON_KEY, (value) => { + this.source.restoreState(restoreSegmentSelection(value)); + }); + verifyOptionalObjectProperty(x, TARGET_JSON_KEY, (value) => { + this.target.restoreState(restoreSegmentSelection(value)); + }); + verifyOptionalObjectProperty(x, CENTROIDS_JSON_KEY, (value) => { + this.centroids.restoreState(value); + }); + verifyOptionalObjectProperty(x, PRECISION_MODE_JSON_KEY, (value) => { + this.precisionMode.restoreState(value); + }); + } +} + class MulticutState extends RefCounted implements Trackable { changed = new NullarySignal(); @@ -996,8 +1203,30 @@ class MulticutState extends RefCounted implements Trackable { this.registerDisposer(this.sources.changed.add(this.changed.dispatch)); } + replaceSegments(oldValues: Uint64Set, newValues: Uint64Set) { + const newValue = newValues.size === 1 ? [...newValues][0] : undefined; + const { + focusSegment: { value: focusSegment }, + } = this; + + if (focusSegment && oldValues.has(focusSegment)) { + if (newValue) { + this.focusSegment.value = newValue; + for (const sink of this.sinks) { + sink.rootId = newValue; + } + for (const source of this.sources) { + source.rootId = newValue; + } + this.changed.dispatch(); + } else { + this.reset(); + } + } + } + reset() { - this.focusSegment.value = undefined; + this.focusSegment.reset(); this.blueGroup.value = false; this.sinks.clear(); this.sources.clear(); @@ -1005,15 +1234,6 @@ class MulticutState extends RefCounted implements Trackable { toJSON() { const { focusSegment, sinks, sources } = this; - - const segmentSelectionToJSON = (x: SegmentSelection) => { - return { - [SEGMENT_ID_JSON_KEY]: x.segmentId.toJSON(), - [ROOT_ID_JSON_KEY]: x.rootId.toJSON(), - [POSITION_JSON_KEY]: [...x.position], - }; - }; - return { [FOCUS_SEGMENT_JSON_KEY]: focusSegment.toJSON(), [SINKS_JSON_KEY]: [...sinks].map(segmentSelectionToJSON), @@ -1080,6 +1300,7 @@ class MulticutState extends RefCounted implements Trackable { class GraphConnection extends SegmentationGraphSourceConnection { public annotationLayerStates: AnnotationLayerState[] = []; public mergeAnnotationState: AnnotationLayerState; + public findPathAnnotationState: AnnotationLayerState; constructor( public graph: GrapheneGraphSource, @@ -1089,27 +1310,31 @@ class GraphConnection extends SegmentationGraphSourceConnection { ) { super(graph, layer.displayState.segmentationGroupState.value); const segmentsState = layer.displayState.segmentationGroupState.value; - segmentsState.selectedSegments.changed.add( - (segmentIds: Uint64[] | Uint64 | null, add: boolean) => { - if (segmentIds !== null) { - segmentIds = Array().concat(segmentIds); - } - this.selectedSegmentsChanged(segmentIds, add); - }, + this.registerDisposer( + segmentsState.selectedSegments.changed.add( + (segmentIds: Uint64[] | Uint64 | null, add: boolean) => { + if (segmentIds !== null) { + segmentIds = Array().concat(segmentIds); + } + this.selectedSegmentsChanged(segmentIds, add); + }, + ), ); - segmentsState.visibleSegments.changed.add( - (segmentIds: Uint64[] | Uint64 | null, add: boolean) => { - if (segmentIds !== null) { - segmentIds = Array().concat(segmentIds); - } - this.visibleSegmentsChanged(segmentIds, add); - }, + this.registerDisposer( + segmentsState.visibleSegments.changed.add( + (segmentIds: Uint64[] | Uint64 | null, add: boolean) => { + if (segmentIds !== null) { + segmentIds = Array().concat(segmentIds); + } + this.visibleSegmentsChanged(segmentIds, add); + }, + ), ); const { annotationLayerStates, - state: { multicutState }, + state: { multicutState, findPathState }, } = this; const loadedSubsource = getGraphLoadedSubsource(layer)!; const redGroup = makeColoredAnnotationState( @@ -1151,80 +1376,153 @@ class GraphConnection extends SegmentationGraphSourceConnection { } // initialize source changes - mergeAnnotationState.source.childAdded.add((x) => { - const annotation = x as Line; - const relatedSegments = annotation.relatedSegments![0]; - const visibles = relatedSegments.map((x) => visibleSegments.has(x)); - if (visibles[0] === false) { - setTimeout(() => { - const { tool } = layer; - if (tool.value instanceof MergeSegmentsPlaceLineTool) { - tool.value.deactivate(); - } - }, 0); - StatusMessage.showTemporaryMessage("Cannot merge a hidden segment."); - } else if (merges.value.length < MAX_MERGE_COUNT) { - merges.value = [...merges.value, lineToSubmission(annotation, true)]; - } else { - setTimeout(() => { - const { tool } = layer; - if (tool.value instanceof MergeSegmentsPlaceLineTool) { - tool.value.deactivate(); - } - }, 0); - StatusMessage.showTemporaryMessage( - `Maximum of ${MAX_MERGE_COUNT} simultanous merges allowed.`, - ); - } - }); - - mergeAnnotationState.source.childCommitted.add((x) => { - const ref = mergeAnnotationState.source.getReference(x); - const annotation = ref.value as Line | undefined; - if (annotation) { + this.registerDisposer( + mergeAnnotationState.source.childAdded.add((x) => { + const annotation = x as Line; const relatedSegments = annotation.relatedSegments![0]; const visibles = relatedSegments.map((x) => visibleSegments.has(x)); - if (relatedSegments.length < 4) { - mergeAnnotationState.source.delete(ref); + if (visibles[0] === false) { + setTimeout(() => { + const { tool } = layer; + if (tool.value instanceof MergeSegmentsPlaceLineTool) { + tool.value.deactivate(); + } + }, 0); StatusMessage.showTemporaryMessage( - "Cannot merge segment with itself.", + `Cannot merge a hidden segment.`, ); - } - if (visibles[2] === false) { - mergeAnnotationState.source.delete(ref); + } else if (merges.value.length < MAX_MERGE_COUNT) { + merges.value = [ + ...merges.value, + lineToSubmission(annotation, true), + ]; + } else { + setTimeout(() => { + const { tool } = layer; + if (tool.value instanceof MergeSegmentsPlaceLineTool) { + tool.value.deactivate(); + } + }, 0); StatusMessage.showTemporaryMessage( - "Cannot merge a hidden segment.", + `Maximum of ${MAX_MERGE_COUNT} simultanous merges allowed.`, ); } - const existingSubmission = merges.value.find((x) => x.id === ref.id); - if (existingSubmission && !existingSubmission?.locked) { - // how would it be locked? - const newSubmission = lineToSubmission(annotation, false); - existingSubmission.sink = newSubmission.sink; - existingSubmission.source = newSubmission.source; - merges.changed.dispatch(); - if (autoSubmit.value) { - this.bulkMerge([existingSubmission]); + }), + ); + + this.registerDisposer( + mergeAnnotationState.source.childCommitted.add((x) => { + const ref = mergeAnnotationState.source.getReference(x); + const annotation = ref.value as Line | undefined; + if (annotation) { + const relatedSegments = annotation.relatedSegments![0]; + const visibles = relatedSegments.map((x) => visibleSegments.has(x)); + if (relatedSegments.length < 4) { + mergeAnnotationState.source.delete(ref); + StatusMessage.showTemporaryMessage( + `Cannot merge segment with itself.`, + ); + } + if (visibles[2] === false) { + mergeAnnotationState.source.delete(ref); + StatusMessage.showTemporaryMessage( + `Cannot merge a hidden segment.`, + ); + } + const existingSubmission = merges.value.find( + (x) => x.id === ref.id, + ); + if (existingSubmission && !existingSubmission?.locked) { + // how would it be locked? + const newSubmission = lineToSubmission(annotation, false); + existingSubmission.sink = newSubmission.sink; + existingSubmission.source = newSubmission.source; + merges.changed.dispatch(); + if (autoSubmit.value) { + this.bulkMerge([existingSubmission]); + } } } - } - ref.dispose(); - }); + ref.dispose(); + }), + ); - mergeAnnotationState.source.childDeleted.add((id) => { - let changed = false; - const filtered = merges.value.filter((x) => { - const keep = x.id !== id || x.locked; - if (!keep) { - changed = true; + this.registerDisposer( + mergeAnnotationState.source.childDeleted.add((id) => { + let changed = false; + const filtered = merges.value.filter((x) => { + const keep = x.id !== id || x.locked; + if (!keep) { + changed = true; + } + return keep; + }); + if (changed) { + merges.value = filtered; } - return keep; - }); - if (changed) { - merges.value = filtered; - } - }); + }), + ); } + + const findPathGroup = makeColoredAnnotationState( + layer, + loadedSubsource, + "findpath", + WHITE_COLOR, + ); + this.findPathAnnotationState = findPathGroup; + findPathGroup.source.childDeleted.add((annotationId) => { + if ( + findPathState.source.value?.annotationReference?.id === annotationId + ) { + findPathState.source.value = undefined; + } + if ( + findPathState.target.value?.annotationReference?.id === annotationId + ) { + findPathState.target.value = undefined; + } + }); + const findPathChanged = () => { + const { path, source, target } = findPathState; + const annotationSource = findPathGroup.source; + if (source.value && !source.value.annotationReference) { + addSelection(annotationSource, source.value, "find path source"); + } + if (target.value && !target.value.annotationReference) { + addSelection(annotationSource, target.value, "find path target"); + } + for (const annotation of annotationSource) { + if ( + annotation.id !== source.value?.annotationReference?.id && + annotation.id !== target.value?.annotationReference?.id + ) { + annotationSource.delete(annotationSource.getReference(annotation.id)); + } + } + for (const line of path) { + // line.id = ''; // TODO, is it a bug that this is necessary? annotationMap is empty if I + // step through it but logging shows it isn't empty + annotationSource.add(line); + } + }; + this.registerDisposer(findPathState.changed.add(findPathChanged)); + this.registerDisposer( + findPathState.triggerPathUpdate.add(() => { + const loadedSubsource = getGraphLoadedSubsource(this.layer)!; + const annotationToNanometers = + loadedSubsource.loadedDataSource.transform.inputSpace.value.scales.map( + (x) => x / 1e-9, + ); + this.submitFindPath( + findPathState.precisionMode.value, + annotationToNanometers, + ).then((success) => { + success; + }); + }), + ); + findPathChanged(); // initial state } createRenderLayers( @@ -1347,7 +1645,7 @@ class GraphConnection extends SegmentationGraphSourceConnection { const graphSubsource = subsources.filter( (subsource) => subsource.id === "graph", )[0]; - if (graphSubsource?.subsource.segmentationGraph) { + if (graphSubsource && graphSubsource.subsource.segmentationGraph) { if (graphSubsource.subsource.segmentationGraph !== this.graph) { continue; } @@ -1386,27 +1684,35 @@ class GraphConnection extends SegmentationGraphSourceConnection { 7000, ); return false; + } else { + const splitRoots = await this.graph.graphServer.splitSegments( + [...sinks].map((x) => selectionInNanometers(x, annotationToNanometers)), + [...sources].map((x) => + selectionInNanometers(x, annotationToNanometers), + ), + ); + if (splitRoots.length === 0) { + StatusMessage.showTemporaryMessage(`No split found.`, 3000); + return false; + } else { + const focusSegment = multicutState.focusSegment.value!; + multicutState.reset(); // need to clear the focus segment before deleting the multicut segment + const { segmentsState } = this; + segmentsState.selectedSegments.delete(focusSegment); + for (const segment of [...sinks, ...sources]) { + segmentsState.selectedSegments.delete(segment.rootId); + } + this.meshAddNewSegments(splitRoots); + segmentsState.selectedSegments.add(splitRoots); + segmentsState.visibleSegments.add(splitRoots); + const oldValues = new Uint64Set(); + oldValues.add(focusSegment); + const newValues = new Uint64Set(); + newValues.add(splitRoots); + this.state.replaceSegments(oldValues, newValues); + return true; + } } - const splitRoots = await this.graph.graphServer.splitSegments( - [...sinks], - [...sources], - annotationToNanometers, - ); - if (splitRoots.length === 0) { - StatusMessage.showTemporaryMessage("No split found.", 3000); - return false; - } - const focusSegment = multicutState.focusSegment.value!; - multicutState.reset(); // need to clear the focus segment before deleting the multicut segment - const { segmentsState } = this; - segmentsState.selectedSegments.delete(focusSegment); - for (const segment of [...sinks, ...sources]) { - segmentsState.selectedSegments.delete(segment.rootId); - } - this.meshAddNewSegments(splitRoots); - segmentsState.selectedSegments.add(splitRoots); - segmentsState.visibleSegments.add(splitRoots); - return true; } deleteMergeSubmission = (submission: MergeSubmission) => { @@ -1421,7 +1727,6 @@ class GraphConnection extends SegmentationGraphSourceConnection { submission: MergeSubmission, attempts = 1, ): Promise => { - this.graph; const loadedSubsource = getGraphLoadedSubsource(this.layer)!; const annotationToNanometers = loadedSubsource.loadedDataSource.transform.inputSpace.value.scales.map( @@ -1430,11 +1735,17 @@ class GraphConnection extends SegmentationGraphSourceConnection { submission.error = undefined; for (let i = 1; i <= attempts; i++) { try { - return await this.graph.graphServer.mergeSegments( - submission.sink, - submission.source!, - annotationToNanometers, + const newRoot = await this.graph.graphServer.mergeSegments( + selectionInNanometers(submission.sink, annotationToNanometers), + selectionInNanometers(submission.source!, annotationToNanometers), ); + const oldValues = new Uint64Set(); + oldValues.add(submission.sink.rootId); + oldValues.add(submission.source!.rootId); + const newValues = new Uint64Set(); + newValues.add(newRoot); + this.state.replaceSegments(oldValues, newValues); + return newRoot; } catch (err) { if (i === attempts) { submission.error = err.message || "unknown"; @@ -1457,20 +1768,6 @@ class GraphConnection extends SegmentationGraphSourceConnection { return; } const segmentsToRemove: Uint64[] = []; - const replaceSegment = (a: Uint64, b: Uint64) => { - segmentsToRemove.push(a); - for (const submission of submissions) { - if ( - submission.source && - Uint64.equal(submission.source.rootId, a) - ) { - submission.source.rootId = b; - } - if (Uint64.equal(submission.sink.rootId, a)) { - submission.sink.rootId = b; - } - } - }; let completed = 0; let activeLoops = 0; const loop = (completedAt: number, pending: MergeSubmission[]) => { @@ -1491,10 +1788,13 @@ class GraphConnection extends SegmentationGraphSourceConnection { submission.locked = true; submission.status = "trying..."; merges.changed.dispatch(); + const segments = [ + submission.source!.rootId, + submission.sink.rootId, + ]; this.submitMerge(submission, 3) .then((mergedRoot) => { - replaceSegment(submission.source!.rootId, mergedRoot); - replaceSegment(submission.sink.rootId, mergedRoot); + segmentsToRemove.push(...segments); submission.status = "done"; submission.mergedRoot = mergedRoot; merges.changed.dispatch(); @@ -1531,24 +1831,53 @@ class GraphConnection extends SegmentationGraphSourceConnection { } else if (submission.mergedRoot) { segmentsToAdd.push(submission.mergedRoot); } + const latestRoots = + await this.graph.graphServer.filterLatestRoots(segmentsToAdd); + const segmentsState = + this.layer.displayState.segmentationGroupState.value; + const { visibleSegments, selectedSegments } = segmentsState; + selectedSegments.delete(segmentsToRemove); + this.meshAddNewSegments(latestRoots); + selectedSegments.add(latestRoots); + visibleSegments.add(latestRoots); + merges.changed.dispatch(); } - const latestRoots = - await this.graph.graphServer.filterLatestRoots(segmentsToAdd); const segmentsState = this.layer.displayState.segmentationGroupState.value; const { visibleSegments, selectedSegments } = segmentsState; selectedSegments.delete(segmentsToRemove); - this.meshAddNewSegments(latestRoots); + const latestRoots = + await this.graph.graphServer.filterLatestRoots(segmentsToAdd); selectedSegments.add(latestRoots); visibleSegments.add(latestRoots); merges.changed.dispatch(); } + + async submitFindPath( + precisionMode: boolean, + annotationToNanometers: Float64Array, + ): Promise { + const { + state: { findPathState }, + } = this; + const { source, target } = findPathState; + if (!source.value || !target.value) return false; + const centroids = await this.graph.findPath( + source.value, + target.value, + precisionMode, + annotationToNanometers, + ); + StatusMessage.showTemporaryMessage("Path found!", 5000); + findPathState.centroids.value = centroids; + return true; + } } async function parseGrapheneError(e: HttpError) { if (e.response) { let msg: string; if (e.response.headers.get("content-type") === "application/json") { - msg = (await e.response.json()).message; + msg = (await e.response.json())["message"]; } else { msg = await e.response.text(); } @@ -1578,20 +1907,30 @@ async function withErrorMessageHTTP( } catch (e) { if (e instanceof HttpError && e.response) { const { errorPrefix = "" } = options; - const msg = await parseGrapheneError(e); - if (msg) { - if (!status) { - status = new StatusMessage(true); - } - status.setErrorMessage(errorPrefix + msg); - status.setVisible(true); - throw new Error(`[${e.response.status}] ${errorPrefix}${msg}`); + const msg = (await parseGrapheneError(e)) || "unknown error"; + if (!status) { + status = new StatusMessage(true); } + status.setErrorMessage(errorPrefix + msg); + status.setVisible(true); + throw new Error(`[${e.response.status}] ${errorPrefix}${msg}`); } throw e; } } +const selectionInNanometers = ( + selection: SegmentSelection, + annotationToNanometers: Float64Array, +): SegmentSelection => { + const { rootId, segmentId, position } = selection; + return { + rootId, + segmentId, + position: position.map((val, i) => val * annotationToNanometers[i]), + }; +}; + export const GRAPH_SERVER_NOT_SPECIFIED = Symbol("Graph Server Not Specified."); class GrapheneGraphServerInterface { @@ -1616,16 +1955,15 @@ class GrapheneGraphServerInterface { const response = await withErrorMessageHTTP(promise, { initialMessage: `Retrieving root for segment ${segment}`, - errorPrefix: "Could not fetch root: ", + errorPrefix: `Could not fetch root: `, }); const jsonResp = await response.json(); - return Uint64.parseString(jsonResp.root_id); + return Uint64.parseString(jsonResp["root_id"]); } async mergeSegments( first: SegmentSelection, second: SegmentSelection, - annotationToNanometers: Float64Array, ): Promise { const { url } = this; if (url === "") { @@ -1638,14 +1976,8 @@ class GrapheneGraphServerInterface { { method: "POST", body: JSON.stringify([ - [ - String(first.segmentId), - ...first.position.map((val, i) => val * annotationToNanometers[i]), - ], - [ - String(second.segmentId), - ...second.position.map((val, i) => val * annotationToNanometers[i]), - ], + [String(first.segmentId), ...first.position], + [String(second.segmentId), ...second.position], ]), }, responseIdentity, @@ -1654,7 +1986,7 @@ class GrapheneGraphServerInterface { try { const response = await promise; const jsonResp = await response.json(); - return Uint64.parseString(jsonResp.new_root_ids[0]); + return Uint64.parseString(jsonResp["new_root_ids"][0]); } catch (e) { if (e instanceof HttpError) { const msg = await parseGrapheneError(e); @@ -1667,7 +1999,6 @@ class GrapheneGraphServerInterface { async splitSegments( first: SegmentSelection[], second: SegmentSelection[], - annotationToNanometers: Float64Array, ): Promise { const { url } = this; if (url === "") { @@ -1680,14 +2011,8 @@ class GrapheneGraphServerInterface { { method: "POST", body: JSON.stringify({ - sources: first.map((x) => [ - String(x.segmentId), - ...x.position.map((val, i) => val * annotationToNanometers[i]), - ]), - sinks: second.map((x) => [ - String(x.segmentId), - ...x.position.map((val, i) => val * annotationToNanometers[i]), - ]), + sources: first.map((x) => [String(x.segmentId), ...x.position]), + sinks: second.map((x) => [String(x.segmentId), ...x.position]), }), }, responseIdentity, @@ -1698,9 +2023,9 @@ class GrapheneGraphServerInterface { errorPrefix: "Split failed: ", }); const jsonResp = await response.json(); - const final: Uint64[] = new Array(jsonResp.new_root_ids.length); + const final: Uint64[] = new Array(jsonResp["new_root_ids"].length); for (let i = 0; i < final.length; ++i) { - final[i] = Uint64.parseString(jsonResp.new_root_ids[i]); + final[i] = Uint64.parseString(jsonResp["new_root_ids"][i]); } return final; } @@ -1713,35 +2038,87 @@ class GrapheneGraphServerInterface { url, { method: "POST", - body: JSON.stringify({ - node_ids: segments.map((x) => x.toJSON()), - }), + body: JSON.stringify({ node_ids: segments.map((x) => x.toJSON()) }), }, responseIdentity, ); const response = await withErrorMessageHTTP(promise, { - errorPrefix: "Could not check latest: ", + errorPrefix: `Could not check latest: `, }); const jsonResp = await response.json(); const res: Uint64[] = []; - for (const [i, isLatest] of jsonResp.is_latest.entries()) { + for (const [i, isLatest] of jsonResp["is_latest"].entries()) { if (isLatest) { res.push(segments[i]); } } return res; } + + async findPath( + first: SegmentSelection, + second: SegmentSelection, + precisionMode: boolean, + ) { + const { url } = this; + if (url === "") { + return Promise.reject(GRAPH_SERVER_NOT_SPECIFIED); + } + const promise = cancellableFetchSpecialOk( + this.credentialsProvider, + `${url}/graph/find_path?int64_as_str=1&precision_mode=${Number( + precisionMode, + )}`, + { + method: "POST", + body: JSON.stringify([ + [String(first.rootId), ...first.position], + [String(second.rootId), ...second.position], + ]), + }, + responseIdentity, + ); + + const response = await withErrorMessageHTTP(promise, { + initialMessage: `Finding path between ${first.segmentId} and ${second.segmentId}`, + errorPrefix: "Path finding failed: ", + }); + const jsonResponse = await response.json(); + const supervoxelCentroidsKey = "centroids_list"; + const centroids = verifyObjectProperty( + jsonResponse, + supervoxelCentroidsKey, + (x) => parseArray(x, verifyFloatArray), + ); + const missingL2IdsKey = "failed_l2_ids"; + const missingL2Ids = jsonResponse[missingL2IdsKey]; + if (missingL2Ids && missingL2Ids.length > 0) { + StatusMessage.showTemporaryMessage( + "Some level 2 meshes are missing, so the path shown may have a poor level of detail.", + ); + } + const l2_path = verifyOptionalObjectProperty( + jsonResponse, + "l2_path", + verifyStringArray, + ); + return { + centroids, + l2_path, + }; + } } class GrapheneGraphSource extends SegmentationGraphSource { private connections = new Set(); public graphServer: GrapheneGraphServerInterface; + private l2CacheAvailable: boolean | undefined = undefined; constructor( public info: GrapheneMultiscaleVolumeInfo, - credentialsProvider: SpecialProtocolCredentialsProvider, + private credentialsProvider: SpecialProtocolCredentialsProvider, private chunkSource: GrapheneMultiscaleVolumeChunkSource, public state: GrapheneState, ) { @@ -1777,10 +2154,81 @@ class GrapheneGraphSource extends SegmentationGraphSource { ); } + async isL2CacheUrlAvailable() { + if (this.l2CacheAvailable !== undefined) { + return this.l2CacheAvailable; + } + try { + const { l2CacheUrl, table } = this.info.app; + const tableMapping = await cancellableFetchSpecialOk( + undefined, + `${l2CacheUrl}/table_mapping`, + {}, + responseJson, + ); + this.l2CacheAvailable = !!(tableMapping && tableMapping[table]); + return this.l2CacheAvailable; + } catch (e) { + console.error("e", e); + return false; + } + } + getRoot(segment: Uint64) { return this.graphServer.getRoot(segment); } + async findPath( + first: SegmentSelection, + second: SegmentSelection, + precisionMode: boolean, + annotationToNanometers: Float64Array, + ): Promise { + const { l2CacheUrl, table } = this.info.app; + const l2CacheAvailable = + precisionMode && (await this.isL2CacheUrlAvailable()); // don't check if available if we don't need it + let { centroids, l2_path } = await this.graphServer.findPath( + selectionInNanometers(first, annotationToNanometers), + selectionInNanometers(second, annotationToNanometers), + precisionMode && !l2CacheAvailable, + ); + if (precisionMode && l2CacheAvailable && l2_path) { + const repCoordinatesUrl = `${l2CacheUrl}/table/${table}/attributes`; + try { + const res = await cancellableFetchSpecialOk( + this.credentialsProvider, + repCoordinatesUrl, + { + method: "POST", + body: JSON.stringify({ + l2_ids: l2_path, + }), + }, + responseJson, + ); + + // many reasons why an l2 id might not have info + // l2 cache has a process that takes time for new ids (even hours) + // maybe a small fraction have no info + // sometime l2 is so small (single voxel), it is ignored by l2 + // best to just drop those points + centroids = l2_path + .map((id) => { + return verifyOptionalObjectProperty(res, id, (x) => { + return verifyIntegerArray(x["rep_coord_nm"]); + }); + }) + .filter((x): x is number[] => x !== undefined); + } catch (e) { + console.log("e", e); + } + } + const centroidsTransformed = centroids.map((point: number[]) => { + return point.map((val, i) => val / annotationToNanometers[i]); + }); + return centroidsTransformed; + } + tabContents( layer: SegmentationUserLayer, context: DependentViewContext, @@ -1804,6 +2252,13 @@ class GrapheneGraphSource extends SegmentationGraphSource { title: "Merge segments", }), ); + toolbox.appendChild( + makeToolButton(context, layer.toolBinder, { + toolJson: GRAPHENE_FIND_PATH_TOOL_ID, + label: "Find Path", + title: "Find Path", + }), + ); parent.appendChild(toolbox); parent.appendChild( context.registerDisposer( @@ -1844,6 +2299,15 @@ class ChunkedGraphChunkSource { spec: ChunkedGraphChunkSpecification; OPTIONS: { spec: ChunkedGraphChunkSpecification }; + + constructor( + chunkManager: ChunkManager, + options: { + spec: ChunkedGraphChunkSpecification; + }, + ) { + super(chunkManager, options); + } } class GrapheneChunkedGraphChunkSource extends WithParameters( @@ -1993,6 +2457,7 @@ class SliceViewPanelChunkedGraphLayer extends SliceViewPanelRenderLayer { const GRAPHENE_MULTICUT_SEGMENTS_TOOL_ID = "grapheneMulticutSegments"; const GRAPHENE_MERGE_SEGMENTS_TOOL_ID = "grapheneMergeSegments"; +const GRAPHENE_FIND_PATH_TOOL_ID = "grapheneFindPath"; class MulticutAnnotationLayerView extends AnnotationLayerView { private _annotationStates: MergedAnnotationStates; @@ -2023,51 +2488,50 @@ class MulticutAnnotationLayerView extends AnnotationLayerView { } } +const addSelection = ( + source: AnnotationSource | MultiscaleAnnotationSource, + selection: SegmentSelection, + description?: string, +) => { + const annotation: Point = { + id: "", + point: selection.position, + type: AnnotationType.POINT, + properties: [], + relatedSegments: [[selection.segmentId, selection.rootId]], + description, + }; + const ref = source.add(annotation); + selection.annotationReference = ref; +}; + const synchronizeAnnotationSource = ( source: WatchableSet, state: AnnotationLayerState, ) => { const annotationSource = state.source; - annotationSource.childDeleted.add((annotationId) => { const selection = [...source].find( (selection) => selection.annotationReference?.id === annotationId, ); if (selection) source.delete(selection); }); - - const addSelection = (selection: SegmentSelection) => { - const annotation: Point = { - id: "", - point: selection.position, - type: AnnotationType.POINT, - properties: [], - relatedSegments: [[selection.segmentId, selection.rootId]], - }; - const ref = annotationSource.add(annotation); - selection.annotationReference = ref; - }; - source.changed.add((x, add) => { if (x === null) { for (const annotation of annotationSource) { - // using .clear does not remove annotations from the list - // (this.blueGroupAnnotationState.source as LocalAnnotationSource).clear(); annotationSource.delete(annotationSource.getReference(annotation.id)); } return; } - if (add) { - addSelection(x); + addSelection(annotationSource, x); } else if (x.annotationReference) { annotationSource.delete(x.annotationReference); } }); - // load initial state for (const selection of source) { - addSelection(selection); + addSelection(annotationSource, selection); } }; @@ -2138,7 +2602,7 @@ class MulticutSegmentsTool extends LayerTool { const { body, header } = makeToolActivationStatusMessageWithHeader(activation); header.textContent = "Multicut segments"; - body.classList.add("graphene-multicut-status"); + body.classList.add("graphene-tool-status", "graphene-multicut"); body.appendChild( makeIcon({ text: "Swap", @@ -2424,8 +2888,6 @@ function mergeToLine(submission: MergeSubmission): Line { const MAX_MERGE_COUNT = 10; -// on error, copy (also clean up error message) - const MERGE_SEGMENTS_INPUT_EVENT_MAP = EventActionMap.fromObject({ "at:shift?+enter": { action: "submit" }, }); @@ -2455,7 +2917,7 @@ class MergeSegmentsTool extends LayerTool { const { body, header } = makeToolActivationStatusMessageWithHeader(activation); header.textContent = "Merge segments"; - body.classList.add("graphene-merge-segments-status"); + body.classList.add("graphene-tool-status", "graphene-merge-segments"); activation.bindInputEventMap(MERGE_SEGMENTS_INPUT_EVENT_MAP); const submitAction = async () => { if (merges.value.filter((x) => x.locked).length) return; @@ -2570,6 +3032,143 @@ class MergeSegmentsTool extends LayerTool { } } +const FIND_PATH_INPUT_EVENT_MAP = EventActionMap.fromObject({ + "at:shift?+enter": { action: "submit" }, + "at:shift?+control+mousedown0": { action: "add-point" }, +}); + +class FindPathTool extends LayerTool { + activate(activation: ToolActivation) { + const { layer } = this; + const { + graphConnection: { value: graphConnection }, + } = layer; + if (!graphConnection || !(graphConnection instanceof GraphConnection)) + return; + const { + state: { findPathState }, + findPathAnnotationState, + } = graphConnection; + const { source, target, precisionMode } = findPathState; + // Ensure we use the same segmentationGroupState while activated. + const segmentationGroupState = + this.layer.displayState.segmentationGroupState.value; + const { body, header } = + makeToolActivationStatusMessageWithHeader(activation); + header.textContent = "Find Path"; + body.classList.add("graphene-tool-status", "graphene-find-path"); + const submitAction = () => { + findPathState.triggerPathUpdate.dispatch(); + }; + body.appendChild( + makeIcon({ + text: "Submit", + title: "Submit Find Path", + onClick: () => { + submitAction(); + }, + }), + ); + body.appendChild( + makeIcon({ + text: "Clear", + title: "Clear Find Path", + onClick: () => { + findPathState.source.reset(); + findPathState.target.reset(); + findPathState.centroids.reset(); + }, + }), + ); + const checkbox = activation.registerDisposer( + new TrackableBooleanCheckbox(precisionMode), + ); + const label = document.createElement("label"); + const labelText = document.createElement("span"); + labelText.textContent = "Precision mode: "; + label.appendChild(labelText); + label.title = + "Precision mode returns a more accurate path, but takes longer."; + label.appendChild(checkbox.element); + body.appendChild(label); + const annotationElements = document.createElement("div"); + annotationElements.classList.add("find-path-annotations"); + body.appendChild(annotationElements); + const bindings = getDefaultAnnotationListBindings(); + this.registerDisposer(new MouseEventBinder(annotationElements, bindings)); + const updateAnnotationElements = () => { + removeChildren(annotationElements); + const maxColumnWidths = [0, 0, 0]; + const globalDimensionIndices = [0, 1, 2]; + const localDimensionIndices: number[] = []; + const template = + "[symbol] 2ch [dim] var(--neuroglancer-column-0-width) [dim] var(--neuroglancer-column-1-width) [dim] var(--neuroglancer-column-2-width) [delete] min-content"; + const endpoints = [source, target]; + const endpointAnnotations = endpoints + .map((x) => x.value?.annotationReference?.value) + .filter((x) => x) as Annotation[]; + for (const annotation of endpointAnnotations) { + const [element, elementColumnWidths] = makeAnnotationListElement( + this.layer, + annotation, + findPathAnnotationState, + template, + globalDimensionIndices, + localDimensionIndices, + ); + for (const [column, width] of elementColumnWidths.entries()) { + maxColumnWidths[column] = width; + } + annotationElements.appendChild(element); + } + for (const [column, width] of maxColumnWidths.entries()) { + annotationElements.style.setProperty( + `--neuroglancer-column-${column}-width`, + `${width + 2}ch`, + ); + } + }; + findPathState.changed.add(updateAnnotationElements); + updateAnnotationElements(); + activation.bindInputEventMap(FIND_PATH_INPUT_EVENT_MAP); + activation.bindAction("submit", (event) => { + event.stopPropagation(); + submitAction(); + }); + activation.bindAction("add-point", (event) => { + event.stopPropagation(); + (async () => { + if (!source.value) { + // first selection + const selection = maybeGetSelection( + this, + segmentationGroupState.visibleSegments, + ); + if (selection) { + source.value = selection; + } + } else if (!target.value) { + const selection = maybeGetSelection( + this, + segmentationGroupState.visibleSegments, + ); + if (selection) { + target.value = selection; + } + } + })(); + }); + } + + toJSON() { + return GRAPHENE_FIND_PATH_TOOL_ID; + } + + get description() { + return "find path"; + } +} + registerTool( SegmentationUserLayer, GRAPHENE_MULTICUT_SEGMENTS_TOOL_ID, @@ -2586,6 +3185,10 @@ registerTool( }, ); +registerTool(SegmentationUserLayer, GRAPHENE_FIND_PATH_TOOL_ID, (layer) => { + return new FindPathTool(layer, true); +}); + const ANNOTATE_MERGE_LINE_TOOL_ID = "annotateMergeLine"; registerLegacyTool( diff --git a/src/datasource/graphene/graphene.css b/src/datasource/graphene/graphene.css index 7de9f02ad2..9696385327 100644 --- a/src/datasource/graphene/graphene.css +++ b/src/datasource/graphene/graphene.css @@ -14,12 +14,12 @@ color: #4444ff; } -.graphene-multicut-status { +.graphene-tool-status { display: flex; - flex-direction: row; + gap: 10px; } -.graphene-multicut-status > .activeGroupIndicator { +.graphene-multicut > .activeGroupIndicator { padding: 2px; margin: auto 0; background-color: red; @@ -27,15 +27,15 @@ font-weight: 900; } -.graphene-multicut-status > .activeGroupIndicator.blueGroup { +.graphene-multicut > .activeGroupIndicator.blueGroup { background-color: blue; } -.graphene-multicut-status > .activeGroupIndicator::after { +.graphene-multicut > .activeGroupIndicator::after { content: "Red"; } -.graphene-multicut-status > .activeGroupIndicator.blueGroup::after { +.graphene-multicut > .activeGroupIndicator.blueGroup::after { content: "Blue"; } @@ -49,13 +49,7 @@ gap: 10px; } -.graphene-merge-segments-status { - display: flex; - gap: 10px; -} - -.graphene-merge-segments-status .neuroglancer-icon, -.graphene-multicut-status .neuroglancer-icon { +.graphene-tool-status .neuroglancer-icon { height: 100%; } @@ -78,3 +72,22 @@ .graphene-merge-segments-point .neuroglancer-segment-list-entry-copy-container { display: none; } + +.graphene-find-path > label { + display: flex; +} + +.graphene-find-path > label > span { + display: flex; + align-content: center; + flex-wrap: wrap; +} + +.find-path-annotations { + display: flex; + gap: 10px; +} + +.find-path-annotations > .neuroglancer-annotation-list-entry { + background-color: black; +} diff --git a/src/status.ts b/src/status.ts index 519df57449..b8657c87c1 100644 --- a/src/status.ts +++ b/src/status.ts @@ -33,6 +33,7 @@ export class StatusMessage { if (statusContainer === null) { statusContainer = document.createElement("ul"); statusContainer.id = "statusContainer"; + statusContainer.tabIndex = -1; const el: HTMLElement | null = document.getElementById( "neuroglancer-container", ); diff --git a/src/ui/annotations.ts b/src/ui/annotations.ts index c799c90be6..f84ccf2617 100644 --- a/src/ui/annotations.ts +++ b/src/ui/annotations.ts @@ -609,7 +609,39 @@ export class AnnotationLayerView extends Tab { private render(index: number) { const { annotation, state } = this.listElements[index]; - return this.makeAnnotationListElement(annotation, state); + const { + layer, + gridTemplate, + globalDimensionIndices, + localDimensionIndices, + } = this; + const [element, elementColumnWidths] = makeAnnotationListElement( + layer, + annotation, + state, + gridTemplate, + globalDimensionIndices, + localDimensionIndices, + ); + for (const [column, width] of elementColumnWidths.entries()) { + this.setColumnWidth(column, width); + } + element.addEventListener("mouseenter", () => { + this.displayState.hoverState.value = { + id: annotation.id, + partIndex: 0, + annotationLayerState: state, + }; + }); + const selectionState = this.selectedAnnotationState.value; + if ( + selectionState !== undefined && + selectionState.annotationLayerState === state && + selectionState.annotationId === annotation.id + ) { + element.classList.add("neuroglancer-annotation-selected"); + } + return element; } private setColumnWidth(column: number, width: number) { @@ -829,141 +861,6 @@ export class AnnotationLayerView extends Tab { this.updateHoverView(); this.updateSelectionView(); } - - private makeAnnotationListElement( - annotation: Annotation, - state: AnnotationLayerState, - ) { - const chunkTransform = state.chunkTransform - .value as ChunkTransformParameters; - const element = document.createElement("div"); - element.classList.add("neuroglancer-annotation-list-entry"); - element.dataset.color = state.displayState.color.toString(); - element.style.gridTemplateColumns = this.gridTemplate; - const icon = document.createElement("div"); - icon.className = "neuroglancer-annotation-icon"; - icon.textContent = annotationTypeHandlers[annotation.type].icon; - element.appendChild(icon); - - let deleteButton: HTMLElement | undefined; - - const maybeAddDeleteButton = () => { - if (state.source.readonly) return; - if (deleteButton !== undefined) return; - deleteButton = makeDeleteButton({ - title: "Delete annotation", - onClick: (event) => { - event.stopPropagation(); - event.preventDefault(); - const ref = state.source.getReference(annotation.id); - try { - state.source.delete(ref); - } finally { - ref.dispose(); - } - }, - }); - deleteButton.classList.add("neuroglancer-annotation-list-entry-delete"); - element.appendChild(deleteButton); - }; - - let numRows = 0; - visitTransformedAnnotationGeometry( - annotation, - chunkTransform, - (layerPosition, isVector) => { - isVector; - ++numRows; - const position = document.createElement("div"); - position.className = "neuroglancer-annotation-position"; - element.appendChild(position); - let i = 0; - const addDims = ( - viewDimensionIndices: readonly number[], - layerDimensionIndices: readonly number[], - ) => { - for (const viewDim of viewDimensionIndices) { - const layerDim = layerDimensionIndices[viewDim]; - if (layerDim !== -1) { - const coord = Math.floor(layerPosition[layerDim]); - const coordElement = document.createElement("div"); - const text = coord.toString(); - coordElement.textContent = text; - coordElement.classList.add("neuroglancer-annotation-coordinate"); - coordElement.style.gridColumn = `dim ${i + 1}`; - this.setColumnWidth(i, text.length); - position.appendChild(coordElement); - } - ++i; - } - }; - addDims( - this.globalDimensionIndices, - chunkTransform.modelTransform.globalToRenderLayerDimensions, - ); - addDims( - this.localDimensionIndices, - chunkTransform.modelTransform.localToRenderLayerDimensions, - ); - maybeAddDeleteButton(); - }, - ); - if (annotation.description) { - ++numRows; - const description = document.createElement("div"); - description.classList.add("neuroglancer-annotation-description"); - description.textContent = annotation.description; - element.appendChild(description); - } - icon.style.gridRow = `span ${numRows}`; - if (deleteButton !== undefined) { - deleteButton.style.gridRow = `span ${numRows}`; - } - element.addEventListener("mouseenter", () => { - this.displayState.hoverState.value = { - id: annotation.id, - partIndex: 0, - annotationLayerState: state, - }; - this.layer.selectAnnotation(state, annotation.id, false); - }); - element.addEventListener("action:select-position", (event) => { - event.stopPropagation(); - this.layer.selectAnnotation(state, annotation.id, "toggle"); - }); - - element.addEventListener("action:pin-annotation", (event) => { - event.stopPropagation(); - this.layer.selectAnnotation(state, annotation.id, true); - }); - - element.addEventListener("action:move-to-annotation", (event) => { - event.stopPropagation(); - event.preventDefault(); - const { layerRank } = chunkTransform; - const chunkPosition = new Float32Array(layerRank); - const layerPosition = new Float32Array(layerRank); - getCenterPosition(chunkPosition, annotation); - matrix.transformPoint( - layerPosition, - chunkTransform.chunkToLayerTransform, - layerRank + 1, - chunkPosition, - layerRank, - ); - setLayerPosition(this.layer, chunkTransform, layerPosition); - }); - - const selectionState = this.selectedAnnotationState.value; - if ( - selectionState !== undefined && - selectionState.annotationLayerState === state && - selectionState.annotationId === annotation.id - ) { - element.classList.add("neuroglancer-annotation-selected"); - } - return element; - } } export class AnnotationTab extends Tab { @@ -2153,3 +2050,128 @@ export function UserLayerWithAnnotationsMixin< export type UserLayerWithAnnotations = InstanceType< ReturnType >; + +export function makeAnnotationListElement( + layer: UserLayerWithAnnotations, + annotation: Annotation, + state: AnnotationLayerState, + gridTemplate: string, + globalDimensionIndices: number[], + localDimensionIndices: number[], +): [HTMLDivElement, number[]] { + const chunkTransform = state.chunkTransform.value as ChunkTransformParameters; + const element = document.createElement("div"); + element.classList.add("neuroglancer-annotation-list-entry"); + element.dataset.color = state.displayState.color.toString(); + element.style.gridTemplateColumns = gridTemplate; + const icon = document.createElement("div"); + icon.className = "neuroglancer-annotation-icon"; + icon.textContent = annotationTypeHandlers[annotation.type].icon; + element.appendChild(icon); + + let deleteButton: HTMLElement | undefined; + + const maybeAddDeleteButton = () => { + if (state.source.readonly) return; + if (deleteButton !== undefined) return; + deleteButton = makeDeleteButton({ + title: "Delete annotation", + onClick: (event) => { + event.stopPropagation(); + event.preventDefault(); + const ref = state.source.getReference(annotation.id); + try { + state.source.delete(ref); + } finally { + ref.dispose(); + } + }, + }); + deleteButton.classList.add("neuroglancer-annotation-list-entry-delete"); + element.appendChild(deleteButton); + }; + + const columnWidths: number[] = []; + + let numRows = 0; + visitTransformedAnnotationGeometry( + annotation, + chunkTransform, + (layerPosition, isVector) => { + isVector; + ++numRows; + const position = document.createElement("div"); + position.className = "neuroglancer-annotation-position"; + element.appendChild(position); + let i = 0; + + const addDims = ( + viewDimensionIndices: readonly number[], + layerDimensionIndices: readonly number[], + ) => { + for (const viewDim of viewDimensionIndices) { + const layerDim = layerDimensionIndices[viewDim]; + if (layerDim !== -1) { + const coord = Math.floor(layerPosition[layerDim]); + const coordElement = document.createElement("div"); + const text = coord.toString(); + coordElement.textContent = text; + coordElement.classList.add("neuroglancer-annotation-coordinate"); + coordElement.style.gridColumn = `dim ${i + 1}`; + columnWidths[i] = Math.max(columnWidths[i] || 0, text.length); + position.appendChild(coordElement); + } + ++i; + } + }; + addDims( + globalDimensionIndices, + chunkTransform.modelTransform.globalToRenderLayerDimensions, + ); + addDims( + localDimensionIndices, + chunkTransform.modelTransform.localToRenderLayerDimensions, + ); + maybeAddDeleteButton(); + }, + ); + if (annotation.description) { + ++numRows; + const description = document.createElement("div"); + description.classList.add("neuroglancer-annotation-description"); + description.textContent = annotation.description; + element.appendChild(description); + } + icon.style.gridRow = `span ${numRows}`; + if (deleteButton !== undefined) { + deleteButton.style.gridRow = `span ${numRows}`; + } + element.addEventListener("mouseenter", () => { + layer.selectAnnotation(state, annotation.id, false); + }); + element.addEventListener("action:select-position", (event) => { + event.stopPropagation(); + layer.selectAnnotation(state, annotation.id, "toggle"); + }); + element.addEventListener("action:pin-annotation", (event) => { + event.stopPropagation(); + layer.selectAnnotation(state, annotation.id, true); + }); + element.addEventListener("action:move-to-annotation", (event) => { + event.stopPropagation(); + event.preventDefault(); + const { layerRank } = chunkTransform; + const chunkPosition = new Float32Array(layerRank); + const layerPosition = new Float32Array(layerRank); + getCenterPosition(chunkPosition, annotation); + matrix.transformPoint( + layerPosition, + chunkTransform.chunkToLayerTransform, + layerRank + 1, + chunkPosition, + layerRank, + ); + setLayerPosition(layer, chunkTransform, layerPosition); + }); + return [element, columnWidths]; +} diff --git a/src/util/json.ts b/src/util/json.ts index 793ad16041..ad3f0ec586 100644 --- a/src/util/json.ts +++ b/src/util/json.ts @@ -702,6 +702,16 @@ export function verifyIntegerArray(a: unknown) { return a; } +export function verifyFloatArray(a: unknown) { + if (!Array.isArray(a)) { + throw new Error(`Expected array, received: ${JSON.stringify(a)}.`); + } + for (const x of a) { + verifyFloat(x); + } + return a; +} + export function verifyBoolean(x: any) { if (typeof x !== "boolean") { throw new Error(`Expected boolean, received: ${JSON.stringify(x)}`); From 623bfc89d777bdbf273cb16705107dd14edfbaf3 Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Fri, 12 Jan 2024 15:01:49 -0500 Subject: [PATCH 06/24] added custom error handler support for credentials provider so middleauth can catch terms of service errors and prompt the user to agree to the terms of service fixed lint errors and minor cleanup --- src/credentials_provider/http_request.ts | 9 +- src/credentials_provider/index.ts | 23 +++ src/credentials_provider/oauth2.ts | 18 +- .../middleauth/credentials_provider.ts | 158 +++++++++++------- 4 files changed, 131 insertions(+), 77 deletions(-) diff --git a/src/credentials_provider/http_request.ts b/src/credentials_provider/http_request.ts index 825dd5b705..1a406e11d2 100644 --- a/src/credentials_provider/http_request.ts +++ b/src/credentials_provider/http_request.ts @@ -38,7 +38,10 @@ export async function fetchWithCredentials( credentials: Credentials, requestInit: RequestInit, ) => RequestInit, - errorHandler: (httpError: HttpError, credentials: Credentials) => "refresh", + errorHandler: ( + httpError: HttpError, + credentials: Credentials, + ) => "refresh" | Promise<"refresh">, cancellationToken: CancellationToken = uncancelableToken, ): Promise { let credentials: CredentialsWithGeneration | undefined; @@ -62,7 +65,9 @@ export async function fetchWithCredentials( ); } catch (error) { if (error instanceof HttpError) { - if (errorHandler(error, credentials.credentials) === "refresh") { + if ( + (await errorHandler(error, credentials.credentials)) === "refresh" + ) { if (++credentialsAttempt === maxCredentialsAttempts) throw error; continue; } diff --git a/src/credentials_provider/index.ts b/src/credentials_provider/index.ts index c0aab02d6f..4f65b5c5d1 100644 --- a/src/credentials_provider/index.ts +++ b/src/credentials_provider/index.ts @@ -18,10 +18,12 @@ * @file Generic facility for providing authentication/authorization credentials. */ +import type { OAuth2Credentials } from "#src/credentials_provider/oauth2.js"; import type { CancellationToken } from "#src/util/cancellation.js"; import { MultipleConsumerCancellationTokenSource } from "#src/util/cancellation.js"; import type { Owned } from "#src/util/disposable.js"; import { RefCounted } from "#src/util/disposable.js"; +import type { HttpError } from "#src/util/http_request.js"; import { StringMemoize } from "#src/util/memoize.js"; /** @@ -45,6 +47,27 @@ export abstract class CredentialsProvider extends RefCounted { invalidCredentials?: CredentialsWithGeneration, cancellationToken?: CancellationToken, ) => Promise>; + + errorHandler? = async ( + error: HttpError, + credentials: OAuth2Credentials, + ): Promise<"refresh"> => { + const { status } = error; + if (status === 401) { + // 401: Authorization needed. OAuth2 token may have expired. + return "refresh"; + } + if (status === 403 && !credentials.accessToken) { + // Anonymous access denied. Request credentials. + return "refresh"; + } + if (error instanceof Error && credentials.email !== undefined) { + error.message += ` (Using credentials for ${JSON.stringify( + credentials.email, + )})`; + } + throw error; + }; } export function makeCachedCredentialsGetter( diff --git a/src/credentials_provider/oauth2.ts b/src/credentials_provider/oauth2.ts index 444b5fd95a..8e4d07174e 100644 --- a/src/credentials_provider/oauth2.ts +++ b/src/credentials_provider/oauth2.ts @@ -59,23 +59,7 @@ export function fetchWithOAuth2Credentials( ); return { ...init, headers }; }, - (error, credentials) => { - const { status } = error; - if (status === 401) { - // 401: Authorization needed. OAuth2 token may have expired. - return "refresh"; - } - if (status === 403 && !credentials.accessToken) { - // Anonymous access denied. Request credentials. - return "refresh"; - } - if (error instanceof Error && credentials.email !== undefined) { - error.message += ` (Using credentials for ${JSON.stringify( - credentials.email, - )})`; - } - throw error; - }, + credentialsProvider.errorHandler!, cancellationToken, ); } diff --git a/src/datasource/middleauth/credentials_provider.ts b/src/datasource/middleauth/credentials_provider.ts index c5a6ab9995..59328f015b 100644 --- a/src/datasource/middleauth/credentials_provider.ts +++ b/src/datasource/middleauth/credentials_provider.ts @@ -22,6 +22,7 @@ import { CredentialsProvider, makeCredentialsGetter, } from "#src/credentials_provider/index.js"; +import type { OAuth2Credentials } from "#src/credentials_provider/oauth2.js"; import { StatusMessage } from "#src/status.js"; import { verifyObject, @@ -29,6 +30,7 @@ import { verifyString, verifyStringArray, } from "#src/util/json.js"; +import type { HttpError } from "src/util/http_request"; export type MiddleAuthToken = { tokenType: string; @@ -51,79 +53,49 @@ function openPopupCenter(url: string, width: number, height: number) { ); } -async function waitForLogin(serverUrl: string): Promise { +async function waitForRemoteFlow( + url: string, + startMessage: string, + startAction: string, + retryMessage: string, + closedMessage: string, +): Promise { const status = new StatusMessage(/*delay=*/ false, /*modal=*/ true); - const res: Promise = new Promise((f) => { - function writeLoginStatus(message: string, buttonMessage: string) { + function writeStatus(message: string, buttonMessage: string) { status.element.textContent = message + " "; const button = document.createElement("button"); button.textContent = buttonMessage; status.element.appendChild(button); button.addEventListener("click", () => { - writeLoginStatus( - `Waiting for login to middleauth server ${serverUrl}...`, - "Retry", - ); - - const auth_popup = openPopupCenter( - `${serverUrl}/api/v1/authorize`, - 400, - 650, - ); - - const closeAuthPopup = () => { - auth_popup?.close(); + writeStatus(retryMessage, "Retry"); + const popup = openPopupCenter(url, 400, 650); + const closePopup = () => { + popup?.close(); }; - - window.addEventListener("beforeunload", closeAuthPopup); + window.addEventListener("beforeunload", closePopup); const checkClosed = setInterval(() => { - if (auth_popup?.closed) { + if (popup?.closed) { clearInterval(checkClosed); - writeLoginStatus( - `Login window closed for middleauth server ${serverUrl}.`, - "Retry", - ); + writeStatus(closedMessage, "Retry"); } }, 1000); - const tokenListener = async (ev: MessageEvent) => { - if (ev.source === auth_popup) { + const messageListener = async (ev: MessageEvent) => { + if (ev.source === popup) { clearInterval(checkClosed); - window.removeEventListener("message", tokenListener); - window.removeEventListener("beforeunload", closeAuthPopup); - closeAuthPopup(); - - verifyObject(ev.data); - const accessToken = verifyObjectProperty( - ev.data, - "token", - verifyString, - ); - const appUrls = verifyObjectProperty( - ev.data, - "app_urls", - verifyStringArray, - ); - - const token: MiddleAuthToken = { - tokenType: "Bearer", - accessToken, - url: serverUrl, - appUrls, - }; - f(token); + window.removeEventListener("message", messageListener); + window.removeEventListener("beforeunload", closePopup); + closePopup(); + f(ev.data); } }; - - window.addEventListener("message", tokenListener); + window.addEventListener("message", messageListener); }); } - - writeLoginStatus(`middleauth server ${serverUrl} login required.`, "Login"); + writeStatus(startMessage, startAction); }); - try { return await res; } finally { @@ -131,6 +103,37 @@ async function waitForLogin(serverUrl: string): Promise { } } +async function waitForLogin(serverUrl: string): Promise { + const data = await waitForRemoteFlow( + `${serverUrl}/api/v1/authorize`, + `middleauth server ${serverUrl} login required.`, + "Login", + `Waiting for login to middleauth server ${serverUrl}...`, + `Login window closed for middleauth server ${serverUrl}.`, + ); + verifyObject(data); + const accessToken = verifyObjectProperty(data, "token", verifyString); + const appUrls = verifyObjectProperty(data, "app_urls", verifyStringArray); + const token: MiddleAuthToken = { + tokenType: "Bearer", + accessToken, + url: serverUrl, + appUrls, + }; + return token; +} + +async function showTosForm(url: string, tosName: string) { + const data = await waitForRemoteFlow( + url, + `Before you can access ${tosName}, you need to accept its Terms of Service.`, + "Open", + "Waiting for Terms of Service agreement...", + `Terms of Service closed for ${tosName}.`, + ); + return data === "success"; +} + const LOCAL_STORAGE_AUTH_KEY = "auth_token_v2"; function getAuthTokenFromLocalStorage(authURL: string) { @@ -156,17 +159,14 @@ export class MiddleAuthCredentialsProvider extends CredentialsProvider { let token = undefined; - if (!this.alreadyTriedLocalStorage) { this.alreadyTriedLocalStorage = true; token = getAuthTokenFromLocalStorage(this.serverUrl); } - if (!token) { token = await waitForLogin(this.serverUrl); saveAuthTokenToLocalStorage(this.serverUrl, token); } - return token; }); } @@ -183,6 +183,7 @@ export class UnverifiedApp extends Error { export class MiddleAuthAppCredentialsProvider extends CredentialsProvider { private credentials: CredentialsWithGeneration | undefined = undefined; + agreedToTos = false; constructor( private serverUrl: string, @@ -192,6 +193,10 @@ export class MiddleAuthAppCredentialsProvider extends CredentialsProvider { + if (this.credentials && this.agreedToTos) { + return this.credentials.credentials; + } + this.agreedToTos = false; const authInfo = await fetch(`${this.serverUrl}/auth_info`).then((res) => res.json(), ); @@ -199,9 +204,7 @@ export class MiddleAuthAppCredentialsProvider extends CredentialsProvider => { + const { status } = error; + if (status === 401) { + // 401: Authorization needed. OAuth2 token may have expired. + return "refresh"; + } + if (status === 403) { + const { response } = error; + if (response) { + const { headers } = response; + const contentType = headers.get("content-type"); + if (contentType === "application/json") { + const json = await response.json(); + if (json.error && json.error === "missing_tos") { + // Missing terms of service agreement. Prompt user. + const url = new URL(json.data.tos_form_url); + url.searchParams.set("client", "ng"); + const success = await showTosForm( + url.toString(), + json.data.tos_name, + ); + if (success) { + this.agreedToTos = true; + return "refresh"; + } + } + } + } + if (!credentials.accessToken) { + // Anonymous access denied. Request credentials. + return "refresh"; + } + } + throw error; + }; } From a9f04069773f38cc10d68f8de029b77f7d4b0786 Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Wed, 21 Feb 2024 19:15:02 -0500 Subject: [PATCH 07/24] feat(segmentation_user_layer): add individual segment color picker tool --- src/layer/segmentation/style.css | 16 ++++ src/segmentation_display_state/frontend.ts | 85 ++++++++++++++++++---- src/widget/color.ts | 29 ++++++-- 3 files changed, 109 insertions(+), 21 deletions(-) diff --git a/src/layer/segmentation/style.css b/src/layer/segmentation/style.css index dd92be0194..359f45ee7b 100644 --- a/src/layer/segmentation/style.css +++ b/src/layer/segmentation/style.css @@ -136,3 +136,19 @@ + .neuroglancer-tool-button { margin-left: 1em; } + +.neuroglancer-segment-list-entry .neuroglancer-color-widget { + border: none; + border-color: transparent; + appearance: none; + background-color: transparent; + padding: 0; + margin: 0; + margin-left: 3px; + height: 19px; + width: 20px; +} + +.neuroglancer-segment-list-entry .neuroglancer-color-widget.overridden { + background-color: white; +} diff --git a/src/segmentation_display_state/frontend.ts b/src/segmentation_display_state/frontend.ts index df36d12496..ab4f210ff2 100644 --- a/src/segmentation_display_state/frontend.ts +++ b/src/segmentation_display_state/frontend.ts @@ -47,14 +47,19 @@ import { observeWatchable, registerNestedSync } from "#src/trackable_value.js"; import { isWithinSelectionPanel } from "#src/ui/selection_details.js"; import type { Uint64Map } from "#src/uint64_map.js"; import { setClipboard } from "#src/util/clipboard.js"; -import { useWhiteBackground } from "#src/util/color.js"; +import { + packColor, + serializeColor, + TrackableRGB, + useWhiteBackground, +} from "#src/util/color.js"; import { RefCounted } from "#src/util/disposable.js"; import { measureElementClone } from "#src/util/dom.js"; -import type { vec3 } from "#src/util/geom.js"; -import { kOneVec, vec4 } from "#src/util/geom.js"; +import { kOneVec, vec3, vec4 } from "#src/util/geom.js"; import { NullarySignal } from "#src/util/signal.js"; import { Uint64 } from "#src/util/uint64.js"; import { withSharedVisibility } from "#src/visibility_priority/frontend.js"; +import { ColorWidget } from "#src/widget/color.js"; import { makeCopyButton } from "#src/widget/copy_button.js"; import { makeEyeButton } from "#src/widget/eye_button.js"; import { makeFilterButton } from "#src/widget/filter_button.js"; @@ -347,6 +352,8 @@ const segmentWidgetTemplate = (() => { filterElement.classList.add("neuroglancer-segment-list-entry-filter"); const filterIndex = template.childElementCount; template.appendChild(filterElement); + const colorWidgetIndex = template.childElementCount; + template.appendChild(ColorWidget.template()); return { template, copyContainerIndex, @@ -357,6 +364,7 @@ const segmentWidgetTemplate = (() => { labelIndex, filterIndex, starIndex, + colorWidgetIndex, unmappedIdIndex: -1, unmappedCopyIndex: -1, }; @@ -426,7 +434,7 @@ function makeRegisterSegmentWidgetEventHandlers( const onMouseEnter = (event: Event) => { const entryElement = event.currentTarget as HTMLElement; const idString = entryElement.dataset.id!; - const id = tempStatedColor; + const id = tempObjectId; id.tryParseString(idString); displayState.segmentSelectionState.set(id); if (!isWithinSelectionPanel(entryElement)) { @@ -437,7 +445,7 @@ function makeRegisterSegmentWidgetEventHandlers( const selectHandler = (event: Event) => { const entryElement = event.currentTarget as HTMLElement; const idString = entryElement.dataset.id!; - const id = tempStatedColor; + const id = tempObjectId; id.tryParseString(idString); displayState.selectSegment( id, @@ -470,7 +478,7 @@ function makeRegisterSegmentWidgetEventHandlers( const visibleCheckboxHandler = (event: Event) => { const entryElement = getEntryElement(event); const idString = entryElement.dataset.id!; - const id = tempStatedColor; + const id = tempObjectId; id.tryParseString(idString); const { selectedSegments, visibleSegments } = displayState.segmentationGroupState.value; @@ -486,7 +494,7 @@ function makeRegisterSegmentWidgetEventHandlers( const filterHandler = (event: Event) => { const entryElement = getEntryElement(event); const idString = entryElement.dataset.id!; - const id = tempStatedColor; + const id = tempObjectId; id.tryParseString(idString); displayState.filterBySegmentLabel(id); event.stopPropagation(); @@ -504,7 +512,7 @@ function makeRegisterSegmentWidgetEventHandlers( } const entryElement = event.currentTarget as HTMLElement; const idString = entryElement.dataset.id!; - const id = tempStatedColor; + const id = tempObjectId; id.tryParseString(idString); displayState.moveToSegment(id); }; @@ -539,11 +547,35 @@ function makeRegisterSegmentWidgetEventHandlers( starButton.addEventListener("click", (event: MouseEvent) => { const entryElement = getEntryElement(event); const idString = entryElement.dataset.id!; - const id = tempStatedColor; + const id = tempObjectId; id.tryParseString(idString); const { selectedSegments } = displayState.segmentationGroupState.value; selectedSegments.set(id, !selectedSegments.has(id)); }); + + const trackableRGB = new TrackableRGB(vec3.fromValues(0, 0, 0)); + trackableRGB.changed.add(() => { + const testU = new Uint64(packColor(trackableRGB.value)); + const idString = element.dataset.id!; + const id = tempObjectId; + id.tryParseString(idString); + displayState.segmentStatedColors.value.delete(id); + displayState.segmentStatedColors.value.set(id, testU); + }); + + // TODO, need to register disposer? + new ColorWidget( + trackableRGB, + undefined, + children[template.colorWidgetIndex] as HTMLInputElement, + () => { + const idString = element.dataset.id!; + const id = tempObjectId; + id.tryParseString(idString); + displayState.segmentStatedColors.value.delete(id); + }, + false, + ); }; } @@ -641,7 +673,7 @@ export class SegmentWidgetFactory