add flat 783 segmentation + skeletons

fafbseg/data/ngl_scenes.json

@@ -65,14 +65,14 @@
                 },
                 "enableDefaultSubsources": false
             },
-            "precomputed://https://flyem.mrc-lmb.cam.ac.uk/flyconnectome/ann/flytable-info"
+            "precomputed://https://flyem.mrc-lmb.cam.ac.uk/flyconnectome/dynann/flytable-info"
         ],
         "tab": "segments",
         "segments": [],
         "colorSeed": 45225755,
         "name": "fly_v31"
     },
-    "FLYWIRE_SEG_LAYER_FLAT": {
+    "FLYWIRE_SEG_LAYER_FLAT_630": {
         "type": "segmentation",
         "source": {
             "url": "precomputed://gs://flywire_v141_m630",
@@ -88,6 +88,22 @@
         "colorSeed": 963379830,
         "name": "flywire_v141_m630"
     },
+    "FLYWIRE_SEG_LAYER_FLAT_783": {
+        "type": "segmentation",
+        "source": {
+            "url": "precomputed://gs://flywire_v141_m783",
+            "subsources": {
+                "default": true,
+                "bounds": true,
+                "mesh": true
+            },
+            "enableDefaultSubsources": false
+        },
+        "tab": "segments",
+        "meshRenderScale": 315182315218263.7,
+        "colorSeed": 963379830,
+        "name": "flywire_v141_m783"
+    },
     "FLYWIRE_BRAIN_LAYER": {
         "source": "precomputed://gs://flywire_neuropil_meshes/whole_neuropil/brain_mesh_v141.surf",
         "type": "segmentation",

fafbseg/flywire/annotations.py

@@ -384,7 +384,7 @@ def get_annotation_table_info(table_name: str,
     return client.annotation.get_table_metadata(table_name)
 
 
-@inject_dataset(disallowed=['flat_630', 'flat_571'])
+@inject_dataset()
 def get_annotations(table_name: str,
                     materialization='latest',
                     split_positions: bool = False,
@@ -592,7 +592,7 @@ def upload_annotations(table_name: str,
     return resp
 
 
-@inject_dataset(disallowed=['flat_630', 'flat_571'])
+@inject_dataset()
 def get_somas(x=None,
               materialization='auto',
               raise_missing=True,
@@ -919,14 +919,13 @@ def search_annotations(x,
     Returns
     -------
     pandas.DataFrame
-                If `x` is root IDs
-                Pandas DataFrame with annotations matching ``x``. Coordinates
+                DataFrame with annotations matching ``x``. Coordinates
                 are in 4x4x40nm voxel space.
 
     See Also
     --------
     :func:`~fafbseg.flywire.get_hierarchical_annotations`
-                Use this to load get a table with all hierarchical annotations.
+                Use this to load a table with all hierarchical annotations.
     :func:`~fafbseg.flywire.search_community_annotations`
                 Use this to search the community annotations.
 
@@ -1264,7 +1263,7 @@ def search_community_annotations(x,
     Parameter
     ---------
     x :         str | int | Neuron/List | list of ints | None
-                Term (str) or root ID(s) to search for. Set to `None` to fetch
+                Term (str) or root ID(s) to search for. Set to ``None`` to fetch
                 all annotations.
     exact :     bool
                 Whether term must be an exact match. For example, if
@@ -1290,16 +1289,15 @@ def search_community_annotations(x,
     Returns
     -------
     pandas.DataFrame
-                If `x` is root IDs
-                Pandas DataFrame with annotations matching ``x``. Coordinates
+                DataFrame with annotations matching ``x``. Coordinates
                 are in 4x4x40nm voxel space.
 
     See Also
     --------
     :func:`~fafbseg.flywire.search_annotations`
-                Use this to search through the hiearchical annotations.
+                Use this to search through the hierarchical annotations.
     :func:`~fafbseg.flywire.get_hierarchical_annotations`
-                Use this to load get a table with all hierarchical annotations.
+                Use this to load a table with all hierarchical annotations.
 
     Examples
     --------
@@ -1369,7 +1367,7 @@ def search_community_annotations(x,
                                  split_positions=True,
                                  materialization=materialization)
 
-    # If no query term, we'll just return the whole
+    # If no query term, we'll just return the whole table
     if x is None:
         return ct
 

fafbseg/flywire/neuroglancer.py

@@ -758,13 +758,13 @@ def _load_ngl_scenes():
 
 def _find_flywire_layer(layers, raise_not_found=True):
     """Find the FlyWire segmentation layer among given layers."""
-    for i, l in enumerate(layers):
-        if l["type"] == "segmentation_with_graph":
-            return i
-        if l["type"] == "segmentation" and l["name"] in list(
-            utils.FLYWIRE_DATASETS.values()
-        ):
+    poss_names = list(utils.FLYWIRE_DATASETS.values())
+    for i, layer in enumerate(layers):
+        if layer["type"] == "segmentation_with_graph":
             return i
+        if layer["type"] == "segmentation":
+            if (layer["name"] in poss_names or 'flywire' in layer['name']):
+                return i
     if raise_not_found:
         raise ValueError("Unable to identify flywire segmentation among layers")
 
@@ -806,7 +806,7 @@ def construct_scene(
                     requires changes to the source for the segmentation.
     layout :        "3d" | "xy-3d" | "xy"
                     Layout to show.
-    dataset :       "public" | "production" | "sandbox" | "flat_630"
+    dataset :       "public" | "production" | "sandbox" | "flat_630" | "flat_783"
                     Which segmentation dataset to use.
 
     Returns
@@ -835,7 +835,9 @@ def construct_scene(
     # Add segmentation layer
     if segmentation:
         if dataset == "flat_630":
-            scene["layers"].append(NGL_SCENES["FLYWIRE_SEG_LAYER_FLAT"])
+            scene["layers"].append(NGL_SCENES["FLYWIRE_SEG_LAYER_FLAT_630"])
+        elif dataset == "flat_783":
+            scene["layers"].append(NGL_SCENES["FLYWIRE_SEG_LAYER_FLAT_783"])
         else:
             if not base_neuroglancer:
                 scene["layers"].append(NGL_SCENES["FLYWIRE_SEG_LAYER"])

fafbseg/flywire/skeletonize.py

@@ -35,7 +35,8 @@
 from .utils import get_cloudvolume, silence_find_mat_version, inject_dataset
 from .annotations import get_somas
 
-SKELETON_BASE_URL = "https://flyem.mrc-lmb.cam.ac.uk/flyconnectome/flywire_skeletons"
+SKELETON_BASE_URL = {'630': "https://flyem.mrc-lmb.cam.ac.uk/flyconnectome/flywire_skeletons_630",
+                     '783': "https://flyem.mrc-lmb.cam.ac.uk/flyconnectome/flywire_skeletons_783",}
 SKELETON_INFO = {"@type": "neuroglancer_skeletons", "transform": [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0], "vertex_attributes": [{"id": "radius", "data_type": "float32", "num_components": 1}]}
 
 
@@ -542,25 +543,27 @@ def _worker_wrapper(x):
 
 
 def get_skeletons(root_id, threads=2, omit_failures=None, max_threads=6,
-                  progress=True):
+                  dataset=783, progress=True):
     """Fetch precomputed skeletons.
 
-    Currently this only works for 630 roots (i.e. the first public release
-    of FlyWire).
+    Currently this only works for proofread 630 and 783 roots (i.e. the first
+    two public releases of FlyWire).
 
     Parameters
     ----------
     root_id  :          int | list of ints
                         Root ID(s) of the FlyWire neuron(s) you want to
                         skeletonize. Must be root IDs that existed at
-                        materialization 630.
+                        materialization 630 or 783 (see dataset parameter).
     omit_failures :     bool, optional
                         Determine behaviour when skeleton generation fails
                         (e.g. if the neuron has only a single chunk):
                          - ``None`` (default) will raise an exception
                          - ``True`` will skip the offending neuron (might result
                            in an empty ``NeuronList``)
                          - ``False`` will return an empty ``TreeNeuron``
+    dataset :           630 | 783
+                        Which dataset to query.
     progress :          bool
                         Whether to show a progress bar.
     max_threads :       int
@@ -577,21 +580,25 @@ def get_skeletons(root_id, threads=2, omit_failures=None, max_threads=6,
     >>> n = flywire.fetch_skeleton(720575940614131061)
 
     """
+    if str(dataset) not in SKELETON_BASE_URL:
+        raise ValueError('Currently we only provide precomputed skeletons for the '
+                         '630 and 783 data releases.')
+
     if omit_failures not in (None, True, False):
         raise ValueError('`omit_failures` must be either None, True or False. '
                          f'Got "{omit_failures}".')
 
     if navis.utils.is_iterable(root_id):
         root_id = np.asarray(root_id, dtype=np.int64)
 
-        il = is_latest_root(root_id, timestamp='mat_630')
+        il = is_latest_root(root_id, timestamp=f'mat_{dataset}')
         if np.any(~il):
-            msg = (f'{(~il).sum()} root ID(s) did not exists at materialization 630')
+            msg = (f'{(~il).sum()} root ID(s) did not exists at materialization {dataset}')
             if omit_failures is None:
                 raise ValueError(msg)
             navis.config.logger.warning(msg)
 
-        get_skels = partial(get_skeletons, omit_failures=omit_failures)
+        get_skels = partial(get_skeletons, omit_failures=omit_failures, dataset=dataset)
         if (max_threads > 1) and (len(root_id) > 1):
             with ThreadPoolExecutor(max_workers=max_threads) as pool:
                 futures = pool.map(get_skels, root_id)
@@ -621,7 +628,7 @@ def get_skeletons(root_id, threads=2, omit_failures=None, max_threads=6,
     root_id = np.int64(root_id)
 
     try:
-        tn = navis.read_precomputed(f'{SKELETON_BASE_URL}/{root_id}',
+        tn = navis.read_precomputed(f'{SKELETON_BASE_URL[str(dataset)]}/{root_id}',
                                     datatype='skeleton',
                                     info=SKELETON_INFO)
         # Force integer (navis.read_precomputed will turn Id into string)

fafbseg/flywire/utils.py

@@ -49,10 +49,12 @@
                 'sandbox': 'graphene://https://prod.flywire-daf.com/segmentation/1.0/fly_v26',
                 'public': 'graphene://https://prodv1.flywire-daf.com/segmentation/1.0/flywire_public',
                 'flat_630': 'precomputed://gs://flywire_v141_m630',
-                'flat_571': 'precomputed://gs://flywire_v141_m526'}
+                'flat_571': 'precomputed://gs://flywire_v141_m526',
+                'flat_783': 'precomputed://gs://flywire_v141_m783'}
 
 CAVE_DATASETS = {'production': 'flywire_fafb_production',
-                 'flat_630': 'flywire_fafb_production',
+                 'flat_783': 'flywire_fafb_production',
+                 'flat_630': 'flywire_fafb_public',
                  'flat_571': 'flywire_fafb_production',
                  'sandbox': 'flywire_fafb_sandbox',
                  'public': 'flywire_fafb_public'}
@@ -190,7 +192,7 @@ def get_neuropil_volumes(neuropils):
 
 
 def get_synapse_areas(ind):
-    """Lazy-load synapse areas.
+    """Lazy-load synapse areas (neuropils).
 
     Parameters
     ----------