tests passing!

src/generate_omero_objects.py

@@ -224,7 +224,6 @@ def parse_xml_metadata(ann: XMLAnnotation,
                        metadata: List[str],
                        hash: str) -> List[List[str]]:
     kv_data = []
-    print("metadata: ", metadata)
     tree = ETree.fromstring(to_xml(ann.value, canonicalize=True))
     for el in tree:
         if el.tag.rpartition('}')[2] == "CLITransferMetadata":
@@ -251,7 +250,6 @@ def parse_xml_metadata(ann: XMLAnnotation,
                     kv_data.append([item, val])
                 if item == "database_id" and "db_id" in metadata:
                     kv_data.append([item, val])
-    print("kv_data: ", kv_data)
     return kv_data
 
 
@@ -317,10 +315,22 @@ def create_plate_map(ome: OME, img_map: dict, conn: BlitzGateway
         for ann in ome.structured_annotations:
             if (ann.id in ann_ids and
                     isinstance(ann, XMLAnnotation)):
-                newome.structured_annotations.remove(ann)
-                map_ref_ids.append(ann.id)
-                file_path = get_server_path(plate.annotation_refs,
-                                            ome.structured_annotations)
+                tree = ETree.fromstring(to_xml(ann.value,
+                                               canonicalize=True))
+                is_metadata = False
+                for el in tree:
+                    if el.tag.rpartition('}')[2] == "CLITransferMetadata":
+                        is_metadata = True
+                if not is_metadata:
+                    newome.structured_annotations.remove(ann)
+                    map_ref_ids.append(ann.id)
+                    file_path = get_server_path(plate.annotation_refs,
+                                                ome.structured_annotations)
+                    annref = next(filter(lambda x: x.id == ann.id,
+                                         plate.annotation_refs))
+                    newplate = next(filter(lambda x: x.id == plate.id,
+                                           newome.plates))
+                    newplate.annotation_refs.remove(annref)
         q = conn.getQueryService()
         params = Parameters()
         if not file_path:
@@ -564,7 +574,6 @@ def link_images(ome: OME, ds_map: dict, img_map: dict, conn: BlitzGateway):
 def link_annotations(ome: OME, proj_map: dict, ds_map: dict, img_map: dict,
                      ann_map: dict, scr_map: dict, pl_map: dict,
                      conn: BlitzGateway):
-    print(ann_map)
     for proj in ome.projects:
         proj_id = proj_map[proj.id]
         proj_obj = conn.getObject("Project", proj_id)
@@ -585,9 +594,7 @@ def link_annotations(ome: OME, proj_map: dict, ds_map: dict, img_map: dict,
             img_obj = conn.getObject("Image", img_id)
             anns = ome.structured_annotations
             for annref in img.annotation_refs:
-                print("image annref:", annref)
                 ann = next(filter(lambda x: x.id == annref.id, anns))
-                print("annotation:", ann)
                 link_one_annotation(img_obj, ann, ann_map, conn)
         except KeyError:
             continue

src/generate_xml.py

@@ -413,6 +413,7 @@ def create_filepath_annotations(id: str, conn: BlitzGateway,
                 anref = AnnotationRef(id=an.id)
                 anrefs.append(anref)
             else:
+                f = f'pixel_images/{clean_id}.tiff'
                 if simple:
                     f = f'{clean_id}.tiff'
                     f = Path(common_root) / proj / ds / f
@@ -424,7 +425,6 @@ def create_filepath_annotations(id: str, conn: BlitzGateway,
                     ann_count += 1
                     anref = AnnotationRef(id=an.id)
                     anrefs.append(anref)
-                f = f'pixel_images/{clean_id}.tiff'
                 xml = create_path_xml(str(f))
                 an, anref = create_xml_and_ref(id=ann_count,
                                                namespace=ns,
@@ -859,12 +859,7 @@ def populate_plate(obj: PlateI, ome: OME, conn: BlitzGateway,
 
     # this will need some changing to tackle XMLs
     last_image_anns = ome.images[-1].annotation_refs
-    last_image_anns_ids = [i.id for i in last_image_anns]
-    for ann in ome.structured_annotations:
-        if (ann.id in last_image_anns_ids and
-                isinstance(ann, CommentAnnotation) and
-                int(ann.id.split(":")[-1]) < 0):
-            plate_path = ann.value
+    plate_path = get_server_path(last_image_anns, ome.structured_annotations)
     filepath_anns, refs = create_filepath_annotations(pl.id, conn,
                                                       simple=False,
                                                       plate_path=plate_path)
@@ -972,6 +967,12 @@ def list_file_ids(ome: OME) -> dict:
     for img in ome.images:
         path = get_server_path(img.annotation_refs, ome.structured_annotations)
         id_list[img.id] = path
+    for ann in ome.structured_annotations:
+        if isinstance(ann, FileAnnotation):
+            if ann.namespace != "omero.web.figure.json":
+                path = get_server_path(ann.annotation_refs,
+                                       ome.structured_annotations)
+            id_list[ann.id] = path
     return id_list
 
 

src/omero_cli_transfer.py

@@ -304,30 +304,32 @@ def _copy_files(self, id_list: Dict[str, Any], folder: str,
         for id in id_list:
             clean_id = int(id.split(":")[-1])
             dtype = id.split(":")[0]
-            if clean_id not in downloaded_ids:
-                path = id_list[id]
-                rel_path = path
-                if dtype == "Image":
+            if (dtype == "Image"):
+                if (clean_id not in downloaded_ids):
+                    path = id_list[id]
+                    rel_path = path
                     rel_path = str(Path(rel_path).parent)
-                subfolder = os.path.join(str(Path(folder)), rel_path)
-                if dtype == "Image":
+                    subfolder = os.path.join(str(Path(folder)), rel_path)
                     os.makedirs(subfolder, mode=DIR_PERM, exist_ok=True)
-                else:
-                    ann_folder = str(Path(subfolder).parent)
-                    os.makedirs(ann_folder, mode=DIR_PERM, exist_ok=True)
-                if dtype == "Annotation":
-                    id = "File" + id
-                if rel_path == "pixel_images":
-                    filepath = str(Path(subfolder) / (str(clean_id) + ".tiff"))
-                    cli.invoke(['export', '--file', filepath, id])
-                    downloaded_ids.append(id)
-                else:
-                    cli.invoke(['download', id, subfolder])
-                    if dtype == "Image":
-                        obj = conn.getObject("Image", clean_id)
-                        fileset = obj.getFileset()
+                    obj = conn.getObject("Image", clean_id)
+                    fileset = obj.getFileset()
+                    if rel_path == "pixel_images" or fileset is None:
+                        filepath = str(Path(subfolder) /
+                                       (str(clean_id) + ".tiff"))
+                        cli.invoke(['export', '--file', filepath, id])
+                        downloaded_ids.append(id)
+                    else:
+                        cli.invoke(['download', id, subfolder])
                         for fs_image in fileset.copyImages():
                             downloaded_ids.append(fs_image.getId())
+            else:
+                path = id_list[id]
+                rel_path = path
+                subfolder = os.path.join(str(Path(folder)), rel_path)
+                ann_folder = str(Path(subfolder).parent)
+                os.makedirs(ann_folder, mode=DIR_PERM, exist_ok=True)
+                id = "File" + id
+                cli.invoke(['download', id, subfolder])
 
     def _package_files(self, tar_path: str, zip: bool, folder: str):
         if zip:
@@ -475,7 +477,6 @@ def __unpack(self, args):
                                           ln_s, args.skip, self.gateway)
         self._delete_all_rois(dest_img_map, self.gateway)
         print("Matching source and destination images...")
-        print(src_img_map, dest_img_map)
         img_map = self._make_image_map(src_img_map, dest_img_map, self.gateway)
         print("Creating and linking OMERO objects...")
         populate_omero(ome, img_map, self.gateway,
@@ -533,7 +534,7 @@ def _create_image_map(self, ome: OME
             else:
                 filelist.append(fpath)
             for anref in img.annotation_refs:
-                for an in ome.structured_annotations:
+                for an in newome.structured_annotations:
                     if anref.id == an.id and isinstance(an, XMLAnnotation):
                         tree = ETree.fromstring(to_xml(an.value,
                                                        canonicalize=True))

test/data/transfer.xml

@@ -10,7 +10,7 @@
         <AnnotationRef ID="Annotation:2209" />
         <AnnotationRef ID="Annotation:2210" />
         <AnnotationRef ID="Annotation:2551" />
-        <AnnotationRef ID="Annotation:-242965625845933864455559976333404928513" />
+        <AnnotationRef ID="Annotation:18350171370993896932" />
     </Image>
     <Image ID="Image:1679" Name="combined_result.tiff">
         <Description />
@@ -23,7 +23,7 @@
         <AnnotationRef ID="Annotation:2209" />
         <AnnotationRef ID="Annotation:2210" />
         <AnnotationRef ID="Annotation:2551" />
-        <AnnotationRef ID="Annotation:-242965625845933864455559976333404928514" />
+        <AnnotationRef ID="Annotation:18350171370993896933" />
     </Image>
     <StructuredAnnotations>
         <MapAnnotation ID="Annotation:2209" Namespace="openmicroscopy.org/omero/client/mapAnnotation">
@@ -37,12 +37,20 @@
         <TagAnnotation ID="Annotation:2551">
             <Value>simple_tag</Value>
         </TagAnnotation>
-        <CommentAnnotation ID="Annotation:-242965625845933864455559976333404928513" Namespace="Image:1678">
-            <Value>root_0/2022-01/14/18-30-55.264/combined_result.tiff</Value>
-        </CommentAnnotation>
-        <CommentAnnotation ID="Annotation:-242965625845933864455559976333404928514" Namespace="Image:1679">
-            <Value>root_0/2022-01/14/18-30-55.264/combined_result.tiff</Value>
-        </CommentAnnotation>
+        <XMLAnnotation ID="Annotation:18350171370993896932" Namespace="openmicroscopy.org/cli/transfer">
+            <Value>
+              <CLITransferServerPath xsi:schemaLocation="https://raw.githubusercontent.com/ome/omero-cli-transfer/main/schemas/serverpath.xsd">
+                <Path>root_0/2022-01/14/18-30-55.264/combined_result.tiff</Path>
+              </CLITransferServerPath>
+            </Value>
+          </XMLAnnotation>
+          <XMLAnnotation ID="Annotation:18350171370993896933" Namespace="openmicroscopy.org/cli/transfer">
+            <Value>
+              <CLITransferServerPath xsi:schemaLocation="https://raw.githubusercontent.com/ome/omero-cli-transfer/main/schemas/serverpath.xsd">
+                <Path>root_0/2022-01/14/18-30-55.264/combined_result.tiff</Path>
+              </CLITransferServerPath>
+            </Value>
+          </XMLAnnotation>
         <MapAnnotation ID="Annotation:2210" Namespace="jax.org/jax/example/namespace">
             <Value>
                 <M K="species"> </M>

test/integration/test_prepare.py

@@ -15,6 +15,8 @@
 from ome_types.model import AnnotationRef, ROI, ROIRef, Rectangle
 from ome_types.model.screen import PlateRef
 from ome_types.model.map import M, Map
+from uuid import uuid4
+
 
 import ezomero
 import pytest
@@ -168,6 +170,7 @@ def run_asserts_edited(self):
             elif img_name == "edited image name":
                 kvs = ezomero.get_map_annotation_ids(self.gw, "Image",
                                                      img.getId())
+                kvs = sorted(kvs)
                 assert len(kvs) == 2
                 kv = ezomero.get_map_annotation(self.gw, kvs[-1])
                 assert len(kv) == 2
@@ -196,10 +199,15 @@ def run_asserts_edited(self):
 
     def edit_xml(self, filename):
         ome = from_xml(filename)
+        ann_count = uuid4().int >> 64
         new_proj = Project(id="Project:1", name="edited project")
         new_ds = Dataset(id="Dataset:1", name="edited dataset")
-        newtag1 = TagAnnotation(id="Annotation:1", value="tag for proj")
-        newtag2 = TagAnnotation(id="Annotation:2", value="tag for img")
+        newtag1 = TagAnnotation(id=f"Annotation:{ann_count}",
+                                value="tag for proj")
+        ann_count += 1
+        newtag2 = TagAnnotation(id=f"Annotation:{ann_count}",
+                                value="tag for img")
+        ann_count += 1
         new_proj.annotation_refs.append(AnnotationRef(id=newtag1.id))
         md_dict = {"key1": "value1", "key2": 2}
         mmap = []
@@ -208,7 +216,8 @@ def edit_xml(self, filename):
                 mmap.append(M(k=_key, value=str(_value)))
             else:
                 mmap.append(M(k=_key, value=''))
-        mapann = MapAnnotation(id="Annotation:3", value=Map(ms=mmap))
+        mapann = MapAnnotation(id=f"Annotation:{ann_count}",
+                               value=Map(ms=mmap))
         rect = Rectangle(id="Shape:1", x=1, y=2, width=3, height=4)
         roi = ROI(id="ROI:1", union=[rect])
         ome.rois.append(roi)

test/integration/test_transfer.py

@@ -294,6 +294,8 @@ def test_unpack(self, package_name):
             ds_ids.sort()
             assert len(ds_ids) == 2
             im_ids = ezomero.get_image_ids(self.gw, dataset=ds_ids[0])
+            assert len(im_ids) == 2
+            im_ids = ezomero.get_image_ids(self.gw, dataset=ds_ids[1])
             assert len(im_ids) == 1
             assert len(ezomero.get_map_annotation_ids(
                             self.gw, "Project", pjs[-1])) == 1
@@ -381,14 +383,16 @@ def test_unpack_merge(self):
         assert len(orphan) == 1
         scr_args = self.args + ['unpack', "test/data/simple_screen.zip"]
         self.cli.invoke(scr_args, strict=True)
-        scr_args += ['--merge']
-        self.cli.invoke(scr_args, strict=True)
         scr_ids = []
         for screen in self.gw.getObjects("Screen"):
             scr_ids.append(screen.getId())
+        screen = self.gw.getObject("Screen", scr_ids[0])
+        for plate in screen.listChildren():
+            print(plate.getId())
+        scr_args += ['--merge']
+        self.cli.invoke(scr_args, strict=True)
         assert len(scr_ids) == 1
         pl_ids = []
-        screen = self.gw.getObject("Screen", scr_ids[0])
         for plate in screen.listChildren():
             pl_ids.append(plate.getId())
         assert len(pl_ids) == 4

test/unit/test_set.py

@@ -74,7 +74,7 @@ def test_load_pack(self):
                 111, 'test/data/output_folder')
         hash, ome, folder = self.transfer._load_from_pack(
             "test/data/valid_single_image.zip", "tmp_folder")
-        assert hash == "ac050c218f01bf189f9b3bdc9cab4f37"
+        assert hash == "6bc8b78eb85f8244f86eded682f95feb"
         assert len(ome.images) == 1
         assert str(folder.resolve()) == "/omero-cli-transfer/tmp_folder"
         hash, ome, folder = self.transfer._load_from_pack(