YDA-6082: specify vault resource on copy-to-vault

If a vault resource has been configured, use it
when copying data from a research collection to the
vault collection. Also use configuration for number
of threads when copying data.

setup.cfg

@@ -5,4 +5,4 @@ strictness=short
 docstring_style=sphinx
 max-line-length=127
 exclude=__init__.py,tools,tests
-application-import-names=avu_json,conftest,util,api,config,constants,data_access_token,datacite,datarequest,data_object,epic,error,folder,groups,groups_import,intake,intake_dataset,intake_lock,intake_scan,intake_utils,intake_vault,json_datacite,json_landing_page,jsonutil,log,mail,meta,meta_form,msi,notifications,schema,schema_transformation,schema_transformations,settings,pathutil,provenance,policies_intake,policies_datamanager,policies_datapackage_status,policies_folder_status,policies_datarequest_status,publication,query,replication,revisions,revision_strategies,revision_utils,rule,user,vault,sram,arb_data_manager,cached_data_manager,resource,yoda_names,policies_utils
+application-import-names=avu_json,conftest,util,api,config,constants,data_access_token,datacite,datarequest,data_object,epic,error,folder,groups,groups_import,intake,intake_dataset,intake_lock,intake_scan,intake_utils,intake_vault,json_datacite,json_landing_page,jsonutil,log,mail,meta,meta_form,msi,notifications,schema,schema_transformation,schema_transformations,settings,pathutil,provenance,policies_intake,policies_datamanager,policies_datapackage_status,policies_folder_status,policies_datarequest_status,publication,query,replication,revisions,revision_strategies,revision_utils,rule,user,vault,sram,arb_data_manager,cached_data_manager,resource,yoda_names,policies_utils,vault_utils

unit-tests/test_vault.py

@@ -0,0 +1,26 @@
+"""Unit tests for the vault functions"""
+
+__copyright__ = 'Copyright (c) 2023-2024, Utrecht University'
+__license__   = 'GPLv3, see LICENSE'
+
+import sys
+from unittest import TestCase
+
+sys.path.append('..')
+
+from vault_utils import get_copy_folder_to_vault_irsync_command
+
+
+class VaultTest(TestCase):
+
+    def test_get_copy_folder_to_vault_irsync_command_with_vault_resc(self):
+        output = get_copy_folder_to_vault_irsync_command("/zoneName/home/research-foo/abc", "/zoneName/home/vault-foo/abc", "vaultResc", True)
+        self.assertEqual(output, ["irsync", "-rK", "-R", "vaultResc", "i:/zoneName/home/research-foo/abc/", "i:/zoneName/home/vault-foo/abc/original"])
+
+    def test_get_copy_folder_to_vault_irsync_command_without_vault_resc(self):
+        output = get_copy_folder_to_vault_irsync_command("/zoneName/home/research-foo/abc", "/zoneName/home/vault-foo/abc", None, True)
+        self.assertEqual(output, ["irsync", "-rK", "i:/zoneName/home/research-foo/abc/", "i:/zoneName/home/vault-foo/abc/original"])
+
+    def test_get_copy_folder_to_vault_irsync_command_no_multithreading(self):
+        output = get_copy_folder_to_vault_irsync_command("/zoneName/home/research-foo/abc", "/zoneName/home/vault-foo/abc", "vaultResc", False)
+        self.assertEqual(output, ["irsync", "-rK", "-R", "vaultResc", "-N", "0", "i:/zoneName/home/research-foo/abc/", "i:/zoneName/home/vault-foo/abc/original"])

unit-tests/unit_tests.py

@@ -13,6 +13,7 @@
 from test_util_misc import UtilMiscTest
 from test_util_pathutil import UtilPathutilTest
 from test_util_yoda_names import UtilYodaNamesTest
+from test_vault import VaultTest
 
 
 def suite():
@@ -27,4 +28,5 @@ def suite():
     test_suite.addTest(makeSuite(UtilMiscTest))
     test_suite.addTest(makeSuite(UtilPathutilTest))
     test_suite.addTest(makeSuite(UtilYodaNamesTest))
+    test_suite.addTest(makeSuite(VaultTest))
     return test_suite

vault.py

@@ -22,6 +22,7 @@
 import policies_datamanager
 import policies_datapackage_status
 from util import *
+from vault_utils import get_copy_folder_to_vault_irsync_command
 
 __all__ = ['api_vault_submit',
            'api_vault_approve',
@@ -954,8 +955,13 @@ def copy_folder_to_vault(ctx, coll, target):
     :returns: True for successful copy
     """
     returncode = 0
+    irsync_command = get_copy_folder_to_vault_irsync_command(coll,
+                                                             target,
+                                                             config.resource_vault,
+                                                             config.vault_copy_multithread_enabled)
+
     try:
-        returncode = subprocess.call(["irsync", "-rK", "i:{}/".format(coll), "i:{}/original".format(target)])
+        returncode = subprocess.call(irsync_command)
     except Exception as e:
         log.write(ctx, "irsync failure: " + e)
         log.write(ctx, "irsync failure for coll <{}> and target <{}>".format(coll, target))

vault_utils.py

@@ -0,0 +1,28 @@
+"""Utility functions for vault module."""
+
+__copyright__ = 'Copyright (c) 2019-2024, Utrecht University'
+__license__   = 'GPLv3, see LICENSE'
+
+
+def get_copy_folder_to_vault_irsync_command(coll, target, vault_resource, multi_threading):
+    """Internal function to determine rsync command for copy-to-vault
+
+       :param coll: source collection
+       :param target: target collection
+       :param vault_resource: resource to store vault data on (can be None)
+       :param multi_threading: if set to false, disable multi threading,
+                               otherwise use server default
+
+       :returns: irsync command with parameters in list format
+    """
+
+    irsync_command = ["irsync", "-rK"]
+
+    if vault_resource is not None:
+        irsync_command.extend(["-R", vault_resource])
+
+    if not multi_threading:
+        irsync_command.extend(["-N", "0"])  # 0 means no multi threading
+
+    irsync_command.extend(["i:{}/".format(coll), "i:{}/original".format(target)])
+    return irsync_command