crab recover - new command (#5250)

* crab recover - first implementation using crab report

This commit provide a new crab command: crab recover.
run with:

crab recover --task=taskname

It aims at recovering tasks such that

- jobtype: analysis
- splitting: lumibased, eventawarelumibased, automatic
- publication: both on or off

What does not cover

- lumibased splitting

* crab recover - defaults to --strategy=notPublished

* crab recover - suppress debug messages

* crab recover - wa's review

* crab recover - silence log in all subcommand, apart from report

* crab recover - submit.py need to be aware of copycat and recover jobtypes

* crab recover - crab report prints proper message if the task does not need to be recovered

* crab recover - replace assert with exceptions

* crab recover - final touches to log messages

* crab recover - jobtype Recover manipulates only a single task configuration

* recover - if task can not be recovered, return exitcode 0 and commandStatus FAILED

* crab recover - adapt to both --dir and --task

src/python/CRABClient/ClientMapping.py

@@ -79,6 +79,7 @@
                                      {'default': False,      'config': ['JobType.disableAutomaticOutputCollection'],'type': 'BooleanType', 'required': False},
                                      {'default': None,       'config': ['JobType.copyCatTaskname'],         'type': 'StringType',  'required': False},
                                      {'default': 'prod',     'config': ['JobType.copyCatInstance'],         'type': 'StringType',  'required': False},
+                                     {'default': None,       'config': ['JobType.copyCatWorkdir'],          'type': 'StringType',  'required': False},
                                      {'default': [],         'config': ['JobType.inputFiles'],              'type': 'ListType',    'required': False}
                            ]
 }
@@ -152,6 +153,8 @@
     'tasks'         : {'acceptsArguments': False, 'requiresREST': True,  'requiresRucio': False, 'requiresDirOption': False, 'useCache': False, 'requiresProxyVOOptions': False, 'requiresLocalCache': False},
     'uploadlog'     : {'acceptsArguments': False, 'requiresREST': True,  'requiresRucio': False, 'requiresDirOption': True,  'useCache': True,  'requiresProxyVOOptions': False, 'requiresLocalCache': False},
     'preparelocal'  : {'acceptsArguments': False, 'requiresREST': True,  'requiresRucio': False, 'requiresDirOption': True,  'useCache': True,  'requiresProxyVOOptions': False, 'requiresLocalCache': True},
+    'recover'       : {'acceptsArguments': False, 'requiresREST': True,  'requiresRucio': False, 'requiresDirOption': True, 'useCache': False, 'requiresProxyVOOptions': False, 'requiresLocalCache': False},
+    'getsandbox'    : {'acceptsArguments': False, 'requiresREST': True,  'requiresRucio': False,  'requiresDirOption': True,  'useCache': True,  'requiresProxyVOOptions': False, 'requiresLocalCache': True },
 }
 
 

src/python/CRABClient/ClientUtilities.py

@@ -356,7 +356,7 @@ def getRequestName(requestName=None):
     prefix = 'crab_'
     postfix = str(datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
 
-    if requestName is None or not isinstance(requestName, str) or len(requestName) == 0:
+    if requestName is None or requestName == "None" or not isinstance(requestName, str) or len(requestName) == 0:
         return prefix + postfix
     elif '/' in requestName:
         msg = "%sError%s: The '/' character is not accepted in the requestName parameter." % (colors.RED, colors.NORMAL)

src/python/CRABClient/Commands/SubCommand.py

@@ -270,6 +270,7 @@ def __init__(self, logger, cmdargs=None, disable_interspersed_args=False):
 
         # Parse the command options/arguments.
         cmdargs = cmdargs or []
+        self.cmdargs = cmdargs
         (self.options, self.args) = self.parser.parse_args(cmdargs)
 
         self.transferringIds = None

src/python/CRABClient/Commands/getsandbox.py

@@ -0,0 +1,95 @@
+import os
+
+from CRABClient.Commands.SubCommand import SubCommand
+
+from CRABClient.UserUtilities import curlGetFileFromURL, getColumn
+
+from ServerUtilities import downloadFromS3, getProxiedWebDir
+
+
+class getsandbox(SubCommand):
+    """
+    given a projdir, downloads locally the user sandbox.
+    It will try s3 first, otherwise it will fall back to the schedd WEBDIR.
+    """
+
+    name = "getsandbox"
+
+    def __call__(self):
+
+        # init. debug. print useful info
+        self.logger.debug("requestarea: %s", self.requestarea)
+        self.logger.debug("cachedinfo: %s", self.cachedinfo)
+
+        # get information necessary for next steps
+        # Get all of the columns from the database for a certain task
+        self.taskname = self.cachedinfo['RequestName']
+        self.crabDBInfo, _, _ = self.crabserver.get(api='task', data={'subresource':'search', 'workflow':self.taskname})
+        self.logger.debug("Got information from server oracle database: %s", self.crabDBInfo)
+
+        # arguments used by following functions
+        self.downloadDir = os.path.join(self.requestarea, "taskconfig")
+
+        # download files: user sandbox, debug sandbox
+        filelist = []
+        # usersandbox = self.downloadUserSandbox()
+        usersandbox = self.downloadSandbox(
+            remotefile=getColumn(self.crabDBInfo, 'tm_user_sandbox'),
+            localfile='sandbox.tar.gz')
+        filelist.append(usersandbox)
+        # debugfiles = self.downloadDebug()
+        debugfiles = self.downloadSandbox(
+            remotefile=getColumn(self.crabDBInfo, 'tm_debug_files'),
+            localfile='debug_files.tar.gz')
+        filelist.append(debugfiles)
+
+        returnDict = {"commandStatus": "FAILED"}
+        if filelist:
+            returnDict = {"commandStatus": "SUCCESS", "sandbox_paths": filelist }
+
+        return returnDict
+
+    def downloadSandbox(self, remotefile, localfile):
+        """
+        Copy remotefile from s3 to localfile on local disk.
+
+        If remotefile is not s3, then as a fallback we look for the corresponding
+        localfile in the schedd webdir.
+        """
+        username = getColumn(self.crabDBInfo, 'tm_username')
+        sandboxFilename = remotefile
+
+        self.logger.debug("will download sandbox from s3: %s",sandboxFilename)
+
+        if not os.path.isdir(self.downloadDir):
+            os.mkdir(self.downloadDir)
+        localSandboxPath = os.path.join(self.downloadDir, localfile)
+
+        try:
+            downloadFromS3(crabserver=self.crabserver,
+                        filepath=localSandboxPath,
+                        objecttype='sandbox', logger=self.logger,
+                        tarballname=sandboxFilename,
+                        username=username
+                        )
+        except Exception as e:
+            self.logger.info("Sandbox download failed with %s", e)
+            self.logger.info("We will look for the sandbox on the webdir of the schedd")
+
+            webdir = getProxiedWebDir(crabserver=self.crabserver, task=self.taskname,
+                                    logFunction=self.logger.debug)
+            if not webdir:
+                webdir = getColumn(self.crabDBInfo, 'tm_user_webdir')
+            self.logger.debug("Downloading %s from %s", localfile, webdir)
+            httpCode = curlGetFileFromURL(webdir + '/' + localfile,
+                                        localSandboxPath, self.proxyfilename,
+                                        logger=self.logger)
+            if httpCode != 200:
+                self.logger.error("Failed to download %s from %s", localfile, webdir)
+                raise Exception("We could not locate the sandbox in the webdir neither.")
+                # we should use
+                # raise Exception("We could not locate the sandbox in the webdir neither.") from e
+                # but that is not py2 compatible...
+
+        return localSandboxPath
+