Merge pull request #42 from tmaeno/master

* allowing ramCount in newOpts of pbook.retry
PanDAWMS · Dec 11, 2024 · afbc596 · afbc596
2 parents c97a41f + 9979f22
commit afbc596
Show file tree

Hide file tree

Showing 5 changed files with 76 additions and 39 deletions.
diff --git a/ChangeLog.txt b/ChangeLog.txt
@@ -1,5 +1,10 @@
 ** Release Notes
 
+1.5.81
+  * Fix --appendStrToExtStream, and add other options for --trf stream names https://github.com/PanDAWMS/panda-client/pull/41
+  * allowing ramCount in newOpts of pbook.retry
+  * added --tarBallViaDDM to prun/pathena
+
 1.5.80
   * added framework option to prun and pathena
 

diff --git a/pandaclient/PBookScript.py b/pandaclient/PBookScript.py
@@ -239,10 +239,10 @@ def retry(taskIDs, newOpts=None, days=14, limit=1000, **kwargs):
         It is possible to specify newOpts, which is a map of options and new arguments like
         {'nFilesPerJob':10,'excludedSite':'ABC,XYZ'}, to overwrite task parameters for new attempts. The list of
         changeable parameters is site, excludedSite, includedSite, nFilesPerJob, nGBPerJob, nFiles, nEvents,
-        loopingCheck, nMaxFilesPerJob, ramCount, avoidVP. It is also possible to specify those parameters as named
-        arguments of the retry function, e.g. nFilesPerJob=10, excludedSite='ABC,XYZ'.
+        loopingCheck, nMaxFilesPerJob, memory, ramCount, avoidVP, ignoreMissingInDS. It is also possible to specify those
+        parameters as named arguments of the retry function, e.g. nFilesPerJob=10, excludedSite='ABC,XYZ'.
         If input files were used or are being used by other jobs for the same
-        output dataset container, those file are skipped to avoid job duplication when retrying failed subjobs.
+        output dataset container, those files are skipped to avoid job duplication when retrying failed subjobs.
 
         If taskIDs is 'all', it retries 1000 tasks at most that have finished for the last 14 days. It is possible
         to retry more tasks by setting the days and limit options. If newOpts and/or named arguments are specified,
@@ -266,6 +266,12 @@ def retry(taskIDs, newOpts=None, days=14, limit=1000, **kwargs):
                 from pandaclient import PrunScript
 
                 client_options = PrunScript.main(get_options=True)
+                client_options.update(
+                    {
+                        "ramCount": None,
+                        "ignoreMissingInDS": None,
+                    }
+                )
             # check options
             for key in list(newOpts):
                 if key == "memory":

diff --git a/pandaclient/PandaToolsPkgInfo.py b/pandaclient/PandaToolsPkgInfo.py
@@ -1 +1 @@
-release_version = "1.5.80"
+release_version = "1.5.81"
diff --git a/pandaclient/PathenaScript.py b/pandaclient/PathenaScript.py
@@ -533,9 +533,9 @@
     help="use the first part of filenames as output stream names when using --trf. E.g., if this option is used %%OUT.AOD.pool.root will be contained in an AOD dataset instead of an EXT0 dataset",
 )
 group_output.add_argument(
-    '--outputStreamNames',
-    nargs='+',
-    help='use custom output stream names when using --trf, instead of the default EXT# suffixes; must specify one stream name for each job output (%%OUT instance, or --extOutFile entries)',
+    "--outputStreamNames",
+    nargs="+",
+    help="use custom output stream names when using --trf, instead of the default EXT# suffixes; must specify one stream name for each job output (%%OUT instance, or --extOutFile entries)",
 )
 group_output.add_argument(
     "--mergeOutput",
@@ -1143,6 +1143,13 @@
     type=str,
     help="Use a gzipped tarball of local files as input to buildXYZ. Generall the tarball is created by using --outTarBall",
 )
+group_build.add_argument(
+    "--tarBallViaDDM",
+    action="store",
+    dest="tarBallViaDDM",
+    default="",
+    help="Use a gzipped tarball pre-uploaded to DDM as input for buildXYZ. --tarBallViaDDM=datasetName:fileName",
+)
 group_config.add_argument(
     "--outRunConfig",
     action="store",
@@ -2216,7 +2223,7 @@ def _onExit(dir, files, del_command):
 # archive sources and send it to HTTP-reachable location
 
 if True:
-    if options.inTarBall == "":
+    if options.inTarBall == "" and options.tarBallViaDDM == "":
         # extract jobOs with full pathnames
         for tmpItem in jobO.split():
             if re.search("^/.*\.py$", tmpItem) is not None:
@@ -2311,6 +2318,11 @@ def _onExit(dir, files, del_command):
             tmpLog.warning(tmpStr)
             for symlink in symlinks:
                 print("  %s" % symlink)
+    elif options.tarBallViaDDM:
+        # go to tmp dir
+        os.chdir(tmpDir)
+        # use sandbox pre-uploaded to DDM
+        archiveName = options.tarBallViaDDM.split(":")[-1]
     else:
         # go to tmp dir
         os.chdir(tmpDir)
@@ -2329,7 +2341,7 @@ def _onExit(dir, files, del_command):
         shutil.copy(archiveName, options.outTarBall)
 
     # put sources/jobO via HTTP POST
-    if not options.noSubmit:
+    if not options.noSubmit and not options.tarBallViaDDM:
         tmpLog.info("uploading sandbox")
         status, out = Client.putFile(archiveName, options.verbose, useCacheSrv=True, reuseSandbox=True)
         if out.startswith("NewFileName:"):
@@ -2527,19 +2539,15 @@ def _onExit(dir, files, del_command):
 
 # build
 if options.noBuild and not options.noCompile:
-    taskParamMap["jobParameters"] += [
-        {
-            "type": "constant",
-            "value": "-a {0}".format(archiveName),
-        },
-    ]
+    tmp_str = "-a {0}".format(archiveName)
+    if options.tarBallViaDDM:
+        tmp_str += " --noTarballDownload"
 else:
-    taskParamMap["jobParameters"] += [
-        {
-            "type": "constant",
-            "value": "-l ${LIB}",
-        },
-    ]
+    tmp_str = "-l ${LIB}"
+taskParamMap["jobParameters"] += [
+    {"type": "constant", "value": tmp_str},
+]
+
 # pre execution string
 pStr1 = ""
 if runConfig.other.rndmStream != []:
@@ -3090,7 +3098,8 @@ def _onExit(dir, files, del_command):
 
 # build step
 if options.noBuild and not options.noCompile:
-    pass
+    if options.tarBallViaDDM:
+        taskParamMap["tarBallViaDDM"] = options.tarBallViaDDM
 else:
     jobParameters = "-i ${IN} -o ${OUT} --sourceURL ${SURL} "
     # no compile
@@ -3108,6 +3117,9 @@ def _onExit(dir, files, del_command):
         "archiveName": archiveName,
         "jobParameters": jobParameters,
     }
+    if options.tarBallViaDDM:
+        taskParamMap["buildSpec"]["tarBallViaDDM"] = options.tarBallViaDDM
+        taskParamMap["buildSpec"]["jobParameters"] += "--noTarballDownload "
     if options.prodSourceLabel != "":
         taskParamMap["buildSpec"]["prodSourceLabel"] = options.prodSourceLabel
 

diff --git a/pandaclient/PrunScript.py b/pandaclient/PrunScript.py
@@ -838,14 +838,21 @@ def main(get_taskparams=False, ext_args=None, dry_mode=False, get_options=False)
         action="store",
         dest="outTarBall",
         default="",
-        help="Save a gzipped tarball of local files which is the input to buildXYZ",
+        help="Save a gzipped tarball of local files which is the input for buildXYZ",
     )
     group_build.add_argument(
         "--inTarBall",
         action="store",
         dest="inTarBall",
         default="",
-        help="Use a gzipped tarball of local files as input to buildXYZ. Generall the tarball is created by using --outTarBall",
+        help="Use a gzipped tarball of local files as input for buildXYZ. Generall the tarball is created by using --outTarBall",
+    )
+    group_build.add_argument(
+        "--tarBallViaDDM",
+        action="store",
+        dest="tarBallViaDDM",
+        default="",
+        help="Use a gzipped tarball pre-uploaded to DDM as input for buildXYZ. --tarBallViaDDM=datasetName:fileName",
     )
     group_build.add_argument(
         "--bexec",
@@ -1777,7 +1784,7 @@ def _onExit(dir, files, del_command):
     # create archive
     archiveName = None
     if (options.containerImage == "" or options.useSandbox) and not dry_mode:
-        if options.inTarBall == "":
+        if options.inTarBall == "" and options.tarBallViaDDM == "":
             # copy RootCore packages
             if options.useRootCore:
                 # check $ROOTCOREDIR
@@ -2044,6 +2051,11 @@ def _onExit(dir, files, del_command):
                     tmpLog.warning(tmpStr)
                     for symlink in symlinks:
                         print("  %s" % symlink)
+        elif options.tarBallViaDDM:
+            # go to tmp dir
+            os.chdir(tmpDir)
+            # use sandbox pre-uploaded to DDM
+            archiveName = options.tarBallViaDDM.split(":")[-1]
         else:
             # go to tmp dir
             os.chdir(tmpDir)
@@ -2062,7 +2074,7 @@ def _onExit(dir, files, del_command):
             shutil.copy(archiveName, options.outTarBall)
 
         # upload source files
-        if not options.noSubmit:
+        if not options.noSubmit and not options.tarBallViaDDM:
             # upload sources via HTTP POST
             tmpLog.info("upload sandbox")
             if options.vo is None:
@@ -2319,19 +2331,17 @@ def _onExit(dir, files, del_command):
     # build
     if options.containerImage == "" or options.useSandbox:
         if options.noBuild and not options.noCompile:
-            taskParamMap["jobParameters"] += [
-                {
-                    "type": "constant",
-                    "value": "-a {0}".format(archiveName),
-                },
-            ]
+            tmp_str = "-a {0}".format(archiveName)
+            if options.tarBallViaDDM:
+                tmp_str += " --noTarballDownload"
         else:
-            taskParamMap["jobParameters"] += [
-                {
-                    "type": "constant",
-                    "value": "-l ${LIB}",
-                },
-            ]
+            tmp_str = "-l ${LIB}"
+        taskParamMap["jobParameters"] += [
+            {
+                "type": "constant",
+                "value": tmp_str,
+            },
+        ]
     # output
     if options.outputs != "":
         outMap = {}
@@ -2631,7 +2641,8 @@ def _onExit(dir, files, del_command):
 
     # build step
     if options.noBuild and not options.noCompile:
-        pass
+        if options.tarBallViaDDM:
+            taskParamMap["tarBallViaDDM"] = options.tarBallViaDDM
     else:
         jobParameters = "-i ${IN} -o ${OUT} --sourceURL ${SURL} "
         jobParameters += "-r {0} ".format(runDir)
@@ -2679,6 +2690,9 @@ def _onExit(dir, files, del_command):
             "archiveName": archiveName,
             "jobParameters": jobParameters,
         }
+        if options.tarBallViaDDM:
+            taskParamMap["buildSpec"]["tarBallViaDDM"] = options.tarBallViaDDM
+            taskParamMap["buildSpec"]["jobParameters"] += "--noTarballDownload "
         if options.prodSourceLabel != "":
             taskParamMap["buildSpec"]["prodSourceLabel"] = options.prodSourceLabel
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		release_version = "1.5.80"
		release_version = "1.5.81"