[WIP] Batch related functions separed to a module

bin/fccanalysis

@@ -5,18 +5,21 @@ Starting point (executable) for fccanalysis command
 
 import argparse
 import logging
+import sys
 
 from parsers import setup_subparsers
 from init_analysis import init_analysis
 from build_analysis import build_analysis
 from test_fccanalyses import test_fccanalyses
 from pin_analysis import PinAnalysis
+from submit import submit_analysis
 from run_analysis import run
 from run_final_analysis import run_final
 from do_plots import do_plots
 from do_combine import do_combine
 
 
+# _____________________________________________________________________________
 class MultiLineFormatter(logging.Formatter):
     '''
     Multi-line formatter.
@@ -42,6 +45,25 @@ class MultiLineFormatter(logging.Formatter):
         return head + ''.join(indent + line for line in trailing)
 
 
+# _____________________________________________________________________________
+class MaxLevelFilter(logging.Filter):
+    '''
+    Filters (lets through) all messages with level < LEVEL
+
+    Found here:
+      https://stackoverflow.com/questions/1383254/
+    '''
+    def __init__(self, level):
+        super().__init__()
+        self.level = level
+
+    def filter(self, record):
+        # "<" instead of "<=": since logger.setLevel is inclusive,
+        # this should be exclusive
+        return record.levelno < self.level
+
+
+# _____________________________________________________________________________
 def main():
     '''
     Starting point for fccanalysis command
@@ -78,10 +100,16 @@ def main():
     logger = logging.getLogger('FCCAnalyses')
     logger.setLevel(verbosity_level)
     formatter = MultiLineFormatter(fmt='----> %(levelname)s: %(message)s')
-    stream_handler = logging.StreamHandler()
-    stream_handler.setLevel(verbosity_level)
-    stream_handler.setFormatter(formatter)
-    logger.addHandler(stream_handler)
+    stdout_stream_handler = logging.StreamHandler(sys.stdout)
+    stderr_stream_handler = logging.StreamHandler(sys.stderr)
+    lower_than_warning_filter = MaxLevelFilter(logging.WARNING)
+    stdout_stream_handler.addFilter(lower_than_warning_filter)
+    stdout_stream_handler.setLevel(verbosity_level)
+    stderr_stream_handler.setLevel(logging.WARNING)
+    stdout_stream_handler.setFormatter(formatter)
+    stderr_stream_handler.setFormatter(formatter)
+    logger.addHandler(stdout_stream_handler)
+    logger.addHandler(stderr_stream_handler)
 
     if args.command == 'init':
         init_analysis(parser)
@@ -91,6 +119,8 @@ def main():
         test_fccanalyses(parser)
     elif args.command == 'pin':
         PinAnalysis(parser)
+    elif args.command == 'submit':
+        submit_analysis(parser)
     elif args.command == 'final':
         run_final(parser)
     elif args.command == 'plots':
@@ -101,5 +131,6 @@ def main():
         run(parser)
 
 
+# _____________________________________________________________________________
 if __name__ == "__main__":
     main()

examples/FCCee/higgs/mH-recoil/ee/analysis_stage1_batch.py

@@ -1,4 +1,4 @@
-electron#Mandatory: List of processes
+# Mandatory: List of processes
 processList = {
     'p8_ee_ZZ_ecm240':{'chunks':20},#Run the full statistics in 10 jobs in output dir <outputDir>/p8_ee_ZZ_ecm240/chunk<N>.root
     'p8_ee_WW_ecm240':{'chunks':20},#Run the full statistics in 10 jobs in output dir <outputDir>/p8_ee_WW_ecm240/chunk<N>.root

examples/FCCee/higgs/mH-recoil/mumu/analysis_stage1_batch.py

@@ -12,15 +12,15 @@ class Analysis():
     Higgs mass recoil analysis in Z(mumu)H.
     '''
     def __init__(self, cmdline_args):
+        # Parse additional arguments not known to the FCCAnalyses parsers.
+        # All command line arguments are provided in the `cmdline_arg`
+        # dictionary and arguments after "--" are stored under "remaining" key.
         parser = ArgumentParser(
             description='Additional analysis arguments',
-            usage='Provide additional arguments after analysis script path')
+            usage='Provided after "--"')
         parser.add_argument('--muon-pt', default='10.', type=float,
                             help='Minimal pT of the mouns.')
-        # Parse additional arguments not known to the FCCAnalyses parsers
-        # All command line arguments know to fccanalysis are provided in the
-        # `cmdline_arg` dictionary.
-        self.ana_args, _ = parser.parse_known_args(cmdline_args['unknown'])
+        self.ana_args, _ = parser.parse_known_args(cmdline_args['remaining'])
 
         # Mandatory: List of processes to run over
         self.process_list = {
@@ -42,27 +42,25 @@ def __init__(self, cmdline_args):
         # self.analysis_name = 'My Analysis'
 
         # Optional: number of threads to run on, default is 'all available'
-        # self.n_threads = 4
-
-        # Optional: running on HTCondor, default is False
-        self.run_batch = True
+        self.n_threads = 4
 
         # Optional: batch queue name when running on HTCondor, default is
-        # 'workday'
-        self.batch_queue = 'workday'
+        # 'longlunch'
+        # self.batch_queue = 'workday'
 
         # Optional: computing account when running on CERN's HTCondor, default
         # is 'group_u_FCC.local_gen'
         self.comp_group = 'group_u_FCC.local_gen'
 
         # Optional: output directory on eos, if specified files will be copied
         # there once the batch job is done, default is empty
-        self.output_dir_eos = '/eos/experiment/fcc/ee/analyses/case-studies/' \
-                              f'higgs/mH-recoil/stage1_{self.ana_args.muon_pt}'
+        # self.output_dir_eos = '/eos/experiment/fcc/ee/analyses/case-studies/' \
+        #                       f'higgs/mH-recoil/stage1_{self.ana_args.muon_pt}'
+        self.output_dir_eos = '/eos/user/j/jsmiesko/FCCAnalyses/output'
 
         # Optional: type for eos, needed when <outputDirEos> is specified. The
         # default is FCC EOS, which is eospublic
-        self.eos_type = 'eospublic'
+        # self.eos_type = 'eospublic'
 
         # Optional: test file
         self.test_file = 'root://eospublic.cern.ch//eos/experiment/fcc/ee/' \

man/man7/fccanalysis-script.7

@@ -125,12 +125,7 @@ Default value: empty string
 \fBnCPUS\fR (optional)
 Number of threads the RDataFrame will use\&.
 .br
-Default value: 4
-.TP
-\fBrunBatch\fR (optional)
-Run the analysis on the HTCondor batch system.
-.br
-Default value: False
+Default value: 1
 .TP
 \fBbatchQueue\fR (optional)
 Batch queue name when running on HTCondor.