Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2025-02-13 02:58:03

0001 #!/usr/bin/env python3
0002 import sys, os
0003 
0004 from Configuration.PyReleaseValidation.MatrixReader import MatrixReader
0005 from Configuration.PyReleaseValidation.MatrixRunner import MatrixRunner
0006 from Configuration.PyReleaseValidation.MatrixInjector import MatrixInjector,performInjectionOptionTest
0007 
0008 # ================================================================================
0009 
0010 def showRaw(opt):
0011 
0012     mrd = MatrixReader(opt)
0013     mrd.showRaw(opt.useInput, opt.refRel, opt.fromScratch, opt.raw, opt.step1Only, selected=opt.testList)
0014 
0015     return 0
0016 
0017 # ================================================================================
0018 
0019 def runSelected(opt):
0020 
0021     mrd = MatrixReader(opt)
0022     mrd.prepare(opt.useInput, opt.refRel, opt.fromScratch)
0023 
0024     # test for wrong input workflows
0025     if opt.testList:
0026         definedWf = [dwf.numId for dwf in mrd.workFlows]
0027         definedSet = set(definedWf)
0028         testSet = set(opt.testList)
0029         undefSet = testSet - definedSet
0030         if len(undefSet)>0: raise ValueError('Undefined workflows: '+', '.join(map(str,list(undefSet))))
0031         duplicates = [wf for wf in testSet if definedWf.count(wf)>1 ]
0032         if len(duplicates)>0: raise ValueError('Duplicated workflows: '+', '.join(map(str,list(duplicates))))
0033 
0034     ret = 0
0035     if opt.show:
0036         mrd.show(opt.testList, opt.extended, opt.cafVeto)
0037         if opt.testList : print('selected items:', opt.testList)
0038     else:
0039         mRunnerHi = MatrixRunner(mrd.workFlows, opt.nProcs, opt.nThreads)
0040         ret = mRunnerHi.runTests(opt)
0041 
0042     if opt.wmcontrol:
0043         if ret!=0:
0044             print('Cannot go on with wmagent injection with failing workflows')
0045         else:
0046             wfInjector = MatrixInjector(opt,mode=opt.wmcontrol,options=opt.wmoptions)
0047             ret= wfInjector.prepare(mrd,
0048                                     mRunnerHi.runDirs)
0049             if ret==0:
0050                 wfInjector.upload()
0051                 wfInjector.submit()
0052     return ret
0053 
0054 # ================================================================================
0055 
0056 if __name__ == '__main__':
0057 
0058     #this can get out of here
0059     predefinedSet={
0060         # See README for further details
0061         'run1_run2' : [
0062             ###### MC (generated from scratch or from RelVal)
0063             # Run1
0064             5.1,        # TTbar_8TeV_TuneCUETP8M1       FastSim
0065             8,          # RelValBeamHalo                Cosmics
0066             9.0,        # RelValHiggs200ChargedTaus
0067             25,         # RelValTTbar
0068             101.0,      # SingleElectronE120EHCAL       + ECALHCAL.customise + fullMixCustomize_cff.setCrossingFrameOn
0069 
0070             # Run2
0071             7.3,        # UndergroundCosmicSPLooseMu
0072             1306.0,     # RelValSingleMuPt1_UP15
0073             1330,       # RelValZMM_13
0074             135.4,      # ZEE_13TeV_TuneCUETP8M1
0075             25202.0,    # RelValTTbar_13                PU = AVE_35_BX_25ns
0076             250202.181, # RelValTTbar_13                PREMIX
0077 
0078             ###### pp Data
0079             ## Run1
0080             4.22,       # Run2011A  Cosmics
0081             4.53,       # Run2012B  Photon                      miniAODs
0082             1000,       # Run2011A  MinimumBias Prompt          RecoTLR.customisePrompt
0083             1001,       # Run2011A  MinimumBias                 Data+Express
0084             ## Run2
0085             136.731,    # Run2016B SinglePhoton
0086             136.793,    # Run2017C DoubleEG
0087             136.874,    # Run2018C EGamma
0088         ],
0089 
0090         'run3' : [
0091             ###### MC (generated from scratch or from RelVals)
0092             # Run3
0093             11634.0,    # TTbar_14TeV                   2021
0094             13234.0,    # RelValTTbar_14TeV             2021 FastsSim
0095             12434.0,    # RelValTTbar_14TeV             2023
0096             12834.0,    # RelValTTbar_14TeV             2024
0097             12846.0,    # RelValZEE_13                  2024
0098             13034.0,    # RelValTTbar_14TeV             2024 PU = Run3_Flat55To75_PoissonOOTPU
0099             16834.0,    # RelValTTbar_14TeV             2025
0100             17034.0,    # RelValTTbar_14TeV     2025 PU = Run3_Flat55To75_PoissonOOTPU
0101             14034.0,    # RelValTTbar_14TeV             Run3_2023_FastSim
0102             14234.0,    # RelValTTbar_14TeV             Run3_2023_FastSim   PU = Run3_Flat55To75_PoissonOOTPU
0103             2500.201,   # RelValTTbar_14TeV             NanoAOD from existing MINI
0104 
0105             ###### pp Data
0106             ## Run3
0107             # 2021
0108             139.001,    # Run2021  MinimumBias                  Commissioning2021
0109 
0110             # 2022
0111             140.045,    # Run2022C JetHT
0112 
0113             # 2023
0114             141.042,    # Run2023D ZeroBias
0115 
0116             # 2024
0117             145.014,      # Run2024B ZeroBias
0118             145.104,      # Run2024C JetMet0
0119             145.202,      # Run2024D EGamma0
0120             145.301,      # Run2024E DisplacedJet
0121             145.408,      # Run2024F ParkingDoubleMuonLowMass0
0122             145.500,      # Run2024G BTagMu
0123             145.604,      # Run2024H JetMET0
0124             145.713,      # Run2024I Tau
0125         ],
0126 
0127         'phase2' : [
0128             ###### MC (generated from scratch or from RelVals)
0129             # Phase2
0130             29634.0,    # RelValTTbar_14TeV                     phase2_realistic_T33        ExtendedRun4D110         (Phase-2 baseline)
0131             24834.911,  # Previous DD4hep baseline for monitoring the stability of DD4hep workflow
0132             29634.911,  # TTbar_14TeV_TuneCP5                   phase2_realistic_T33        DD4hepExtendedRun4D110   DD4Hep (HLLHC14TeV BeamSpot)
0133             29834.999,  # RelValTTbar_14TeV (PREMIX)            phase2_realistic_T33        ExtendedRun4D110         AVE_50_BX_25ns_m3p3
0134             29696.0,    # RelValCloseByPGun_CE_E_Front_120um    phase2_realistic_T33        ExtendedRun4D110
0135             29700.0,    # RelValCloseByPGun_CE_H_Coarse_Scint   phase2_realistic_T33        ExtendedRun4D110
0136             #23234.0,   # Need new workflow with HFNose
0137             29634.75,   # RelValTTbar_14TeV                     phase2_realistic_T33        ExtendedRun4D110         (Phase-2 baseline -  but using timing menu, and only up to step 2)
0138         ],
0139 
0140         'heavyIons' : [
0141             ###### Heavy Ions
0142             ## Data
0143             # Run2
0144             140.56,    # HIRun2018A HIHardProbes                    Run2_2018_pp_on_AA
0145             ## MC
0146             312.0,     # Pyquen_ZeemumuJets_pt10_2760GeV            PU : HiMixGEN
0147         ],
0148 
0149         'jetmc': [5.1, 13, 15, 25, 38, 39], #MC
0150         'metmc' : [5.1, 15, 25, 37, 38, 39], #MC
0151         'muonmc' : [5.1, 124.4, 124.5, 20, 21, 22, 23, 25, 30], #MC
0152     }
0153 
0154     predefinedSet['limited'] = (
0155         predefinedSet['run1_run2'] +
0156         predefinedSet['run3'] +
0157         predefinedSet['phase2'] +
0158         predefinedSet['heavyIons']
0159     )
0160 
0161     import argparse
0162     usage = 'usage: runTheMatrix.py --show -s '
0163 
0164     parser = argparse.ArgumentParser(usage,formatter_class=argparse.ArgumentDefaultsHelpFormatter)
0165 
0166     parser.add_argument('-b','--batchName',
0167                         help='relval batch: suffix to be appended to Campaign name',
0168                         dest='batchName',
0169                         default='')
0170 
0171     parser.add_argument('-m','--memoryOffset',
0172                         help='memory of the wf for single core',
0173                         dest='memoryOffset',
0174                         type=int,
0175                         default=3000)
0176 
0177     parser.add_argument('--addMemPerCore',
0178                         help='increase of memory per each n > 1 core:  memory(n_core) = memoryOffset + (n_core-1) * memPerCore',
0179                         dest='memPerCore',
0180                         type=int,
0181                         default=1500)
0182 
0183     parser.add_argument('-j','--nproc',
0184                         help='number of processes. 0 Will use 4 processes, not execute anything but create the wfs',
0185                         dest='nProcs',
0186                         type=int,
0187                         default=4)
0188 
0189     parser.add_argument('-t','--nThreads',
0190                         help='number of threads per process to use in cmsRun.',
0191                         dest='nThreads',
0192                         type=int,
0193                         default=1)
0194 
0195     parser.add_argument('--nStreams',
0196                         help='number of streams to use in cmsRun.',
0197                         dest='nStreams',
0198                         type=int,
0199                         default=0)
0200 
0201     parser.add_argument('--nEvents',
0202                         help='number of events to process in cmsRun. If 0 will use the standard 10 events.',
0203                         dest='nEvents',
0204                         type=int,
0205                         default=0)
0206 
0207     parser.add_argument('--numberEventsInLuminosityBlock',
0208                         help='number of events in a luminosity block',
0209                         dest='numberEventsInLuminosityBlock',
0210                         type=int,
0211                         default=-1)
0212 
0213     parser.add_argument('-n','--showMatrix',
0214                         help='Only show the worflows. Use --ext to show more',
0215                         dest='show',
0216                         default=False,
0217                         action='store_true')
0218 
0219     parser.add_argument('-e','--extended',
0220                         help='Show details of workflows, used with --show',
0221                         dest='extended',
0222                         default=False,
0223                         action='store_true')
0224 
0225     parser.add_argument('-s','--selected',
0226                         help='Run a pre-defined selected matrix of wf. Deprecated, please use -l limited',
0227                         dest='restricted',
0228                         default=False,
0229                         action='store_true')
0230 
0231     parser.add_argument('-l','--list',
0232                         help='Comma separated list of workflow to be shown or ran. Possible keys are also '+str(predefinedSet.keys())+'. and wild card like muon, or mc',
0233                         dest='testList',
0234                         default=None)
0235 
0236     parser.add_argument('-f','--failed-from',
0237                         help='Provide a matrix report to specify the workflows to be run again. Augments the -l option if specified already',
0238                         dest='failed_from',
0239                         default=None)
0240 
0241     parser.add_argument('-r','--raw',
0242                         help='Temporary dump the .txt needed for prodAgent interface. To be discontinued soon. Argument must be the name of the set (standard, pileup,...)',
0243                         dest='raw')
0244 
0245     parser.add_argument('-i','--useInput',
0246                         help='Use recyling where available. Either all, or a comma separated list of wf number.',
0247                         dest='useInput',
0248                         type=lambda x: x.split(','),
0249                         default=None)
0250 
0251     parser.add_argument('-w','--what',
0252                         help='Specify the set to be used. Argument must be the name of a set (standard, pileup,...) or multiple sets separated by commas (--what standard,pileup )',
0253                         dest='what',
0254                         default='all')
0255 
0256     parser.add_argument('--step1',
0257                         help='Used with --raw. Limit the production to step1',
0258                         dest='step1Only',
0259                         default=False)
0260 
0261     parser.add_argument('--maxSteps',
0262                         help='Only run maximum on maxSteps. Used when we are only interested in first n steps.',
0263                         dest='maxSteps',
0264                         default=9999,
0265                         type=int)
0266 
0267     parser.add_argument('--fromScratch',
0268                         help='Comma separated list of wf to be run without recycling. all is not supported as default.',
0269                         dest='fromScratch',
0270                         type=lambda x: x.split(','),
0271                         default=None)
0272 
0273     parser.add_argument('--refRelease',
0274                         help='Allow to modify the recycling dataset version',
0275                         dest='refRel',
0276                         default=None)
0277 
0278     parser.add_argument('--wmcontrol',
0279                         help='Create the workflows for injection to WMAgent. In the WORKING. -wmcontrol init will create the the workflows, -wmcontrol test will dryRun a test, -wmcontrol submit will submit to wmagent',
0280                         choices=['init','test','submit','force'],
0281                         dest='wmcontrol',
0282                         default=None)
0283 
0284     parser.add_argument('--revertDqmio',
0285                         help='When submitting workflows to wmcontrol, force DQM outout to use pool and not DQMIO',
0286                         choices=['yes','no'],
0287                         dest='revertDqmio',
0288                         default='no')
0289 
0290     parser.add_argument('--optionswm',
0291                         help='Specify a few things for wm injection',
0292                         default='',
0293                         dest='wmoptions')
0294 
0295     parser.add_argument('--keep',
0296                         help='allow to specify for which comma separated steps the output is needed',
0297                         default=None)
0298 
0299     parser.add_argument('--label',
0300                         help='allow to give a special label to the output dataset name',
0301                         default='')
0302 
0303     parser.add_argument('--command',
0304                         help='provide a way to add additional command to all of the cmsDriver commands in the matrix',
0305                         dest='command',
0306                         action='append',
0307                         default=None)
0308 
0309     parser.add_argument('--apply',
0310                         help='allow to use the --command only for 1 comma separeated',
0311                         dest='apply',
0312                         default=None)
0313 
0314     parser.add_argument('--workflow',
0315                         help='define a workflow to be created or altered from the matrix',
0316                         action='append',
0317                         dest='workflow',
0318                         default=None)
0319 
0320     parser.add_argument('--dryRun',
0321                         help='do not run the wf at all',
0322                         action='store_true',
0323                         dest='dryRun',
0324                         default=False)
0325 
0326     parser.add_argument('--testbed',
0327                         help='workflow injection to cmswebtest (you need dedicated rqmgr account)',
0328                         dest='testbed',
0329                         default=False,
0330                         action='store_true')
0331 
0332     parser.add_argument('--noCafVeto',
0333                         help='Run from any source, ignoring the CAF label',
0334                         dest='cafVeto',
0335                         default=True,
0336                         action='store_false')
0337 
0338     parser.add_argument('--overWrite',
0339                         help='Change the content of a step for another. List of pairs.',
0340                         dest='overWrite',
0341                         default=None)
0342 
0343     parser.add_argument('--noRun',
0344                         help='Remove all run list selection from wfs',
0345                         dest='noRun',
0346                         default=False,
0347                         action='store_true')
0348 
0349     parser.add_argument('--das-options',
0350                         help='Options to be passed to dasgoclient.',
0351                         dest='dasOptions',
0352                         default="--limit 0",
0353                         action='store')
0354 
0355     parser.add_argument('--job-reports',
0356                         help='Dump framework job reports',
0357                         dest='jobReports',
0358                         default=False,
0359                         action='store_true')
0360 
0361     parser.add_argument('--ibeos',
0362                         help='Use IB EOS site configuration',
0363                         dest='IBEos',
0364                         default=False,
0365                         action='store_true')
0366 
0367     parser.add_argument('--sites',
0368                         help='Run DAS query to get data from a specific site. Set it to empty string to search all sites.',
0369                         dest='dasSites',
0370                         default='T2_CH_CERN',
0371                         action='store')
0372 
0373     parser.add_argument('--interactive',
0374                         help="Open the Matrix interactive shell",
0375                         action='store_true',
0376                         default=False)
0377 
0378     parser.add_argument('--dbs-url',
0379                         help='Overwrite DbsUrl value in JSON submitted to ReqMgr2',
0380                         dest='dbsUrl',
0381                         default=None,
0382                         action='store')
0383 
0384     gpugroup = parser.add_argument_group('GPU-related options','These options are only meaningful when --gpu is used, and is not set to forbidden.')
0385 
0386     gpugroup.add_argument('--gpu','--requires-gpu',
0387                           help='Enable GPU workflows. Possible options are "forbidden" (default), "required" (implied if no argument is given), or "optional".',
0388                           dest='gpu',
0389                           choices=['forbidden', 'optional', 'required'],
0390                           nargs='?',
0391                           const='required',
0392                           default='forbidden',
0393                           action='store')
0394 
0395     gpugroup.add_argument('--gpu-memory',
0396                           help='Specify the minimum amount of GPU memory required by the job, in MB.',
0397                           dest='GPUMemoryMB',
0398                           type=int,
0399                           default=8000)
0400 
0401     gpugroup.add_argument('--cuda-capabilities',
0402                           help='Specify a comma-separated list of CUDA "compute capabilities", or GPU hardware architectures, that the job can use.',
0403                           dest='CUDACapabilities',
0404                           type=lambda x: x.split(','),
0405                           default='6.0,6.1,6.2,7.0,7.2,7.5,8.0,8.6')
0406 
0407     # read the CUDA runtime version included in CMSSW
0408     cudart_version = None
0409     libcudart = os.path.realpath(os.path.expandvars('$CMSSW_RELEASE_BASE/external/$SCRAM_ARCH/lib/libcudart.so'))
0410     if os.path.isfile(libcudart):
0411         cudart_basename = os.path.basename(libcudart)
0412         cudart_version = '.'.join(cudart_basename.split('.')[2:4])
0413     gpugroup.add_argument('--cuda-runtime',
0414                           help='Specify major and minor version of the CUDA runtime used to build the application.',
0415                           dest='CUDARuntime',
0416                           default=cudart_version)
0417 
0418     gpugroup.add_argument('--force-gpu-name',
0419                           help='Request a specific GPU model, e.g. "Tesla T4" or "NVIDIA GeForce RTX 2080". The default behaviour is to accept any supported GPU.',
0420                           dest='GPUName',
0421                           default='')
0422 
0423     gpugroup.add_argument('--force-cuda-driver-version',
0424                           help='Request a specific CUDA driver version, e.g. 470.57.02. The default behaviour is to accept any supported CUDA driver version.',
0425                           dest='CUDADriverVersion',
0426                           default='')
0427 
0428     gpugroup.add_argument('--force-cuda-runtime-version',
0429                           help='Request a specific CUDA runtime version, e.g. 11.4. The default behaviour is to accept any supported CUDA runtime version.',
0430                           dest='CUDARuntimeVersion',
0431                           default='')
0432 
0433     opt = parser.parse_args()
0434     if opt.command: opt.command = ' '.join(opt.command)
0435     os.environ["CMSSW_DAS_QUERY_SITES"]=opt.dasSites
0436     if opt.failed_from:
0437         rerunthese=[]
0438         with open(opt.failed_from,'r') as report:
0439             for report_line in report:
0440                 if 'FAILED' in report_line:
0441                     to_run,_=report_line.split('_',1)
0442                     rerunthese.append(to_run)
0443         if opt.testList:
0444             opt.testList+=','.join(['']+rerunthese)
0445         else:
0446             opt.testList = ','.join(rerunthese)
0447 
0448     if opt.IBEos:
0449       os.environ["CMSSW_USE_IBEOS"]="true"
0450     if opt.restricted:
0451         print('Deprecated, please use -l limited')
0452         if opt.testList:            opt.testList+=',limited'
0453         else:            opt.testList='limited'
0454 
0455     def stepOrIndex(s):
0456         if s.isdigit():
0457             return int(s)
0458         else:
0459             return s
0460     if opt.apply:
0461         opt.apply=map(stepOrIndex,opt.apply.split(','))
0462     if opt.keep:
0463         opt.keep=map(stepOrIndex,opt.keep.split(','))
0464 
0465     if opt.testList:
0466         testList=[]
0467         for entry in opt.testList.split(','):
0468             if not entry: continue
0469             mapped=False
0470             for k in predefinedSet:
0471                 if k.lower().startswith(entry.lower()) or k.lower().endswith(entry.lower()):
0472                     testList.extend(predefinedSet[k])
0473                     mapped=True
0474                     break
0475             if not mapped:
0476                 try:
0477                     testList.append(float(entry))
0478                 except:
0479                     print(entry,'is not a possible selected entry')
0480 
0481         opt.testList = list(set(testList))
0482 
0483     if opt.wmcontrol:
0484         performInjectionOptionTest(opt)
0485     if opt.overWrite:
0486         opt.overWrite=eval(opt.overWrite)
0487     if opt.interactive:
0488         import cmd
0489         from colorama import Fore, Style
0490         from os import isatty
0491         import subprocess
0492         import time
0493 
0494         class TheMatrix(cmd.Cmd):
0495             intro = "Welcome to the Matrix (? for help)"
0496             prompt = "matrix> "
0497 
0498             def __init__(self, opt):
0499                 cmd.Cmd.__init__(self)
0500                 self.opt_ = opt
0501                 self.matrices_ = {}
0502                 tmp = MatrixReader(self.opt_)
0503                 self.processes_ = dict()
0504                 for what in tmp.files:
0505                     what = what.replace('relval_','')
0506                     self.opt_.what = what
0507                     self.matrices_[what] = MatrixReader(self.opt_)
0508                     self.matrices_[what].prepare(self.opt_.useInput, self.opt_.refRel,
0509                                                 self.opt_.fromScratch)
0510                 os.system("clear")
0511 
0512             def do_clear(self, arg):
0513                 """Clear the screen, put prompt at the top"""
0514                 os.system("clear")
0515 
0516             def do_exit(self, arg):
0517                 print("Leaving the Matrix")
0518                 return True
0519 
0520             def default(self, inp):
0521                 if inp == 'x' or inp == 'q':
0522                     return self.do_exit(inp)
0523                 else:
0524                     is_pipe = not isatty(sys.stdin.fileno())
0525                     print(Fore.RED + "Error: " + Fore.RESET + "unrecognized command.")
0526                     # Quit only if given a piped command.
0527                     if is_pipe:
0528                       sys.exit(1)
0529 
0530             def help_predefined(self):
0531                 print("\n".join(["predefined [predef1 [...]]\n",
0532                 "Run w/o argument, it will print the list of known predefined workflows.",
0533                 "Run with space-separated predefined workflows, it will print the workflow-ids registered to them"]))
0534 
0535             def complete_predefined(self, text, line, start_idx, end_idx):
0536                 if text and len(text) > 0:
0537                     return [t for t in predefinedSet.keys() if t.startswith(text)]
0538                 else:
0539                     return predefinedSet.keys()
0540 
0541             def do_predefined(self, arg):
0542                 """Print the list of predefined workflows"""
0543                 print("List of predefined workflows")
0544                 if arg:
0545                     for w in arg.split():
0546                         if w in predefinedSet.keys():
0547                             print("Predefined Set: %s" % w)
0548                             print(predefinedSet[w])
0549                         else:
0550                             print("Unknown Set: %s" % w)
0551                 else:
0552                     print("[ " + Fore.RED + ", ".join([str(k) for k in predefinedSet.keys()]) + Fore.RESET + " ]")
0553 
0554             def help_showWorkflow(self):
0555                 print("\n".join(["showWorkflow [workflow1 [...]]\n",
0556                     "Run w/o arguments, it will print the list of registered macro-workflows.",
0557                     "Run with space-separated workflows, it will print the full list of workflow-ids registered to them"]))
0558 
0559             def complete_showWorkflow(self, text, line, start_idx, end_idx):
0560                 if text and len(text) > 0:
0561                     return [t for t in self.matrices_.keys() if t.startswith(text)]
0562                 else:
0563                     return self.matrices_.keys()
0564 
0565             def do_showWorkflow(self, arg):
0566                 if arg == '':
0567                     print("Available workflows:")
0568                     for k in self.matrices_.keys():
0569                         print(Fore.RED + Style.BRIGHT + k)
0570                     print(Style.RESET_ALL)
0571                 else:
0572                     selected = arg.split()
0573                     for k in selected:
0574                         if k not in self.matrices_.keys():
0575                             print("Unknown workflow %s: skipping" % k)
0576                         else:
0577                             for wfl in self.matrices_[k].workFlows:
0578                                 print("%s %s" % (Fore.BLUE + str(wfl.numId) + Fore.RESET,
0579                                                               Fore.GREEN + wfl.nameId + Fore.RESET))
0580                             print("%s contains %d workflows" % (Fore.RED + k + Fore.RESET, len(self.matrices_[k].workFlows)))
0581 
0582             def do_runWorkflow(self, arg):
0583                 # Split the input arguments into a list
0584                 args = arg.split()
0585                 if len(args) < 2:
0586                     print(Fore.RED + Style.BRIGHT + "Wrong number of parameters passed")
0587                     print(Style.RESET_ALL)
0588                     return
0589                 workflow_class = args[0]
0590                 workflow_id = args[1]
0591                 passed_down_args = list()
0592                 if len(args) > 2:
0593                   passed_down_args = args[2:]
0594                 print(Fore.YELLOW + Style.BRIGHT + "Running with the following options:\n")
0595                 print(Fore.GREEN + Style.BRIGHT + "Workflow class: {}".format(workflow_class))
0596                 print(Fore.GREEN + Style.BRIGHT + "Workflow ID:    {}".format(workflow_id))
0597                 print(Fore.GREEN + Style.BRIGHT + "Additional runTheMatrix options: {}".format(passed_down_args))
0598                 print(Style.RESET_ALL)
0599                 if workflow_class not in self.matrices_.keys():
0600                     print(Fore.RED + Style.BRIGHT + "Unknown workflow selected: {}".format(workflow_class))
0601                     print("Available workflows:")
0602                     for k in self.matrices_.keys():
0603                          print(Fore.RED + Style.BRIGHT + k)
0604                     print(Style.RESET_ALL)
0605                     return
0606                 wflnums = [x.numId for x in self.matrices_[workflow_class].workFlows]
0607                 if float(workflow_id) not in wflnums:
0608                     print(Fore.RED + Style.BRIGHT + "Unknown workflow {}".format(workflow_id))
0609                     print(Fore.GREEN + Style.BRIGHT)
0610                     print(wflnums)
0611                     print(Style.RESET_ALL)
0612                     return
0613                 if workflow_id in self.processes_.keys():
0614                     # Check if the process is still active
0615                     if self.processes_[workflow_id][0].poll() is None:
0616                         print(Fore.RED + Style.BRIGHT + "Workflow {} already running!".format(workflow_id))
0617                         print(Style.RESET_ALL)
0618                         return
0619                 # If it was there but it's gone, proceeed and update the value for the same key
0620                 # run a job, redirecting standard output and error to files
0621                 lognames = ['stdout', 'stderr']
0622                 logfiles = tuple('%s_%s_%s.log' % (workflow_class, workflow_id, name) for name in lognames)
0623                 stdout = open(logfiles[0], 'w')
0624                 stderr = open(logfiles[1], 'w')
0625                 command = ('runTheMatrix.py', '-w', workflow_class, '-l', workflow_id)
0626                 if len(passed_down_args) > 0:
0627                   command += tuple(passed_down_args)
0628                 print(command)
0629                 p = subprocess.Popen(command,
0630                     stdout = stdout,
0631                     stderr = stderr)
0632                 self.processes_[workflow_id] = (p, time.time())
0633 
0634 
0635             def complete_runWorkflow(self, text, line, start_idx, end_idx):
0636                 if text and len(text) > 0:
0637                     return [t for t in self.matrices_.keys() if t.startswith(text)]
0638                 else:
0639                     return self.matrices_.keys()
0640 
0641             def help_runWorkflow(self):
0642               print("\n".join(["runWorkflow workflow_class workflow_id\n",
0643                 "This command will launch a new and independent process that invokes",
0644                 "the command:\n",
0645                 "runTheMatrix.py -w workflow_class -l workflow_id [runTheMatrix.py options]",
0646                 "\nYou can specify just one workflow_class and workflow_id per invocation.",
0647                 "The job will continue even after quitting the interactive session.",
0648                 "stdout and stderr of the new process will be automatically",
0649                 "redirected to 2 logfiles whose names contain the workflow_class",
0650                 "and workflow_id. Mutiple command can be issued one after the other.",
0651                 "The working directory of the new process will be the directory",
0652                 "from which the interactive session has started.",
0653                 "Autocompletion is available for workflow_class, but",
0654                 "not for workflow_id. Supplying a wrong workflow_class or",
0655                 "a non-existing workflow_id for a valid workflow_class",
0656                 "will trigger an error and no process will be invoked.",
0657                 "The interactive shell will keep track of all active processes",
0658                 "and will prevent the accidental resubmission of an already",
0659                 "active jobs."]))
0660 
0661             def do_jobs(self, args):
0662                 print(Fore.GREEN + Style.BRIGHT + "List of jobs:")
0663                 for w in self.processes_.keys():
0664                     if self.processes_[w][0].poll() is None:
0665                       print(Fore.YELLOW + Style.BRIGHT + "Active job: {} since {:.2f} seconds.".format(w, time.time() - self.processes_[w][1]))
0666                     else:
0667                         print(Fore.RED + Style.BRIGHT + "Done job: {}".format(w))
0668                 print(Style.RESET_ALL)
0669 
0670             def help_jobs(self):
0671               print("\n".join(["Print a full list of active and done jobs submitted",
0672                 "in the ongoing interactive session"]))
0673 
0674             def help_searchInWorkflow(self):
0675                 print("\n".join(["searchInWorkflow wfl_name search_regexp\n",
0676                     "This command will search for a match within all workflows registered to wfl_name.",
0677                     "The search is done on both the workflow name and the names of steps registered to it."]))
0678 
0679             def complete_searchInWorkflow(self, text, line, start_idx, end_idx):
0680                 if text and len(text) > 0:
0681                     return [t for t in self.matrices_.keys() if t.startswith(text)]
0682                 else:
0683                     return self.matrices_.keys()
0684 
0685             def do_searchInWorkflow(self, arg):
0686                 args = arg.split()
0687                 if len(args) < 2:
0688                     print("searchInWorkflow name regexp")
0689                     return
0690                 if args[0] not in self.matrices_.keys():
0691                     print("Unknown workflow")
0692                     return
0693                 import re
0694                 pattern = None
0695                 try:
0696                     pattern = re.compile(args[1])
0697                 except:
0698                     print("Failed to compile regexp %s" % args[1])
0699                     return
0700                 counter = 0
0701                 for wfl in self.matrices_[args[0]].workFlows:
0702                     if re.match(pattern, wfl.nameId):
0703                       print("%s %s" % (Fore.BLUE + str(wfl.numId) + Fore.RESET,
0704                                        Fore.GREEN + wfl.nameId + Fore.RESET))
0705                       counter +=1
0706                 print("Found %s compatible workflows inside %s" % (Fore.RED + str(counter) + Fore.RESET,
0707                                                                    Fore.YELLOW + str(args[0])) + Fore.RESET)
0708 
0709             def help_search(self):
0710                 print("\n".join(["search search_regexp\n",
0711                     "This command will search for a match within all workflows registered.",
0712                     "The search is done on both the workflow name and the names of steps registered to it."]))
0713 
0714             def do_search(self, arg):
0715                 args = arg.split()
0716                 if len(args) < 1:
0717                     print("search regexp")
0718                     return
0719                 for wfl in self.matrices_.keys():
0720                     self.do_searchInWorkflow(' '.join([wfl, args[0]]))
0721 
0722             def help_dumpWorkflowId(self):
0723                 print("\n".join(["dumpWorkflowId [wfl-id1 [...]]\n",
0724                     "Dumps the details (cmsDriver commands for all steps) of the space-separated workflow-ids in input."]))
0725 
0726             def do_dumpWorkflowId(self, arg):
0727                 wflids = arg.split()
0728                 if len(wflids) == 0:
0729                     print("dumpWorkflowId [wfl-id1 [...]]")
0730                     return
0731 
0732                 fmt   = "[%s]: %s\n"
0733                 maxLen = 100
0734                 for wflid in wflids:
0735                     dump = True
0736                     for key, mrd in self.matrices_.items():
0737                         for wfl in mrd.workFlows:
0738                             if wfl.numId == float(wflid):
0739                                 if dump:
0740                                     dump = False
0741                                     print(Fore.GREEN + str(wfl.numId) + Fore.RESET + " " + Fore.YELLOW + wfl.nameId + Fore.RESET)
0742                                     for i,s in enumerate(wfl.cmds):
0743                                         print(fmt % (Fore.RED + str(i+1) + Fore.RESET,
0744                                           (str(s)+' ')))
0745                                     print("\nWorkflow found in %s." % key)
0746                                 else:
0747                                     print("Workflow also found in %s." % key)
0748 
0749             do_EOF = do_exit
0750 
0751         TheMatrix(opt).cmdloop()
0752         sys.exit(0)
0753 
0754     if opt.raw and opt.show: ###prodAgent to be discontinued
0755         ret = showRaw(opt)
0756     else:
0757         ret = runSelected(opt)
0758 
0759 
0760     sys.exit(ret)