PyReleaseValidation/scripts/runTheMatrix.py

0001 #!/usr/bin/env python3
0002 from __future__ import print_function
0003 import sys, os
0004
0005 from Configuration.PyReleaseValidation.MatrixReader import MatrixReader
0006 from Configuration.PyReleaseValidation.MatrixRunner import MatrixRunner
0007 from Configuration.PyReleaseValidation.MatrixInjector import MatrixInjector,performInjectionOptionTest
0008
0009 # ================================================================================
0010
0011 def showRaw(opt):
0012
0013     mrd = MatrixReader(opt)
0014     mrd.showRaw(opt.useInput, opt.refRel, opt.fromScratch, opt.raw, opt.step1Only, selected=opt.testList)
0015
0016     return 0
0017
0018 # ================================================================================
0019
0020 def runSelected(opt):
0021
0022     mrd = MatrixReader(opt)
0023     mrd.prepare(opt.useInput, opt.refRel, opt.fromScratch)
0024
0025     # test for wrong input workflows
0026     if opt.testList:
0027         definedWf = [dwf.numId for dwf in mrd.workFlows]
0028         definedSet = set(definedWf)
0029         testSet = set(opt.testList)
0030         undefSet = testSet - definedSet
0031         if len(undefSet)>0: raise ValueError('Undefined workflows: '+', '.join(map(str,list(undefSet))))
0032         duplicates = [wf for wf in testSet if definedWf.count(wf)>1 ]
0033         if len(duplicates)>0: raise ValueError('Duplicated workflows: '+', '.join(map(str,list(duplicates))))
0034
0035     ret = 0
0036     if opt.show:
0037         mrd.show(opt.testList, opt.extended, opt.cafVeto)
0038         if opt.testList : print('testListected items:', opt.testList)
0039     else:
0040         mRunnerHi = MatrixRunner(mrd.workFlows, opt.nProcs, opt.nThreads)
0041         ret = mRunnerHi.runTests(opt)
0042
0043     if opt.wmcontrol:
0044         if ret!=0:
0045             print('Cannot go on with wmagent injection with failing workflows')
0046         else:
0047             wfInjector = MatrixInjector(opt,mode=opt.wmcontrol,options=opt.wmoptions)
0048             ret= wfInjector.prepare(mrd,
0049                                     mRunnerHi.runDirs)
0050             if ret==0:
0051                 wfInjector.upload()
0052                 wfInjector.submit()
0053     return ret
0054
0055 # ================================================================================
0056
0057 if __name__ == '__main__':
0058
0059     #this can get out of here
0060     predefinedSet={
0061         'limited' : [5.1, #FastSim ttbar
0062                      7.3, #CosmicsSPLoose_UP17
0063                      8, #BH/Cosmic MC
0064                      25, #MC ttbar
0065                      4.22, #cosmic data
0066                      4.53, #run1 data + miniAOD
0067                      9.0, #Higgs200 charged taus
0068                      1000, #data+prompt
0069                      1001, #data+express
0070                      101.0, #SingleElectron120E120EHCAL
0071                      136.731, #2016B Photon data
0072                      136.7611, #2016E JetHT reMINIAOD from 80X legacy
0073                      136.8311, #2017F JetHT reMINIAOD from 94X reprocessing
0074                      136.88811,#2018D JetHT reMINIAOD from UL processing
0075                      136.793, #2017C DoubleEG
0076                      136.874, #2018C EGamma
0077                      138.4, #2021 MinimumBias prompt reco
0078                      138.5, #2021 MinimumBias express
0079                      139.001, #2021 MinimumBias offline with HLT step
0080                      140.53, #2011 HI data
0081                      140.56, #2018 HI data
0082                      158.01, #reMiniAOD of 2018 HI MC with pp-like reco
0083                      312.0, #2021/Run3 HI MC Pyquen_ZeemumuJets_pt10 with pp-like reco
0084                      1306.0, #SingleMu Pt1 UP15
0085                      1325.81, #test NanoAOD from existing MINI UL 106Xv1
0086                      136.8523, #test NanoAOD from existing reMINI UL 106Xv2
0087                      1330, #Run2 2015/2016 MC Zmm
0088                      135.4, #Run 2 2015/2016 Zee ttbar fastsim
0089                      10042.0, #2017 ZMM
0090                      10024.0, #2017 ttbar
0091                      10824.0, #2018 ttbar
0092                      2018.1, #2018 ttbar fastsim
0093                      11634.911, #2021 DD4hep ttbar reading geometry from XML
0094                      11634.914, #2021 DDD ttbar reading geometry from the DB
0095                      11634.0, #2021 ttbar (switching to DD4hep by default)
0096                      11634.301, #2021 ttbar fastsim
0097                      11634.7, #2021 ttbar mkFit
0098                      11834.0, #2021 ttbar PU
0099                      12434.0, #2023 ttbar
0100                      23234.0, #2026D49 ttbar (HLT TDR baseline w/ HGCal v11)
0101                      28234.0, #2026D60 (exercise HF nose)
0102                      35034.0, #2026D77 ttbar
0103                      39434.0, #2026D88 ttbar (2022 new baseline)
0104                      39434.75, #2026D88 ttbar with HLT75e33
0105                      #39434.911, #2026D88 ttbar DD4hep XML
0106                      39634.999, #2026D88 ttbar premixing stage1+stage2, PU50
0107                      39496.0, #CE_E_Front_120um D88
0108                      39500.0, #CE_H_Coarse_Scint D88
0109                      25202.0, #2016 ttbar UP15 PU
0110                      250202.181, #2018 ttbar stage1 + stage2 premix
0111                      ],
0112         'jetmc': [5.1, 13, 15, 25, 38, 39], #MC
0113         'metmc' : [5.1, 15, 25, 37, 38, 39], #MC
0114         'muonmc' : [5.1, 124.4, 124.5, 20, 21, 22, 23, 25, 30], #MC
0115         }
0116
0117
0118     import argparse
0119     usage = 'usage: runTheMatrix.py --show -s '
0120
0121     parser = argparse.ArgumentParser(usage,formatter_class=argparse.ArgumentDefaultsHelpFormatter)
0122
0123     parser.add_argument('-b','--batchName',
0124                         help='relval batch: suffix to be appended to Campaign name',
0125                         dest='batchName',
0126                         default='')
0127
0128     parser.add_argument('-m','--memoryOffset',
0129                         help='memory of the wf for single core',
0130                         dest='memoryOffset',
0131                         type=int,
0132                         default=3000)
0133
0134     parser.add_argument('--addMemPerCore',
0135                         help='increase of memory per each n > 1 core:  memory(n_core) = memoryOffset + (n_core-1) * memPerCore',
0136                         dest='memPerCore',
0137                         type=int,
0138                         default=1500)
0139
0140     parser.add_argument('-j','--nproc',
0141                         help='number of processes. 0 Will use 4 processes, not execute anything but create the wfs',
0142                         dest='nProcs',
0143                         type=int,
0144                         default=4)
0145
0146     parser.add_argument('-t','--nThreads',
0147                         help='number of threads per process to use in cmsRun.',
0148                         dest='nThreads',
0149                         type=int,
0150                         default=1)
0151
0152     parser.add_argument('--nStreams',
0153                         help='number of streams to use in cmsRun.',
0154                         dest='nStreams',
0155                         type=int,
0156                         default=0)
0157
0158     parser.add_argument('--numberEventsInLuminosityBlock',
0159                         help='number of events in a luminosity block',
0160                         dest='numberEventsInLuminosityBlock',
0161                         type=int,
0162                         default=-1)
0163
0164     parser.add_argument('-n','--showMatrix',
0165                         help='Only show the worflows. Use --ext to show more',
0166                         dest='show',
0167                         default=False,
0168                         action='store_true')
0169
0170     parser.add_argument('-e','--extended',
0171                         help='Show details of workflows, used with --show',
0172                         dest='extended',
0173                         default=False,
0174                         action='store_true')
0175
0176     parser.add_argument('-s','--selected',
0177                         help='Run a pre-defined selected matrix of wf. Deprecated, please use -l limited',
0178                         dest='restricted',
0179                         default=False,
0180                         action='store_true')
0181
0182     parser.add_argument('-l','--list',
0183                         help='Comma separated list of workflow to be shown or ran. Possible keys are also '+str(predefinedSet.keys())+'. and wild card like muon, or mc',
0184                         dest='testList',
0185                         default=None)
0186
0187     parser.add_argument('-r','--raw',
0188                         help='Temporary dump the .txt needed for prodAgent interface. To be discontinued soon. Argument must be the name of the set (standard, pileup,...)',
0189                         dest='raw')
0190
0191     parser.add_argument('-i','--useInput',
0192                         help='Use recyling where available. Either all, or a comma separated list of wf number.',
0193                         dest='useInput',
0194                         type=lambda x: x.split(','),
0195                         default=None)
0196
0197     parser.add_argument('-w','--what',
0198                         help='Specify the set to be used. Argument must be the name of a set (standard, pileup,...) or multiple sets separated by commas (--what standard,pileup )',
0199                         dest='what',
0200                         default='all')
0201
0202     parser.add_argument('--step1',
0203                         help='Used with --raw. Limit the production to step1',
0204                         dest='step1Only',
0205                         default=False)
0206
0207     parser.add_argument('--maxSteps',
0208                         help='Only run maximum on maxSteps. Used when we are only interested in first n steps.',
0209                         dest='maxSteps',
0210                         default=9999,
0211                         type=int)
0212
0213     parser.add_argument('--fromScratch',
0214                         help='Comma separated list of wf to be run without recycling. all is not supported as default.',
0215                         dest='fromScratch',
0216                         type=lambda x: x.split(','),
0217                         default=None)
0218
0219     parser.add_argument('--refRelease',
0220                         help='Allow to modify the recycling dataset version',
0221                         dest='refRel',
0222                         default=None)
0223
0224     parser.add_argument('--wmcontrol',
0225                         help='Create the workflows for injection to WMAgent. In the WORKING. -wmcontrol init will create the the workflows, -wmcontrol test will dryRun a test, -wmcontrol submit will submit to wmagent',
0226                         choices=['init','test','submit','force'],
0227                         dest='wmcontrol',
0228                         default=None)
0229
0230     parser.add_argument('--revertDqmio',
0231                         help='When submitting workflows to wmcontrol, force DQM outout to use pool and not DQMIO',
0232                         choices=['yes','no'],
0233                         dest='revertDqmio',
0234                         default='no')
0235
0236     parser.add_argument('--optionswm',
0237                         help='Specify a few things for wm injection',
0238                         default='',
0239                         dest='wmoptions')
0240
0241     parser.add_argument('--keep',
0242                         help='allow to specify for which comma separated steps the output is needed',
0243                         default=None)
0244
0245     parser.add_argument('--label',
0246                         help='allow to give a special label to the output dataset name',
0247                         default='')
0248
0249     parser.add_argument('--command',
0250                         help='provide a way to add additional command to all of the cmsDriver commands in the matrix',
0251                         dest='command',
0252                         action='append',
0253                         default=None)
0254
0255     parser.add_argument('--apply',
0256                         help='allow to use the --command only for 1 comma separeated',
0257                         dest='apply',
0258                         default=None)
0259
0260     parser.add_argument('--workflow',
0261                         help='define a workflow to be created or altered from the matrix',
0262                         action='append',
0263                         dest='workflow',
0264                         default=None)
0265
0266     parser.add_argument('--dryRun',
0267                         help='do not run the wf at all',
0268                         action='store_true',
0269                         dest='dryRun',
0270                         default=False)
0271
0272     parser.add_argument('--testbed',
0273                         help='workflow injection to cmswebtest (you need dedicated rqmgr account)',
0274                         dest='testbed',
0275                         default=False,
0276                         action='store_true')
0277
0278     parser.add_argument('--noCafVeto',
0279                         help='Run from any source, ignoring the CAF label',
0280                         dest='cafVeto',
0281                         default=True,
0282                         action='store_false')
0283
0284     parser.add_argument('--overWrite',
0285                         help='Change the content of a step for another. List of pairs.',
0286                         dest='overWrite',
0287                         default=None)
0288
0289     parser.add_argument('--noRun',
0290                         help='Remove all run list selection from wfs',
0291                         dest='noRun',
0292                         default=False,
0293                         action='store_true')
0294
0295     parser.add_argument('--das-options',
0296                         help='Options to be passed to dasgoclient.',
0297                         dest='dasOptions',
0298                         default="--limit 0",
0299                         action='store')
0300
0301     parser.add_argument('--job-reports',
0302                         help='Dump framework job reports',
0303                         dest='jobReports',
0304                         default=False,
0305                         action='store_true')
0306
0307     parser.add_argument('--ibeos',
0308                         help='Use IB EOS site configuration',
0309                         dest='IBEos',
0310                         default=False,
0311                         action='store_true')
0312
0313     parser.add_argument('--sites',
0314                         help='Run DAS query to get data from a specific site. Set it to empty string to search all sites.',
0315                         dest='dasSites',
0316                         default='T2_CH_CERN',
0317                         action='store')
0318
0319     parser.add_argument('--interactive',
0320                         help="Open the Matrix interactive shell",
0321                         action='store_true',
0322                         default=False)
0323
0324     parser.add_argument('--dbs-url',
0325                         help='Overwrite DbsUrl value in JSON submitted to ReqMgr2',
0326                         dest='dbsUrl',
0327                         default=None,
0328                         action='store')
0329
0330     gpugroup = parser.add_argument_group('GPU-related options','These options are only meaningful when --gpu is used, and is not set to forbidden.')
0331
0332     gpugroup.add_argument('--gpu','--requires-gpu',
0333                           help='Enable GPU workflows. Possible options are "forbidden" (default), "required" (implied if no argument is given), or "optional".',
0334                           dest='gpu',
0335                           choices=['forbidden', 'optional', 'required'],
0336                           nargs='?',
0337                           const='required',
0338                           default='forbidden',
0339                           action='store')
0340
0341     gpugroup.add_argument('--gpu-memory',
0342                           help='Specify the minimum amount of GPU memory required by the job, in MB.',
0343                           dest='GPUMemoryMB',
0344                           type=int,
0345                           default=8000)
0346
0347     gpugroup.add_argument('--cuda-capabilities',
0348                           help='Specify a comma-separated list of CUDA "compute capabilities", or GPU hardware architectures, that the job can use.',
0349                           dest='CUDACapabilities',
0350                           type=lambda x: x.split(','),
0351                           default='6.0,6.1,6.2,7.0,7.2,7.5,8.0,8.6')
0352
0353     # read the CUDA runtime version included in CMSSW
0354     cudart_version = None
0355     libcudart = os.path.realpath(os.path.expandvars('$CMSSW_RELEASE_BASE/external/$SCRAM_ARCH/lib/libcudart.so'))
0356     if os.path.isfile(libcudart):
0357         cudart_basename = os.path.basename(libcudart)
0358         cudart_version = '.'.join(cudart_basename.split('.')[2:4])
0359     gpugroup.add_argument('--cuda-runtime',
0360                           help='Specify major and minor version of the CUDA runtime used to build the application.',
0361                           dest='CUDARuntime',
0362                           default=cudart_version)
0363
0364     gpugroup.add_argument('--force-gpu-name',
0365                           help='Request a specific GPU model, e.g. "Tesla T4" or "NVIDIA GeForce RTX 2080". The default behaviour is to accept any supported GPU.',
0366                           dest='GPUName',
0367                           default='')
0368
0369     gpugroup.add_argument('--force-cuda-driver-version',
0370                           help='Request a specific CUDA driver version, e.g. 470.57.02. The default behaviour is to accept any supported CUDA driver version.',
0371                           dest='CUDADriverVersion',
0372                           default='')
0373
0374     gpugroup.add_argument('--force-cuda-runtime-version',
0375                           help='Request a specific CUDA runtime version, e.g. 11.4. The default behaviour is to accept any supported CUDA runtime version.',
0376                           dest='CUDARuntimeVersion',
0377                           default='')
0378
0379     opt = parser.parse_args()
0380     if opt.command: opt.command = ' '.join(opt.command)
0381     os.environ["CMSSW_DAS_QUERY_SITES"]=opt.dasSites
0382     if opt.IBEos:
0383       from subprocess import getstatusoutput as run_cmd
0384
0385       ibeos_cache = os.path.join(os.getenv("LOCALRT"), "ibeos_cache.txt")
0386       if not os.path.exists(ibeos_cache):
0387         err, out = run_cmd("curl -L -s -o %s https://raw.githubusercontent.com/cms-sw/cms-sw.github.io/master/das_queries/ibeos.txt" % ibeos_cache)
0388         if err:
0389           run_cmd("rm -f %s" % ibeos_cache)
0390           print("Error: Unable to download ibeos cache information")
0391           print(out)
0392           sys.exit(err)
0393
0394       for cmssw_env in [ "CMSSW_BASE", "CMSSW_RELEASE_BASE" ]:
0395         cmssw_base = os.getenv(cmssw_env,None)
0396         if not cmssw_base: continue
0397         cmssw_base = os.path.join(cmssw_base,"src/Utilities/General/ibeos")
0398         if os.path.exists(cmssw_base):
0399           os.environ["PATH"]=cmssw_base+":"+os.getenv("PATH")
0400           os.environ["CMS_PATH"]="/cvmfs/cms-ib.cern.ch"
0401           os.environ["CMSSW_USE_IBEOS"]="true"
0402           print(">> WARNING: You are using SITECONF from /cvmfs/cms-ib.cern.ch")
0403           break
0404     if opt.restricted:
0405         print('Deprecated, please use -l limited')
0406         if opt.testList:            opt.testList+=',limited'
0407         else:            opt.testList='limited'
0408
0409     def stepOrIndex(s):
0410         if s.isdigit():
0411             return int(s)
0412         else:
0413             return s
0414     if opt.apply:
0415         opt.apply=map(stepOrIndex,opt.apply.split(','))
0416     if opt.keep:
0417         opt.keep=map(stepOrIndex,opt.keep.split(','))
0418
0419     if opt.testList:
0420         testList=[]
0421         for entry in opt.testList.split(','):
0422             if not entry: continue
0423             mapped=False
0424             for k in predefinedSet:
0425                 if k.lower().startswith(entry.lower()) or k.lower().endswith(entry.lower()):
0426                     testList.extend(predefinedSet[k])
0427                     mapped=True
0428                     break
0429             if not mapped:
0430                 try:
0431                     testList.append(float(entry))
0432                 except:
0433                     print(entry,'is not a possible selected entry')
0434
0435         opt.testList = list(set(testList))
0436
0437     if opt.wmcontrol:
0438         performInjectionOptionTest(opt)
0439     if opt.overWrite:
0440         opt.overWrite=eval(opt.overWrite)
0441     if opt.interactive:
0442         import cmd
0443
0444         class TheMatrix(cmd.Cmd):
0445             intro = "Welcome to the Matrix (? for help)"
0446             prompt = "matrix> "
0447
0448             def __init__(self, opt):
0449                 cmd.Cmd.__init__(self)
0450                 self.opt_ = opt
0451                 self.matrices_ = {}
0452                 tmp = MatrixReader(self.opt_)
0453                 for what in tmp.files:
0454                     what = what.replace('relval_','')
0455                     self.opt_.what = what
0456                     self.matrices_[what] = MatrixReader(self.opt_)
0457                     self.matrices_[what].prepare(self.opt_.useInput, self.opt_.refRel,
0458                                                 self.opt_.fromScratch)
0459                 os.system("clear")
0460
0461             def do_clear(self, arg):
0462                 """Clear the screen, put prompt at the top"""
0463                 os.system("clear")
0464
0465             def do_exit(self, arg):
0466                 print("Leaving the Matrix")
0467                 return True
0468
0469             def default(self, inp):
0470                 if inp == 'x' or inp == 'q':
0471                     return self.do_exit(inp)
0472
0473             def help_predefined(self):
0474                 print("\n".join(["predefined [predef1 [...]]\n",
0475                 "Run w/o argument, it will print the list of known predefined workflows.",
0476                 "Run with space-separated predefined workflows, it will print the workflow-ids registered to them"]))
0477
0478             def complete_predefined(self, text, line, start_idx, end_idx):
0479                 if text and len(text) > 0:
0480                     return [t for t in predefinedSet.keys() if t.startswith(text)]
0481                 else:
0482                     return predefinedSet.keys()
0483
0484             def do_predefined(self, arg):
0485                 """Print the list of predefined workflows"""
0486                 print("List of predefined workflows")
0487                 if arg:
0488                     for w in arg.split():
0489                         if w in predefinedSet.keys():
0490                             print("Predefined Set: %s" % w)
0491                             print(predefinedSet[w])
0492                         else:
0493                             print("Unknown Set: %s" % w)
0494                 else:
0495                     print(predefinedSet.keys())
0496
0497             def help_showWorkflow(self):
0498                 print("\n".join(["showWorkflow [workflow1 [...]]\n",
0499                     "Run w/o arguments, it will print the list of registered macro-workflows.",
0500                     "Run with space-separated workflows, it will print the full list of workflow-ids registered to them"]))
0501
0502             def complete_showWorkflow(self, text, line, start_idx, end_idx):
0503                 if text and len(text) > 0:
0504                     return [t for t in self.matrices_.keys() if t.startswith(text)]
0505                 else:
0506                     return self.matrices_.keys()
0507
0508             def do_showWorkflow(self, arg):
0509                 if arg == '':
0510                     print("Available workflows:")
0511                     for k in self.matrices_.keys():
0512                         print(k)
0513                 else:
0514                     selected = arg.split()
0515                     for k in selected:
0516                         if k not in self.matrices_.keys():
0517                             print("Unknown workflow %s: skipping" % k)
0518                         else:
0519                             for wfl in self.matrices_[k].workFlows:
0520                                 wfName, stepNames = wfl.nameId.split('+',1)
0521                                 print("%s %s %s" % (wfl.numId, wfName, stepNames))
0522                             print("%s contains %d workflows" % (k, len(self.matrices_[k].workFlows)))
0523
0524             def help_searchInWorkflow(self):
0525                 print("\n".join(["searchInWorkflow wfl_name search_regexp\n",
0526                     "This command will search for a match within all workflows registered to wfl_name.",
0527                     "The search is done on both the workflow name and the names of steps registered to it."]))
0528
0529             def complete_searchInWorkflow(self, text, line, start_idx, end_idx):
0530                 if text and len(text) > 0:
0531                     return [t for t in self.matrices_.keys() if t.startswith(text)]
0532                 else:
0533                     return self.matrices_.keys()
0534
0535             def do_searchInWorkflow(self, arg):
0536                 args = arg.split()
0537                 if len(args) < 2:
0538                     print("searchInWorkflow name regexp")
0539                     return
0540                 if args[0] not in self.matrices_.keys():
0541                     print("Unknown workflow")
0542                     return
0543                 import re
0544                 pattern = None
0545                 try:
0546                     pattern = re.compile(args[1])
0547                 except:
0548                     print("Failed to compile regexp %s" % args[1])
0549                     return
0550                 counter = 0
0551                 for wfl in self.matrices_[args[0]].workFlows:
0552                     wfName, stepNames = wfl.nameId.split('+',1)
0553                     if re.match(pattern, wfName) or re.match(pattern, stepNames):
0554                         print("%s %s %s" % (wfl.numId, wfName, stepNames))
0555                         counter += 1
0556                 print("Found %d compatible workflows inside %s" % (counter, args[0]))
0557
0558             def help_search(self):
0559                 print("\n".join(["search search_regexp\n",
0560                     "This command will search for a match within all workflows registered.",
0561                     "The search is done on both the workflow name and the names of steps registered to it."]))
0562
0563             def do_search(self, arg):
0564                 args = arg.split()
0565                 if len(args) < 1:
0566                     print("search regexp")
0567                     return
0568                 for wfl in self.matrices_.keys():
0569                     self.do_searchInWorkflow(' '.join([wfl, args[0]]))
0570
0571             def help_dumpWorkflowId(self):
0572                 print("\n".join(["dumpWorkflowId [wfl-id1 [...]]\n",
0573                     "Dumps the details (cmsDriver commands for all steps) of the space-separated workflow-ids in input."]))
0574
0575             def do_dumpWorkflowId(self, arg):
0576                 wflids = arg.split()
0577                 if len(wflids) == 0:
0578                     print("dumpWorkflowId [wfl-id1 [...]]")
0579                     return
0580
0581                 fmt   = "[%d]: %s\n"
0582                 maxLen = 100
0583                 for wflid in wflids:
0584                     dump = True
0585                     for key, mrd in self.matrices_.items():
0586                         for wfl in mrd.workFlows:
0587                             if wfl.numId == float(wflid):
0588                                 wfName, stepNames = wfl.nameId.split('+',1)
0589                                 if dump:
0590                                     dump = False
0591                                     print(wfl.numId, stepNames)
0592                                     for i,s in enumerate(wfl.cmds):
0593                                         print(fmt % (i+1, (str(s)+' ')))
0594                                     print("\nWorkflow found in %s." % key)
0595                                 else:
0596                                     print("Workflow also found in %s." % key)
0597
0598             do_EOF = do_exit
0599
0600         TheMatrix(opt).cmdloop()
0601         sys.exit(0)
0602
0603     if opt.raw and opt.show: ###prodAgent to be discontinued
0604         ret = showRaw(opt)
0605     else:
0606         ret = runSelected(opt)
0607
0608
0609     sys.exit(ret)