File indexing completed on 2022-05-15 22:20:24
0001
0002 from __future__ import print_function
0003 import sys, os
0004
0005 from Configuration.PyReleaseValidation.MatrixReader import MatrixReader
0006 from Configuration.PyReleaseValidation.MatrixRunner import MatrixRunner
0007 from Configuration.PyReleaseValidation.MatrixInjector import MatrixInjector,performInjectionOptionTest
0008
0009
0010
0011 def showRaw(opt):
0012
0013 mrd = MatrixReader(opt)
0014 mrd.showRaw(opt.useInput, opt.refRel, opt.fromScratch, opt.raw, opt.step1Only, selected=opt.testList)
0015
0016 return 0
0017
0018
0019
0020 def runSelected(opt):
0021
0022 mrd = MatrixReader(opt)
0023 mrd.prepare(opt.useInput, opt.refRel, opt.fromScratch)
0024
0025
0026 if opt.testList:
0027 definedWf = [dwf.numId for dwf in mrd.workFlows]
0028 definedSet = set(definedWf)
0029 testSet = set(opt.testList)
0030 undefSet = testSet - definedSet
0031 if len(undefSet)>0: raise ValueError('Undefined workflows: '+', '.join(map(str,list(undefSet))))
0032 duplicates = [wf for wf in testSet if definedWf.count(wf)>1 ]
0033 if len(duplicates)>0: raise ValueError('Duplicated workflows: '+', '.join(map(str,list(duplicates))))
0034
0035 ret = 0
0036 if opt.show:
0037 mrd.show(opt.testList, opt.extended, opt.cafVeto)
0038 if opt.testList : print('testListected items:', opt.testList)
0039 else:
0040 mRunnerHi = MatrixRunner(mrd.workFlows, opt.nProcs, opt.nThreads)
0041 ret = mRunnerHi.runTests(opt)
0042
0043 if opt.wmcontrol:
0044 if ret!=0:
0045 print('Cannot go on with wmagent injection with failing workflows')
0046 else:
0047 wfInjector = MatrixInjector(opt,mode=opt.wmcontrol,options=opt.wmoptions)
0048 ret= wfInjector.prepare(mrd,
0049 mRunnerHi.runDirs)
0050 if ret==0:
0051 wfInjector.upload()
0052 wfInjector.submit()
0053 return ret
0054
0055
0056
0057 if __name__ == '__main__':
0058
0059
0060 predefinedSet={
0061 'limited' : [5.1,
0062 7.3,
0063 8,
0064 25,
0065 4.22,
0066 4.53,
0067 9.0,
0068 1000,
0069 1001,
0070 101.0,
0071 136.731,
0072 136.7611,
0073 136.8311,
0074 136.88811,
0075 136.793,
0076 136.874,
0077 138.4,
0078 138.5,
0079 139.001,
0080 140.53,
0081 140.56,
0082 158.01,
0083 312.0,
0084 1306.0,
0085 1325.81,
0086 136.8523,
0087 1330,
0088 135.4,
0089 10042.0,
0090 10024.0,
0091 10824.0,
0092 2018.1,
0093 11634.911,
0094 11634.914,
0095 11634.0,
0096 11634.301,
0097 11634.7,
0098 11834.0,
0099 12434.0,
0100 23234.0,
0101 28234.0,
0102 35034.0,
0103 39434.0,
0104 39434.75,
0105
0106 39634.999,
0107 39496.0,
0108 39500.0,
0109 25202.0,
0110 250202.181,
0111 ],
0112 'jetmc': [5.1, 13, 15, 25, 38, 39],
0113 'metmc' : [5.1, 15, 25, 37, 38, 39],
0114 'muonmc' : [5.1, 124.4, 124.5, 20, 21, 22, 23, 25, 30],
0115 }
0116
0117
0118 import argparse
0119 usage = 'usage: runTheMatrix.py --show -s '
0120
0121 parser = argparse.ArgumentParser(usage,formatter_class=argparse.ArgumentDefaultsHelpFormatter)
0122
0123 parser.add_argument('-b','--batchName',
0124 help='relval batch: suffix to be appended to Campaign name',
0125 dest='batchName',
0126 default='')
0127
0128 parser.add_argument('-m','--memoryOffset',
0129 help='memory of the wf for single core',
0130 dest='memoryOffset',
0131 type=int,
0132 default=3000)
0133
0134 parser.add_argument('--addMemPerCore',
0135 help='increase of memory per each n > 1 core: memory(n_core) = memoryOffset + (n_core-1) * memPerCore',
0136 dest='memPerCore',
0137 type=int,
0138 default=1500)
0139
0140 parser.add_argument('-j','--nproc',
0141 help='number of processes. 0 Will use 4 processes, not execute anything but create the wfs',
0142 dest='nProcs',
0143 type=int,
0144 default=4)
0145
0146 parser.add_argument('-t','--nThreads',
0147 help='number of threads per process to use in cmsRun.',
0148 dest='nThreads',
0149 type=int,
0150 default=1)
0151
0152 parser.add_argument('--nStreams',
0153 help='number of streams to use in cmsRun.',
0154 dest='nStreams',
0155 type=int,
0156 default=0)
0157
0158 parser.add_argument('--numberEventsInLuminosityBlock',
0159 help='number of events in a luminosity block',
0160 dest='numberEventsInLuminosityBlock',
0161 type=int,
0162 default=-1)
0163
0164 parser.add_argument('-n','--showMatrix',
0165 help='Only show the worflows. Use --ext to show more',
0166 dest='show',
0167 default=False,
0168 action='store_true')
0169
0170 parser.add_argument('-e','--extended',
0171 help='Show details of workflows, used with --show',
0172 dest='extended',
0173 default=False,
0174 action='store_true')
0175
0176 parser.add_argument('-s','--selected',
0177 help='Run a pre-defined selected matrix of wf. Deprecated, please use -l limited',
0178 dest='restricted',
0179 default=False,
0180 action='store_true')
0181
0182 parser.add_argument('-l','--list',
0183 help='Comma separated list of workflow to be shown or ran. Possible keys are also '+str(predefinedSet.keys())+'. and wild card like muon, or mc',
0184 dest='testList',
0185 default=None)
0186
0187 parser.add_argument('-r','--raw',
0188 help='Temporary dump the .txt needed for prodAgent interface. To be discontinued soon. Argument must be the name of the set (standard, pileup,...)',
0189 dest='raw')
0190
0191 parser.add_argument('-i','--useInput',
0192 help='Use recyling where available. Either all, or a comma separated list of wf number.',
0193 dest='useInput',
0194 type=lambda x: x.split(','),
0195 default=None)
0196
0197 parser.add_argument('-w','--what',
0198 help='Specify the set to be used. Argument must be the name of a set (standard, pileup,...) or multiple sets separated by commas (--what standard,pileup )',
0199 dest='what',
0200 default='all')
0201
0202 parser.add_argument('--step1',
0203 help='Used with --raw. Limit the production to step1',
0204 dest='step1Only',
0205 default=False)
0206
0207 parser.add_argument('--maxSteps',
0208 help='Only run maximum on maxSteps. Used when we are only interested in first n steps.',
0209 dest='maxSteps',
0210 default=9999,
0211 type=int)
0212
0213 parser.add_argument('--fromScratch',
0214 help='Comma separated list of wf to be run without recycling. all is not supported as default.',
0215 dest='fromScratch',
0216 type=lambda x: x.split(','),
0217 default=None)
0218
0219 parser.add_argument('--refRelease',
0220 help='Allow to modify the recycling dataset version',
0221 dest='refRel',
0222 default=None)
0223
0224 parser.add_argument('--wmcontrol',
0225 help='Create the workflows for injection to WMAgent. In the WORKING. -wmcontrol init will create the the workflows, -wmcontrol test will dryRun a test, -wmcontrol submit will submit to wmagent',
0226 choices=['init','test','submit','force'],
0227 dest='wmcontrol',
0228 default=None)
0229
0230 parser.add_argument('--revertDqmio',
0231 help='When submitting workflows to wmcontrol, force DQM outout to use pool and not DQMIO',
0232 choices=['yes','no'],
0233 dest='revertDqmio',
0234 default='no')
0235
0236 parser.add_argument('--optionswm',
0237 help='Specify a few things for wm injection',
0238 default='',
0239 dest='wmoptions')
0240
0241 parser.add_argument('--keep',
0242 help='allow to specify for which comma separated steps the output is needed',
0243 default=None)
0244
0245 parser.add_argument('--label',
0246 help='allow to give a special label to the output dataset name',
0247 default='')
0248
0249 parser.add_argument('--command',
0250 help='provide a way to add additional command to all of the cmsDriver commands in the matrix',
0251 dest='command',
0252 action='append',
0253 default=None)
0254
0255 parser.add_argument('--apply',
0256 help='allow to use the --command only for 1 comma separeated',
0257 dest='apply',
0258 default=None)
0259
0260 parser.add_argument('--workflow',
0261 help='define a workflow to be created or altered from the matrix',
0262 action='append',
0263 dest='workflow',
0264 default=None)
0265
0266 parser.add_argument('--dryRun',
0267 help='do not run the wf at all',
0268 action='store_true',
0269 dest='dryRun',
0270 default=False)
0271
0272 parser.add_argument('--testbed',
0273 help='workflow injection to cmswebtest (you need dedicated rqmgr account)',
0274 dest='testbed',
0275 default=False,
0276 action='store_true')
0277
0278 parser.add_argument('--noCafVeto',
0279 help='Run from any source, ignoring the CAF label',
0280 dest='cafVeto',
0281 default=True,
0282 action='store_false')
0283
0284 parser.add_argument('--overWrite',
0285 help='Change the content of a step for another. List of pairs.',
0286 dest='overWrite',
0287 default=None)
0288
0289 parser.add_argument('--noRun',
0290 help='Remove all run list selection from wfs',
0291 dest='noRun',
0292 default=False,
0293 action='store_true')
0294
0295 parser.add_argument('--das-options',
0296 help='Options to be passed to dasgoclient.',
0297 dest='dasOptions',
0298 default="--limit 0",
0299 action='store')
0300
0301 parser.add_argument('--job-reports',
0302 help='Dump framework job reports',
0303 dest='jobReports',
0304 default=False,
0305 action='store_true')
0306
0307 parser.add_argument('--ibeos',
0308 help='Use IB EOS site configuration',
0309 dest='IBEos',
0310 default=False,
0311 action='store_true')
0312
0313 parser.add_argument('--sites',
0314 help='Run DAS query to get data from a specific site. Set it to empty string to search all sites.',
0315 dest='dasSites',
0316 default='T2_CH_CERN',
0317 action='store')
0318
0319 parser.add_argument('--interactive',
0320 help="Open the Matrix interactive shell",
0321 action='store_true',
0322 default=False)
0323
0324 parser.add_argument('--dbs-url',
0325 help='Overwrite DbsUrl value in JSON submitted to ReqMgr2',
0326 dest='dbsUrl',
0327 default=None,
0328 action='store')
0329
0330 gpugroup = parser.add_argument_group('GPU-related options','These options are only meaningful when --gpu is used, and is not set to forbidden.')
0331
0332 gpugroup.add_argument('--gpu','--requires-gpu',
0333 help='Enable GPU workflows. Possible options are "forbidden" (default), "required" (implied if no argument is given), or "optional".',
0334 dest='gpu',
0335 choices=['forbidden', 'optional', 'required'],
0336 nargs='?',
0337 const='required',
0338 default='forbidden',
0339 action='store')
0340
0341 gpugroup.add_argument('--gpu-memory',
0342 help='Specify the minimum amount of GPU memory required by the job, in MB.',
0343 dest='GPUMemoryMB',
0344 type=int,
0345 default=8000)
0346
0347 gpugroup.add_argument('--cuda-capabilities',
0348 help='Specify a comma-separated list of CUDA "compute capabilities", or GPU hardware architectures, that the job can use.',
0349 dest='CUDACapabilities',
0350 type=lambda x: x.split(','),
0351 default='6.0,6.1,6.2,7.0,7.2,7.5,8.0,8.6')
0352
0353
0354 cudart_version = None
0355 libcudart = os.path.realpath(os.path.expandvars('$CMSSW_RELEASE_BASE/external/$SCRAM_ARCH/lib/libcudart.so'))
0356 if os.path.isfile(libcudart):
0357 cudart_basename = os.path.basename(libcudart)
0358 cudart_version = '.'.join(cudart_basename.split('.')[2:4])
0359 gpugroup.add_argument('--cuda-runtime',
0360 help='Specify major and minor version of the CUDA runtime used to build the application.',
0361 dest='CUDARuntime',
0362 default=cudart_version)
0363
0364 gpugroup.add_argument('--force-gpu-name',
0365 help='Request a specific GPU model, e.g. "Tesla T4" or "NVIDIA GeForce RTX 2080". The default behaviour is to accept any supported GPU.',
0366 dest='GPUName',
0367 default='')
0368
0369 gpugroup.add_argument('--force-cuda-driver-version',
0370 help='Request a specific CUDA driver version, e.g. 470.57.02. The default behaviour is to accept any supported CUDA driver version.',
0371 dest='CUDADriverVersion',
0372 default='')
0373
0374 gpugroup.add_argument('--force-cuda-runtime-version',
0375 help='Request a specific CUDA runtime version, e.g. 11.4. The default behaviour is to accept any supported CUDA runtime version.',
0376 dest='CUDARuntimeVersion',
0377 default='')
0378
0379 opt = parser.parse_args()
0380 if opt.command: opt.command = ' '.join(opt.command)
0381 os.environ["CMSSW_DAS_QUERY_SITES"]=opt.dasSites
0382 if opt.IBEos:
0383 from subprocess import getstatusoutput as run_cmd
0384
0385 ibeos_cache = os.path.join(os.getenv("LOCALRT"), "ibeos_cache.txt")
0386 if not os.path.exists(ibeos_cache):
0387 err, out = run_cmd("curl -L -s -o %s https://raw.githubusercontent.com/cms-sw/cms-sw.github.io/master/das_queries/ibeos.txt" % ibeos_cache)
0388 if err:
0389 run_cmd("rm -f %s" % ibeos_cache)
0390 print("Error: Unable to download ibeos cache information")
0391 print(out)
0392 sys.exit(err)
0393
0394 for cmssw_env in [ "CMSSW_BASE", "CMSSW_RELEASE_BASE" ]:
0395 cmssw_base = os.getenv(cmssw_env,None)
0396 if not cmssw_base: continue
0397 cmssw_base = os.path.join(cmssw_base,"src/Utilities/General/ibeos")
0398 if os.path.exists(cmssw_base):
0399 os.environ["PATH"]=cmssw_base+":"+os.getenv("PATH")
0400 os.environ["CMS_PATH"]="/cvmfs/cms-ib.cern.ch"
0401 os.environ["CMSSW_USE_IBEOS"]="true"
0402 print(">> WARNING: You are using SITECONF from /cvmfs/cms-ib.cern.ch")
0403 break
0404 if opt.restricted:
0405 print('Deprecated, please use -l limited')
0406 if opt.testList: opt.testList+=',limited'
0407 else: opt.testList='limited'
0408
0409 def stepOrIndex(s):
0410 if s.isdigit():
0411 return int(s)
0412 else:
0413 return s
0414 if opt.apply:
0415 opt.apply=map(stepOrIndex,opt.apply.split(','))
0416 if opt.keep:
0417 opt.keep=map(stepOrIndex,opt.keep.split(','))
0418
0419 if opt.testList:
0420 testList=[]
0421 for entry in opt.testList.split(','):
0422 if not entry: continue
0423 mapped=False
0424 for k in predefinedSet:
0425 if k.lower().startswith(entry.lower()) or k.lower().endswith(entry.lower()):
0426 testList.extend(predefinedSet[k])
0427 mapped=True
0428 break
0429 if not mapped:
0430 try:
0431 testList.append(float(entry))
0432 except:
0433 print(entry,'is not a possible selected entry')
0434
0435 opt.testList = list(set(testList))
0436
0437 if opt.wmcontrol:
0438 performInjectionOptionTest(opt)
0439 if opt.overWrite:
0440 opt.overWrite=eval(opt.overWrite)
0441 if opt.interactive:
0442 import cmd
0443
0444 class TheMatrix(cmd.Cmd):
0445 intro = "Welcome to the Matrix (? for help)"
0446 prompt = "matrix> "
0447
0448 def __init__(self, opt):
0449 cmd.Cmd.__init__(self)
0450 self.opt_ = opt
0451 self.matrices_ = {}
0452 tmp = MatrixReader(self.opt_)
0453 for what in tmp.files:
0454 what = what.replace('relval_','')
0455 self.opt_.what = what
0456 self.matrices_[what] = MatrixReader(self.opt_)
0457 self.matrices_[what].prepare(self.opt_.useInput, self.opt_.refRel,
0458 self.opt_.fromScratch)
0459 os.system("clear")
0460
0461 def do_clear(self, arg):
0462 """Clear the screen, put prompt at the top"""
0463 os.system("clear")
0464
0465 def do_exit(self, arg):
0466 print("Leaving the Matrix")
0467 return True
0468
0469 def default(self, inp):
0470 if inp == 'x' or inp == 'q':
0471 return self.do_exit(inp)
0472
0473 def help_predefined(self):
0474 print("\n".join(["predefined [predef1 [...]]\n",
0475 "Run w/o argument, it will print the list of known predefined workflows.",
0476 "Run with space-separated predefined workflows, it will print the workflow-ids registered to them"]))
0477
0478 def complete_predefined(self, text, line, start_idx, end_idx):
0479 if text and len(text) > 0:
0480 return [t for t in predefinedSet.keys() if t.startswith(text)]
0481 else:
0482 return predefinedSet.keys()
0483
0484 def do_predefined(self, arg):
0485 """Print the list of predefined workflows"""
0486 print("List of predefined workflows")
0487 if arg:
0488 for w in arg.split():
0489 if w in predefinedSet.keys():
0490 print("Predefined Set: %s" % w)
0491 print(predefinedSet[w])
0492 else:
0493 print("Unknown Set: %s" % w)
0494 else:
0495 print(predefinedSet.keys())
0496
0497 def help_showWorkflow(self):
0498 print("\n".join(["showWorkflow [workflow1 [...]]\n",
0499 "Run w/o arguments, it will print the list of registered macro-workflows.",
0500 "Run with space-separated workflows, it will print the full list of workflow-ids registered to them"]))
0501
0502 def complete_showWorkflow(self, text, line, start_idx, end_idx):
0503 if text and len(text) > 0:
0504 return [t for t in self.matrices_.keys() if t.startswith(text)]
0505 else:
0506 return self.matrices_.keys()
0507
0508 def do_showWorkflow(self, arg):
0509 if arg == '':
0510 print("Available workflows:")
0511 for k in self.matrices_.keys():
0512 print(k)
0513 else:
0514 selected = arg.split()
0515 for k in selected:
0516 if k not in self.matrices_.keys():
0517 print("Unknown workflow %s: skipping" % k)
0518 else:
0519 for wfl in self.matrices_[k].workFlows:
0520 wfName, stepNames = wfl.nameId.split('+',1)
0521 print("%s %s %s" % (wfl.numId, wfName, stepNames))
0522 print("%s contains %d workflows" % (k, len(self.matrices_[k].workFlows)))
0523
0524 def help_searchInWorkflow(self):
0525 print("\n".join(["searchInWorkflow wfl_name search_regexp\n",
0526 "This command will search for a match within all workflows registered to wfl_name.",
0527 "The search is done on both the workflow name and the names of steps registered to it."]))
0528
0529 def complete_searchInWorkflow(self, text, line, start_idx, end_idx):
0530 if text and len(text) > 0:
0531 return [t for t in self.matrices_.keys() if t.startswith(text)]
0532 else:
0533 return self.matrices_.keys()
0534
0535 def do_searchInWorkflow(self, arg):
0536 args = arg.split()
0537 if len(args) < 2:
0538 print("searchInWorkflow name regexp")
0539 return
0540 if args[0] not in self.matrices_.keys():
0541 print("Unknown workflow")
0542 return
0543 import re
0544 pattern = None
0545 try:
0546 pattern = re.compile(args[1])
0547 except:
0548 print("Failed to compile regexp %s" % args[1])
0549 return
0550 counter = 0
0551 for wfl in self.matrices_[args[0]].workFlows:
0552 wfName, stepNames = wfl.nameId.split('+',1)
0553 if re.match(pattern, wfName) or re.match(pattern, stepNames):
0554 print("%s %s %s" % (wfl.numId, wfName, stepNames))
0555 counter += 1
0556 print("Found %d compatible workflows inside %s" % (counter, args[0]))
0557
0558 def help_search(self):
0559 print("\n".join(["search search_regexp\n",
0560 "This command will search for a match within all workflows registered.",
0561 "The search is done on both the workflow name and the names of steps registered to it."]))
0562
0563 def do_search(self, arg):
0564 args = arg.split()
0565 if len(args) < 1:
0566 print("search regexp")
0567 return
0568 for wfl in self.matrices_.keys():
0569 self.do_searchInWorkflow(' '.join([wfl, args[0]]))
0570
0571 def help_dumpWorkflowId(self):
0572 print("\n".join(["dumpWorkflowId [wfl-id1 [...]]\n",
0573 "Dumps the details (cmsDriver commands for all steps) of the space-separated workflow-ids in input."]))
0574
0575 def do_dumpWorkflowId(self, arg):
0576 wflids = arg.split()
0577 if len(wflids) == 0:
0578 print("dumpWorkflowId [wfl-id1 [...]]")
0579 return
0580
0581 fmt = "[%d]: %s\n"
0582 maxLen = 100
0583 for wflid in wflids:
0584 dump = True
0585 for key, mrd in self.matrices_.items():
0586 for wfl in mrd.workFlows:
0587 if wfl.numId == float(wflid):
0588 wfName, stepNames = wfl.nameId.split('+',1)
0589 if dump:
0590 dump = False
0591 print(wfl.numId, stepNames)
0592 for i,s in enumerate(wfl.cmds):
0593 print(fmt % (i+1, (str(s)+' ')))
0594 print("\nWorkflow found in %s." % key)
0595 else:
0596 print("Workflow also found in %s." % key)
0597
0598 do_EOF = do_exit
0599
0600 TheMatrix(opt).cmdloop()
0601 sys.exit(0)
0602
0603 if opt.raw and opt.show:
0604 ret = showRaw(opt)
0605 else:
0606 ret = runSelected(opt)
0607
0608
0609 sys.exit(ret)