File indexing completed on 2023-05-17 02:07:24
0001
0002 from __future__ import print_function
0003 import sys, os
0004
0005 from Configuration.PyReleaseValidation.MatrixReader import MatrixReader
0006 from Configuration.PyReleaseValidation.MatrixRunner import MatrixRunner
0007 from Configuration.PyReleaseValidation.MatrixInjector import MatrixInjector,performInjectionOptionTest
0008
0009
0010
0011 def showRaw(opt):
0012
0013 mrd = MatrixReader(opt)
0014 mrd.showRaw(opt.useInput, opt.refRel, opt.fromScratch, opt.raw, opt.step1Only, selected=opt.testList)
0015
0016 return 0
0017
0018
0019
0020 def runSelected(opt):
0021
0022 mrd = MatrixReader(opt)
0023 mrd.prepare(opt.useInput, opt.refRel, opt.fromScratch)
0024
0025
0026 if opt.testList:
0027 definedWf = [dwf.numId for dwf in mrd.workFlows]
0028 definedSet = set(definedWf)
0029 testSet = set(opt.testList)
0030 undefSet = testSet - definedSet
0031 if len(undefSet)>0: raise ValueError('Undefined workflows: '+', '.join(map(str,list(undefSet))))
0032 duplicates = [wf for wf in testSet if definedWf.count(wf)>1 ]
0033 if len(duplicates)>0: raise ValueError('Duplicated workflows: '+', '.join(map(str,list(duplicates))))
0034
0035 ret = 0
0036 if opt.show:
0037 mrd.show(opt.testList, opt.extended, opt.cafVeto)
0038 if opt.testList : print('testListected items:', opt.testList)
0039 else:
0040 mRunnerHi = MatrixRunner(mrd.workFlows, opt.nProcs, opt.nThreads)
0041 ret = mRunnerHi.runTests(opt)
0042
0043 if opt.wmcontrol:
0044 if ret!=0:
0045 print('Cannot go on with wmagent injection with failing workflows')
0046 else:
0047 wfInjector = MatrixInjector(opt,mode=opt.wmcontrol,options=opt.wmoptions)
0048 ret= wfInjector.prepare(mrd,
0049 mRunnerHi.runDirs)
0050 if ret==0:
0051 wfInjector.upload()
0052 wfInjector.submit()
0053 return ret
0054
0055
0056
0057 if __name__ == '__main__':
0058
0059
0060 predefinedSet={
0061 'limited' : [5.1,
0062 7.3,
0063 8,
0064 25,
0065 4.22,
0066 4.53,
0067 9.0,
0068 1000,
0069 1001,
0070 101.0,
0071 136.731,
0072 136.7611,
0073 136.8311,
0074 136.88811,
0075 136.793,
0076 136.874,
0077 138.4,
0078 138.5,
0079 139.001,
0080 140.53,
0081 140.56,
0082 158.01,
0083 312.0,
0084 1306.0,
0085 2500.601,
0086 1330,
0087 135.4,
0088 10042.0,
0089 10024.0,
0090 10824.0,
0091 2018.1,
0092 11634.911,
0093 11634.914,
0094 11634.0,
0095 11834.0,
0096 13234.0,
0097 13434.0,
0098 12434.0,
0099 12434.7,
0100 23634.0,
0101 23634.911,
0102 23834.999,
0103 23696.0,
0104 23700.0,
0105 23234.0,
0106 25202.0,
0107 250202.181,
0108 ],
0109 'jetmc': [5.1, 13, 15, 25, 38, 39],
0110 'metmc' : [5.1, 15, 25, 37, 38, 39],
0111 'muonmc' : [5.1, 124.4, 124.5, 20, 21, 22, 23, 25, 30],
0112 }
0113
0114
0115 import argparse
0116 usage = 'usage: runTheMatrix.py --show -s '
0117
0118 parser = argparse.ArgumentParser(usage,formatter_class=argparse.ArgumentDefaultsHelpFormatter)
0119
0120 parser.add_argument('-b','--batchName',
0121 help='relval batch: suffix to be appended to Campaign name',
0122 dest='batchName',
0123 default='')
0124
0125 parser.add_argument('-m','--memoryOffset',
0126 help='memory of the wf for single core',
0127 dest='memoryOffset',
0128 type=int,
0129 default=3000)
0130
0131 parser.add_argument('--addMemPerCore',
0132 help='increase of memory per each n > 1 core: memory(n_core) = memoryOffset + (n_core-1) * memPerCore',
0133 dest='memPerCore',
0134 type=int,
0135 default=1500)
0136
0137 parser.add_argument('-j','--nproc',
0138 help='number of processes. 0 Will use 4 processes, not execute anything but create the wfs',
0139 dest='nProcs',
0140 type=int,
0141 default=4)
0142
0143 parser.add_argument('-t','--nThreads',
0144 help='number of threads per process to use in cmsRun.',
0145 dest='nThreads',
0146 type=int,
0147 default=1)
0148
0149 parser.add_argument('--nStreams',
0150 help='number of streams to use in cmsRun.',
0151 dest='nStreams',
0152 type=int,
0153 default=0)
0154
0155 parser.add_argument('--nEvents',
0156 help='number of events to process in cmsRun. If 0 will use the standard 10 events.',
0157 dest='nEvents',
0158 type=int,
0159 default=0)
0160
0161 parser.add_argument('--numberEventsInLuminosityBlock',
0162 help='number of events in a luminosity block',
0163 dest='numberEventsInLuminosityBlock',
0164 type=int,
0165 default=-1)
0166
0167 parser.add_argument('-n','--showMatrix',
0168 help='Only show the worflows. Use --ext to show more',
0169 dest='show',
0170 default=False,
0171 action='store_true')
0172
0173 parser.add_argument('-e','--extended',
0174 help='Show details of workflows, used with --show',
0175 dest='extended',
0176 default=False,
0177 action='store_true')
0178
0179 parser.add_argument('-s','--selected',
0180 help='Run a pre-defined selected matrix of wf. Deprecated, please use -l limited',
0181 dest='restricted',
0182 default=False,
0183 action='store_true')
0184
0185 parser.add_argument('-l','--list',
0186 help='Comma separated list of workflow to be shown or ran. Possible keys are also '+str(predefinedSet.keys())+'. and wild card like muon, or mc',
0187 dest='testList',
0188 default=None)
0189
0190 parser.add_argument('-f','--failed-from',
0191 help='Provide a matrix report to specify the workflows to be run again. Augments the -l option if specified already',
0192 dest='failed_from',
0193 default=None)
0194
0195 parser.add_argument('-r','--raw',
0196 help='Temporary dump the .txt needed for prodAgent interface. To be discontinued soon. Argument must be the name of the set (standard, pileup,...)',
0197 dest='raw')
0198
0199 parser.add_argument('-i','--useInput',
0200 help='Use recyling where available. Either all, or a comma separated list of wf number.',
0201 dest='useInput',
0202 type=lambda x: x.split(','),
0203 default=None)
0204
0205 parser.add_argument('-w','--what',
0206 help='Specify the set to be used. Argument must be the name of a set (standard, pileup,...) or multiple sets separated by commas (--what standard,pileup )',
0207 dest='what',
0208 default='all')
0209
0210 parser.add_argument('--step1',
0211 help='Used with --raw. Limit the production to step1',
0212 dest='step1Only',
0213 default=False)
0214
0215 parser.add_argument('--maxSteps',
0216 help='Only run maximum on maxSteps. Used when we are only interested in first n steps.',
0217 dest='maxSteps',
0218 default=9999,
0219 type=int)
0220
0221 parser.add_argument('--fromScratch',
0222 help='Comma separated list of wf to be run without recycling. all is not supported as default.',
0223 dest='fromScratch',
0224 type=lambda x: x.split(','),
0225 default=None)
0226
0227 parser.add_argument('--refRelease',
0228 help='Allow to modify the recycling dataset version',
0229 dest='refRel',
0230 default=None)
0231
0232 parser.add_argument('--wmcontrol',
0233 help='Create the workflows for injection to WMAgent. In the WORKING. -wmcontrol init will create the the workflows, -wmcontrol test will dryRun a test, -wmcontrol submit will submit to wmagent',
0234 choices=['init','test','submit','force'],
0235 dest='wmcontrol',
0236 default=None)
0237
0238 parser.add_argument('--revertDqmio',
0239 help='When submitting workflows to wmcontrol, force DQM outout to use pool and not DQMIO',
0240 choices=['yes','no'],
0241 dest='revertDqmio',
0242 default='no')
0243
0244 parser.add_argument('--optionswm',
0245 help='Specify a few things for wm injection',
0246 default='',
0247 dest='wmoptions')
0248
0249 parser.add_argument('--keep',
0250 help='allow to specify for which comma separated steps the output is needed',
0251 default=None)
0252
0253 parser.add_argument('--label',
0254 help='allow to give a special label to the output dataset name',
0255 default='')
0256
0257 parser.add_argument('--command',
0258 help='provide a way to add additional command to all of the cmsDriver commands in the matrix',
0259 dest='command',
0260 action='append',
0261 default=None)
0262
0263 parser.add_argument('--apply',
0264 help='allow to use the --command only for 1 comma separeated',
0265 dest='apply',
0266 default=None)
0267
0268 parser.add_argument('--workflow',
0269 help='define a workflow to be created or altered from the matrix',
0270 action='append',
0271 dest='workflow',
0272 default=None)
0273
0274 parser.add_argument('--dryRun',
0275 help='do not run the wf at all',
0276 action='store_true',
0277 dest='dryRun',
0278 default=False)
0279
0280 parser.add_argument('--testbed',
0281 help='workflow injection to cmswebtest (you need dedicated rqmgr account)',
0282 dest='testbed',
0283 default=False,
0284 action='store_true')
0285
0286 parser.add_argument('--noCafVeto',
0287 help='Run from any source, ignoring the CAF label',
0288 dest='cafVeto',
0289 default=True,
0290 action='store_false')
0291
0292 parser.add_argument('--overWrite',
0293 help='Change the content of a step for another. List of pairs.',
0294 dest='overWrite',
0295 default=None)
0296
0297 parser.add_argument('--noRun',
0298 help='Remove all run list selection from wfs',
0299 dest='noRun',
0300 default=False,
0301 action='store_true')
0302
0303 parser.add_argument('--das-options',
0304 help='Options to be passed to dasgoclient.',
0305 dest='dasOptions',
0306 default="--limit 0",
0307 action='store')
0308
0309 parser.add_argument('--job-reports',
0310 help='Dump framework job reports',
0311 dest='jobReports',
0312 default=False,
0313 action='store_true')
0314
0315 parser.add_argument('--ibeos',
0316 help='Use IB EOS site configuration',
0317 dest='IBEos',
0318 default=False,
0319 action='store_true')
0320
0321 parser.add_argument('--sites',
0322 help='Run DAS query to get data from a specific site. Set it to empty string to search all sites.',
0323 dest='dasSites',
0324 default='T2_CH_CERN',
0325 action='store')
0326
0327 parser.add_argument('--interactive',
0328 help="Open the Matrix interactive shell",
0329 action='store_true',
0330 default=False)
0331
0332 parser.add_argument('--dbs-url',
0333 help='Overwrite DbsUrl value in JSON submitted to ReqMgr2',
0334 dest='dbsUrl',
0335 default=None,
0336 action='store')
0337
0338 gpugroup = parser.add_argument_group('GPU-related options','These options are only meaningful when --gpu is used, and is not set to forbidden.')
0339
0340 gpugroup.add_argument('--gpu','--requires-gpu',
0341 help='Enable GPU workflows. Possible options are "forbidden" (default), "required" (implied if no argument is given), or "optional".',
0342 dest='gpu',
0343 choices=['forbidden', 'optional', 'required'],
0344 nargs='?',
0345 const='required',
0346 default='forbidden',
0347 action='store')
0348
0349 gpugroup.add_argument('--gpu-memory',
0350 help='Specify the minimum amount of GPU memory required by the job, in MB.',
0351 dest='GPUMemoryMB',
0352 type=int,
0353 default=8000)
0354
0355 gpugroup.add_argument('--cuda-capabilities',
0356 help='Specify a comma-separated list of CUDA "compute capabilities", or GPU hardware architectures, that the job can use.',
0357 dest='CUDACapabilities',
0358 type=lambda x: x.split(','),
0359 default='6.0,6.1,6.2,7.0,7.2,7.5,8.0,8.6')
0360
0361
0362 cudart_version = None
0363 libcudart = os.path.realpath(os.path.expandvars('$CMSSW_RELEASE_BASE/external/$SCRAM_ARCH/lib/libcudart.so'))
0364 if os.path.isfile(libcudart):
0365 cudart_basename = os.path.basename(libcudart)
0366 cudart_version = '.'.join(cudart_basename.split('.')[2:4])
0367 gpugroup.add_argument('--cuda-runtime',
0368 help='Specify major and minor version of the CUDA runtime used to build the application.',
0369 dest='CUDARuntime',
0370 default=cudart_version)
0371
0372 gpugroup.add_argument('--force-gpu-name',
0373 help='Request a specific GPU model, e.g. "Tesla T4" or "NVIDIA GeForce RTX 2080". The default behaviour is to accept any supported GPU.',
0374 dest='GPUName',
0375 default='')
0376
0377 gpugroup.add_argument('--force-cuda-driver-version',
0378 help='Request a specific CUDA driver version, e.g. 470.57.02. The default behaviour is to accept any supported CUDA driver version.',
0379 dest='CUDADriverVersion',
0380 default='')
0381
0382 gpugroup.add_argument('--force-cuda-runtime-version',
0383 help='Request a specific CUDA runtime version, e.g. 11.4. The default behaviour is to accept any supported CUDA runtime version.',
0384 dest='CUDARuntimeVersion',
0385 default='')
0386
0387 opt = parser.parse_args()
0388 if opt.command: opt.command = ' '.join(opt.command)
0389 os.environ["CMSSW_DAS_QUERY_SITES"]=opt.dasSites
0390 if opt.failed_from:
0391 rerunthese=[]
0392 with open(opt.failed_from,'r') as report:
0393 for report_line in report:
0394 if 'FAILED' in report_line:
0395 to_run,_=report_line.split('_',1)
0396 rerunthese.append(to_run)
0397 if opt.testList:
0398 opt.testList+=','.join(['']+rerunthese)
0399 else:
0400 opt.testList = ','.join(rerunthese)
0401
0402 if opt.IBEos:
0403 from subprocess import getstatusoutput as run_cmd
0404
0405 ibeos_cache = os.path.join(os.getenv("LOCALRT"), "ibeos_cache.txt")
0406 if not os.path.exists(ibeos_cache):
0407 err, out = run_cmd("curl -L -s -o %s https://raw.githubusercontent.com/cms-sw/cms-sw.github.io/master/das_queries/ibeos.txt" % ibeos_cache)
0408 if err:
0409 run_cmd("rm -f %s" % ibeos_cache)
0410 print("Error: Unable to download ibeos cache information")
0411 print(out)
0412 sys.exit(err)
0413
0414 for cmssw_env in [ "CMSSW_BASE", "CMSSW_RELEASE_BASE" ]:
0415 cmssw_base = os.getenv(cmssw_env,None)
0416 if not cmssw_base: continue
0417 cmssw_base = os.path.join(cmssw_base,"src/Utilities/General/ibeos")
0418 if os.path.exists(cmssw_base):
0419 os.environ["PATH"]=cmssw_base+":"+os.getenv("PATH")
0420 os.environ["CMS_PATH"]="/cvmfs/cms-ib.cern.ch"
0421 os.environ["SITECONFIG_PATH"]="/cvmfs/cms-ib.cern.ch/SITECONF/local"
0422 os.environ["CMSSW_USE_IBEOS"]="true"
0423 print(">> WARNING: You are using SITECONF from /cvmfs/cms-ib.cern.ch")
0424 break
0425 if opt.restricted:
0426 print('Deprecated, please use -l limited')
0427 if opt.testList: opt.testList+=',limited'
0428 else: opt.testList='limited'
0429
0430 def stepOrIndex(s):
0431 if s.isdigit():
0432 return int(s)
0433 else:
0434 return s
0435 if opt.apply:
0436 opt.apply=map(stepOrIndex,opt.apply.split(','))
0437 if opt.keep:
0438 opt.keep=map(stepOrIndex,opt.keep.split(','))
0439
0440 if opt.testList:
0441 testList=[]
0442 for entry in opt.testList.split(','):
0443 if not entry: continue
0444 mapped=False
0445 for k in predefinedSet:
0446 if k.lower().startswith(entry.lower()) or k.lower().endswith(entry.lower()):
0447 testList.extend(predefinedSet[k])
0448 mapped=True
0449 break
0450 if not mapped:
0451 try:
0452 testList.append(float(entry))
0453 except:
0454 print(entry,'is not a possible selected entry')
0455
0456 opt.testList = list(set(testList))
0457
0458 if opt.wmcontrol:
0459 performInjectionOptionTest(opt)
0460 if opt.overWrite:
0461 opt.overWrite=eval(opt.overWrite)
0462 if opt.interactive:
0463 import cmd
0464 from colorama import Fore, Style
0465 from os import isatty
0466
0467 class TheMatrix(cmd.Cmd):
0468 intro = "Welcome to the Matrix (? for help)"
0469 prompt = "matrix> "
0470
0471 def __init__(self, opt):
0472 cmd.Cmd.__init__(self)
0473 self.opt_ = opt
0474 self.matrices_ = {}
0475 tmp = MatrixReader(self.opt_)
0476 for what in tmp.files:
0477 what = what.replace('relval_','')
0478 self.opt_.what = what
0479 self.matrices_[what] = MatrixReader(self.opt_)
0480 self.matrices_[what].prepare(self.opt_.useInput, self.opt_.refRel,
0481 self.opt_.fromScratch)
0482 os.system("clear")
0483
0484 def do_clear(self, arg):
0485 """Clear the screen, put prompt at the top"""
0486 os.system("clear")
0487
0488 def do_exit(self, arg):
0489 print("Leaving the Matrix")
0490 return True
0491
0492 def default(self, inp):
0493 if inp == 'x' or inp == 'q':
0494 return self.do_exit(inp)
0495 else:
0496 is_pipe = not isatty(sys.stdin.fileno())
0497 print(Fore.RED + "Error: " + Fore.RESET + "unrecognized command.")
0498
0499 if is_pipe:
0500 sys.exit(1)
0501
0502 def help_predefined(self):
0503 print("\n".join(["predefined [predef1 [...]]\n",
0504 "Run w/o argument, it will print the list of known predefined workflows.",
0505 "Run with space-separated predefined workflows, it will print the workflow-ids registered to them"]))
0506
0507 def complete_predefined(self, text, line, start_idx, end_idx):
0508 if text and len(text) > 0:
0509 return [t for t in predefinedSet.keys() if t.startswith(text)]
0510 else:
0511 return predefinedSet.keys()
0512
0513 def do_predefined(self, arg):
0514 """Print the list of predefined workflows"""
0515 print("List of predefined workflows")
0516 if arg:
0517 for w in arg.split():
0518 if w in predefinedSet.keys():
0519 print("Predefined Set: %s" % w)
0520 print(predefinedSet[w])
0521 else:
0522 print("Unknown Set: %s" % w)
0523 else:
0524 print("[ " + Fore.RED + ", ".join([str(k) for k in predefinedSet.keys()]) + Fore.RESET + " ]")
0525
0526 def help_showWorkflow(self):
0527 print("\n".join(["showWorkflow [workflow1 [...]]\n",
0528 "Run w/o arguments, it will print the list of registered macro-workflows.",
0529 "Run with space-separated workflows, it will print the full list of workflow-ids registered to them"]))
0530
0531 def complete_showWorkflow(self, text, line, start_idx, end_idx):
0532 if text and len(text) > 0:
0533 return [t for t in self.matrices_.keys() if t.startswith(text)]
0534 else:
0535 return self.matrices_.keys()
0536
0537 def do_showWorkflow(self, arg):
0538 if arg == '':
0539 print("Available workflows:")
0540 for k in self.matrices_.keys():
0541 print(Fore.RED + Style.BRIGHT + k)
0542 print(Style.RESET_ALL)
0543 else:
0544 selected = arg.split()
0545 for k in selected:
0546 if k not in self.matrices_.keys():
0547 print("Unknown workflow %s: skipping" % k)
0548 else:
0549 for wfl in self.matrices_[k].workFlows:
0550 print("%s %s" % (Fore.BLUE + str(wfl.numId) + Fore.RESET,
0551 Fore.GREEN + wfl.nameId + Fore.RESET))
0552 print("%s contains %d workflows" % (Fore.RED + k + Fore.RESET, len(self.matrices_[k].workFlows)))
0553
0554 def help_searchInWorkflow(self):
0555 print("\n".join(["searchInWorkflow wfl_name search_regexp\n",
0556 "This command will search for a match within all workflows registered to wfl_name.",
0557 "The search is done on both the workflow name and the names of steps registered to it."]))
0558
0559 def complete_searchInWorkflow(self, text, line, start_idx, end_idx):
0560 if text and len(text) > 0:
0561 return [t for t in self.matrices_.keys() if t.startswith(text)]
0562 else:
0563 return self.matrices_.keys()
0564
0565 def do_searchInWorkflow(self, arg):
0566 args = arg.split()
0567 if len(args) < 2:
0568 print("searchInWorkflow name regexp")
0569 return
0570 if args[0] not in self.matrices_.keys():
0571 print("Unknown workflow")
0572 return
0573 import re
0574 pattern = None
0575 try:
0576 pattern = re.compile(args[1])
0577 except:
0578 print("Failed to compile regexp %s" % args[1])
0579 return
0580 counter = 0
0581 for wfl in self.matrices_[args[0]].workFlows:
0582 if re.match(pattern, wfl.nameId):
0583 print("%s %s" % (Fore.BLUE + str(wfl.numId) + Fore.RESET,
0584 Fore.GREEN + wfl.nameId + Fore.RESET))
0585 counter +=1
0586 print("Found %s compatible workflows inside %s" % (Fore.RED + str(counter) + Fore.RESET,
0587 Fore.YELLOW + str(args[0])) + Fore.RESET)
0588
0589 def help_search(self):
0590 print("\n".join(["search search_regexp\n",
0591 "This command will search for a match within all workflows registered.",
0592 "The search is done on both the workflow name and the names of steps registered to it."]))
0593
0594 def do_search(self, arg):
0595 args = arg.split()
0596 if len(args) < 1:
0597 print("search regexp")
0598 return
0599 for wfl in self.matrices_.keys():
0600 self.do_searchInWorkflow(' '.join([wfl, args[0]]))
0601
0602 def help_dumpWorkflowId(self):
0603 print("\n".join(["dumpWorkflowId [wfl-id1 [...]]\n",
0604 "Dumps the details (cmsDriver commands for all steps) of the space-separated workflow-ids in input."]))
0605
0606 def do_dumpWorkflowId(self, arg):
0607 wflids = arg.split()
0608 if len(wflids) == 0:
0609 print("dumpWorkflowId [wfl-id1 [...]]")
0610 return
0611
0612 fmt = "[%s]: %s\n"
0613 maxLen = 100
0614 for wflid in wflids:
0615 dump = True
0616 for key, mrd in self.matrices_.items():
0617 for wfl in mrd.workFlows:
0618 if wfl.numId == float(wflid):
0619 if dump:
0620 dump = False
0621 print(Fore.GREEN + str(wfl.numId) + Fore.RESET + " " + Fore.YELLOW + wfl.nameId + Fore.RESET)
0622 for i,s in enumerate(wfl.cmds):
0623 print(fmt % (Fore.RED + str(i+1) + Fore.RESET,
0624 (str(s)+' ')))
0625 print("\nWorkflow found in %s." % key)
0626 else:
0627 print("Workflow also found in %s." % key)
0628
0629 do_EOF = do_exit
0630
0631 TheMatrix(opt).cmdloop()
0632 sys.exit(0)
0633
0634 if opt.raw and opt.show:
0635 ret = showRaw(opt)
0636 else:
0637 ret = runSelected(opt)
0638
0639
0640 sys.exit(ret)