File indexing completed on 2022-05-13 01:31:53
0001 from __future__ import print_function
0002 import sys, os
0003
0004 from Configuration.PyReleaseValidation.WorkFlow import WorkFlow
0005 from Configuration.PyReleaseValidation.MatrixUtil import InputInfo
0006
0007
0008
0009 class MatrixException(Exception):
0010 def __init__(self, msg):
0011 self.msg = msg
0012 def __str__(self):
0013 return self.msg
0014
0015
0016
0017 class MatrixReader(object):
0018
0019 def __init__(self, opt):
0020
0021 self.reset(opt.what)
0022
0023 self.wm=opt.wmcontrol
0024 self.revertDqmio=opt.revertDqmio
0025 self.addCommand=opt.command
0026 self.apply=opt.apply
0027 self.commandLineWf=opt.workflow
0028 self.overWrite=opt.overWrite
0029
0030 self.noRun = opt.noRun
0031 return
0032
0033 def reset(self, what='all'):
0034
0035 self.what = what
0036
0037
0038 self.workFlowSteps = {}
0039
0040 self.workFlows = []
0041 self.nameList = {}
0042
0043 self.filesPrefMap = {'relval_standard' : 'std-' ,
0044 'relval_highstats': 'hi-' ,
0045 'relval_pileup': 'PU-' ,
0046 'relval_generator': 'gen-',
0047 'relval_extendedgen': 'genExt-',
0048 'relval_production': 'prod-' ,
0049 'relval_ged': 'ged-',
0050 'relval_upgrade':'upg-',
0051 'relval_cleanedupgrade':'clnupg-',
0052 'relval_gpu':'gpu-',
0053 'relval_2017':'2017-',
0054 'relval_2026':'2026-',
0055 'relval_identity':'id-',
0056 'relval_machine': 'mach-',
0057 'relval_premix': 'premix-'
0058 }
0059
0060 self.files = ['relval_standard' ,
0061 'relval_highstats',
0062 'relval_pileup',
0063 'relval_generator',
0064 'relval_extendedgen',
0065 'relval_production',
0066 'relval_ged',
0067 'relval_upgrade',
0068 'relval_cleanedupgrade',
0069 'relval_gpu',
0070 'relval_2017',
0071 'relval_2026',
0072 'relval_identity',
0073 'relval_machine',
0074 'relval_premix'
0075 ]
0076 self.filesDefault = {'relval_standard':True ,
0077 'relval_highstats':True ,
0078 'relval_pileup':True,
0079 'relval_generator':True,
0080 'relval_extendedgen':True,
0081 'relval_production':True,
0082 'relval_ged':True,
0083 'relval_upgrade':False,
0084 'relval_cleanedupgrade':False,
0085 'relval_gpu':False,
0086 'relval_2017':True,
0087 'relval_2026':True,
0088 'relval_identity':False,
0089 'relval_machine':True,
0090 'relval_premix':True
0091 }
0092
0093 self.relvalModule = None
0094
0095 return
0096
0097 def makeCmd(self, step):
0098
0099 cmd = ''
0100 cfg = None
0101 input = None
0102 for k,v in step.items():
0103 if 'no_exec' in k : continue
0104 if k.lower() == 'cfg':
0105 cfg = v
0106 continue
0107 if k.lower() == 'input':
0108 input = v
0109 continue
0110
0111
0112
0113
0114 cmd += ' ' + k + ' ' + str(v)
0115 return cfg, input, cmd
0116
0117 def makeStep(self,step,overrides):
0118 from Configuration.PyReleaseValidation.relval_steps import merge
0119 if len(overrides) > 0:
0120 copyStep=merge([overrides]+[step])
0121 return copyStep
0122 else:
0123 return step
0124
0125 def readMatrix(self, fileNameIn, useInput=None, refRel=None, fromScratch=None):
0126
0127 prefix = self.filesPrefMap[fileNameIn]
0128
0129 print("processing", fileNameIn)
0130
0131 try:
0132 _tmpMod = __import__( 'Configuration.PyReleaseValidation.'+fileNameIn )
0133 self.relvalModule = sys.modules['Configuration.PyReleaseValidation.'+fileNameIn]
0134 except Exception as e:
0135 print("ERROR importing file ", fileNameIn, str(e))
0136 return
0137
0138 if useInput is not None:
0139 print("request for INPUT for ", useInput)
0140
0141
0142 fromInput={}
0143
0144 if useInput:
0145 for i in useInput:
0146 if ':' in i:
0147 (ik,il)=i.split(':')
0148 if ik=='all':
0149 for k in self.relvalModule.workflows.keys():
0150 fromInput[float(k)]=int(il)
0151 else:
0152 fromInput[float(ik)]=int(il)
0153 else:
0154 if i=='all':
0155 for k in self.relvalModule.workflows.keys():
0156 fromInput[float(k)]=0
0157 else:
0158 fromInput[float(i)]=0
0159
0160 if fromScratch:
0161 fromScratch=map(float,fromScratch)
0162 for num in fromScratch:
0163 if num in fromInput:
0164 fromInput.pop(num)
0165
0166 if self.overWrite:
0167 for p in self.overWrite:
0168 self.relvalModule.steps.overwrite(p)
0169
0170
0171 if refRel:
0172 if ',' in refRel:
0173 refRels=refRel.split(',')
0174 if len(refRels)!=len(self.relvalModule.baseDataSetRelease):
0175 return
0176 self.relvalModule.changeRefRelease(
0177 self.relvalModule.steps,
0178 list(zip(self.relvalModule.baseDataSetRelease,refRels))
0179 )
0180 else:
0181 self.relvalModule.changeRefRelease(
0182 self.relvalModule.steps,
0183 [(x,refRel) for x in self.relvalModule.baseDataSetRelease]
0184 )
0185
0186
0187 for num, wfInfo in self.relvalModule.workflows.items():
0188 commands=[]
0189 wfName = wfInfo[0]
0190 stepList = wfInfo[1]
0191 stepOverrides=wfInfo.overrides
0192
0193 wfKey = ""
0194 wfSuffix = ""
0195 if isinstance(wfName, list) and len(wfName)>1:
0196 if len(wfName)>2: wfSuffix = wfName[2]
0197 wfKey = wfName[1]
0198 wfName = wfName[0]
0199
0200 if wfName.strip() == '': wfName = stepList[0]
0201
0202 addTo=None
0203 addCom=None
0204 if len(wfInfo)>=3:
0205 addCom=wfInfo[2]
0206 if not isinstance(addCom, list): addCom=[addCom]
0207
0208 if len(wfInfo)>=4:
0209 addTo=wfInfo[3]
0210
0211 while len(addTo)!=len(stepList):
0212 addTo.append(0)
0213
0214 name=wfName
0215
0216 if len(wfKey)>0:
0217 name = name+'+'+wfKey
0218 if len(wfSuffix)>0: name = name+wfSuffix
0219 stepIndex=0
0220 ranStepList=[]
0221
0222
0223 if num in fromInput:
0224 ilevel=fromInput[num]
0225
0226 for (stepIr,step) in enumerate(reversed(stepList)):
0227 stepName=step
0228 stepI=(len(stepList)-stepIr)-1
0229
0230 if stepI>ilevel:
0231
0232 continue
0233 if stepI!=0:
0234 testName='__'.join(stepList[0:stepI+1])+'INPUT'
0235 else:
0236 testName=step+'INPUT'
0237
0238 if testName in self.relvalModule.steps:
0239
0240 stepList[stepI]=testName
0241
0242
0243 for p in range(stepI):
0244 stepList.pop(0)
0245
0246 break
0247
0248
0249 for (stepI,step) in enumerate(stepList):
0250 stepName=step
0251 if self.relvalModule.steps[stepName] is None:
0252 continue
0253 if self.wm:
0254
0255 if stepName in ['SKIMD','SKIMCOSD','SKIMDreHLT']:
0256 continue
0257
0258
0259
0260 if len(name) > 0 : name += '+'
0261
0262
0263 """
0264 if num in fromInput:
0265 if step+'INPUT' in self.relvalModule.steps.keys():
0266 stepName = step+"INPUT"
0267 stepList.remove(step)
0268 stepList.insert(stepIndex,stepName)
0269 """
0270 stepNameTmp = stepName
0271 if len(wfKey)>0: stepNameTmp = stepNameTmp.replace('_'+wfKey,"")
0272 if len(wfSuffix)>0: stepNameTmp = stepNameTmp.replace(wfSuffix,"")
0273 name += stepNameTmp
0274 if addCom and (not addTo or addTo[stepIndex]==1):
0275 from Configuration.PyReleaseValidation.relval_steps import merge
0276 copyStep=merge(addCom+[self.makeStep(self.relvalModule.steps[stepName],stepOverrides)])
0277 cfg, input, opts = self.makeCmd(copyStep)
0278 else:
0279 cfg, input, opts = self.makeCmd(self.makeStep(self.relvalModule.steps[stepName],stepOverrides))
0280
0281 if input and cfg :
0282 msg = "FATAL ERROR: found both cfg and input for workflow "+str(num)+' step '+stepName
0283 raise MatrixException(msg)
0284
0285 if input:
0286 cmd = input
0287 if self.noRun:
0288 cmd.run=[]
0289 else:
0290 if cfg:
0291 cmd = 'cmsDriver.py '+cfg+' '+opts
0292 else:
0293 cmd = 'cmsDriver.py step'+str(stepIndex+1)+' '+opts
0294 if self.wm:
0295 cmd+=' --io %s.io --python %s.py'%(stepName,stepName)
0296 if self.addCommand:
0297 if self.apply:
0298 if stepIndex in self.apply or stepName in self.apply:
0299 cmd +=' '+self.addCommand
0300 else:
0301 cmd +=' '+self.addCommand
0302 if self.wm and self.revertDqmio=='yes':
0303 cmd=cmd.replace('DQMIO','DQM')
0304 cmd=cmd.replace('--filetype DQM','')
0305 commands.append(cmd)
0306 ranStepList.append(stepName)
0307 stepIndex+=1
0308
0309 self.workFlowSteps[(num,prefix)] = (num, name, commands, ranStepList)
0310
0311 return
0312
0313
0314 def showRaw(self, useInput, refRel=None, fromScratch=None, what='all',step1Only=False,selected=None):
0315
0316 if selected:
0317 selected=map(float,selected)
0318 for matrixFile in self.files:
0319
0320 self.reset(what)
0321
0322 if self.what != 'all' and not any('_'+el in matrixFile for el in self.what.split(",")):
0323 print("ignoring non-requested file",matrixFile)
0324 continue
0325
0326 if self.what == 'all' and not self.filesDefault[matrixFile]:
0327 print("ignoring file not used by default (enable with -w)",matrixFile)
0328 continue
0329
0330 try:
0331 self.readMatrix(matrixFile, useInput, refRel, fromScratch)
0332 except Exception as e:
0333 print("ERROR reading file:", matrixFile, str(e))
0334 raise
0335
0336 if not self.workFlowSteps: continue
0337
0338 dataFileName = matrixFile.replace('relval_', 'cmsDriver_')+'_hlt.txt'
0339 outFile = open(dataFileName,'w')
0340
0341 print("found ", len(self.workFlowSteps), ' workflows for ', dataFileName)
0342 ids = sorted(self.workFlowSteps.keys())
0343 indexAndSteps=[]
0344
0345 writtenWF=0
0346 for key in ids:
0347 if selected and not (key[0] in selected):
0348 continue
0349
0350 if key[0]==203.1 or key[0]==204.1 or key[0]==205.1 or key[0]==4.51 or key[0]==4.52: continue
0351 num, name, commands, stepList = self.workFlowSteps[key]
0352
0353 wfName,stepNames= name.split('+',1)
0354
0355 stepNames=stepNames.replace('+SKIMCOSD','')
0356 stepNames=stepNames.replace('+SKIMD','')
0357 if 'HARVEST' in stepNames:
0358
0359 exactb=stepNames.index('+HARVEST')
0360 exacte=stepNames.index('+',exactb+1) if ('+' in stepNames[exactb+1:]) else (len(stepNames))
0361 stepNames=stepNames.replace(stepNames[exactb:exacte],'')
0362 otherSteps = None
0363 if '+' in stepNames:
0364 step1,otherSteps = stepNames.split('+',1)
0365
0366 line = str(num) + ' ++ '+ wfName
0367 if otherSteps and not step1Only:
0368 line += ' ++ ' +otherSteps.replace('+',',')
0369 else:
0370 line += ' ++ none'
0371 inputInfo=None
0372 if not isinstance(commands[0],str):
0373 inputInfo=commands[0]
0374 if otherSteps:
0375 for (i,c) in enumerate(otherSteps.split('+')):
0376
0377 for p in range(len(indexAndSteps),i+2):
0378 indexAndSteps.append(set())
0379 indexAndSteps[i+1].add((c,commands[i+1]))
0380
0381 if inputInfo :
0382
0383 if step1Only: continue
0384 line += ' ++ REALDATA: '+inputInfo.dataSet
0385 if inputInfo.run!=[]: line += ', RUN:'+'|'.join(map(str,inputInfo.run))
0386 line += ', FILES: ' +str(inputInfo.files)
0387 line += ', EVENTS: '+str(inputInfo.events)
0388 if inputInfo.label!='':
0389 line += ', LABEL: ' +inputInfo.label
0390 line += ', LOCATION:'+inputInfo.location
0391 line += ' @@@'
0392 else:
0393 line += ' @@@ '+commands[0]
0394 if self.revertDqmio=='yes':
0395 line=line.replace('DQMIO','DQM')
0396 writtenWF+=1
0397 outFile.write(line+'\n')
0398
0399
0400 outFile.write('\n'+'\n')
0401 if step1Only: continue
0402
0403 for (index,s) in enumerate(indexAndSteps):
0404 for (stepName,cmd) in s:
0405 stepIndex=index+1
0406 if 'dasquery.log' in cmd: continue
0407 line = 'STEP%d ++ '%(stepIndex,) +stepName + ' @@@ '+cmd
0408 if self.revertDqmio=='yes':
0409 line=line.replace('DQMIO','DQM')
0410 outFile.write(line+'\n')
0411 outFile.write('\n'+'\n')
0412 outFile.close()
0413 print("wrote ",writtenWF, ' workflow'+('s' if (writtenWF!=1) else ''),' to ', outFile.name)
0414 return
0415
0416 def workFlowsByLocation(self, cafVeto=True):
0417
0418 onCAF = False
0419 if 'cms/caf/cms' in os.environ['CMS_PATH']:
0420 onCAF = True
0421
0422 workflows = []
0423 for workflow in self.workFlows:
0424 if isinstance(workflow.cmds[0], InputInfo):
0425 if cafVeto and (workflow.cmds[0].location == 'CAF' and not onCAF):
0426 continue
0427 workflows.append(workflow)
0428
0429 return workflows
0430
0431 def showWorkFlows(self, selected=None, extended=True, cafVeto=True):
0432 if selected: selected = list(map(float,selected))
0433 wfs = self.workFlowsByLocation(cafVeto)
0434 maxLen = 100
0435 fmt1 = "%-6s %-35s [1]: %s ..."
0436 fmt2 = " %35s [%d]: %s ..."
0437 print("\nfound a total of ", len(wfs), ' workflows:')
0438 if selected:
0439 print(" of which the following", len(selected), 'were selected:')
0440
0441 maxLen = -1
0442 fmt1 = "%-6s %-35s [1]: %s "
0443 fmt2 = " %35s [%d]: %s"
0444
0445 N=[]
0446 for wf in wfs:
0447 if selected and float(wf.numId) not in selected: continue
0448 if extended: print('')
0449
0450 for i in range(len(N),len(wf.cmds)): N.append(0)
0451 N[len(wf.cmds)-1]+=1
0452 wfName, stepNames = wf.nameId.split('+',1)
0453 for i,s in enumerate(wf.cmds):
0454 if extended:
0455 if i==0:
0456 print(fmt1 % (wf.numId, stepNames, (str(s)+' ')[:maxLen]))
0457 else:
0458 print(fmt2 % ( ' ', i+1, (str(s)+' ')[:maxLen]))
0459 else:
0460 print("%-6s %-35s "% (wf.numId, stepNames))
0461 break
0462 print('')
0463 for i,n in enumerate(N):
0464 if n: print(n,'workflows with',i+1,'steps')
0465
0466 return
0467
0468 def createWorkFlows(self, fileNameIn):
0469
0470 prefixIn = self.filesPrefMap[fileNameIn]
0471
0472
0473 keyList = self.workFlowSteps.keys()
0474 ids = []
0475 for item in keyList:
0476 id, pref = item
0477 if pref != prefixIn : continue
0478 ids.append(id)
0479 ids.sort()
0480 for key in ids:
0481 val = self.workFlowSteps[(key,prefixIn)]
0482 num, name, commands, stepList = val
0483 nameId = str(num)+'_'+name
0484 if nameId in self.nameList:
0485 print("==> duplicate name found for ", nameId)
0486 print(' keeping : ', self.nameList[nameId])
0487 print(' ignoring : ', val)
0488 else:
0489 self.nameList[nameId] = val
0490
0491 self.workFlows.append(WorkFlow(num, name, commands=commands))
0492
0493 return
0494
0495 def prepare(self, useInput=None, refRel='', fromScratch=None):
0496
0497 for matrixFile in self.files:
0498 if self.what != 'all' and not any('_'+el in matrixFile for el in self.what.split(",")):
0499 print("ignoring non-requested file",matrixFile)
0500 continue
0501 if self.what == 'all' and not self.filesDefault[matrixFile]:
0502 print("ignoring",matrixFile,"from default matrix")
0503 continue
0504
0505 try:
0506 self.readMatrix(matrixFile, useInput, refRel, fromScratch)
0507 except Exception as e:
0508 print("ERROR reading file:", matrixFile, str(e))
0509 raise
0510
0511 try:
0512 self.createWorkFlows(matrixFile)
0513 except Exception as e:
0514 print("ERROR creating workflows :", str(e))
0515 raise
0516
0517
0518 def show(self, selected=None, extended=True, cafVeto=True):
0519
0520 self.showWorkFlows(selected, extended, cafVeto)
0521 print('\n','-'*80,'\n')
0522
0523
0524 def updateDB(self):
0525
0526 import pickle
0527 pickle.dump(self.workFlows, open('theMatrix.pkl', 'w') )
0528
0529 return
0530