File indexing completed on 2024-04-06 12:12:53
0001
0002
0003
0004 from __future__ import print_function
0005 from __future__ import absolute_import
0006 import os
0007 from .Options import Options
0008 options = Options()
0009
0010
0011
0012 import sys
0013 from typing import Union
0014 from .Mixins import PrintOptions,_ParameterTypeBase,_SimpleParameterTypeBase, _Parameterizable, _ConfigureComponent, _TypedParameterizable, _Labelable, _Unlabelable, _ValidatingListBase, _modifyParametersFromDict
0015 from .Mixins import *
0016 from .Types import *
0017 from .Modules import *
0018 from .Modules import _Module
0019 from .SequenceTypes import *
0020 from .SequenceTypes import _ModuleSequenceType, _Sequenceable
0021 from .SequenceVisitors import PathValidator, EndPathValidator, FinalPathValidator, ScheduleTaskValidator, NodeVisitor, CompositeVisitor, ModuleNamesFromGlobalsVisitor
0022 from .MessageLogger import MessageLogger
0023 from . import DictTypes
0024
0025 from .ExceptionHandling import *
0026
0027
0028 if sys.getrecursionlimit()<5000:
0029 sys.setrecursionlimit(5000)
0030
0031 class edm(object):
0032 class errors(object):
0033
0034 Configuration = "{Configuration}"
0035 UnavailableAccelerator = "{UnavailableAccelerator}"
0036
0037 class EDMException(Exception):
0038 def __init__(self, error:str, message: str):
0039 super().__init__(error+"\n"+message)
0040
0041
0042 def checkImportPermission(minLevel: int = 2, allowedPatterns = []):
0043 """
0044 Raise an exception if called by special config files. This checks
0045 the call or import stack for the importing file. An exception is raised if
0046 the importing module is not in allowedPatterns and if it is called too deeply:
0047 minLevel = 2: inclusion by top lvel cfg only
0048 minLevel = 1: No inclusion allowed
0049 allowedPatterns = ['Module1','Module2/SubModule1'] allows import
0050 by any module in Module1 or Submodule1
0051 """
0052
0053 import inspect
0054 import os
0055
0056 ignorePatterns = ['FWCore/ParameterSet/Config.py', 'FWCore/ParameterSet/python/Config.py','<string>','<frozen ']
0057 CMSSWPath = [os.environ['CMSSW_BASE'],os.environ['CMSSW_RELEASE_BASE']]
0058
0059
0060 trueStack = []
0061 for item in inspect.stack():
0062 inPath = False
0063 ignore = False
0064
0065 for pattern in CMSSWPath:
0066 if item[1].find(pattern) != -1:
0067 inPath = True
0068 break
0069 if item[1].find('/') == -1:
0070 inPath = True
0071
0072 for pattern in ignorePatterns:
0073 if item[1].find(pattern) != -1:
0074 ignore = True
0075 break
0076
0077 if inPath and not ignore:
0078 trueStack.append(item[1])
0079
0080 importedFile = trueStack[0]
0081 importedBy = ''
0082 if len(trueStack) > 1:
0083 importedBy = trueStack[1]
0084
0085 for pattern in allowedPatterns:
0086 if importedBy.find(pattern) > -1:
0087 return True
0088
0089 if len(trueStack) <= minLevel:
0090 return True
0091
0092 raise ImportError("Inclusion of %s is allowed only by cfg or specified cfi files."
0093 % importedFile)
0094
0095 def findProcess(module):
0096 """Look inside the module and find the Processes it contains"""
0097 class Temp(object):
0098 pass
0099 process = None
0100 if isinstance(module,dict):
0101 if 'process' in module:
0102 p = module['process']
0103 module = Temp()
0104 module.process = p
0105 if hasattr(module,'process'):
0106 if isinstance(module.process,Process):
0107 process = module.process
0108 else:
0109 raise RuntimeError("The attribute named 'process' does not inherit from the Process class")
0110 else:
0111 raise RuntimeError("no 'process' attribute found in the module, please add one")
0112 return process
0113
0114 class Process(object):
0115 """Root class for a CMS configuration process"""
0116 _firstProcess = True
0117 def __init__(self,name: str, *Mods):
0118 """The argument 'name' will be the name applied to this Process
0119 Can optionally pass as additional arguments cms.Modifier instances
0120 that will be used to modify the Process as it is built
0121 """
0122 self.__dict__['_Process__name'] = name
0123 if not name.isalnum():
0124 raise RuntimeError("Error: The process name is an empty string or contains non-alphanumeric characters")
0125 self.__dict__['_Process__filters'] = {}
0126 self.__dict__['_Process__producers'] = {}
0127 self.__dict__['_Process__switchproducers'] = {}
0128 self.__dict__['_Process__source'] = None
0129 self.__dict__['_Process__looper'] = None
0130 self.__dict__['_Process__subProcesses'] = []
0131 self.__dict__['_Process__schedule'] = None
0132 self.__dict__['_Process__analyzers'] = {}
0133 self.__dict__['_Process__outputmodules'] = {}
0134 self.__dict__['_Process__paths'] = DictTypes.SortedKeysDict()
0135 self.__dict__['_Process__endpaths'] = DictTypes.SortedKeysDict()
0136 self.__dict__['_Process__finalpaths'] = DictTypes.SortedKeysDict()
0137 self.__dict__['_Process__sequences'] = {}
0138 self.__dict__['_Process__tasks'] = {}
0139 self.__dict__['_Process__conditionaltasks'] = {}
0140 self.__dict__['_Process__services'] = {}
0141 self.__dict__['_Process__essources'] = {}
0142 self.__dict__['_Process__esproducers'] = {}
0143 self.__dict__['_Process__esprefers'] = {}
0144 self.__dict__['_Process__aliases'] = {}
0145 self.__dict__['_Process__psets']={}
0146 self.__dict__['_Process__vpsets']={}
0147 self.__dict__['_cloneToObjectDict'] = {}
0148
0149 self.__dict__['_Process__InExtendCall'] = False
0150 self.__dict__['_Process__partialschedules'] = {}
0151 self.__isStrict = False
0152 self.__dict__['_Process__modifiers'] = Mods
0153 self.__dict__['_Process__accelerators'] = {}
0154 self.__injectValidValue('options', Process.defaultOptions_())
0155 self.__injectValidValue('maxEvents', Process.defaultMaxEvents_())
0156 self.maxLuminosityBlocks = Process.defaultMaxLuminosityBlocks_()
0157
0158
0159
0160
0161 self.MessageLogger = MessageLogger
0162 if Process._firstProcess:
0163 Process._firstProcess = False
0164 else:
0165 if len(Mods) > 0:
0166 for m in self.__modifiers:
0167 if not m._isChosen():
0168 raise RuntimeError("The Process {} tried to redefine which Modifiers to use after another Process was already started".format(name))
0169 for m in self.__modifiers:
0170 m._setChosen()
0171
0172 def setStrict(self, value: bool):
0173 self.__isStrict = value
0174 _Module.__isStrict__ = True
0175
0176
0177 def producerNames(self):
0178 """Returns a string containing all the EDProducer labels separated by a blank"""
0179 return ' '.join(self.producers_().keys())
0180 def switchProducerNames(self):
0181 """Returns a string containing all the SwitchProducer labels separated by a blank"""
0182 return ' '.join(self.switchProducers_().keys())
0183 def analyzerNames(self):
0184 """Returns a string containing all the EDAnalyzer labels separated by a blank"""
0185 return ' '.join(self.analyzers_().keys())
0186 def filterNames(self):
0187 """Returns a string containing all the EDFilter labels separated by a blank"""
0188 return ' '.join(self.filters_().keys())
0189 def pathNames(self):
0190 """Returns a string containing all the Path names separated by a blank"""
0191 return ' '.join(self.paths_().keys())
0192
0193 def __setstate__(self, pkldict):
0194 """
0195 Unpickling hook.
0196
0197 Since cloneToObjectDict stores a hash of objects by their
0198 id() it needs to be updated when unpickling to use the
0199 new object id values instantiated during the unpickle.
0200
0201 """
0202 self.__dict__.update(pkldict)
0203 tmpDict = {}
0204 for value in self._cloneToObjectDict.values():
0205 tmpDict[id(value)] = value
0206 self.__dict__['_cloneToObjectDict'] = tmpDict
0207
0208
0209
0210 def filters_(self):
0211 """returns a dict of the filters that have been added to the Process"""
0212 return DictTypes.FixedKeysDict(self.__filters)
0213 filters = property(filters_, doc="dictionary containing the filters for the process")
0214 def name_(self) -> str:
0215 return self.__name
0216 def setName_(self,name: str):
0217 if not name.isalnum():
0218 raise RuntimeError("Error: The process name is an empty string or contains non-alphanumeric characters")
0219 self.__dict__['_Process__name'] = name
0220 process = property(name_,setName_, doc="name of the process")
0221 def producers_(self):
0222 """returns a dict of the producers that have been added to the Process"""
0223 return DictTypes.FixedKeysDict(self.__producers)
0224 producers = property(producers_,doc="dictionary containing the producers for the process")
0225 def switchProducers_(self):
0226 """returns a dict of the SwitchProducers that have been added to the Process"""
0227 return DictTypes.FixedKeysDict(self.__switchproducers)
0228 switchProducers = property(switchProducers_,doc="dictionary containing the SwitchProducers for the process")
0229 def source_(self):
0230 """returns the source that has been added to the Process or None if none have been added"""
0231 return self.__source
0232 def setSource_(self,src):
0233 self._placeSource('source',src)
0234 source = property(source_,setSource_,doc='the main source or None if not set')
0235 def looper_(self):
0236 """returns the looper that has been added to the Process or None if none have been added"""
0237 return self.__looper
0238 def setLooper_(self,lpr):
0239 self._placeLooper('looper',lpr)
0240 looper = property(looper_,setLooper_,doc='the main looper or None if not set')
0241 @staticmethod
0242 def defaultOptions_():
0243 return untracked.PSet(numberOfThreads = untracked.uint32(1),
0244 numberOfStreams = untracked.uint32(0),
0245 numberOfConcurrentRuns = untracked.uint32(1),
0246 numberOfConcurrentLuminosityBlocks = untracked.uint32(0),
0247 eventSetup = untracked.PSet(
0248 numberOfConcurrentIOVs = untracked.uint32(0),
0249 forceNumberOfConcurrentIOVs = untracked.PSet(
0250 allowAnyLabel_ = required.untracked.uint32
0251 )
0252 ),
0253 accelerators = untracked.vstring('*'),
0254 wantSummary = untracked.bool(False),
0255 fileMode = untracked.string('FULLMERGE'),
0256 forceEventSetupCacheClearOnNewRun = untracked.bool(False),
0257 throwIfIllegalParameter = untracked.bool(True),
0258 printDependencies = untracked.bool(False),
0259 deleteNonConsumedUnscheduledModules = untracked.bool(True),
0260 sizeOfStackForThreadsInKB = optional.untracked.uint32,
0261 Rethrow = untracked.vstring(),
0262 TryToContinue = untracked.vstring(),
0263 IgnoreCompletely = untracked.vstring(),
0264 modulesToCallForTryToContinue = untracked.vstring(),
0265 canDeleteEarly = untracked.vstring(),
0266 holdsReferencesToDeleteEarly = untracked.VPSet(),
0267 modulesToIgnoreForDeleteEarly = untracked.vstring(),
0268 dumpOptions = untracked.bool(False),
0269 allowUnscheduled = obsolete.untracked.bool,
0270 emptyRunLumiMode = obsolete.untracked.string,
0271 makeTriggerResults = obsolete.untracked.bool,
0272 )
0273 def __updateOptions(self,opt):
0274 newOpts = self.defaultOptions_()
0275 if isinstance(opt,dict):
0276 for k,v in opt.items():
0277 setattr(newOpts,k,v)
0278 else:
0279 for p in opt.parameters_():
0280 setattr(newOpts, p, getattr(opt,p))
0281 return newOpts
0282 @staticmethod
0283 def defaultMaxEvents_():
0284 return untracked.PSet(input=optional.untracked.int32,
0285 output=optional.untracked.allowed(int32,PSet))
0286 def __updateMaxEvents(self,ps: Union[dict,PSet]):
0287 newMax = self.defaultMaxEvents_()
0288 if isinstance(ps,dict):
0289 for k,v in ps.items():
0290 setattr(newMax,k,v)
0291 else:
0292 for p in ps.parameters_():
0293 setattr(newMax, p, getattr(ps,p))
0294 return newMax
0295 @staticmethod
0296 def defaultMaxLuminosityBlocks_():
0297 return untracked.PSet(input=untracked.int32(-1))
0298 def subProcesses_(self):
0299 """returns a list of the subProcesses that have been added to the Process"""
0300 return self.__subProcesses
0301 subProcesses = property(subProcesses_,doc='the SubProcesses that have been added to the Process')
0302 def analyzers_(self):
0303 """returns a dict of the analyzers that have been added to the Process"""
0304 return DictTypes.FixedKeysDict(self.__analyzers)
0305 analyzers = property(analyzers_,doc="dictionary containing the analyzers for the process")
0306 def outputModules_(self):
0307 """returns a dict of the output modules that have been added to the Process"""
0308 return DictTypes.FixedKeysDict(self.__outputmodules)
0309 outputModules = property(outputModules_,doc="dictionary containing the output_modules for the process")
0310 def paths_(self):
0311 """returns a dict of the paths that have been added to the Process"""
0312 return DictTypes.SortedAndFixedKeysDict(self.__paths)
0313 paths = property(paths_,doc="dictionary containing the paths for the process")
0314 def endpaths_(self):
0315 """returns a dict of the endpaths that have been added to the Process"""
0316 return DictTypes.SortedAndFixedKeysDict(self.__endpaths)
0317 endpaths = property(endpaths_,doc="dictionary containing the endpaths for the process")
0318 def finalpaths_(self):
0319 """returns a dict of the finalpaths that have been added to the Process"""
0320 return DictTypes.SortedAndFixedKeysDict(self.__finalpaths)
0321 finalpaths = property(finalpaths_,doc="dictionary containing the finalpaths for the process")
0322 def sequences_(self):
0323 """returns a dict of the sequences that have been added to the Process"""
0324 return DictTypes.FixedKeysDict(self.__sequences)
0325 sequences = property(sequences_,doc="dictionary containing the sequences for the process")
0326 def tasks_(self):
0327 """returns a dict of the tasks that have been added to the Process"""
0328 return DictTypes.FixedKeysDict(self.__tasks)
0329 tasks = property(tasks_,doc="dictionary containing the tasks for the process")
0330 def conditionaltasks_(self):
0331 """returns a dict of the conditionaltasks that have been added to the Process"""
0332 return DictTypes.FixedKeysDict(self.__conditionaltasks)
0333 conditionaltasks = property(conditionaltasks_,doc="dictionary containing the conditionatasks for the process")
0334 def schedule_(self):
0335 """returns the schedule that has been added to the Process or None if none have been added"""
0336 return self.__schedule
0337 def setPartialSchedule_(self,sch: Schedule,label: str):
0338 if label == "schedule":
0339 self.setSchedule_(sch)
0340 else:
0341 self._place(label, sch, self.__partialschedules)
0342 def setSchedule_(self,sch: Schedule):
0343
0344 index = 0
0345 try:
0346 for p in sch:
0347 p.label_()
0348 index +=1
0349 except:
0350 raise RuntimeError("The path at index "+str(index)+" in the Schedule was not attached to the process.")
0351 self.__dict__['_Process__schedule'] = sch
0352 schedule = property(schedule_,setSchedule_,doc='the schedule or None if not set')
0353 def services_(self):
0354 """returns a dict of the services that have been added to the Process"""
0355 return DictTypes.FixedKeysDict(self.__services)
0356 services = property(services_,doc="dictionary containing the services for the process")
0357 def processAccelerators_(self):
0358 """returns a dict of the ProcessAccelerators that have been added to the Process"""
0359 return DictTypes.FixedKeysDict(self.__accelerators)
0360 processAccelerators = property(processAccelerators_,doc="dictionary containing the ProcessAccelerators for the process")
0361 def es_producers_(self):
0362 """returns a dict of the esproducers that have been added to the Process"""
0363 return DictTypes.FixedKeysDict(self.__esproducers)
0364 es_producers = property(es_producers_,doc="dictionary containing the es_producers for the process")
0365 def es_sources_(self):
0366 """returns a the es_sources that have been added to the Process"""
0367 return DictTypes.FixedKeysDict(self.__essources)
0368 es_sources = property(es_sources_,doc="dictionary containing the es_sources for the process")
0369 def es_prefers_(self):
0370 """returns a dict of the es_prefers that have been added to the Process"""
0371 return DictTypes.FixedKeysDict(self.__esprefers)
0372 es_prefers = property(es_prefers_,doc="dictionary containing the es_prefers for the process")
0373 def aliases_(self):
0374 """returns a dict of the aliases that have been added to the Process"""
0375 return DictTypes.FixedKeysDict(self.__aliases)
0376 aliases = property(aliases_,doc="dictionary containing the aliases for the process")
0377 def psets_(self):
0378 """returns a dict of the PSets that have been added to the Process"""
0379 return DictTypes.FixedKeysDict(self.__psets)
0380 psets = property(psets_,doc="dictionary containing the PSets for the process")
0381 def vpsets_(self):
0382 """returns a dict of the VPSets that have been added to the Process"""
0383 return DictTypes.FixedKeysDict(self.__vpsets)
0384 vpsets = property(vpsets_,doc="dictionary containing the PSets for the process")
0385
0386 def isUsingModifier(self,mod) -> bool:
0387 """returns True if the Modifier is in used by this Process"""
0388 if mod._isChosen():
0389 for m in self.__modifiers:
0390 if m._isOrContains(mod):
0391 return True
0392 return False
0393
0394 def __setObjectLabel(self, object, newLabel:str) :
0395 if not object.hasLabel_() :
0396 object.setLabel(newLabel)
0397 return
0398 if newLabel == object.label_() :
0399 return
0400 if newLabel is None :
0401 object.setLabel(None)
0402 return
0403 if (hasattr(self, object.label_()) and id(getattr(self, object.label_())) == id(object)) :
0404 msg100 = "Attempting to change the label of an attribute of the Process\n"
0405 msg101 = "Old label = "+object.label_()+" New label = "+newLabel+"\n"
0406 msg102 = "Type = "+str(type(object))+"\n"
0407 msg103 = "Some possible solutions:\n"
0408 msg104 = " 1. Clone modules instead of using simple assignment. Cloning is\n"
0409 msg105 = " also preferred for other types when possible.\n"
0410 msg106 = " 2. Declare new names starting with an underscore if they are\n"
0411 msg107 = " for temporaries you do not want propagated into the Process. The\n"
0412 msg108 = " underscore tells \"from x import *\" and process.load not to import\n"
0413 msg109 = " the name.\n"
0414 msg110 = " 3. Reorganize so the assigment is not necessary. Giving a second\n"
0415 msg111 = " name to the same object usually causes confusion and problems.\n"
0416 msg112 = " 4. Compose Sequences: newName = cms.Sequence(oldName)\n"
0417 raise ValueError(msg100+msg101+msg102+msg103+msg104+msg105+msg106+msg107+msg108+msg109+msg110+msg111+msg112)
0418 object.setLabel(None)
0419 object.setLabel(newLabel)
0420
0421 def __setattr__(self,name:str,value):
0422
0423 if not name.replace('_','').isalnum():
0424 raise ValueError('The label '+name+' contains forbiden characters')
0425
0426 if name == 'options':
0427 value = self.__updateOptions(value)
0428 if name == 'maxEvents':
0429 value = self.__updateMaxEvents(value)
0430
0431
0432 if name.startswith('_Process__'):
0433 self.__dict__[name]=value
0434 return
0435 if not isinstance(value,_ConfigureComponent):
0436 raise TypeError("can only assign labels to an object that inherits from '_ConfigureComponent'\n"
0437 +"an instance of "+str(type(value))+" will not work - requested label is "+name)
0438 if not isinstance(value,_Labelable) and not isinstance(value,Source) and not isinstance(value,Looper) and not isinstance(value,Schedule):
0439 if name == value.type_():
0440 if hasattr(self,name) and (getattr(self,name)!=value):
0441 self._replaceInTasks(name, value)
0442 self._replaceInConditionalTasks(name, value)
0443
0444 self.add_(value)
0445 return
0446 else:
0447 raise TypeError("an instance of "+str(type(value))+" can not be assigned the label '"+name+"'.\n"+
0448 "Please either use the label '"+value.type_()+" or use the 'add_' method instead.")
0449
0450 if self.__isStrict:
0451 newValue =value.copy()
0452 try:
0453 newValue._filename = value._filename
0454 except:
0455 pass
0456 value.setIsFrozen()
0457 else:
0458 newValue =value
0459 if not self._okToPlace(name, value, self.__dict__):
0460 newFile='top level config'
0461 if hasattr(value,'_filename'):
0462 newFile = value._filename
0463 oldFile='top level config'
0464 oldValue = getattr(self,name)
0465 if hasattr(oldValue,'_filename'):
0466 oldFile = oldValue._filename
0467 msg = "Trying to override definition of process."+name
0468 msg += "\n new object defined in: "+newFile
0469 msg += "\n existing object defined in: "+oldFile
0470 raise ValueError(msg)
0471
0472 if hasattr(self,name) and not (getattr(self,name)==newValue):
0473
0474
0475
0476 if newValue._isTaskComponent():
0477 if not self.__InExtendCall:
0478 self._replaceInTasks(name, newValue)
0479 self._replaceInConditionalTasks(name, newValue)
0480 self._replaceInSchedule(name, newValue)
0481 else:
0482 if not isinstance(newValue, Task):
0483
0484 newFile='top level config'
0485 if hasattr(value,'_filename'):
0486 newFile = value._filename
0487 oldFile='top level config'
0488 oldValue = getattr(self,name)
0489 if hasattr(oldValue,'_filename'):
0490 oldFile = oldValue._filename
0491 msg1 = "Trying to override definition of "+name+" while it is used by the task "
0492 msg2 = "\n new object defined in: "+newFile
0493 msg2 += "\n existing object defined in: "+oldFile
0494 s = self.__findFirstUsingModule(self.tasks,oldValue)
0495 if s is not None:
0496 raise ValueError(msg1+s.label_()+msg2)
0497
0498 if isinstance(newValue, _Sequenceable) or newValue._isTaskComponent() or isinstance(newValue, ConditionalTask):
0499 if not self.__InExtendCall:
0500 if isinstance(newValue, ConditionalTask):
0501 self._replaceInConditionalTasks(name, newValue)
0502 self._replaceInSequences(name, newValue)
0503 else:
0504
0505 newFile='top level config'
0506 if hasattr(value,'_filename'):
0507 newFile = value._filename
0508 oldFile='top level config'
0509 oldValue = getattr(self,name)
0510 if hasattr(oldValue,'_filename'):
0511 oldFile = oldValue._filename
0512 msg1 = "Trying to override definition of "+name+" while it is used by the "
0513 msg2 = "\n new object defined in: "+newFile
0514 msg2 += "\n existing object defined in: "+oldFile
0515 s = self.__findFirstUsingModule(self.sequences,oldValue)
0516 if s is not None:
0517 raise ValueError(msg1+"sequence "+s.label_()+msg2)
0518 s = self.__findFirstUsingModule(self.paths,oldValue)
0519 if s is not None:
0520 raise ValueError(msg1+"path "+s.label_()+msg2)
0521 s = self.__findFirstUsingModule(self.endpaths,oldValue)
0522 if s is not None:
0523 raise ValueError(msg1+"endpath "+s.label_()+msg2)
0524 s = self.__findFirstUsingModule(self.finalpaths,oldValue)
0525 if s is not None:
0526 raise ValueError(msg1+"finalpath "+s.label_()+msg2)
0527
0528
0529 if isinstance(newValue, EDAlias):
0530 oldValue = getattr(self, name)
0531
0532 newFile='top level config'
0533 if hasattr(value,'_filename'):
0534 newFile = value._filename
0535 oldFile='top level config'
0536 if hasattr(oldValue,'_filename'):
0537 oldFile = oldValue._filename
0538 msg1 = "Trying to override definition of "+name+" with an EDAlias while it is used by the "
0539 msg2 = "\n new object defined in: "+newFile
0540 msg2 += "\n existing object defined in: "+oldFile
0541 s = self.__findFirstUsingModule(self.tasks,oldValue)
0542 if s is not None:
0543 raise ValueError(msg1+"task "+s.label_()+msg2)
0544 s = self.__findFirstUsingModule(self.sequences,oldValue)
0545 if s is not None:
0546 raise ValueError(msg1+"sequence "+s.label_()+msg2)
0547 s = self.__findFirstUsingModule(self.paths,oldValue)
0548 if s is not None:
0549 raise ValueError(msg1+"path "+s.label_()+msg2)
0550 s = self.__findFirstUsingModule(self.endpaths,oldValue)
0551 if s is not None:
0552 raise ValueError(msg1+"endpath "+s.label_()+msg2)
0553 s = self.__findFirstUsingModule(self.finalpaths,oldValue)
0554 if s is not None:
0555 raise ValueError(msg1+"finalpath "+s.label_()+msg2)
0556
0557 if not self.__InExtendCall and (Schedule._itemIsValid(newValue) or isinstance(newValue, Task)):
0558 self._replaceInScheduleDirectly(name, newValue)
0559
0560 self._delattrFromSetattr(name)
0561 self.__injectValidValue(name, value, newValue)
0562 def __injectValidValue(self, name:str, value, newValue = None):
0563 if newValue is None:
0564 newValue = value
0565 self.__dict__[name]=newValue
0566 if isinstance(newValue,_Labelable):
0567 self.__setObjectLabel(newValue, name)
0568 self._cloneToObjectDict[id(value)] = newValue
0569 self._cloneToObjectDict[id(newValue)] = newValue
0570
0571 newValue._place(name,self)
0572 def __findFirstUsingModule(self, seqsOrTasks, mod):
0573 """Given a container of sequences or tasks, find the first sequence or task
0574 containing mod and return it. If none is found, return None"""
0575 from FWCore.ParameterSet.SequenceTypes import ModuleNodeVisitor
0576 l = list()
0577 for seqOrTask in seqsOrTasks.values():
0578 l[:] = []
0579 v = ModuleNodeVisitor(l)
0580 seqOrTask.visit(v)
0581 if mod in l:
0582 return seqOrTask
0583 return None
0584
0585 def _delHelper(self,name:str):
0586 if not hasattr(self,name):
0587 raise KeyError('process does not know about '+name)
0588 elif name.startswith('_Process__'):
0589 raise ValueError('this attribute cannot be deleted')
0590
0591
0592 dicts = [item for item in self.__dict__.values() if (isinstance(item, dict) or isinstance(item, DictTypes.SortedKeysDict))]
0593 for reg in dicts:
0594 if name in reg: del reg[name]
0595
0596 obj = getattr(self,name)
0597 if isinstance(obj,_Labelable):
0598 obj.setLabel(None)
0599 if isinstance(obj,Service):
0600 obj._inProcess = False
0601
0602 def __delattr__(self,name:str):
0603 self._delHelper(name)
0604 obj = getattr(self,name)
0605 if not obj is None:
0606 if not isinstance(obj, Sequence) and not isinstance(obj, Task) and not isinstance(obj,ConditionalTask):
0607
0608
0609
0610
0611
0612
0613
0614
0615
0616 if obj._isTaskComponent():
0617 self._replaceInTasks(name, None)
0618 self._replaceInConditionalTasks(name, None)
0619 self._replaceInSchedule(name, None)
0620 if isinstance(obj, _Sequenceable) or obj._isTaskComponent():
0621 self._replaceInSequences(name, None)
0622 if Schedule._itemIsValid(obj) or isinstance(obj, Task):
0623 self._replaceInScheduleDirectly(name, None)
0624
0625 try:
0626 del self.__dict__[name]
0627 except:
0628 pass
0629
0630 def _delattrFromSetattr(self,name:str):
0631 """Similar to __delattr__ but we need different behavior when called from __setattr__"""
0632 self._delHelper(name)
0633
0634 try:
0635 del self.__dict__[name]
0636 except:
0637 pass
0638
0639 def add_(self,value):
0640 """Allows addition of components that do not have to have a label, e.g. Services"""
0641 if not isinstance(value,_ConfigureComponent):
0642 raise TypeError
0643 if not isinstance(value,_Unlabelable):
0644 raise TypeError
0645
0646 if self.__isStrict:
0647 newValue =value.copy()
0648 value.setIsFrozen()
0649 else:
0650 newValue =value
0651 newValue._place('',self)
0652
0653 def _okToPlace(self, name:str, mod, d) -> bool:
0654 if not self.__InExtendCall:
0655
0656 return True
0657 elif not self.__isStrict:
0658 return True
0659 elif name in d:
0660
0661
0662
0663
0664 if mod._isModified:
0665 if d[name]._isModified:
0666 return False
0667 else:
0668 return True
0669 else:
0670 return True
0671 else:
0672 return True
0673
0674 def _place(self, name:str, mod, d):
0675 if self._okToPlace(name, mod, d):
0676 if self.__isStrict and isinstance(mod, _ModuleSequenceType):
0677 d[name] = mod._postProcessFixup(self._cloneToObjectDict)
0678 else:
0679 d[name] = mod
0680 if isinstance(mod,_Labelable):
0681 self.__setObjectLabel(mod, name)
0682 def _placeOutputModule(self,name:str,mod):
0683 self._place(name, mod, self.__outputmodules)
0684 def _placeProducer(self,name:str,mod):
0685 self._place(name, mod, self.__producers)
0686 def _placeSwitchProducer(self,name:str,mod):
0687 self._place(name, mod, self.__switchproducers)
0688 def _placeFilter(self,name:str,mod):
0689 self._place(name, mod, self.__filters)
0690 def _placeAnalyzer(self,name:str,mod):
0691 self._place(name, mod, self.__analyzers)
0692 def _placePath(self,name:str,mod):
0693 self._validateSequence(mod, name)
0694 try:
0695 self._place(name, mod, self.__paths)
0696 except ModuleCloneError as msg:
0697 context = format_outerframe(4)
0698 raise Exception("%sThe module %s in path %s is unknown to the process %s." %(context, msg, name, self._Process__name))
0699 def _placeEndPath(self,name:str,mod):
0700 self._validateSequence(mod, name)
0701 try:
0702 self._place(name, mod, self.__endpaths)
0703 except ModuleCloneError as msg:
0704 context = format_outerframe(4)
0705 raise Exception("%sThe module %s in endpath %s is unknown to the process %s." %(context, msg, name, self._Process__name))
0706 def _placeFinalPath(self,name:str,mod):
0707 self._validateSequence(mod, name)
0708 try:
0709 self._place(name, mod, self.__finalpaths)
0710 except ModuleCloneError as msg:
0711 context = format_outerframe(4)
0712 raise Exception("%sThe module %s in finalpath %s is unknown to the process %s." %(context, msg, name, self._Process__name))
0713 def _placeSequence(self,name:str,mod):
0714 self._validateSequence(mod, name)
0715 self._place(name, mod, self.__sequences)
0716 def _placeESProducer(self,name:str,mod):
0717 self._place(name, mod, self.__esproducers)
0718 def _placeESPrefer(self,name:str,mod):
0719 self._place(name, mod, self.__esprefers)
0720 def _placeESSource(self,name:str,mod):
0721 self._place(name, mod, self.__essources)
0722 def _placeTask(self,name:str,task):
0723 self._validateTask(task, name)
0724 self._place(name, task, self.__tasks)
0725 def _placeConditionalTask(self,name:str,task):
0726 self._validateConditionalTask(task, name)
0727 self._place(name, task, self.__conditionaltasks)
0728 def _placeAlias(self,name:str,mod):
0729 self._place(name, mod, self.__aliases)
0730 def _placePSet(self,name:str,mod):
0731 self._place(name, mod, self.__psets)
0732 def _placeVPSet(self,name:str,mod):
0733 self._place(name, mod, self.__vpsets)
0734 def _placeSource(self,name:str,mod):
0735 """Allow the source to be referenced by 'source' or by type name"""
0736 if name != 'source':
0737 raise ValueError("The label '"+name+"' can not be used for a Source. Only 'source' is allowed.")
0738 if self.__dict__['_Process__source'] is not None :
0739 del self.__dict__[self.__dict__['_Process__source'].type_()]
0740 self.__dict__['_Process__source'] = mod
0741 self.__dict__[mod.type_()] = mod
0742 def _placeLooper(self,name:str,mod):
0743 if name != 'looper':
0744 raise ValueError("The label '"+name+"' can not be used for a Looper. Only 'looper' is allowed.")
0745 self.__dict__['_Process__looper'] = mod
0746 self.__dict__[mod.type_()] = mod
0747 def _placeSubProcess(self,name:str,mod):
0748 self.__dict__['_Process__subProcess'] = mod
0749 self.__dict__[mod.type_()] = mod
0750 def addSubProcess(self,mod):
0751 self.__subProcesses.append(mod)
0752 def _placeService(self,typeName:str,mod):
0753 self._place(typeName, mod, self.__services)
0754 if typeName in self.__dict__:
0755 self.__dict__[typeName]._inProcess = False
0756 self.__dict__[typeName]=mod
0757 def _placeAccelerator(self,typeName:str,mod):
0758 self._place(typeName, mod, self.__accelerators)
0759 self.__dict__[typeName]=mod
0760 def load(self, moduleName:str):
0761 moduleName = moduleName.replace("/",".")
0762 module = __import__(moduleName)
0763 self.extend(sys.modules[moduleName])
0764 def extend(self,other,items=()):
0765 """Look in other and find types that we can use"""
0766
0767 self.__dict__['_Process__InExtendCall'] = True
0768
0769 seqs = dict()
0770 tasksToAttach = dict()
0771 mods = []
0772 for name in dir(other):
0773
0774 if name.startswith('_'):
0775 continue
0776 item = getattr(other,name)
0777 if name == "source" or name == "looper":
0778
0779 if item is not None:
0780 self.__setattr__(name,item)
0781 elif isinstance(item,_ModuleSequenceType):
0782 seqs[name]=item
0783 elif isinstance(item,Task) or isinstance(item, ConditionalTask):
0784 tasksToAttach[name] = item
0785 elif isinstance(item,_Labelable):
0786 self.__setattr__(name,item)
0787 if not item.hasLabel_() :
0788 item.setLabel(name)
0789 elif isinstance(item,Schedule):
0790 self.__setattr__(name,item)
0791 elif isinstance(item,_Unlabelable):
0792 self.add_(item)
0793 elif isinstance(item,ProcessModifier):
0794 mods.append(item)
0795 elif isinstance(item,ProcessFragment):
0796 self.extend(item)
0797
0798
0799 for name,seq in seqs.items():
0800 if id(seq) not in self._cloneToObjectDict:
0801 self.__setattr__(name,seq)
0802 else:
0803 newSeq = self._cloneToObjectDict[id(seq)]
0804 self.__dict__[name]=newSeq
0805 self.__setObjectLabel(newSeq, name)
0806
0807 newSeq._place(name,self)
0808
0809 for name, task in tasksToAttach.items():
0810 self.__setattr__(name, task)
0811
0812
0813 for item in mods:
0814 item.apply(self)
0815
0816 self.__dict__['_Process__InExtendCall'] = False
0817
0818 def _dumpConfigNamedList(self,items,typeName:str,options:PrintOptions) -> str:
0819 returnValue = ''
0820 for name,item in items:
0821 returnValue +=options.indentation()+typeName+' '+name+' = '+item.dumpConfig(options)
0822 return returnValue
0823
0824 def _dumpConfigUnnamedList(self,items,typeName:str,options:PrintOptions) -> str:
0825 returnValue = ''
0826 for name,item in items:
0827 returnValue +=options.indentation()+typeName+' = '+item.dumpConfig(options)
0828 return returnValue
0829
0830 def _dumpConfigOptionallyNamedList(self,items,typeName:str,options:PrintOptions) -> str:
0831 returnValue = ''
0832 for name,item in items:
0833 if name == item.type_():
0834 name = ''
0835 returnValue +=options.indentation()+typeName+' '+name+' = '+item.dumpConfig(options)
0836 return returnValue
0837
0838 def dumpConfig(self, options:PrintOptions=PrintOptions()) -> str:
0839 """return a string containing the equivalent process defined using the old configuration language"""
0840 config = "process "+self.__name+" = {\n"
0841 options.indent()
0842 if self.source_():
0843 config += options.indentation()+"source = "+self.source_().dumpConfig(options)
0844 if self.looper_():
0845 config += options.indentation()+"looper = "+self.looper_().dumpConfig(options)
0846
0847 config+=self._dumpConfigNamedList(self.subProcesses_(),
0848 'subProcess',
0849 options)
0850 config+=self._dumpConfigNamedList(self.producers_().items(),
0851 'module',
0852 options)
0853 config+=self._dumpConfigNamedList(self.switchProducers_().items(),
0854 'module',
0855 options)
0856 config+=self._dumpConfigNamedList(self.filters_().items(),
0857 'module',
0858 options)
0859 config+=self._dumpConfigNamedList(self.analyzers_().items(),
0860 'module',
0861 options)
0862 config+=self._dumpConfigNamedList(self.outputModules_().items(),
0863 'module',
0864 options)
0865 config+=self._dumpConfigNamedList(self.sequences_().items(),
0866 'sequence',
0867 options)
0868 config+=self._dumpConfigNamedList(self.paths_().items(),
0869 'path',
0870 options)
0871 config+=self._dumpConfigNamedList(self.endpaths_().items(),
0872 'endpath',
0873 options)
0874 config+=self._dumpConfigNamedList(self.finalpaths_().items(),
0875 'finalpath',
0876 options)
0877 config+=self._dumpConfigUnnamedList(self.services_().items(),
0878 'service',
0879 options)
0880 config+=self._dumpConfigNamedList(self.aliases_().items(),
0881 'alias',
0882 options)
0883 config+=self._dumpConfigOptionallyNamedList(
0884 self.es_producers_().items(),
0885 'es_module',
0886 options)
0887 config+=self._dumpConfigOptionallyNamedList(
0888 self.es_sources_().items(),
0889 'es_source',
0890 options)
0891 config += self._dumpConfigESPrefers(options)
0892 for name,item in self.psets.items():
0893 config +=options.indentation()+item.configTypeName()+' '+name+' = '+item.configValue(options)
0894 for name,item in self.vpsets.items():
0895 config +=options.indentation()+'VPSet '+name+' = '+item.configValue(options)
0896 if self.schedule:
0897 pathNames = [p.label_() for p in self.schedule]
0898 config +=options.indentation()+'schedule = {'+','.join(pathNames)+'}\n'
0899
0900
0901
0902
0903 config += "}\n"
0904 options.unindent()
0905 return config
0906
0907 def _dumpConfigESPrefers(self, options:PrintOptions) -> str:
0908 result = ''
0909 for item in self.es_prefers_().values():
0910 result +=options.indentation()+'es_prefer '+item.targetLabel_()+' = '+item.dumpConfig(options)
0911 return result
0912
0913 def _dumpPythonSubProcesses(self, l, options:PrintOptions) -> str:
0914 returnValue = ''
0915 for item in l:
0916 returnValue += item.dumpPython(options)+'\n\n'
0917 return returnValue
0918
0919 def _dumpPythonList(self, d, options:PrintOptions) -> str:
0920 returnValue = ''
0921 if isinstance(d, DictTypes.SortedKeysDict):
0922 for name,item in d.items():
0923 returnValue +='process.'+name+' = '+item.dumpPython(options)+'\n\n'
0924 else:
0925 for name,item in sorted(d.items()):
0926 returnValue +='process.'+name+' = '+item.dumpPython(options)+'\n\n'
0927 return returnValue
0928
0929 def _splitPythonList(self, subfolder, d, options:PrintOptions) -> str:
0930 parts = DictTypes.SortedKeysDict()
0931 for name, item in d.items() if isinstance(d, DictTypes.SortedKeysDict) else sorted(d.items()):
0932 code = ''
0933 dependencies = item.directDependencies()
0934 for module_subfolder, module in dependencies:
0935 module = module + '_cfi'
0936 if options.useSubdirectories and module_subfolder:
0937 module = module_subfolder + '.' + module
0938 if options.targetDirectory is not None:
0939 if options.useSubdirectories and subfolder:
0940 module = '..' + module
0941 else:
0942 module = '.' + module
0943 code += 'from ' + module + ' import *\n'
0944 if dependencies:
0945 code += '\n'
0946 code += name + ' = ' + item.dumpPython(options)
0947 parts[name] = subfolder, code
0948 return parts
0949
0950 def _validateSequence(self, sequence, label):
0951
0952 try:
0953 l = set()
0954 visitor = NodeNameVisitor(l)
0955 sequence.visit(visitor)
0956 except Exception as e:
0957 raise RuntimeError("An entry in sequence {} has no label\n Seen entries: {}\n Error: {}".format(label, l, e))
0958
0959 def _validateTask(self, task, label:str):
0960
0961 try:
0962 l = set()
0963 visitor = NodeNameVisitor(l)
0964 task.visit(visitor)
0965 except:
0966 raise RuntimeError("An entry in task " + label + ' has not been attached to the process')
0967 def _validateConditionalTask(self, task, label:str):
0968
0969 try:
0970 l = set()
0971 visitor = NodeNameVisitor(l)
0972 task.visit(visitor)
0973 except:
0974 raise RuntimeError("An entry in task " + label + ' has not been attached to the process')
0975
0976 def _itemsInDependencyOrder(self, processDictionaryOfItems):
0977
0978
0979
0980
0981 returnValue=DictTypes.SortedKeysDict()
0982
0983
0984
0985 dependencies = {}
0986 for label,item in processDictionaryOfItems.items():
0987 containedItems = []
0988 if isinstance(item, Task):
0989 v = TaskVisitor(containedItems)
0990 elif isinstance(item, ConditionalTask):
0991 v = ConditionalTaskVisitor(containedItems)
0992 else:
0993 v = SequenceVisitor(containedItems)
0994 try:
0995 item.visit(v)
0996 except RuntimeError:
0997 if isinstance(item, Task):
0998 raise RuntimeError("Failed in a Task visitor. Probably " \
0999 "a circular dependency discovered in Task with label " + label)
1000 elif isinstance(item, ConditionalTask):
1001 raise RuntimeError("Failed in a ConditionalTask visitor. Probably " \
1002 "a circular dependency discovered in ConditionalTask with label " + label)
1003 else:
1004 raise RuntimeError("Failed in a Sequence visitor. Probably a " \
1005 "circular dependency discovered in Sequence with label " + label)
1006 for containedItem in containedItems:
1007
1008
1009
1010
1011
1012 if containedItem.hasLabel_():
1013 testItem = processDictionaryOfItems.get(containedItem.label_())
1014 if testItem is None or containedItem != testItem:
1015 if isinstance(item, Task):
1016 raise RuntimeError("Task has a label, but using its label to get an attribute" \
1017 " from the process yields a different object or None\n"+
1018 "label = " + containedItem.label_())
1019 if isinstance(item, ConditionalTask):
1020 raise RuntimeError("ConditionalTask has a label, but using its label to get an attribute" \
1021 " from the process yields a different object or None\n"+
1022 "label = " + containedItem.label_())
1023 else:
1024 raise RuntimeError("Sequence has a label, but using its label to get an attribute" \
1025 " from the process yields a different object or None\n"+
1026 "label = " + containedItem.label_())
1027 dependencies[label]=[dep.label_() for dep in containedItems if dep.hasLabel_()]
1028
1029
1030 while dependencies:
1031 oldDeps = dict(dependencies)
1032 for label,deps in oldDeps.items():
1033 if len(deps)==0:
1034 returnValue[label]=processDictionaryOfItems[label]
1035
1036 del dependencies[label]
1037 for lb2,deps2 in dependencies.items():
1038 while deps2.count(label):
1039 deps2.remove(label)
1040 return returnValue
1041
1042 def _dumpPython(self, d, options:PrintOptions) -> str:
1043 result = ''
1044 for name, value in sorted(d.items()):
1045 result += value.dumpPythonAs(name,options)+'\n'
1046 return result
1047
1048 def _splitPython(self, subfolder, d, options:PrintOptions) -> dict:
1049 result = {}
1050 for name, value in sorted(d.items()):
1051 result[name] = subfolder, value.dumpPythonAs(name, options) + '\n'
1052 return result
1053
1054 def dumpPython(self, options=PrintOptions()) -> str:
1055 """return a string containing the equivalent process defined using python"""
1056 specialImportRegistry._reset()
1057 header = "import FWCore.ParameterSet.Config as cms"
1058 result = "process = cms.Process(\""+self.__name+"\")\n\n"
1059 if self.source_():
1060 result += "process.source = "+self.source_().dumpPython(options)
1061 if self.looper_():
1062 result += "process.looper = "+self.looper_().dumpPython()
1063 result+=self._dumpPythonList(self.psets, options)
1064 result+=self._dumpPythonList(self.vpsets, options)
1065 result+=self._dumpPythonSubProcesses(self.subProcesses_(), options)
1066 result+=self._dumpPythonList(self.producers_(), options)
1067 result+=self._dumpPythonList(self.switchProducers_(), options)
1068 result+=self._dumpPythonList(self.filters_() , options)
1069 result+=self._dumpPythonList(self.analyzers_(), options)
1070 result+=self._dumpPythonList(self.outputModules_(), options)
1071 result+=self._dumpPythonList(self.services_(), options)
1072 result+=self._dumpPythonList(self.processAccelerators_(), options)
1073 result+=self._dumpPythonList(self.es_producers_(), options)
1074 result+=self._dumpPythonList(self.es_sources_(), options)
1075 result+=self._dumpPython(self.es_prefers_(), options)
1076 result+=self._dumpPythonList(self._itemsInDependencyOrder(self.tasks), options)
1077 result+=self._dumpPythonList(self._itemsInDependencyOrder(self.conditionaltasks), options)
1078 result+=self._dumpPythonList(self._itemsInDependencyOrder(self.sequences), options)
1079 result+=self._dumpPythonList(self.paths_(), options)
1080 result+=self._dumpPythonList(self.endpaths_(), options)
1081 result+=self._dumpPythonList(self.finalpaths_(), options)
1082 result+=self._dumpPythonList(self.aliases_(), options)
1083 if not self.schedule_() == None:
1084 result += 'process.schedule = ' + self.schedule.dumpPython(options)
1085 imports = specialImportRegistry.getSpecialImports()
1086 if len(imports) > 0:
1087 header += "\n" + "\n".join(imports)
1088 header += "\n\n"
1089 return header+result
1090
1091 def splitPython(self, options:PrintOptions = PrintOptions()) -> dict:
1092 """return a map of file names to python configuration fragments"""
1093 specialImportRegistry._reset()
1094
1095 options.isCfg = False
1096 header = "import FWCore.ParameterSet.Config as cms"
1097 result = ''
1098 parts = {}
1099 files = {}
1100
1101 result = 'process = cms.Process("' + self.__name + '")\n\n'
1102
1103 if self.source_():
1104 parts['source'] = (None, 'source = ' + self.source_().dumpPython(options))
1105
1106 if self.looper_():
1107 parts['looper'] = (None, 'looper = ' + self.looper_().dumpPython())
1108
1109 parts.update(self._splitPythonList('psets', self.psets, options))
1110 parts.update(self._splitPythonList('psets', self.vpsets, options))
1111
1112
1113 if len(self.subProcesses_()):
1114 sys.stderr.write("error: subprocesses are not supported yet\n\n")
1115 parts.update(self._splitPythonList('modules', self.producers_(), options))
1116 parts.update(self._splitPythonList('modules', self.switchProducers_(), options))
1117 parts.update(self._splitPythonList('modules', self.filters_() , options))
1118 parts.update(self._splitPythonList('modules', self.analyzers_(), options))
1119 parts.update(self._splitPythonList('modules', self.outputModules_(), options))
1120 parts.update(self._splitPythonList('services', self.services_(), options))
1121 parts.update(self._splitPythonList('eventsetup', self.es_producers_(), options))
1122 parts.update(self._splitPythonList('eventsetup', self.es_sources_(), options))
1123 parts.update(self._splitPython('eventsetup', self.es_prefers_(), options))
1124 parts.update(self._splitPythonList('tasks', self._itemsInDependencyOrder(self.tasks), options))
1125 parts.update(self._splitPythonList('sequences', self._itemsInDependencyOrder(self.sequences), options))
1126 parts.update(self._splitPythonList('paths', self.paths_(), options))
1127 parts.update(self._splitPythonList('paths', self.endpaths_(), options))
1128 parts.update(self._splitPythonList('paths', self.finalpaths_(), options))
1129 parts.update(self._splitPythonList('modules', self.aliases_(), options))
1130
1131 if options.targetDirectory is not None:
1132 files[options.targetDirectory + '/__init__.py'] = ''
1133
1134 if options.useSubdirectories:
1135 for sub in 'psets', 'modules', 'services', 'eventsetup', 'tasks', 'sequences', 'paths':
1136 if options.targetDirectory is not None:
1137 sub = options.targetDirectory + '/' + sub
1138 files[sub + '/__init__.py'] = ''
1139
1140
1141 parts = sorted(parts.items(), key = lambda nsc: (nsc[1][0].lower() if nsc[1][0] else '', nsc[0].lower()))
1142
1143 for (name, (subfolder, code)) in parts:
1144 filename = name + '_cfi'
1145 if options.useSubdirectories and subfolder:
1146 filename = subfolder + '/' + filename
1147 if options.targetDirectory is not None:
1148 filename = options.targetDirectory + '/' + filename
1149 result += 'process.load("%s")\n' % filename
1150 files[filename + '.py'] = header + '\n\n' + code
1151
1152 if self.schedule_() is not None:
1153 options.isCfg = True
1154 result += '\nprocess.schedule = ' + self.schedule.dumpPython(options)
1155
1156 imports = specialImportRegistry.getSpecialImports()
1157 if len(imports) > 0:
1158 header += '\n' + '\n'.join(imports)
1159 files['-'] = header + '\n\n' + result
1160 return files
1161
1162 def _replaceInSequences(self, label:str, new):
1163 old = getattr(self,label)
1164
1165
1166
1167
1168
1169 for sequenceable in self.sequences.values():
1170 sequenceable._replaceIfHeldDirectly(old,new)
1171 for sequenceable in self.sequences.values():
1172 sequenceable.replace(old,new)
1173 for sequenceable in self.paths.values():
1174 sequenceable.replace(old,new)
1175 for sequenceable in self.endpaths.values():
1176 sequenceable.replace(old,new)
1177 for sequenceable in self.finalpaths.values():
1178 sequenceable.replace(old,new)
1179 def _replaceInTasks(self, label:str, new):
1180 old = getattr(self,label)
1181 for task in self.tasks.values():
1182 task.replace(old, new)
1183 def _replaceInConditionalTasks(self, label:str, new):
1184 old = getattr(self,label)
1185 for task in self.conditionaltasks.values():
1186 task.replace(old, new)
1187 def _replaceInSchedule(self, label:str, new):
1188 if self.schedule_() == None:
1189 return
1190 old = getattr(self,label)
1191 for task in self.schedule_()._tasks:
1192 task.replace(old, new)
1193 def _replaceInScheduleDirectly(self, label:str, new):
1194 if self.schedule_() == None:
1195 return
1196 old = getattr(self,label)
1197 self.schedule_()._replaceIfHeldDirectly(old, new)
1198 def globalReplace(self,label:str,new):
1199 """ Replace the item with label 'label' by object 'new' in the process and all sequences/paths/tasks"""
1200 if not hasattr(self,label):
1201 raise LookupError("process has no item of label "+label)
1202 setattr(self,label,new)
1203 def _insertInto(self, parameterSet, itemDict):
1204 for name,value in itemDict.items():
1205 value.insertInto(parameterSet, name)
1206 def _insertOneInto(self, parameterSet, label:str, item, tracked:bool):
1207 vitems = []
1208 if not item == None:
1209 newlabel = item.nameInProcessDesc_(label)
1210 vitems = [newlabel]
1211 item.insertInto(parameterSet, newlabel)
1212 parameterSet.addVString(tracked, label, vitems)
1213 def _insertManyInto(self, parameterSet, label:str, itemDict, tracked:bool):
1214 l = []
1215 for name,value in itemDict.items():
1216 value.appendToProcessDescList_(l, name)
1217 value.insertInto(parameterSet, name)
1218
1219 l.sort()
1220 parameterSet.addVString(tracked, label, l)
1221 def _insertSwitchProducersInto(self, parameterSet, labelModules, labelAliases, itemDict, tracked:bool):
1222 modules = parameterSet.getVString(tracked, labelModules)
1223 aliases = parameterSet.getVString(tracked, labelAliases)
1224 accelerators = parameterSet.getVString(False, "@selected_accelerators")
1225 for name,value in itemDict.items():
1226 value.appendToProcessDescLists_(modules, aliases, name)
1227 value.insertInto(parameterSet, name, accelerators)
1228 modules.sort()
1229 aliases.sort()
1230 parameterSet.addVString(tracked, labelModules, modules)
1231 parameterSet.addVString(tracked, labelAliases, aliases)
1232 def _insertSubProcessesInto(self, parameterSet, label:str, itemList, tracked:bool):
1233 l = []
1234 subprocs = []
1235 for value in itemList:
1236 name = value.getProcessName()
1237 newLabel = value.nameInProcessDesc_(name)
1238 l.append(newLabel)
1239 pset = value.getSubProcessPSet(parameterSet)
1240 subprocs.append(pset)
1241
1242 l.sort()
1243 parameterSet.addVString(tracked, label, l)
1244 parameterSet.addVPSet(False,"subProcesses",subprocs)
1245 def _insertPaths(self, processPSet, nodeVisitor):
1246 scheduledPaths = []
1247 triggerPaths = []
1248 endpaths = []
1249 finalpaths = []
1250 if self.schedule_() == None:
1251
1252 for name in self.paths_():
1253 scheduledPaths.append(name)
1254 triggerPaths.append(name)
1255 for name in self.endpaths_():
1256 scheduledPaths.append(name)
1257 endpaths.append(name)
1258 for name in self.finalpaths_():
1259 finalpaths.append(name)
1260 else:
1261 for path in self.schedule_():
1262 pathname = path.label_()
1263 if pathname in self.endpaths_():
1264 endpaths.append(pathname)
1265 scheduledPaths.append(pathname)
1266 elif pathname in self.finalpaths_():
1267 finalpaths.append(pathname)
1268 else:
1269 scheduledPaths.append(pathname)
1270 triggerPaths.append(pathname)
1271 for task in self.schedule_()._tasks:
1272 task.resolve(self.__dict__)
1273 scheduleTaskValidator = ScheduleTaskValidator()
1274 task.visit(scheduleTaskValidator)
1275 task.visit(nodeVisitor)
1276
1277 endPathWithFinalPathModulesName ="@finalPath"
1278 finalPathEndPath = EndPath()
1279 if finalpaths:
1280 endpaths.append(endPathWithFinalPathModulesName)
1281 scheduledPaths.append(endPathWithFinalPathModulesName)
1282 finalpathValidator = FinalPathValidator()
1283 modulesOnFinalPath = []
1284 for finalpathname in finalpaths:
1285 iFinalPath = self.finalpaths_()[finalpathname]
1286 iFinalPath.resolve(self.__dict__)
1287 finalpathValidator.setLabel(finalpathname)
1288 iFinalPath.visit(finalpathValidator)
1289 invalidModules = finalpathValidator.invalidModulesOnFinalpaths
1290 if invalidModules:
1291 raise RuntimeError("FinalPath %s has non OutputModules %s" % (finalpathname, ",".join(invalidModules)))
1292 modulesOnFinalPath.extend(iFinalPath.moduleNames())
1293 for m in modulesOnFinalPath:
1294 mod = getattr(self, m)
1295 setattr(mod, "@onFinalPath", untracked.bool(True))
1296 finalPathEndPath += mod
1297
1298 processPSet.addVString(True, "@end_paths", endpaths)
1299 processPSet.addVString(True, "@paths", scheduledPaths)
1300
1301 p = processPSet.newPSet()
1302 p.addVString(True, "@trigger_paths", triggerPaths)
1303 processPSet.addPSet(True, "@trigger_paths", p)
1304
1305 pathValidator = PathValidator()
1306 endpathValidator = EndPathValidator()
1307 decoratedList = []
1308 lister = DecoratedNodeNameVisitor(decoratedList)
1309 condTaskModules = []
1310 condTaskVistor = ModuleNodeOnConditionalTaskVisitor(condTaskModules)
1311 pathCompositeVisitor = CompositeVisitor(pathValidator, nodeVisitor, lister, condTaskVistor)
1312 endpathCompositeVisitor = CompositeVisitor(endpathValidator, nodeVisitor, lister)
1313 for triggername in triggerPaths:
1314 iPath = self.paths_()[triggername]
1315 iPath.resolve(self.__dict__)
1316 pathValidator.setLabel(triggername)
1317 lister.initialize()
1318 condTaskModules[:] = []
1319 iPath.visit(pathCompositeVisitor)
1320 if condTaskModules:
1321 decoratedList.append("#")
1322 l = list({x.label_() for x in condTaskModules})
1323 l.sort()
1324 decoratedList.extend(l)
1325 decoratedList.append("@")
1326 iPath.insertInto(processPSet, triggername, decoratedList[:])
1327 for endpathname in endpaths:
1328 if endpathname is not endPathWithFinalPathModulesName:
1329 iEndPath = self.endpaths_()[endpathname]
1330 else:
1331 iEndPath = finalPathEndPath
1332 iEndPath.resolve(self.__dict__)
1333 endpathValidator.setLabel(endpathname)
1334 lister.initialize()
1335 iEndPath.visit(endpathCompositeVisitor)
1336 iEndPath.insertInto(processPSet, endpathname, decoratedList[:])
1337 processPSet.addVString(False, "@filters_on_endpaths", endpathValidator.filtersOnEndpaths)
1338
1339
1340 def resolve(self,keepUnresolvedSequencePlaceholders:bool=False):
1341 for x in self.paths.values():
1342 x.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1343 for x in self.endpaths.values():
1344 x.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1345 for x in self.finalpaths.values():
1346 x.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1347 if not self.schedule_() == None:
1348 for task in self.schedule_()._tasks:
1349 task.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1350
1351 def prune(self,verbose=False,keepUnresolvedSequencePlaceholders:bool=False):
1352 """ Remove clutter from the process that we think is unnecessary:
1353 tracked PSets, VPSets and unused modules and sequences. If a Schedule has been set, then Paths and EndPaths
1354 not in the schedule will also be removed, along with an modules and sequences used only by
1355 those removed Paths and EndPaths. The keepUnresolvedSequencePlaceholders keeps also unresolved TaskPlaceholders."""
1356
1357
1358
1359
1360
1361 for name in self.vpsets_():
1362 delattr(self, name)
1363
1364 self.resolve(keepUnresolvedSequencePlaceholders)
1365 usedModules = set()
1366 unneededPaths = set()
1367 tasks = list()
1368 tv = TaskVisitor(tasks)
1369 if self.schedule_():
1370 usedModules=set(self.schedule_().moduleNames())
1371
1372 schedNames = set(( x.label_() for x in self.schedule_()))
1373 names = set(self.paths)
1374 names.update(set(self.endpaths))
1375 names.update(set(self.finalpaths))
1376 unneededPaths = names - schedNames
1377 for n in unneededPaths:
1378 delattr(self,n)
1379 for t in self.schedule_().tasks():
1380 tv.enter(t)
1381 t.visit(tv)
1382 tv.leave(t)
1383 else:
1384 pths = list(self.paths.values())
1385 pths.extend(self.endpaths.values())
1386 pths.extend(self.finalpaths.values())
1387 temp = Schedule(*pths)
1388 usedModules=set(temp.moduleNames())
1389 unneededModules = self._pruneModules(self.producers_(), usedModules)
1390 unneededModules.update(self._pruneModules(self.switchProducers_(), usedModules))
1391 unneededModules.update(self._pruneModules(self.filters_(), usedModules))
1392 unneededModules.update(self._pruneModules(self.analyzers_(), usedModules))
1393
1394 seqs = list()
1395 sv = SequenceVisitor(seqs)
1396 for p in self.paths.values():
1397 p.visit(sv)
1398 p.visit(tv)
1399 for p in self.endpaths.values():
1400 p.visit(sv)
1401 p.visit(tv)
1402 for p in self.finalpaths.values():
1403 p.visit(sv)
1404 p.visit(tv)
1405 def removeUnneeded(seqOrTasks, allSequencesOrTasks):
1406 _keepSet = set(( s for s in seqOrTasks if s.hasLabel_()))
1407 _availableSet = set(allSequencesOrTasks.values())
1408 _unneededSet = _availableSet-_keepSet
1409 _unneededLabels = []
1410 for s in _unneededSet:
1411 _unneededLabels.append(s.label_())
1412 delattr(self,s.label_())
1413 return _unneededLabels
1414 unneededSeqLabels = removeUnneeded(seqs, self.sequences)
1415 unneededTaskLabels = removeUnneeded(tasks, self.tasks)
1416 if verbose:
1417 print("prune removed the following:")
1418 print(" modules:"+",".join(unneededModules))
1419 print(" tasks:"+",".join(unneededTaskLabels))
1420 print(" sequences:"+",".join(unneededSeqLabels))
1421 print(" paths/endpaths/finalpaths:"+",".join(unneededPaths))
1422 def _pruneModules(self, d, scheduledNames):
1423 moduleNames = set(d.keys())
1424 junk = moduleNames - scheduledNames
1425 for name in junk:
1426 delattr(self, name)
1427 return junk
1428
1429 def fillProcessDesc(self, processPSet):
1430 """Used by the framework to convert python to C++ objects"""
1431 class ServiceInjectorAdaptor(object):
1432 def __init__(self,ppset,thelist):
1433 self.__thelist = thelist
1434 self.__processPSet = ppset
1435 def addService(self,pset):
1436 self.__thelist.append(pset)
1437 def newPSet(self):
1438 return self.__processPSet.newPSet()
1439
1440
1441
1442 class TopLevelPSetAcessorAdaptor(object):
1443 def __init__(self,ppset,process):
1444 self.__ppset = ppset
1445 self.__process = process
1446 def __getattr__(self,attr):
1447 return getattr(self.__ppset,attr)
1448 def getTopPSet_(self,label):
1449 return getattr(self.__process,label)
1450 def newPSet(self):
1451 return TopLevelPSetAcessorAdaptor(self.__ppset.newPSet(),self.__process)
1452 def addPSet(self,tracked,name,ppset):
1453 return self.__ppset.addPSet(tracked,name,self.__extractPSet(ppset))
1454 def addVPSet(self,tracked,name,vpset):
1455 return self.__ppset.addVPSet(tracked,name,[self.__extractPSet(x) for x in vpset])
1456 def __extractPSet(self,pset):
1457 if isinstance(pset,TopLevelPSetAcessorAdaptor):
1458 return pset.__ppset
1459 return pset
1460
1461 self.validate()
1462 processPSet.addString(True, "@process_name", self.name_())
1463 self.handleProcessAccelerators(processPSet)
1464 all_modules = self.producers_().copy()
1465 all_modules.update(self.filters_())
1466 all_modules.update(self.analyzers_())
1467 all_modules.update(self.outputModules_())
1468 if hasattr(self.options,"modulesToCallForTryToContinue") :
1469 shouldTryToContinue = set(self.options.modulesToCallForTryToContinue)
1470 for m in all_modules:
1471 if m in shouldTryToContinue:
1472 setattr(getattr(self,m),"@shouldTryToContinue",untracked.bool(True))
1473 missing = shouldTryToContinue.difference(all_modules)
1474 if missing:
1475 print("Warning: The following modules appear in options.modulesToCallForTryToContinue but are not in the Process: {} ".format(",".join(missing)))
1476 adaptor = TopLevelPSetAcessorAdaptor(processPSet,self)
1477 self._insertInto(adaptor, self.psets_())
1478 self._insertInto(adaptor, self.vpsets_())
1479 self._insertOneInto(adaptor, "@all_sources", self.source_(), True)
1480 self._insertOneInto(adaptor, "@all_loopers", self.looper_(), True)
1481 self._insertSubProcessesInto(adaptor, "@all_subprocesses", self.subProcesses_(), False)
1482 self._insertManyInto(adaptor, "@all_esprefers", self.es_prefers_(), True)
1483 self._insertManyInto(adaptor, "@all_aliases", self.aliases_(), True)
1484
1485
1486
1487 nodeVisitor = NodeVisitor()
1488 self._insertPaths(adaptor, nodeVisitor)
1489 all_modules_onTasksOrScheduled = { key:value for key, value in all_modules.items() if value in nodeVisitor.modules }
1490 self._insertManyInto(adaptor, "@all_modules", all_modules_onTasksOrScheduled, True)
1491 all_switches = self.switchProducers_().copy()
1492 all_switches_onTasksOrScheduled = {key:value for key, value in all_switches.items() if value in nodeVisitor.modules }
1493 self._insertSwitchProducersInto(adaptor, "@all_modules", "@all_aliases", all_switches_onTasksOrScheduled, True)
1494
1495
1496 processNodeVisitor = NodeVisitor()
1497 for pTask in self.tasks.values():
1498 pTask.visit(processNodeVisitor)
1499 esProducersToEnable = {}
1500 for esProducerName, esProducer in self.es_producers_().items():
1501 if esProducer in nodeVisitor.esProducers or not (esProducer in processNodeVisitor.esProducers):
1502 esProducersToEnable[esProducerName] = esProducer
1503 self._insertManyInto(adaptor, "@all_esmodules", esProducersToEnable, True)
1504 esSourcesToEnable = {}
1505 for esSourceName, esSource in self.es_sources_().items():
1506 if esSource in nodeVisitor.esSources or not (esSource in processNodeVisitor.esSources):
1507 esSourcesToEnable[esSourceName] = esSource
1508 self._insertManyInto(adaptor, "@all_essources", esSourcesToEnable, True)
1509
1510 services = []
1511 for serviceName, serviceObject in self.services_().items():
1512 if serviceObject in nodeVisitor.services or not (serviceObject in processNodeVisitor.services):
1513 serviceObject.insertInto(ServiceInjectorAdaptor(adaptor,services))
1514 adaptor.addVPSet(False,"services",services)
1515 return processPSet
1516
1517 def validate(self):
1518
1519
1520
1521
1522 pass
1523
1524 def handleProcessAccelerators(self, parameterSet):
1525
1526 allAccelerators = set(["cpu"])
1527 availableAccelerators = set(["cpu"])
1528 for acc in self.__dict__['_Process__accelerators'].values():
1529 allAccelerators.update(acc.labels())
1530 availableAccelerators.update(acc.enabledLabels())
1531 availableAccelerators = sorted(list(availableAccelerators))
1532 parameterSet.addVString(False, "@available_accelerators", availableAccelerators)
1533
1534
1535 selectedAccelerators = []
1536 if "*" in self.options.accelerators:
1537 if len(self.options.accelerators) >= 2:
1538 raise ValueError("process.options.accelerators may contain '*' only as the only element, now it has {} elements".format(len(self.options.accelerators)))
1539 selectedAccelerators = availableAccelerators
1540 else:
1541 import fnmatch
1542 resolved = set()
1543 invalid = []
1544 for pattern in self.options.accelerators:
1545 acc = [a for a in availableAccelerators if fnmatch.fnmatchcase(a, pattern)]
1546 if len(acc) == 0:
1547 if not any(fnmatch.fnmatchcase(a, pattern) for a in allAccelerators):
1548 invalid.append(pattern)
1549 else:
1550 resolved.update(acc)
1551
1552 if len(invalid) != 0:
1553 raise ValueError("Invalid pattern{} of '{}' in process.options.accelerators, valid values are '{}' or a pattern matching some of them.".format(
1554 "s" if len(invalid) > 2 else "",
1555 "', '".join(invalid),
1556 "', '".join(sorted(list(allAccelerators)))))
1557 selectedAccelerators = sorted(list(resolved))
1558 parameterSet.addVString(False, "@selected_accelerators", selectedAccelerators)
1559
1560
1561 moduleTypeResolver = None
1562 moduleTypeResolverPlugin = ""
1563 for acc in self.__dict__['_Process__accelerators'].values():
1564 resolver = acc.moduleTypeResolver(selectedAccelerators)
1565 if resolver is not None:
1566 if moduleTypeResolver is not None:
1567 raise RuntimeError("Module type resolver was already set to {} when {} tried to set it to {}. A job can have at most one ProcessAccelerator that sets module type resolver.".format(
1568 moduleTypeResolver.__class__.__name__,
1569 acc.__class__.__name__,
1570 resolver.__class__.__name__))
1571 moduleTypeResolver = resolver
1572 if moduleTypeResolver is not None:
1573
1574 moduleTypeResolverPlugin = moduleTypeResolver.plugin()
1575
1576
1577 for modlist in [self.producers_, self.filters_, self.analyzers_,
1578 self.es_producers_, self.es_sources_]:
1579 for module in modlist().values():
1580 moduleTypeResolver.setModuleVariant(module)
1581
1582 parameterSet.addString(False, "@module_type_resolver", moduleTypeResolverPlugin)
1583
1584
1585 wrapped = ProcessForProcessAccelerator(self)
1586 for acc in self.__dict__['_Process__accelerators'].values():
1587 acc.apply(wrapped, selectedAccelerators)
1588
1589 def prefer(self, esmodule,*args,**kargs):
1590 """Prefer this ES source or producer. The argument can
1591 either be an object label, e.g.,
1592 process.prefer(process.juicerProducer) (not supported yet)
1593 or a name of an ESSource or ESProducer
1594 process.prefer("juicer")
1595 or a type of unnamed ESSource or ESProducer
1596 process.prefer("JuicerProducer")
1597 In addition, you can pass as a labelled arguments the name of the Record you wish to
1598 prefer where the type passed is a cms.vstring and that vstring can contain the
1599 name of the C++ types in the Record that are being preferred, e.g.,
1600 #prefer all data in record 'OrangeRecord' from 'juicer'
1601 process.prefer("juicer", OrangeRecord=cms.vstring())
1602 or
1603 #prefer only "Orange" data in "OrangeRecord" from "juicer"
1604 process.prefer("juicer", OrangeRecord=cms.vstring("Orange"))
1605 or
1606 #prefer only "Orange" data with label "ExtraPulp" in "OrangeRecord" from "juicer"
1607 ESPrefer("ESJuicerProd", OrangeRecord=cms.vstring("Orange/ExtraPulp"))
1608 """
1609
1610 if isinstance(esmodule, ESSource) or isinstance(esmodule, ESProducer):
1611 raise RuntimeError("Syntax of process.prefer(process.esmodule) not supported yet")
1612 elif self._findPreferred(esmodule, self.es_producers_(),*args,**kargs) or \
1613 self._findPreferred(esmodule, self.es_sources_(),*args,**kargs):
1614 pass
1615 else:
1616 raise RuntimeError("Cannot resolve prefer for "+repr(esmodule))
1617
1618 def _findPreferred(self, esname:str, d,*args,**kargs) -> bool:
1619
1620 if esname in d:
1621 typ = d[esname].type_()
1622 if typ == esname:
1623 self.__setattr__( esname+"_prefer", ESPrefer(typ,*args,**kargs) )
1624 else:
1625 self.__setattr__( esname+"_prefer", ESPrefer(typ, esname,*args,**kargs) )
1626 return True
1627 else:
1628
1629 found = False
1630 for name, value in d.items():
1631 if value.type_() == esname:
1632 if found:
1633 raise RuntimeError("More than one ES module for "+esname)
1634 found = True
1635 self.__setattr__(esname+"_prefer", ESPrefer(d[esname].type_()) )
1636 return found
1637
1638
1639 class ProcessFragment(object):
1640 def __init__(self, process: Union[Process,str]):
1641 if isinstance(process, Process):
1642 self.__process = process
1643 elif isinstance(process, str):
1644 self.__process = Process(process)
1645
1646 del self.__process.options
1647 del self.__process.maxEvents
1648 del self.__process.maxLuminosityBlocks
1649 else:
1650 raise TypeError('a ProcessFragment can only be constructed from an existig Process or from process name')
1651 def __dir__(self):
1652 return [ x for x in dir(self.__process) if isinstance(getattr(self.__process, x), _ConfigureComponent) ]
1653 def __getattribute__(self, name:str):
1654 if name == '_ProcessFragment__process':
1655 return object.__getattribute__(self, '_ProcessFragment__process')
1656 else:
1657 return getattr(self.__process, name)
1658 def __setattr__(self, name:str, value):
1659 if name == '_ProcessFragment__process':
1660 object.__setattr__(self, name, value)
1661 else:
1662 setattr(self.__process, name, value)
1663 def __delattr__(self, name:str):
1664 if name == '_ProcessFragment__process':
1665 pass
1666 else:
1667 return delattr(self.__process, name)
1668
1669
1670 class FilteredStream(dict):
1671 """a dictionary with fixed keys"""
1672 def _blocked_attribute(obj):
1673 raise AttributeError("An FilteredStream defintion cannot be modified after creation.")
1674 _blocked_attribute = property(_blocked_attribute)
1675 __setattr__ = __delitem__ = __setitem__ = clear = _blocked_attribute
1676 pop = popitem = setdefault = update = _blocked_attribute
1677 def __new__(cls, *args, **kw):
1678 new = dict.__new__(cls)
1679 dict.__init__(new, *args, **kw)
1680 keys = sorted(kw.keys())
1681 if keys != ['content', 'dataTier', 'name', 'paths', 'responsible', 'selectEvents']:
1682 raise ValueError("The needed parameters are: content, dataTier, name, paths, responsible, selectEvents")
1683 if not isinstance(kw['name'],str):
1684 raise ValueError("name must be of type string")
1685 if not isinstance(kw['content'], vstring) and not isinstance(kw['content'],str):
1686 raise ValueError("content must be of type vstring or string")
1687 if not isinstance(kw['dataTier'], string):
1688 raise ValueError("dataTier must be of type string")
1689 if not isinstance(kw['selectEvents'], PSet):
1690 raise ValueError("selectEvents must be of type PSet")
1691 if not isinstance(kw['paths'],(tuple, Path)):
1692 raise ValueError("'paths' must be a tuple of paths")
1693 return new
1694 def __init__(self, *args, **kw):
1695 pass
1696 def __repr__(self) -> str:
1697 return "FilteredStream object: %s" %self["name"]
1698 def __getattr__(self,attr):
1699 return self[attr]
1700
1701 class SubProcess(_Unlabelable):
1702 """Allows embedding another process within a parent process. This allows one to
1703 chain processes together directly in one cmsRun job rather than having to run
1704 separate jobs that are connected via a temporary file.
1705 """
1706 def __init__(self,process, SelectEvents = untracked.PSet(), outputCommands = untracked.vstring()):
1707 """
1708 """
1709 if not isinstance(process, Process):
1710 raise ValueError("the 'process' argument must be of type cms.Process")
1711 if not isinstance(SelectEvents,PSet):
1712 raise ValueError("the 'SelectEvents' argument must be of type cms.untracked.PSet")
1713 if not isinstance(outputCommands,vstring):
1714 raise ValueError("the 'outputCommands' argument must be of type cms.untracked.vstring")
1715 self.__process = process
1716 self.__SelectEvents = SelectEvents
1717 self.__outputCommands = outputCommands
1718
1719 if self.__process.MessageLogger is not MessageLogger:
1720 print("""Warning: You have reconfigured service
1721 'edm::MessageLogger' in a subprocess.
1722 This service has already been configured.
1723 This particular service may not be reconfigured in a subprocess.
1724 The reconfiguration will be ignored.""")
1725 del self.__process.MessageLogger
1726 def dumpPython(self, options:PrintOptions=PrintOptions()) -> str:
1727 out = "parentProcess"+str(hash(self))+" = process\n"
1728 out += self.__process.dumpPython()
1729 out += "childProcess = process\n"
1730 out += "process = parentProcess"+str(hash(self))+"\n"
1731 out += "process.addSubProcess(cms.SubProcess(process = childProcess, SelectEvents = "+self.__SelectEvents.dumpPython(options) +", outputCommands = "+self.__outputCommands.dumpPython(options) +"))"
1732 return out
1733 def getProcessName(self) -> str:
1734 return self.__process.name_()
1735 def process(self):
1736 return self.__process
1737 def SelectEvents(self):
1738 return self.__SelectEvents
1739 def outputCommands(self):
1740 return self.__outputCommands
1741 def type_(self):
1742 return 'subProcess'
1743 def nameInProcessDesc_(self,label:str) -> str:
1744 return label
1745 def _place(self,label,process):
1746 process._placeSubProcess('subProcess',self)
1747 def getSubProcessPSet(self,parameterSet):
1748 topPSet = parameterSet.newPSet()
1749 self.__process.fillProcessDesc(topPSet)
1750 subProcessPSet = parameterSet.newPSet()
1751 self.__SelectEvents.insertInto(subProcessPSet,"SelectEvents")
1752 self.__outputCommands.insertInto(subProcessPSet,"outputCommands")
1753 subProcessPSet.addPSet(False,"process",topPSet)
1754 return subProcessPSet
1755
1756 class _ParameterModifier(object):
1757 """Helper class for Modifier that takes key/value pairs and uses them to reset parameters of the object"""
1758 def __init__(self,args):
1759 self.__args = args
1760 def __call__(self,obj):
1761 params = {}
1762 for k in self.__args.keys():
1763 if hasattr(obj,k):
1764 params[k] = getattr(obj,k)
1765 _modifyParametersFromDict(params, self.__args, self._raiseUnknownKey)
1766 for k in self.__args.keys():
1767 if k in params:
1768 setattr(obj,k,params[k])
1769 else:
1770
1771 delattr(obj,k)
1772 @staticmethod
1773 def _raiseUnknownKey(key):
1774 raise KeyError("Unknown parameter name "+key+" specified while calling Modifier")
1775
1776 class _BoolModifierBase(object):
1777 """A helper base class for _AndModifier, _InvertModifier, and _OrModifier to contain the common code"""
1778 def __init__(self, lhs, rhs=None):
1779 self._lhs = lhs
1780 if rhs is not None:
1781 self._rhs = rhs
1782 def toModify(self,obj, func=None,**kw):
1783 Modifier._toModifyCheck(obj,func,**kw)
1784 if self._isChosen():
1785 Modifier._toModify(obj,func,**kw)
1786 return self
1787 def toReplaceWith(self,toObj,fromObj):
1788 Modifier._toReplaceWithCheck(toObj,fromObj)
1789 if self._isChosen():
1790 Modifier._toReplaceWith(toObj,fromObj)
1791 return self
1792 def makeProcessModifier(self,func):
1793 """This is used to create a ProcessModifer that can perform actions on the process as a whole.
1794 This takes as argument a callable object (e.g. function) that takes as its sole argument an instance of Process.
1795 In order to work, the value returned from this function must be assigned to a uniquely named variable."""
1796 return ProcessModifier(self,func)
1797 def __and__(self, other):
1798 return _AndModifier(self,other)
1799 def __invert__(self):
1800 return _InvertModifier(self)
1801 def __or__(self, other):
1802 return _OrModifier(self,other)
1803
1804 class _AndModifier(_BoolModifierBase):
1805 """A modifier which only applies if multiple Modifiers are chosen"""
1806 def __init__(self, lhs, rhs):
1807 super(_AndModifier,self).__init__(lhs, rhs)
1808 def _isChosen(self) -> bool:
1809 return self._lhs._isChosen() and self._rhs._isChosen()
1810
1811 class _InvertModifier(_BoolModifierBase):
1812 """A modifier which only applies if a Modifier is not chosen"""
1813 def __init__(self, lhs):
1814 super(_InvertModifier,self).__init__(lhs)
1815 def _isChosen(self) -> bool:
1816 return not self._lhs._isChosen()
1817
1818 class _OrModifier(_BoolModifierBase):
1819 """A modifier which only applies if at least one of multiple Modifiers is chosen"""
1820 def __init__(self, lhs, rhs):
1821 super(_OrModifier,self).__init__(lhs, rhs)
1822 def _isChosen(self) -> bool:
1823 return self._lhs._isChosen() or self._rhs._isChosen()
1824
1825
1826 class Modifier(object):
1827 """This class is used to define standard modifications to a Process.
1828 An instance of this class is declared to denote a specific modification,e.g. era2017 could
1829 reconfigure items in a process to match our expectation of running in 2017. Once declared,
1830 these Modifier instances are imported into a configuration and items that need to be modified
1831 are then associated with the Modifier and with the action to do the modification.
1832 The registered modifications will only occur if the Modifier was passed to
1833 the cms.Process' constructor.
1834 """
1835 def __init__(self):
1836 self.__processModifiers = []
1837 self.__chosen = False
1838 def makeProcessModifier(self,func):
1839 """This is used to create a ProcessModifer that can perform actions on the process as a whole.
1840 This takes as argument a callable object (e.g. function) that takes as its sole argument an instance of Process.
1841 In order to work, the value returned from this function must be assigned to a uniquely named variable.
1842 """
1843 return ProcessModifier(self,func)
1844 @staticmethod
1845 def _toModifyCheck(obj,func,**kw):
1846 if func is not None and len(kw) != 0:
1847 raise TypeError("toModify takes either two arguments or one argument and key/value pairs")
1848 def toModify(self,obj, func=None,**kw):
1849 """This is used to register an action to be performed on the specific object. Two different forms are allowed
1850 Form 1: A callable object (e.g. function) can be passed as the second. This callable object is expected to take one argument
1851 that will be the object passed in as the first argument.
1852 Form 2: A list of parameter name, value pairs can be passed
1853 mod.toModify(foo, fred=cms.int32(7), barney = cms.double(3.14))
1854 This form can also be used to remove a parameter by passing the value of None
1855 #remove the parameter foo.fred
1856 mod.toModify(foo, fred = None)
1857 Additionally, parameters embedded within PSets can also be modified using a dictionary
1858 #change foo.fred.pebbles to 3 and foo.fred.friend to "barney"
1859 mod.toModify(foo, fred = dict(pebbles = 3, friend = "barney)) )
1860 """
1861 Modifier._toModifyCheck(obj,func,**kw)
1862 if self._isChosen():
1863 Modifier._toModify(obj,func,**kw)
1864 return self
1865 @staticmethod
1866 def _toModify(obj,func,**kw):
1867 if func is not None:
1868 func(obj)
1869 else:
1870 temp =_ParameterModifier(kw)
1871 temp(obj)
1872 @staticmethod
1873 def _toReplaceWithCheck(toObj,fromObj):
1874 if not isinstance(fromObj, type(toObj)):
1875 raise TypeError("toReplaceWith requires both arguments to be the same class type")
1876 def toReplaceWith(self,toObj,fromObj):
1877 """If the Modifier is chosen the internals of toObj will be associated with the internals of fromObj
1878 """
1879 Modifier._toReplaceWithCheck(toObj,fromObj)
1880 if self._isChosen():
1881 Modifier._toReplaceWith(toObj,fromObj)
1882 return self
1883 @staticmethod
1884 def _toReplaceWith(toObj,fromObj):
1885 if isinstance(fromObj,_ModuleSequenceType):
1886 toObj._seq = fromObj._seq
1887 toObj._tasks = fromObj._tasks
1888 elif isinstance(fromObj,Task):
1889 toObj._collection = fromObj._collection
1890 elif isinstance(fromObj,ConditionalTask):
1891 toObj._collection = fromObj._collection
1892 elif isinstance(fromObj,_Parameterizable):
1893
1894 for p in toObj.parameterNames_():
1895 delattr(toObj,p)
1896 for p in fromObj.parameterNames_():
1897 setattr(toObj,p,getattr(fromObj,p))
1898 if isinstance(fromObj,_TypedParameterizable):
1899 toObj._TypedParameterizable__type = fromObj._TypedParameterizable__type
1900
1901 else:
1902 raise TypeError("toReplaceWith does not work with type "+str(type(toObj)))
1903
1904 def _setChosen(self):
1905 """Should only be called by cms.Process instances"""
1906 self.__chosen = True
1907 def _isChosen(self) -> bool:
1908 return self.__chosen
1909 def __and__(self, other):
1910 return _AndModifier(self,other)
1911 def __invert__(self):
1912 return _InvertModifier(self)
1913 def __or__(self, other):
1914 return _OrModifier(self,other)
1915 def _isOrContains(self, other):
1916 return self == other
1917
1918
1919 class ModifierChain(object):
1920 """A Modifier made up of a list of Modifiers
1921 """
1922 def __init__(self, *chainedModifiers):
1923 self.__chosen = False
1924 self.__chain = chainedModifiers
1925 def _applyNewProcessModifiers(self,process):
1926 """Should only be called by cms.Process instances
1927 applies list of accumulated changes to the process"""
1928 for m in self.__chain:
1929 m._applyNewProcessModifiers(process)
1930 def _setChosen(self):
1931 """Should only be called by cms.Process instances"""
1932 self.__chosen = True
1933 for m in self.__chain:
1934 m._setChosen()
1935 def _isChosen(self) -> bool:
1936 return self.__chosen
1937 def copyAndExclude(self, toExclude):
1938 """Creates a new ModifierChain which is a copy of
1939 this ModifierChain but excludes any Modifier or
1940 ModifierChain in the list toExclude.
1941 The exclusion is done recursively down the chain.
1942 """
1943 newMods = []
1944 for m in self.__chain:
1945 if m not in toExclude:
1946 s = m
1947 if isinstance(m,ModifierChain):
1948 s = m.__copyIfExclude(toExclude)
1949 newMods.append(s)
1950 return ModifierChain(*newMods)
1951 def __copyIfExclude(self,toExclude):
1952 shouldCopy = False
1953 for m in toExclude:
1954 if self._isOrContains(m):
1955 shouldCopy = True
1956 break
1957 if shouldCopy:
1958 return self.copyAndExclude(toExclude)
1959 return self
1960 def _isOrContains(self, other):
1961 if self is other:
1962 return True
1963 for m in self.__chain:
1964 if m._isOrContains(other):
1965 return True
1966 return False
1967
1968 class ProcessModifier(object):
1969 """A class used by a Modifier to affect an entire Process instance.
1970 When a Process 'loads' a module containing a ProcessModifier, that
1971 ProcessModifier will be applied to the Process if and only if the
1972 Modifier passed to the constructor has been chosen.
1973 """
1974 def __init__(self, modifier, func):
1975 self.__modifier = modifier
1976 self.__func = func
1977 self.__seenProcesses = set()
1978 def apply(self,process):
1979 if self.__modifier._isChosen():
1980 if process not in self.__seenProcesses:
1981 self.__func(process)
1982 self.__seenProcesses.add(process)
1983
1984 class ProcessAccelerator(_ConfigureComponent,_Unlabelable):
1985 """A class used to specify possible compute accelerators in a Process
1986 instance. It is intended to be derived for any
1987 accelerator/portability technology, and provides hooks such that a
1988 specific customization can be applied to the Process on a worker
1989 node at the point where the python configuration is serialized for C++.
1990
1991 The customization must not change the configuration hash. To
1992 enforce this reuirement, the customization gets a
1993 ProcessForProcessAccelerator wrapper that gives access to only
1994 those parts of the configuration that can be changed. Nevertheless
1995 it would be good to have specific unit test for each deriving
1996 class to ensure that all combinations of the enabled accelerators
1997 give the same configuration hash.
1998
1999 The deriving class must do its checks for hardware availability
2000 only in enabledLabels(), and possibly in apply() if any further
2001 fine-tuning is needed, because those two are the only functions
2002 that are guaranteed to be called at the worker node.
2003 """
2004 def __init__(self):
2005 pass
2006 def _place(self, name, proc):
2007 proc._placeAccelerator(self.type_(), self)
2008 def type_(self):
2009 return type(self).__name__
2010 def dumpPython(self, options:PrintOptions=PrintOptions()) -> str:
2011 specialImportRegistry.registerUse(self)
2012 result = self.__class__.__name__+"("
2013 options.indent()
2014 res = self.dumpPythonImpl(options)
2015 options.unindent()
2016 if len(res) > 0:
2017 result += "\n"+res+"\n"
2018 result += ")\n"
2019 return result
2020
2021
2022 def dumpPythonImpl(self, options) -> str:
2023 """Override if need to add any 'body' content to dumpPython(). Returns a string."""
2024 return ""
2025 def labels(self):
2026 """Override to return a list of strings for the accelerator labels."""
2027 return []
2028 def enabledLabels(self):
2029 """Override to return a list of strings for the accelerator labels
2030 that are enabled in the system the job is being run on."""
2031 return []
2032 def moduleTypeResolver(self, accelerators):
2033 """Override to return an object that implements "module type resolver"
2034 in python. The object should have the following methods
2035 - __init__(self, accelerators)
2036 * accelerators = list of selected accelerators
2037 - plugin(self):
2038 * should return a string for the type resolver plugin name
2039 - setModuleVariant(self, module):
2040 * Called for each ED and ES module. Should act only if
2041 module.type_() contains the magic identifier
2042
2043 At most one of the ProcessAccelerators in a job can return a
2044 non-None object
2045 """
2046 return None
2047 def apply(self, process, accelerators):
2048 """Override if need to customize the Process at worker node. The
2049 selected available accelerator labels are given in the
2050 'accelerators' argument (the patterns, e.g. '*' have been
2051 expanded to concrete labels).
2052
2053 This function may touch only untracked parameters.
2054 """
2055 pass
2056
2057 class ProcessForProcessAccelerator(object):
2058 """This class is inteded to wrap the Process object to constrain the
2059 available functionality for ProcessAccelerator.apply()"""
2060 def __init__(self, process):
2061 self.__process = process
2062 def __getattr__(self, label):
2063 value = getattr(self.__process, label)
2064 if not isinstance(value, Service):
2065 raise TypeError("ProcessAccelerator.apply() can get only Services. Tried to get {} with label {}".format(str(type(value)), label))
2066 return value
2067 def __setattr__(self, label:str, value):
2068 if label == "_ProcessForProcessAccelerator__process":
2069 super().__setattr__(label, value)
2070 else:
2071 if not isinstance(value, Service):
2072 raise TypeError("ProcessAccelerator.apply() can only set Services. Tried to set {} with label {}".format(str(type(value)), label))
2073 setattr(self.__process, label, value)
2074 def __delattr__(self, label:str):
2075 value = getattr(self.__process, label)
2076 if not isinstance(value, Service):
2077 raise TypeError("ProcessAccelerator.apply() can delete only Services. Tried to del {} with label {}".format(str(type(value)), label))
2078 delattr(self.__process, label)
2079 def add_(self, value):
2080 if not isinstance(value, Service):
2081 raise TypeError("ProcessAccelerator.apply() can only add Services. Tried to set {} with label {}".format(str(type(value)), label))
2082 self.__process.add_(value)
2083
2084
2085
2086 def _switchproducer_test2_case1(accelerators):
2087 return ("test1" in accelerators, -10)
2088 def _switchproducer_test2_case2(accelerators):
2089 return ("test2" in accelerators, -9)
2090
2091 if __name__=="__main__":
2092 import unittest
2093 import copy
2094
2095 def _lineDiff(newString, oldString):
2096 newString = ( x for x in newString.split('\n') if len(x) > 0)
2097 oldString = [ x for x in oldString.split('\n') if len(x) > 0]
2098 diff = []
2099 oldStringLine = 0
2100 for l in newString:
2101 if oldStringLine >= len(oldString):
2102 diff.append(l)
2103 continue
2104 if l == oldString[oldStringLine]:
2105 oldStringLine +=1
2106 continue
2107 diff.append(l)
2108 return "\n".join( diff )
2109
2110 class TestMakePSet(object):
2111 """Has same interface as the C++ object that creates PSets
2112 """
2113 def __init__(self):
2114 self.values = dict()
2115 def __insertValue(self,tracked,label,value):
2116 self.values[label]=(tracked,value)
2117 def __getValue(self,tracked,label):
2118 pair = self.values[label]
2119 if pair[0] != tracked:
2120 raise Exception("Asked for %s parameter '%s', but it is %s" % ("tracked" if tracked else "untracked",
2121 label,
2122 "tracked" if pair[0] else "untracked"))
2123 return pair[1]
2124 def addInt32(self,tracked,label,value):
2125 self.__insertValue(tracked,label,value)
2126 def addVInt32(self,tracked,label,value):
2127 self.__insertValue(tracked,label,value)
2128 def addUInt32(self,tracked,label,value):
2129 self.__insertValue(tracked,label,value)
2130 def addVUInt32(self,tracked,label,value):
2131 self.__insertValue(tracked,label,value)
2132 def addInt64(self,tracked,label,value):
2133 self.__insertValue(tracked,label,value)
2134 def addVInt64(self,tracked,label,value):
2135 self.__insertValue(tracked,label,value)
2136 def addUInt64(self,tracked,label,value):
2137 self.__insertValue(tracked,label,value)
2138 def addVUInt64(self,tracked,label,value):
2139 self.__insertValue(tracked,label,value)
2140 def addDouble(self,tracked,label,value):
2141 self.__insertValue(tracked,label,value)
2142 def addVDouble(self,tracked,label,value):
2143 self.__insertValue(tracked,label,value)
2144 def addBool(self,tracked,label,value):
2145 self.__insertValue(tracked,label,value)
2146 def addString(self,tracked,label,value):
2147 self.__insertValue(tracked,label,value)
2148 def addVString(self,tracked,label,value):
2149 self.__insertValue(tracked,label,value)
2150 def getVString(self,tracked,label):
2151 return self.__getValue(tracked, label)
2152 def addInputTag(self,tracked,label,value):
2153 self.__insertValue(tracked,label,value)
2154 def addVInputTag(self,tracked,label,value):
2155 self.__insertValue(tracked,label,value)
2156 def addESInputTag(self,tracked,label,value):
2157 self.__insertValue(tracked,label,value)
2158 def addVESInputTag(self,tracked,label,value):
2159 self.__insertValue(tracked,label,value)
2160 def addEventID(self,tracked,label,value):
2161 self.__insertValue(tracked,label,value)
2162 def addVEventID(self,tracked,label,value):
2163 self.__insertValue(tracked,label,value)
2164 def addLuminosityBlockID(self,tracked,label,value):
2165 self.__insertValue(tracked,label,value)
2166 def addLuminosityBlockID(self,tracked,label,value):
2167 self.__insertValue(tracked,label,value)
2168 def addEventRange(self,tracked,label,value):
2169 self.__insertValue(tracked,label,value)
2170 def addVEventRange(self,tracked,label,value):
2171 self.__insertValue(tracked,label,value)
2172 def addPSet(self,tracked,label,value):
2173 self.__insertValue(tracked,label,value)
2174 def addVPSet(self,tracked,label,value):
2175 self.__insertValue(tracked,label,value)
2176 def addFileInPath(self,tracked,label,value):
2177 self.__insertValue(tracked,label,value)
2178 def newPSet(self):
2179 return TestMakePSet()
2180
2181 class SwitchProducerTest(SwitchProducer):
2182 def __init__(self, **kargs):
2183 super(SwitchProducerTest,self).__init__(
2184 dict(
2185 test1 = lambda accelerators: (True, -10),
2186 test2 = lambda accelerators: (True, -9),
2187 test3 = lambda accelerators: (True, -8),
2188 test4 = lambda accelerators: (True, -7)
2189 ), **kargs)
2190 specialImportRegistry.registerSpecialImportForType(SwitchProducerTest, "from test import SwitchProducerTest")
2191
2192 class SwitchProducerTest2(SwitchProducer):
2193 def __init__(self, **kargs):
2194 super(SwitchProducerTest2,self).__init__(
2195 dict(
2196 test1 = _switchproducer_test2_case1,
2197 test2 = _switchproducer_test2_case2,
2198 ), **kargs)
2199 specialImportRegistry.registerSpecialImportForType(SwitchProducerTest2, "from test import SwitchProducerTest2")
2200
2201 class TestModuleTypeResolver:
2202 def __init__(self, accelerators):
2203
2204 self._valid_backends = []
2205 if "test1" in accelerators:
2206 self._valid_backends.append("test1_backend")
2207 if "test2" in accelerators:
2208 self._valid_backends.append("test2_backend")
2209 if len(self._valid_backends) == 0:
2210 raise EDMException(edm.errors.UnavailableAccelerator, "Machine has no accelerators that Test supports (has {})".format(", ".join(accelerators)))
2211
2212 def plugin(self):
2213 return "TestModuleTypeResolver"
2214
2215 def setModuleVariant(self, module):
2216 if "@test" in module.type_():
2217 defaultBackend = self._valid_backends[0]
2218 if hasattr(module, "test"):
2219 if hasattr(module.test, "backend"):
2220 if module.test.backend.value() not in self._valid_backends:
2221 raise EDMException(edm.errors.UnavailableAccelerator, "Module {} has the Test backend set explicitly, but its accelerator is not available for the job".format(module.label_()))
2222 else:
2223 module.test.backend = untracked.string(defaultBackend)
2224 else:
2225 module.test = untracked.PSet(
2226 backend = untracked.string(defaultBackend)
2227 )
2228
2229 class ProcessAcceleratorTest(ProcessAccelerator):
2230 def __init__(self, enabled=["test1", "test2", "anothertest3"], moduleTypeResolverMaker=None):
2231 super().__init__()
2232 self._labels = ["test1", "test2", "anothertest3"]
2233 self.setEnabled(enabled)
2234 self._moduleTypeResolverMaker = moduleTypeResolverMaker
2235 def setEnabled(self, enabled):
2236 invalid = set(enabled).difference(set(self._labels))
2237 if len(invalid) > 0:
2238 raise Exception("Tried to enabled nonexistent test accelerators {}".format(",".join(invalid)))
2239 self._enabled = enabled[:]
2240 def dumpPythonImpl(self,options):
2241 result = "{}enabled = [{}]".format(options.indentation(),
2242 ", ".join(["'{}'".format(e) for e in self._enabled]))
2243 return result
2244 def labels(self):
2245 return self._labels
2246 def enabledLabels(self):
2247 return self._enabled
2248 def moduleTypeResolver(self, accelerators):
2249 if not self._moduleTypeResolverMaker:
2250 return super().moduleTypeResolver(accelerators)
2251 return self._moduleTypeResolverMaker(accelerators)
2252 def apply(self, process, accelerators):
2253 process.AcceleratorTestService = Service("AcceleratorTestService")
2254 if hasattr(process, "AcceleratorTestServiceRemove"):
2255 del process.AcceleratorTestServiceRemove
2256 specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorTest, "from test import ProcessAcceleratorTest")
2257
2258 class ProcessAcceleratorTest2(ProcessAccelerator):
2259 def __init__(self, enabled=["anothertest3", "anothertest4"], moduleTypeResolverMaker=None):
2260 super().__init__()
2261 self._labels = ["anothertest3", "anothertest4"]
2262 self.setEnabled(enabled)
2263 self._moduleTypeResolverMaker = moduleTypeResolverMaker
2264 def setEnabled(self, enabled):
2265 invalid = set(enabled).difference(set(self._labels))
2266 if len(invalid) > 0:
2267 raise Exception("Tried to enabled nonexistent test accelerators {}".format(",".join(invalid)))
2268 self._enabled = enabled[:]
2269 def dumpPythonImpl(self,options):
2270 result = "{}enabled = [{}]".format(options.indentation(),
2271 ", ".join(["'{}'".format(e) for e in self._enabled]))
2272 return result
2273 def labels(self):
2274 return self._labels
2275 def enabledLabels(self):
2276 return self._enabled
2277 def moduleTypeResolver(self, accelerators):
2278 if not self._moduleTypeResolverMaker:
2279 return super().moduleTypeResolver(accelerators)
2280 return self._moduleTypeResolverMaker(accelerators)
2281 def apply(self, process, accelerators):
2282 pass
2283 specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorTest2, "from test import ProcessAcceleratorTest2")
2284
2285 class TestModuleCommand(unittest.TestCase):
2286 def setUp(self):
2287 """Nothing to do """
2288 None
2289 def testParameterizable(self):
2290 p = _Parameterizable()
2291 self.assertEqual(len(p.parameterNames_()),0)
2292 p.a = int32(1)
2293 self.assertTrue('a' in p.parameterNames_())
2294 self.assertEqual(p.a.value(), 1)
2295 p.a = 10
2296 self.assertEqual(p.a.value(), 10)
2297 p.a = untracked(int32(1))
2298 self.assertEqual(p.a.value(), 1)
2299 self.assertFalse(p.a.isTracked())
2300 p.a = untracked.int32(1)
2301 self.assertEqual(p.a.value(), 1)
2302 self.assertFalse(p.a.isTracked())
2303 p = _Parameterizable(foo=int32(10), bar = untracked(double(1.0)))
2304 self.assertEqual(p.foo.value(), 10)
2305 self.assertEqual(p.bar.value(),1.0)
2306 self.assertFalse(p.bar.isTracked())
2307 self.assertRaises(TypeError,setattr,(p,'c',1))
2308 p = _Parameterizable(a=PSet(foo=int32(10), bar = untracked(double(1.0))))
2309 self.assertEqual(p.a.foo.value(),10)
2310 self.assertEqual(p.a.bar.value(),1.0)
2311 p.b = untracked(PSet(fii = int32(1)))
2312 self.assertEqual(p.b.fii.value(),1)
2313 self.assertFalse(p.b.isTracked())
2314
2315 v = int32(10)
2316 p=_Parameterizable(a=v)
2317 v.setValue(11)
2318 self.assertEqual(p.a.value(),11)
2319 p.a = 12
2320 self.assertEqual(p.a.value(),12)
2321 self.assertEqual(v.value(),12)
2322 def testTypedParameterizable(self):
2323 p = _TypedParameterizable("blah", b=int32(1))
2324
2325 other = p.copy()
2326 other.b = 2
2327 self.assertNotEqual(p.b,other.b)
2328
2329 def testProcessInsertion(self):
2330 p = Process("test")
2331 p.a = EDAnalyzer("MyAnalyzer")
2332 self.assertTrue( 'a' in p.analyzers_() )
2333 self.assertTrue( 'a' in p.analyzers)
2334 p.add_(Service("SomeService"))
2335 self.assertTrue('SomeService' in p.services_())
2336 self.assertEqual(p.SomeService.type_(), "SomeService")
2337 p.Tracer = Service("Tracer")
2338 self.assertTrue('Tracer' in p.services_())
2339 self.assertRaises(TypeError, setattr, *(p,'b',"this should fail"))
2340 self.assertRaises(TypeError, setattr, *(p,'bad',Service("MessageLogger")))
2341 self.assertRaises(ValueError, setattr, *(p,'bad',Source("PoolSource")))
2342 p.out = OutputModule("Outer")
2343 self.assertEqual(p.out.type_(), 'Outer')
2344 self.assertTrue( 'out' in p.outputModules_() )
2345
2346 p.geom = ESSource("GeomProd")
2347 self.assertTrue('geom' in p.es_sources_())
2348 p.add_(ESSource("ConfigDB"))
2349 self.assertTrue('ConfigDB' in p.es_sources_())
2350
2351 p.aliasfoo1 = EDAlias(foo1 = VPSet(PSet(type = string("Foo1"))))
2352 self.assertTrue('aliasfoo1' in p.aliases_())
2353
2354 def testProcessExtend(self):
2355 class FromArg(object):
2356 def __init__(self,*arg,**args):
2357 for name in args.keys():
2358 self.__dict__[name]=args[name]
2359
2360 a=EDAnalyzer("MyAnalyzer")
2361 t=EDAnalyzer("MyAnalyzer")
2362 t.setLabel("foo")
2363 s1 = Sequence(a)
2364 s2 = Sequence(s1)
2365 s3 = Sequence(s2)
2366 d = FromArg(
2367 a=a,
2368 b=Service("Full"),
2369 c=Path(a),
2370 d=s2,
2371 e=s1,
2372 f=s3,
2373 g=Sequence(s1+s2+s3)
2374 )
2375 p = Process("Test")
2376 p.extend(d)
2377 self.assertEqual(p.a.type_(),"MyAnalyzer")
2378 self.assertEqual(p.a.label_(),"a")
2379 self.assertRaises(AttributeError,getattr,p,'b')
2380 self.assertEqual(p.Full.type_(),"Full")
2381 self.assertEqual(str(p.c),'a')
2382 self.assertEqual(str(p.d),'a')
2383
2384 z1 = FromArg(
2385 a=a,
2386 b=Service("Full"),
2387 c=Path(a),
2388 d=s2,
2389 e=s1,
2390 f=s3,
2391 s4=s3,
2392 g=Sequence(s1+s2+s3)
2393 )
2394
2395 p1 = Process("Test")
2396
2397 self.assertRaises(ValueError, p1.extend, z1)
2398
2399 z2 = FromArg(
2400 a=a,
2401 b=Service("Full"),
2402 c=Path(a),
2403 d=s2,
2404 e=s1,
2405 f=s3,
2406 aaa=copy.deepcopy(a),
2407 s4=copy.deepcopy(s3),
2408 g=Sequence(s1+s2+s3),
2409 t=t
2410 )
2411 p2 = Process("Test")
2412 p2.extend(z2)
2413
2414 self.assertEqual(p2.s4.label_(),"s4")
2415
2416 self.assertRaises(ValueError, p2.s4.setLabel, "foo")
2417 p2.s4.setLabel("s4")
2418 p2.s4.setLabel(None)
2419 p2.s4.setLabel("foo")
2420 p2._Process__setObjectLabel(p2.s4, "foo")
2421 p2._Process__setObjectLabel(p2.s4, None)
2422 p2._Process__setObjectLabel(p2.s4, "bar")
2423
2424
2425 p = Process('test')
2426 p.a = EDProducer("MyProducer")
2427 p.t = Task(p.a)
2428 p.p = Path(p.t)
2429 self.assertRaises(ValueError, p.extend, FromArg(a = EDProducer("YourProducer")))
2430 self.assertRaises(ValueError, p.extend, FromArg(a = EDAlias()))
2431 self.assertRaises(ValueError, p.__setattr__, "a", EDAlias())
2432
2433 p = Process('test')
2434 p.a = EDProducer("MyProducer")
2435 p.t = ConditionalTask(p.a)
2436 p.p = Path(p.t)
2437 self.assertRaises(ValueError, p.extend, FromArg(a = EDProducer("YourProducer")))
2438 self.assertRaises(ValueError, p.extend, FromArg(a = EDAlias()))
2439 self.assertRaises(ValueError, p.__setattr__, "a", EDAlias())
2440
2441 p = Process('test')
2442 p.a = EDProducer("MyProducer")
2443 p.s = Sequence(p.a)
2444 p.p = Path(p.s)
2445 self.assertRaises(ValueError, p.extend, FromArg(a = EDProducer("YourProducer")))
2446 self.assertRaises(ValueError, p.extend, FromArg(a = EDAlias()))
2447 self.assertRaises(ValueError, p.__setattr__, "a", EDAlias())
2448
2449 def testProcessDumpPython(self):
2450 self.maxDiff = None
2451 self.assertEqual(Process("test").dumpPython(),
2452 """import FWCore.ParameterSet.Config as cms
2453
2454 process = cms.Process("test")
2455
2456 process.maxEvents = cms.untracked.PSet(
2457 input = cms.optional.untracked.int32,
2458 output = cms.optional.untracked.allowed(cms.int32,cms.PSet)
2459 )
2460
2461 process.maxLuminosityBlocks = cms.untracked.PSet(
2462 input = cms.untracked.int32(-1)
2463 )
2464
2465 process.options = cms.untracked.PSet(
2466 IgnoreCompletely = cms.untracked.vstring(),
2467 Rethrow = cms.untracked.vstring(),
2468 TryToContinue = cms.untracked.vstring(),
2469 accelerators = cms.untracked.vstring('*'),
2470 allowUnscheduled = cms.obsolete.untracked.bool,
2471 canDeleteEarly = cms.untracked.vstring(),
2472 deleteNonConsumedUnscheduledModules = cms.untracked.bool(True),
2473 dumpOptions = cms.untracked.bool(False),
2474 emptyRunLumiMode = cms.obsolete.untracked.string,
2475 eventSetup = cms.untracked.PSet(
2476 forceNumberOfConcurrentIOVs = cms.untracked.PSet(
2477 allowAnyLabel_=cms.required.untracked.uint32
2478 ),
2479 numberOfConcurrentIOVs = cms.untracked.uint32(0)
2480 ),
2481 fileMode = cms.untracked.string('FULLMERGE'),
2482 forceEventSetupCacheClearOnNewRun = cms.untracked.bool(False),
2483 holdsReferencesToDeleteEarly = cms.untracked.VPSet(),
2484 makeTriggerResults = cms.obsolete.untracked.bool,
2485 modulesToCallForTryToContinue = cms.untracked.vstring(),
2486 modulesToIgnoreForDeleteEarly = cms.untracked.vstring(),
2487 numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(0),
2488 numberOfConcurrentRuns = cms.untracked.uint32(1),
2489 numberOfStreams = cms.untracked.uint32(0),
2490 numberOfThreads = cms.untracked.uint32(1),
2491 printDependencies = cms.untracked.bool(False),
2492 sizeOfStackForThreadsInKB = cms.optional.untracked.uint32,
2493 throwIfIllegalParameter = cms.untracked.bool(True),
2494 wantSummary = cms.untracked.bool(False)
2495 )
2496
2497 process.MessageLogger = cms.Service("MessageLogger",
2498 cerr = cms.untracked.PSet(
2499 FwkReport = cms.untracked.PSet(
2500 limit = cms.untracked.int32(10000000),
2501 reportEvery = cms.untracked.int32(1)
2502 ),
2503 FwkSummary = cms.untracked.PSet(
2504 limit = cms.untracked.int32(10000000),
2505 reportEvery = cms.untracked.int32(1)
2506 ),
2507 INFO = cms.untracked.PSet(
2508 limit = cms.untracked.int32(0)
2509 ),
2510 Root_NoDictionary = cms.untracked.PSet(
2511 limit = cms.untracked.int32(0)
2512 ),
2513 default = cms.untracked.PSet(
2514 limit = cms.untracked.int32(10000000)
2515 ),
2516 enable = cms.untracked.bool(True),
2517 enableStatistics = cms.untracked.bool(False),
2518 lineLength = cms.optional.untracked.int32,
2519 noLineBreaks = cms.optional.untracked.bool,
2520 noTimeStamps = cms.untracked.bool(False),
2521 resetStatistics = cms.untracked.bool(False),
2522 statisticsThreshold = cms.untracked.string('WARNING'),
2523 threshold = cms.untracked.string('INFO'),
2524 allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2525 limit = cms.optional.untracked.int32,
2526 reportEvery = cms.untracked.int32(1),
2527 timespan = cms.optional.untracked.int32
2528 )
2529 ),
2530 cout = cms.untracked.PSet(
2531 enable = cms.untracked.bool(False),
2532 enableStatistics = cms.untracked.bool(False),
2533 lineLength = cms.optional.untracked.int32,
2534 noLineBreaks = cms.optional.untracked.bool,
2535 noTimeStamps = cms.optional.untracked.bool,
2536 resetStatistics = cms.untracked.bool(False),
2537 statisticsThreshold = cms.optional.untracked.string,
2538 threshold = cms.optional.untracked.string,
2539 allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2540 limit = cms.optional.untracked.int32,
2541 reportEvery = cms.untracked.int32(1),
2542 timespan = cms.optional.untracked.int32
2543 )
2544 ),
2545 debugModules = cms.untracked.vstring(),
2546 default = cms.untracked.PSet(
2547 limit = cms.optional.untracked.int32,
2548 lineLength = cms.untracked.int32(80),
2549 noLineBreaks = cms.untracked.bool(False),
2550 noTimeStamps = cms.untracked.bool(False),
2551 reportEvery = cms.untracked.int32(1),
2552 statisticsThreshold = cms.untracked.string('INFO'),
2553 threshold = cms.untracked.string('INFO'),
2554 timespan = cms.optional.untracked.int32,
2555 allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2556 limit = cms.optional.untracked.int32,
2557 reportEvery = cms.untracked.int32(1),
2558 timespan = cms.optional.untracked.int32
2559 )
2560 ),
2561 files = cms.untracked.PSet(
2562 allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2563 enableStatistics = cms.untracked.bool(False),
2564 extension = cms.optional.untracked.string,
2565 filename = cms.optional.untracked.string,
2566 lineLength = cms.optional.untracked.int32,
2567 noLineBreaks = cms.optional.untracked.bool,
2568 noTimeStamps = cms.optional.untracked.bool,
2569 output = cms.optional.untracked.string,
2570 resetStatistics = cms.untracked.bool(False),
2571 statisticsThreshold = cms.optional.untracked.string,
2572 threshold = cms.optional.untracked.string,
2573 allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2574 limit = cms.optional.untracked.int32,
2575 reportEvery = cms.untracked.int32(1),
2576 timespan = cms.optional.untracked.int32
2577 )
2578 )
2579 ),
2580 suppressDebug = cms.untracked.vstring(),
2581 suppressFwkInfo = cms.untracked.vstring(),
2582 suppressInfo = cms.untracked.vstring(),
2583 suppressWarning = cms.untracked.vstring(),
2584 allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2585 limit = cms.optional.untracked.int32,
2586 reportEvery = cms.untracked.int32(1),
2587 timespan = cms.optional.untracked.int32
2588 )
2589 )
2590
2591
2592 """)
2593 p = Process("test")
2594 p.a = EDAnalyzer("MyAnalyzer")
2595 p.p = Path(p.a)
2596 p.s = Sequence(p.a)
2597 p.r = Sequence(p.s)
2598 p.p2 = Path(p.s)
2599 p.schedule = Schedule(p.p2,p.p)
2600 d=p.dumpPython()
2601 self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2602 """process.a = cms.EDAnalyzer("MyAnalyzer")
2603 process.s = cms.Sequence(process.a)
2604 process.r = cms.Sequence(process.s)
2605 process.p = cms.Path(process.a)
2606 process.p2 = cms.Path(process.s)
2607 process.schedule = cms.Schedule(*[ process.p2, process.p ])""")
2608
2609 p = Process("test")
2610 p.a = EDAnalyzer("MyAnalyzer")
2611 p.p = Path(p.a)
2612 p.r = Sequence(p.a)
2613 p.s = Sequence(p.r)
2614 p.p2 = Path(p.r)
2615 p.schedule = Schedule(p.p2,p.p)
2616 p.b = EDAnalyzer("YourAnalyzer")
2617 d=p.dumpPython()
2618 self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2619 """process.a = cms.EDAnalyzer("MyAnalyzer")
2620 process.b = cms.EDAnalyzer("YourAnalyzer")
2621 process.r = cms.Sequence(process.a)
2622 process.s = cms.Sequence(process.r)
2623 process.p = cms.Path(process.a)
2624 process.p2 = cms.Path(process.r)
2625 process.schedule = cms.Schedule(*[ process.p2, process.p ])""")
2626
2627 p = Process("test")
2628 p.a = EDAnalyzer("MyAnalyzer")
2629 p.p = Path(p.a)
2630 s = Sequence(p.a)
2631 p.r = Sequence(s)
2632 p.p2 = Path(p.r)
2633 p.schedule = Schedule(p.p2,p.p)
2634 d=p.dumpPython()
2635 self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2636 """process.a = cms.EDAnalyzer("MyAnalyzer")
2637 process.r = cms.Sequence((process.a))
2638 process.p = cms.Path(process.a)
2639 process.p2 = cms.Path(process.r)
2640 process.schedule = cms.Schedule(*[ process.p2, process.p ])""")
2641
2642
2643 p = Process("test")
2644 p.a = EDAnalyzer("MyAnalyzer")
2645 p.b = EDProducer("bProducer")
2646 p.c = EDProducer("cProducer")
2647 p.d = EDProducer("dProducer")
2648 p.e = EDProducer("eProducer")
2649 p.f = EDProducer("fProducer")
2650 p.g = EDProducer("gProducer")
2651 p.task5 = Task()
2652 p.task3 = Task()
2653 p.task2 = Task(p.c, p.task3)
2654 p.task4 = Task(p.f, p.task2)
2655 p.task1 = Task(p.task5)
2656 p.task3.add(p.task1)
2657 p.p = Path(p.a)
2658 s = Sequence(p.a)
2659 p.r = Sequence(s)
2660 p.p2 = Path(p.r, p.task1, p.task2)
2661 p.schedule = Schedule(p.p2,p.p,tasks=[p.task3,p.task4, p.task5])
2662 d=p.dumpPython()
2663 self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2664 """process.b = cms.EDProducer("bProducer")
2665 process.c = cms.EDProducer("cProducer")
2666 process.d = cms.EDProducer("dProducer")
2667 process.e = cms.EDProducer("eProducer")
2668 process.f = cms.EDProducer("fProducer")
2669 process.g = cms.EDProducer("gProducer")
2670 process.a = cms.EDAnalyzer("MyAnalyzer")
2671 process.task5 = cms.Task()
2672 process.task1 = cms.Task(process.task5)
2673 process.task3 = cms.Task(process.task1)
2674 process.task2 = cms.Task(process.c, process.task3)
2675 process.task4 = cms.Task(process.f, process.task2)
2676 process.r = cms.Sequence((process.a))
2677 process.p = cms.Path(process.a)
2678 process.p2 = cms.Path(process.r, process.task1, process.task2)
2679 process.schedule = cms.Schedule(*[ process.p2, process.p ], tasks=[process.task3, process.task4, process.task5])""")
2680
2681 p = Process("test")
2682 p.a = EDAnalyzer("MyAnalyzer")
2683 p.b = EDProducer("bProducer")
2684 p.c = EDProducer("cProducer")
2685 p.d = EDProducer("dProducer")
2686 p.e = EDProducer("eProducer")
2687 p.f = EDProducer("fProducer")
2688 p.g = EDProducer("gProducer")
2689 p.task5 = Task()
2690 p.task3 = Task()
2691 p.task2 = ConditionalTask(p.c, p.task3)
2692 p.task1 = ConditionalTask(p.task5)
2693 p.p = Path(p.a)
2694 s = Sequence(p.a)
2695 p.r = Sequence(s)
2696 p.p2 = Path(p.r, p.task1, p.task2)
2697 p.schedule = Schedule(p.p2,p.p,tasks=[p.task5])
2698 d=p.dumpPython()
2699 self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2700 """process.b = cms.EDProducer("bProducer")
2701 process.c = cms.EDProducer("cProducer")
2702 process.d = cms.EDProducer("dProducer")
2703 process.e = cms.EDProducer("eProducer")
2704 process.f = cms.EDProducer("fProducer")
2705 process.g = cms.EDProducer("gProducer")
2706 process.a = cms.EDAnalyzer("MyAnalyzer")
2707 process.task5 = cms.Task()
2708 process.task3 = cms.Task()
2709 process.task2 = cms.ConditionalTask(process.c, process.task3)
2710 process.task1 = cms.ConditionalTask(process.task5)
2711 process.r = cms.Sequence((process.a))
2712 process.p = cms.Path(process.a)
2713 process.p2 = cms.Path(process.r, process.task1, process.task2)
2714 process.schedule = cms.Schedule(*[ process.p2, process.p ], tasks=[process.task5])""")
2715
2716 p = Process("test")
2717 p.d = EDProducer("dProducer")
2718 p.e = EDProducer("eProducer")
2719 p.f = EDProducer("fProducer")
2720 p.g = EDProducer("gProducer")
2721 p.task1 = Task(p.d, p.e)
2722 task2 = Task(p.f, p.g)
2723 p.schedule = Schedule(tasks=[p.task1,task2])
2724 d=p.dumpPython()
2725 self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2726 """process.d = cms.EDProducer("dProducer")
2727 process.e = cms.EDProducer("eProducer")
2728 process.f = cms.EDProducer("fProducer")
2729 process.g = cms.EDProducer("gProducer")
2730 process.task1 = cms.Task(process.d, process.e)
2731 process.schedule = cms.Schedule(tasks=[cms.Task(process.f, process.g), process.task1])""")
2732
2733 p = Process("test")
2734 p.schedule = Schedule()
2735 d=p.dumpPython()
2736 self.assertEqual(_lineDiff(d,Process('test').dumpPython()),
2737 """process.schedule = cms.Schedule()""")
2738
2739 s = Sequence()
2740 a = EDProducer("A")
2741 s2 = Sequence(a)
2742 s2 += s
2743 process = Process("DUMP")
2744 process.a = a
2745 process.s2 = s2
2746 d=process.dumpPython()
2747 self.assertEqual(_lineDiff(d,Process('DUMP').dumpPython()),
2748 """process.a = cms.EDProducer("A")
2749 process.s2 = cms.Sequence(process.a)""")
2750 s = Sequence()
2751 s1 = Sequence(s)
2752 a = EDProducer("A")
2753 s3 = Sequence(a+a)
2754 s2 = Sequence(a+s3)
2755 s2 += s1
2756 process = Process("DUMP")
2757 process.a = a
2758 process.s2 = s2
2759 d=process.dumpPython()
2760 self.assertEqual(_lineDiff(d,Process('DUMP').dumpPython()),
2761 """process.a = cms.EDProducer("A")
2762 process.s2 = cms.Sequence(process.a+(process.a+process.a))""")
2763
2764 def testSecSource(self):
2765 p = Process('test')
2766 p.a = SecSource("MySecSource")
2767 self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),'process.a = cms.SecSource("MySecSource")')
2768
2769 def testGlobalReplace(self):
2770 p = Process('test')
2771 p.a = EDAnalyzer("MyAnalyzer")
2772 old = p.a
2773 p.b = EDAnalyzer("YourAnalyzer")
2774 p.c = EDAnalyzer("OurAnalyzer")
2775 p.d = EDProducer("MyProducer")
2776 old2 = p.d
2777 p.t1 = Task(p.d)
2778 t2 = Task(p.d)
2779 t3 = Task(p.d)
2780 t4 = Task(p.d)
2781 t5 = Task(p.d)
2782 t6 = Task(p.d)
2783 p.ct1 = ConditionalTask(p.d)
2784 s = Sequence(p.a*p.b)
2785 p.s4 = Sequence(p.a*p.b, p.ct1)
2786 s.associate(t2)
2787 p.s4.associate(t2)
2788 p.p = Path(p.c+s+p.a)
2789 p.p2 = Path(p.c+p.s4+p.a, p.ct1)
2790 p.e3 = EndPath(p.c+s+p.a)
2791 new = EDAnalyzer("NewAnalyzer")
2792 new2 = EDProducer("NewProducer")
2793 visitor1 = NodeVisitor()
2794 p.p.visit(visitor1)
2795 self.assertTrue(visitor1.modules == set([old,old2,p.b,p.c]))
2796 p.schedule = Schedule(tasks=[t6])
2797 p.globalReplace("a",new)
2798 p.globalReplace("d",new2)
2799 visitor2 = NodeVisitor()
2800 p.p.visit(visitor2)
2801 self.assertTrue(visitor2.modules == set([new,new2,p.b,p.c]))
2802 self.assertEqual(p.p.dumpPython()[:-1], "cms.Path(process.c+process.a+process.b+process.a, cms.Task(process.d))")
2803 visitor_p2 = NodeVisitor()
2804 p.p2.visit(visitor_p2)
2805 self.assertTrue(visitor_p2.modules == set([new,new2,p.b,p.c]))
2806 self.assertEqual(p.p2.dumpPython()[:-1], "cms.Path(process.c+process.s4+process.a, process.ct1)")
2807 visitor3 = NodeVisitor()
2808 p.e3.visit(visitor3)
2809 self.assertTrue(visitor3.modules == set([new,new2,p.b,p.c]))
2810 visitor4 = NodeVisitor()
2811 p.s4.visit(visitor4)
2812 self.assertTrue(visitor4.modules == set([new,new2,p.b]))
2813 self.assertEqual(p.s4.dumpPython()[:-1],"cms.Sequence(process.a+process.b, cms.Task(process.d), process.ct1)")
2814 visitor5 = NodeVisitor()
2815 p.t1.visit(visitor5)
2816 self.assertTrue(visitor5.modules == set([new2]))
2817 visitor6 = NodeVisitor()
2818 listOfTasks = list(p.schedule._tasks)
2819 listOfTasks[0].visit(visitor6)
2820 self.assertTrue(visitor6.modules == set([new2]))
2821 visitor7 = NodeVisitor()
2822 p.ct1.visit(visitor7)
2823 self.assertTrue(visitor7.modules == set([new2]))
2824 visitor8 = NodeVisitor()
2825 listOfConditionalTasks = list(p.conditionaltasks.values())
2826 listOfConditionalTasks[0].visit(visitor8)
2827 self.assertTrue(visitor8.modules == set([new2]))
2828
2829
2830 p.d2 = EDProducer("YourProducer")
2831 p.schedule = Schedule(p.p, p.p2, p.e3, tasks=[p.t1])
2832 self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2833 p.p = Path(p.c+s)
2834 self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2835 p.e3 = EndPath(p.c)
2836 self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2837 p.t1 = Task(p.d2)
2838 self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2839
2840 def testSequence(self):
2841 p = Process('test')
2842 p.a = EDAnalyzer("MyAnalyzer")
2843 p.b = EDAnalyzer("YourAnalyzer")
2844 p.c = EDAnalyzer("OurAnalyzer")
2845 p.s = Sequence(p.a*p.b)
2846 self.assertEqual(str(p.s),'a+b')
2847 self.assertEqual(p.s.label_(),'s')
2848 path = Path(p.c+p.s)
2849 self.assertEqual(str(path),'c+a+b')
2850 p._validateSequence(path, 'p1')
2851 notInProcess = EDAnalyzer('NotInProcess')
2852 p2 = Path(p.c+p.s*notInProcess)
2853 self.assertRaises(RuntimeError, p._validateSequence, p2, 'p2')
2854
2855 def testSequence2(self):
2856 p = Process('test')
2857 p.a = EDAnalyzer("MyAnalyzer")
2858 p.b = EDAnalyzer("YourAnalyzer")
2859 p.c = EDAnalyzer("OurAnalyzer")
2860 testseq = Sequence(p.a*p.b)
2861 p.s = testseq
2862
2863 self.assertRaises(ValueError, p.__setattr__, "y", testseq)
2864
2865 def testServiceInProcess(self):
2866 service = Service("d")
2867 self.assertFalse(service._inProcess)
2868 process = Process("test")
2869 process.d = service
2870 self.assertTrue(service._inProcess)
2871 service2 = Service("d")
2872 process.d = service2
2873 self.assertFalse(service._inProcess)
2874 self.assertTrue(service2._inProcess)
2875 del process.d
2876 self.assertFalse(service2._inProcess)
2877
2878 def testTask(self):
2879
2880
2881 edanalyzer = EDAnalyzer("a")
2882 edproducer = EDProducer("b")
2883 edproducer2 = EDProducer("b2")
2884 edproducer3 = EDProducer("b3")
2885 edproducer4 = EDProducer("b4")
2886 edproducer8 = EDProducer("b8")
2887 edproducer9 = EDProducer("b9")
2888 edfilter = EDFilter("c")
2889 service = Service("d")
2890 service3 = Service("d", v = untracked.uint32(3))
2891 essource = ESSource("e")
2892 esproducer = ESProducer("f")
2893 testTask2 = Task()
2894
2895
2896 testTask1 = Task(edproducer, edfilter)
2897 self.assertRaises(RuntimeError, testTask1.add, edanalyzer)
2898 testTask1.add(essource, service)
2899 testTask1.add(essource, esproducer)
2900 testTask1.add(testTask2)
2901 coll = testTask1._collection
2902 self.assertTrue(edproducer in coll)
2903 self.assertTrue(edfilter in coll)
2904 self.assertTrue(service in coll)
2905 self.assertTrue(essource in coll)
2906 self.assertTrue(esproducer in coll)
2907 self.assertTrue(testTask2 in coll)
2908 self.assertTrue(len(coll) == 6)
2909 self.assertTrue(len(testTask2._collection) == 0)
2910
2911 taskContents = []
2912 for i in testTask1:
2913 taskContents.append(i)
2914 self.assertTrue(taskContents == [edproducer, edfilter, essource, service, esproducer, testTask2])
2915
2916
2917 process = Process("test")
2918
2919 process.mproducer = edproducer
2920 process.mproducer2 = edproducer2
2921 process.mfilter = edfilter
2922 process.messource = essource
2923 process.mesproducer = esproducer
2924 process.d = service
2925
2926 testTask3 = Task(edproducer, edproducer2)
2927 testTask1.add(testTask3)
2928 process.myTask1 = testTask1
2929
2930
2931
2932
2933 l = set()
2934 visitor = NodeNameVisitor(l)
2935 testTask1.visit(visitor)
2936 self.assertTrue(l == set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
2937 l2 = testTask1.moduleNames
2938 self.assertTrue(l == set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
2939
2940 testTask4 = Task(edproducer3)
2941 l.clear()
2942 self.assertRaises(RuntimeError, testTask4.visit, visitor)
2943 try:
2944 process.myTask4 = testTask4
2945 self.assertTrue(False)
2946 except RuntimeError:
2947 pass
2948
2949 testTask5 = Task(service3)
2950 l.clear()
2951 self.assertRaises(RuntimeError, testTask5.visit, visitor)
2952 try:
2953 process.myTask5 = testTask5
2954 self.assertTrue(False)
2955 except RuntimeError:
2956 pass
2957
2958 process.d = service3
2959 process.myTask5 = testTask5
2960
2961
2962 expectedDict = { 'myTask1' : testTask1, 'myTask5' : testTask5 }
2963 expectedFixedDict = DictTypes.FixedKeysDict(expectedDict);
2964 self.assertTrue(process.tasks == expectedFixedDict)
2965 self.assertTrue(process.tasks['myTask1'] == testTask1)
2966 self.assertTrue(process.myTask1 == testTask1)
2967
2968
2969
2970 process.mproducer2 = edproducer4
2971 process.d = service
2972 l = list()
2973 visitor1 = ModuleNodeVisitor(l)
2974 testTask1.visit(visitor1)
2975 l.sort(key=lambda mod: mod.__str__())
2976 expectedList = sorted([edproducer,essource,esproducer,service,edfilter,edproducer,edproducer4],key=lambda mod: mod.__str__())
2977 self.assertTrue(expectedList == l)
2978 process.myTask6 = Task()
2979 process.myTask7 = Task()
2980 process.mproducer8 = edproducer8
2981 process.myTask8 = Task(process.mproducer8)
2982 process.myTask6.add(process.myTask7)
2983 process.myTask7.add(process.myTask8)
2984 process.myTask1.add(process.myTask6)
2985 process.myTask8.add(process.myTask5)
2986
2987 testDict = process._itemsInDependencyOrder(process.tasks)
2988 expectedLabels = ["myTask5", "myTask8", "myTask7", "myTask6", "myTask1"]
2989 expectedTasks = [process.myTask5, process.myTask8, process.myTask7, process.myTask6, process.myTask1]
2990 index = 0
2991 for testLabel, testTask in testDict.items():
2992 self.assertTrue(testLabel == expectedLabels[index])
2993 self.assertTrue(testTask == expectedTasks[index])
2994 index += 1
2995
2996 pythonDump = testTask1.dumpPython(PrintOptions())
2997
2998
2999 expectedPythonDump = 'cms.Task(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer, process.mproducer2, process.myTask6)\n'
3000 self.assertTrue(pythonDump == expectedPythonDump)
3001
3002 process.myTask5 = Task()
3003 process.myTask100 = Task()
3004 process.mproducer9 = edproducer9
3005 sequence1 = Sequence(process.mproducer8, process.myTask1, process.myTask5, testTask2, testTask3)
3006 sequence2 = Sequence(process.mproducer8 + process.mproducer9)
3007 process.sequence3 = Sequence((process.mproducer8 + process.mfilter))
3008 sequence4 = Sequence()
3009 process.path1 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+sequence4)
3010 process.path1.associate(process.myTask1, process.myTask5, testTask2, testTask3)
3011 process.path11 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+ sequence4,process.myTask1, process.myTask5, testTask2, testTask3, process.myTask100)
3012 process.path2 = Path(process.mproducer)
3013 process.path3 = Path(process.mproducer9+process.mproducer8,testTask2)
3014
3015 self.assertTrue(process.path1.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)\n')
3016
3017 self.assertTrue(process.path11.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask100, process.myTask5)\n')
3018
3019
3020 l = set()
3021 nameVisitor = NodeNameVisitor(l)
3022 process.path1.visit(nameVisitor)
3023 self.assertTrue(l == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3024 self.assertTrue(process.path1.moduleNames() == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3025
3026
3027 process.mproducer10 = EDProducer("b10")
3028 process.path21 = process.path11.copy()
3029 process.path21.replace(process.mproducer, process.mproducer10)
3030
3031 self.assertTrue(process.path11.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask100, process.myTask5)\n')
3032
3033
3034
3035
3036
3037
3038
3039
3040 self.assertTrue(process.path21.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer10), cms.Task(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), process.myTask100, process.myTask5)\n')
3041
3042 process.path22 = process.path21.copyAndExclude([process.d, process.mesproducer, process.mfilter])
3043 self.assertTrue(process.path22.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer10), cms.Task(process.messource, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), process.myTask100, process.myTask5)\n')
3044
3045 process.path23 = process.path22.copyAndExclude([process.messource, process.mproducer10])
3046 self.assertTrue(process.path23.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.Task(), cms.Task(process.None), cms.Task(process.mproducer2, process.mproducer8, process.myTask5), process.myTask100, process.myTask5)\n')
3047
3048 process.a = EDAnalyzer("MyAnalyzer")
3049 process.b = OutputModule("MyOutputModule")
3050 process.c = EDFilter("MyFilter")
3051 process.d = EDProducer("MyProducer")
3052 process.e = ESProducer("MyESProducer")
3053 process.f = ESSource("MyESSource")
3054 process.g = ESProducer("g")
3055 process.path24 = Path(process.a+process.b+process.c+process.d)
3056 process.path25 = process.path24.copyAndExclude([process.a,process.b,process.c])
3057 self.assertTrue(process.path25.dumpPython() == 'cms.Path(process.d)\n')
3058
3059
3060
3061 process.path200 = EndPath(Sequence(process.c,Task(process.e)))
3062 process.path200.replace(process.c,process.b)
3063 process.path200.replace(process.e,process.f)
3064 self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.b, cms.Task(process.f))\n")
3065 process.path200.replace(process.b,process.c)
3066 process.path200.replace(process.f,process.e)
3067 self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.c, cms.Task(process.e))\n")
3068 process.path200.replace(process.c,process.a)
3069 process.path200.replace(process.e,process.g)
3070 self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.a, cms.Task(process.g))\n")
3071 process.path200.replace(process.a,process.c)
3072 process.path200.replace(process.g,process.e)
3073 self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.c, cms.Task(process.e))\n")
3074
3075 def testConditionalTask(self):
3076
3077
3078 edanalyzer = EDAnalyzer("a")
3079 edproducer = EDProducer("b")
3080 edproducer2 = EDProducer("b2")
3081 edproducer3 = EDProducer("b3")
3082 edproducer4 = EDProducer("b4")
3083 edproducer8 = EDProducer("b8")
3084 edproducer9 = EDProducer("b9")
3085 edfilter = EDFilter("c")
3086 service = Service("d")
3087 service3 = Service("d", v = untracked.uint32(3))
3088 essource = ESSource("e")
3089 esproducer = ESProducer("f")
3090 testTask2 = Task()
3091 testCTask2 = ConditionalTask()
3092
3093
3094 testTask1 = ConditionalTask(edproducer, edfilter)
3095 self.assertRaises(RuntimeError, testTask1.add, edanalyzer)
3096 testTask1.add(essource, service)
3097 testTask1.add(essource, esproducer)
3098 testTask1.add(testTask2)
3099 testTask1.add(testCTask2)
3100 coll = testTask1._collection
3101 self.assertTrue(edproducer in coll)
3102 self.assertTrue(edfilter in coll)
3103 self.assertTrue(service in coll)
3104 self.assertTrue(essource in coll)
3105 self.assertTrue(esproducer in coll)
3106 self.assertTrue(testTask2 in coll)
3107 self.assertTrue(testCTask2 in coll)
3108 self.assertTrue(len(coll) == 7)
3109 self.assertTrue(len(testTask2._collection) == 0)
3110
3111 taskContents = []
3112 for i in testTask1:
3113 taskContents.append(i)
3114 self.assertEqual(taskContents, [edproducer, edfilter, essource, service, esproducer, testTask2, testCTask2])
3115
3116
3117 process = Process("test")
3118
3119 process.mproducer = edproducer
3120 process.mproducer2 = edproducer2
3121 process.mfilter = edfilter
3122 process.messource = essource
3123 process.mesproducer = esproducer
3124 process.d = service
3125
3126 testTask3 = ConditionalTask(edproducer, edproducer2)
3127 testTask1.add(testTask3)
3128 process.myTask1 = testTask1
3129
3130
3131
3132
3133 l = set()
3134 visitor = NodeNameVisitor(l)
3135 testTask1.visit(visitor)
3136 self.assertEqual(l, set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
3137 l2 = testTask1.moduleNames()
3138 self.assertEqual(l2, set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
3139
3140 testTask4 = ConditionalTask(edproducer3)
3141 l.clear()
3142 self.assertRaises(RuntimeError, testTask4.visit, visitor)
3143 try:
3144 process.myTask4 = testTask4
3145 self.assertTrue(False)
3146 except RuntimeError:
3147 pass
3148
3149 testTask5 = ConditionalTask(service3)
3150 l.clear()
3151 self.assertRaises(RuntimeError, testTask5.visit, visitor)
3152 try:
3153 process.myTask5 = testTask5
3154 self.assertTrue(False)
3155 except RuntimeError:
3156 pass
3157
3158 process.d = service3
3159 process.myTask5 = testTask5
3160
3161
3162 expectedDict = { 'myTask1' : testTask1, 'myTask5' : testTask5 }
3163 expectedFixedDict = DictTypes.FixedKeysDict(expectedDict);
3164 self.assertEqual(process.conditionaltasks, expectedFixedDict)
3165 self.assertEqual(process.conditionaltasks['myTask1'], testTask1)
3166 self.assertEqual(process.myTask1, testTask1)
3167
3168
3169
3170 process.mproducer2 = edproducer4
3171 process.d = service
3172 l = list()
3173 visitor1 = ModuleNodeVisitor(l)
3174 testTask1.visit(visitor1)
3175 l.sort(key=lambda mod: mod.__str__())
3176 expectedList = sorted([edproducer,essource,esproducer,service,edfilter,edproducer,edproducer4],key=lambda mod: mod.__str__())
3177 self.assertEqual(expectedList, l)
3178 process.myTask6 = ConditionalTask()
3179 process.myTask7 = ConditionalTask()
3180 process.mproducer8 = edproducer8
3181 process.myTask8 = ConditionalTask(process.mproducer8)
3182 process.myTask6.add(process.myTask7)
3183 process.myTask7.add(process.myTask8)
3184 process.myTask1.add(process.myTask6)
3185 process.myTask8.add(process.myTask5)
3186 self.assertEqual(process.myTask8.dumpPython(), "cms.ConditionalTask(process.mproducer8, process.myTask5)\n")
3187
3188 testDict = process._itemsInDependencyOrder(process.conditionaltasks)
3189 expectedLabels = ["myTask5", "myTask8", "myTask7", "myTask6", "myTask1"]
3190 expectedTasks = [process.myTask5, process.myTask8, process.myTask7, process.myTask6, process.myTask1]
3191 index = 0
3192 for testLabel, testTask in testDict.items():
3193 self.assertEqual(testLabel, expectedLabels[index])
3194 self.assertEqual(testTask, expectedTasks[index])
3195 index += 1
3196
3197 pythonDump = testTask1.dumpPython(PrintOptions())
3198
3199
3200 expectedPythonDump = 'cms.ConditionalTask(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer, process.mproducer2, process.myTask6)\n'
3201 self.assertEqual(pythonDump, expectedPythonDump)
3202
3203 process.myTask5 = ConditionalTask()
3204 self.assertEqual(process.myTask8.dumpPython(), "cms.ConditionalTask(process.mproducer8, process.myTask5)\n")
3205 process.myTask100 = ConditionalTask()
3206 process.mproducer9 = edproducer9
3207 sequence1 = Sequence(process.mproducer8, process.myTask1, process.myTask5, testTask2, testTask3)
3208 sequence2 = Sequence(process.mproducer8 + process.mproducer9)
3209 process.sequence3 = Sequence((process.mproducer8 + process.mfilter))
3210 sequence4 = Sequence()
3211 process.path1 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+sequence4)
3212 process.path1.associate(process.myTask1, process.myTask5, testTask2, testTask3)
3213 process.path11 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+ sequence4,process.myTask1, process.myTask5, testTask2, testTask3, process.myTask100)
3214 process.path2 = Path(process.mproducer)
3215 process.path3 = Path(process.mproducer9+process.mproducer8,testTask2)
3216
3217 self.assertEqual(process.path1.dumpPython(PrintOptions()), 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)\n')
3218
3219 self.assertEqual(process.path11.dumpPython(PrintOptions()), 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask100, process.myTask5)\n')
3220
3221
3222 l = set()
3223 nameVisitor = NodeNameVisitor(l)
3224 process.path1.visit(nameVisitor)
3225 self.assertTrue(l == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3226 self.assertTrue(process.path1.moduleNames() == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3227
3228
3229 process.mproducer10 = EDProducer("b10")
3230 process.path21 = process.path11.copy()
3231 process.path21.replace(process.mproducer, process.mproducer10)
3232
3233 self.assertEqual(process.path11.dumpPython(PrintOptions()), 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask100, process.myTask5)\n')
3234
3235
3236
3237
3238
3239
3240
3241
3242 self.assertEqual(process.path21.dumpPython(PrintOptions()), 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer10), cms.ConditionalTask(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), cms.Task(), process.myTask100, process.myTask5)\n')
3243
3244 process.path22 = process.path21.copyAndExclude([process.d, process.mesproducer, process.mfilter])
3245 self.assertEqual(process.path22.dumpPython(PrintOptions()), 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.ConditionalTask(process.None, process.mproducer10), cms.ConditionalTask(process.messource, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), cms.Task(), process.myTask100, process.myTask5)\n')
3246
3247 process.path23 = process.path22.copyAndExclude([process.messource, process.mproducer10])
3248 self.assertEqual(process.path23.dumpPython(PrintOptions()), 'cms.Path(process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.ConditionalTask(process.None), cms.ConditionalTask(process.mproducer2, process.mproducer8, process.myTask5), cms.Task(), process.myTask100, process.myTask5)\n')
3249
3250 process = Process("Test")
3251
3252 process.b = EDProducer("b")
3253 process.b2 = EDProducer("b2")
3254 process.b3 = EDProducer("b3")
3255 process.p = Path(process.b, ConditionalTask(process.b3, process.b2))
3256 p = TestMakePSet()
3257 process.fillProcessDesc(p)
3258 self.assertEqual(p.values["@all_modules"], (True, ['b', 'b2', 'b3']))
3259 self.assertEqual(p.values["@paths"], (True, ['p']))
3260 self.assertEqual(p.values["p"], (True, ['b','#','b2','b3','@']))
3261
3262
3263 def testPath(self):
3264 p = Process("test")
3265 p.a = EDAnalyzer("MyAnalyzer")
3266 p.b = EDAnalyzer("YourAnalyzer")
3267 p.c = EDAnalyzer("OurAnalyzer")
3268 path = Path(p.a)
3269 path *= p.b
3270 path += p.c
3271 self.assertEqual(str(path),'a+b+c')
3272 path = Path(p.a*p.b+p.c)
3273 self.assertEqual(str(path),'a+b+c')
3274
3275
3276 path = Path(p.a+ p.b*p.c)
3277 self.assertEqual(str(path),'a+b+c')
3278 path = Path(p.a*(p.b+p.c))
3279 self.assertEqual(str(path),'a+b+c')
3280 path = Path(p.a*(p.b+~p.c))
3281 pathx = Path(p.a*(p.b+ignore(p.c)))
3282 self.assertEqual(str(path),'a+b+~c')
3283 p.es = ESProducer("AnESProducer")
3284 self.assertRaises(TypeError,Path,p.es)
3285
3286 t = Path()
3287 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path()\n')
3288
3289 t = Path(p.a)
3290 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a)\n')
3291
3292 t = Path(Task())
3293 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(cms.Task())\n')
3294
3295 t = Path(p.a, Task())
3296 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.Task())\n')
3297
3298 p.prod = EDProducer("prodName")
3299 p.t1 = Task(p.prod)
3300 t = Path(p.a, p.t1, Task(), p.t1)
3301 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.Task(), process.t1)\n')
3302
3303 t = Path(ConditionalTask())
3304 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(cms.ConditionalTask())\n')
3305
3306 t = Path(p.a, ConditionalTask())
3307 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.ConditionalTask())\n')
3308
3309 p.prod = EDProducer("prodName")
3310 p.t1 = ConditionalTask(p.prod)
3311 t = Path(p.a, p.t1, Task(), p.t1)
3312 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.Task(), process.t1)\n')
3313
3314 def testFinalPath(self):
3315 p = Process("test")
3316 p.a = OutputModule("MyOutputModule")
3317 p.b = OutputModule("YourOutputModule")
3318 p.c = OutputModule("OurOutputModule")
3319 path = FinalPath(p.a)
3320 path *= p.b
3321 path += p.c
3322 self.assertEqual(str(path),'a+b+c')
3323 path = FinalPath(p.a*p.b+p.c)
3324 self.assertEqual(str(path),'a+b+c')
3325 path = FinalPath(p.a+ p.b*p.c)
3326 self.assertEqual(str(path),'a+b+c')
3327 path = FinalPath(p.a*(p.b+p.c))
3328 self.assertEqual(str(path),'a+b+c')
3329 p.es = ESProducer("AnESProducer")
3330 self.assertRaises(TypeError,FinalPath, p.es)
3331
3332 t = FinalPath()
3333 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.FinalPath()\n')
3334
3335 t = FinalPath(p.a)
3336 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.FinalPath(process.a)\n')
3337
3338 self.assertRaises(TypeError, FinalPath, Task())
3339 self.assertRaises(TypeError, FinalPath, p.a, Task())
3340
3341 p.prod = EDProducer("prodName")
3342 p.t1 = Task(p.prod)
3343 self.assertRaises(TypeError, FinalPath, p.a, p.t1, Task(), p.t1)
3344
3345 p.prod = EDProducer("prodName")
3346 p.t1 = ConditionalTask(p.prod)
3347 self.assertRaises(TypeError, FinalPath, p.a, p.t1, ConditionalTask(), p.t1)
3348
3349 p.t = FinalPath(p.a)
3350 p.a = OutputModule("ReplacedOutputModule")
3351 self.assertEqual(p.t.dumpPython(PrintOptions()), 'cms.FinalPath(process.a)\n')
3352
3353 p.anal = EDAnalyzer("MyAnalyzer")
3354 p.t = FinalPath(p.anal)
3355 pset = TestMakePSet()
3356 self.assertRaises(RuntimeError, p.fillProcessDesc, pset)
3357
3358 p.prod = EDProducer("MyProducer")
3359 p.t = FinalPath(p.prod)
3360 pset = TestMakePSet()
3361 self.assertRaises(RuntimeError, p.fillProcessDesc, pset)
3362
3363 p.filt = EDFilter("MyFilter")
3364 p.t = FinalPath(p.filt)
3365 pset = TestMakePSet()
3366 self.assertRaises(RuntimeError, p.fillProcessDesc, pset)
3367
3368 p.outp = OutputModule("MyOutputModule")
3369 p.t = FinalPath(p.outp)
3370 pset = TestMakePSet()
3371 p.fillProcessDesc(pset)
3372
3373 def testCloneSequence(self):
3374 p = Process("test")
3375 a = EDAnalyzer("MyAnalyzer")
3376 p.a = a
3377 a.setLabel("a")
3378 b = EDAnalyzer("YOurAnalyzer")
3379 p.b = b
3380 b.setLabel("b")
3381 path = Path(a * b)
3382 p.path = Path(p.a*p.b)
3383 lookuptable = {id(a): p.a, id(b): p.b}
3384
3385
3386
3387 self.assertEqual(str(path),str(p.path))
3388
3389 def testContains(self):
3390
3391 a = EDProducer("a")
3392 b = EDProducer("b")
3393 c = EDProducer("c")
3394 d = EDProducer("d")
3395 e = EDProducer("e")
3396 f = EDProducer("f")
3397 g = EDProducer("g")
3398 h = EDProducer("h")
3399 i = EDProducer("i")
3400 j = EDProducer("j")
3401 k = EDProducer("k")
3402 l = EDProducer("l")
3403 m = EDProducer("m")
3404 n = EDProducer("n")
3405
3406 seq1 = Sequence(e)
3407 task1 = Task(g)
3408 ctask1 = ConditionalTask(h)
3409 path = Path(a * c * seq1, task1, ctask1)
3410
3411 self.assertTrue(path.contains(a))
3412 self.assertFalse(path.contains(b))
3413 self.assertTrue(path.contains(c))
3414 self.assertFalse(path.contains(d))
3415 self.assertTrue(path.contains(e))
3416 self.assertFalse(path.contains(f))
3417 self.assertTrue(path.contains(g))
3418 self.assertTrue(path.contains(h))
3419
3420 endpath = EndPath(h * i)
3421 self.assertFalse(endpath.contains(b))
3422 self.assertTrue(endpath.contains(i))
3423
3424 seq = Sequence(a * c)
3425 self.assertFalse(seq.contains(b))
3426 self.assertTrue(seq.contains(c))
3427
3428 task2 = Task(l)
3429 task = Task(j, k, task2)
3430 self.assertFalse(task.contains(b))
3431 self.assertTrue(task.contains(j))
3432 self.assertTrue(task.contains(k))
3433 self.assertTrue(task.contains(l))
3434
3435 task3 = Task(m)
3436 path2 = Path(n)
3437 sch = Schedule(path, path2, tasks=[task,task3])
3438 self.assertFalse(sch.contains(b))
3439 self.assertTrue(sch.contains(a))
3440 self.assertTrue(sch.contains(c))
3441 self.assertTrue(sch.contains(e))
3442 self.assertTrue(sch.contains(g))
3443 self.assertTrue(sch.contains(n))
3444 self.assertTrue(sch.contains(j))
3445 self.assertTrue(sch.contains(k))
3446 self.assertTrue(sch.contains(l))
3447 self.assertTrue(sch.contains(m))
3448
3449 ctask2 = ConditionalTask(l, task1)
3450 ctask = ConditionalTask(j, k, ctask2)
3451 self.assertFalse(ctask.contains(b))
3452 self.assertTrue(ctask.contains(j))
3453 self.assertTrue(ctask.contains(k))
3454 self.assertTrue(ctask.contains(l))
3455 self.assertTrue(ctask.contains(g))
3456
3457 def testSchedule(self):
3458 p = Process("test")
3459 p.a = EDAnalyzer("MyAnalyzer")
3460 p.b = EDAnalyzer("YourAnalyzer")
3461 p.c = EDAnalyzer("OurAnalyzer")
3462 p.d = EDAnalyzer("OurAnalyzer")
3463 p.path1 = Path(p.a)
3464 p.path2 = Path(p.b)
3465 p.path3 = Path(p.d)
3466
3467 s = Schedule(p.path1,p.path2)
3468 self.assertEqual(s[0],p.path1)
3469 self.assertEqual(s[1],p.path2)
3470 p.schedule = s
3471 self.assertTrue('b' in p.schedule.moduleNames())
3472 self.assertTrue(hasattr(p, 'b'))
3473 self.assertTrue(hasattr(p, 'c'))
3474 self.assertTrue(hasattr(p, 'd'))
3475 self.assertTrue(hasattr(p, 'path1'))
3476 self.assertTrue(hasattr(p, 'path2'))
3477 self.assertTrue(hasattr(p, 'path3'))
3478 p.prune()
3479 self.assertTrue('b' in p.schedule.moduleNames())
3480 self.assertTrue(hasattr(p, 'b'))
3481 self.assertTrue(not hasattr(p, 'c'))
3482 self.assertTrue(not hasattr(p, 'd'))
3483 self.assertTrue(hasattr(p, 'path1'))
3484 self.assertTrue(hasattr(p, 'path2'))
3485 self.assertTrue(not hasattr(p, 'path3'))
3486
3487 self.assertTrue(len(p.schedule._tasks) == 0)
3488
3489 p = Process("test")
3490 p.a = EDAnalyzer("MyAnalyzer")
3491 p.b = EDAnalyzer("YourAnalyzer")
3492 p.c = EDAnalyzer("OurAnalyzer")
3493 p.d = EDAnalyzer("dAnalyzer")
3494 p.e = EDProducer("eProducer")
3495 p.f = EDProducer("fProducer")
3496 p.Tracer = Service("Tracer")
3497 p.path1 = Path(p.a)
3498 p.path2 = Path(p.b)
3499 p.path3 = Path(p.d)
3500 p.task1 = Task(p.e)
3501 p.task2 = Task(p.f, p.Tracer)
3502 s = Schedule(p.path1,p.path2,tasks=[p.task1,p.task2,p.task1])
3503 self.assertEqual(s[0],p.path1)
3504 self.assertEqual(s[1],p.path2)
3505 self.assertTrue(len(s._tasks) == 2)
3506 self.assertTrue(p.task1 in s._tasks)
3507 self.assertTrue(p.task2 in s._tasks)
3508 listOfTasks = list(s._tasks)
3509 self.assertTrue(len(listOfTasks) == 2)
3510 self.assertTrue(p.task1 == listOfTasks[0])
3511 self.assertTrue(p.task2 == listOfTasks[1])
3512 p.schedule = s
3513 self.assertTrue('b' in p.schedule.moduleNames())
3514
3515 process2 = Process("test")
3516 process2.a = EDAnalyzer("MyAnalyzer")
3517 process2.e = EDProducer("eProducer")
3518 process2.path1 = Path(process2.a)
3519 process2.task1 = Task(process2.e)
3520 process2.schedule = Schedule(process2.path1,tasks=process2.task1)
3521 listOfTasks = list(process2.schedule._tasks)
3522 self.assertTrue(listOfTasks[0] == process2.task1)
3523
3524
3525 s2 = s.copy()
3526 self.assertEqual(s2[0],p.path1)
3527 self.assertEqual(s2[1],p.path2)
3528 self.assertTrue(len(s2._tasks) == 2)
3529 self.assertTrue(p.task1 in s2._tasks)
3530 self.assertTrue(p.task2 in s2._tasks)
3531 listOfTasks = list(s2._tasks)
3532 self.assertTrue(len(listOfTasks) == 2)
3533 self.assertTrue(p.task1 == listOfTasks[0])
3534 self.assertTrue(p.task2 == listOfTasks[1])
3535
3536 names = s.moduleNames()
3537 self.assertTrue(names == set(['a', 'b', 'e', 'Tracer', 'f']))
3538
3539 p = Process("test")
3540 p.a = EDAnalyzer("MyAnalyzer")
3541 path1 = Path(p.a)
3542 s = Schedule(path1)
3543 self.assertRaises(RuntimeError, lambda : p.setSchedule_(s) )
3544
3545
3546 p = Process("test")
3547 p.a = EDAnalyzer("MyAnalyzer")
3548 p.b = EDAnalyzer("MyOtherAnalyzer")
3549 p.c = EDProducer("MyProd")
3550 path1 = Path(p.c*Sequence(p.a+p.b))
3551 s = Schedule(path1)
3552 self.assertTrue('a' in s.moduleNames())
3553 self.assertTrue('b' in s.moduleNames())
3554 self.assertTrue('c' in s.moduleNames())
3555 p.path1 = path1
3556 p.schedule = s
3557 p.prune()
3558 self.assertTrue('a' in s.moduleNames())
3559 self.assertTrue('b' in s.moduleNames())
3560 self.assertTrue('c' in s.moduleNames())
3561
3562 def testImplicitSchedule(self):
3563 p = Process("test")
3564 p.a = EDAnalyzer("MyAnalyzer")
3565 p.b = EDAnalyzer("YourAnalyzer")
3566 p.c = EDAnalyzer("OurAnalyzer")
3567 p.path1 = Path(p.a)
3568 p.path2 = Path(p.b)
3569 self.assertTrue(p.schedule is None)
3570 pths = p.paths
3571 keys = pths.keys()
3572 self.assertEqual(pths[keys[0]],p.path1)
3573 self.assertEqual(pths[keys[1]],p.path2)
3574 p.prune()
3575 self.assertTrue(hasattr(p, 'a'))
3576 self.assertTrue(hasattr(p, 'b'))
3577 self.assertTrue(not hasattr(p, 'c'))
3578 self.assertTrue(hasattr(p, 'path1'))
3579 self.assertTrue(hasattr(p, 'path2'))
3580
3581
3582 p = Process("test")
3583 p.a = EDAnalyzer("MyAnalyzer")
3584 p.b = EDAnalyzer("YourAnalyzer")
3585 p.c = EDAnalyzer("OurAnalyzer")
3586 p.path2 = Path(p.b)
3587 p.path1 = Path(p.a)
3588 self.assertTrue(p.schedule is None)
3589 pths = p.paths
3590 keys = pths.keys()
3591 self.assertEqual(pths[keys[1]],p.path1)
3592 self.assertEqual(pths[keys[0]],p.path2)
3593
3594
3595 def testUsing(self):
3596 p = Process('test')
3597 p.block = PSet(a = int32(1))
3598 p.modu = EDAnalyzer('Analyzer', p.block, b = int32(2))
3599 self.assertEqual(p.modu.a.value(),1)
3600 self.assertEqual(p.modu.b.value(),2)
3601
3602 def testOverride(self):
3603 p = Process('test')
3604 a = EDProducer("A", a1=int32(0))
3605 self.assertTrue(not a.isModified())
3606 a.a1 = 1
3607 self.assertTrue(a.isModified())
3608 p.a = a
3609 self.assertEqual(p.a.a1.value(), 1)
3610
3611
3612 p.a = EDProducer("A", a1=int32(2))
3613 self.assertEqual(p.a.a1.value(), 2)
3614
3615
3616
3617 b = EDProducer("A", a1=int32(3))
3618 b.a1 = 4
3619
3620 ps1 = PSet(a = int32(1))
3621 ps2 = PSet(a = int32(2))
3622 self.assertRaises(ValueError, EDProducer, 'C', ps1, ps2)
3623 self.assertRaises(ValueError, EDProducer, 'C', ps1, a=int32(3))
3624
3625 def testOptions(self):
3626 p = Process('test')
3627 self.assertEqual(p.options.numberOfThreads.value(),1)
3628 p.options.numberOfThreads = 8
3629 self.assertEqual(p.options.numberOfThreads.value(),8)
3630 p.options = PSet()
3631 self.assertEqual(p.options.numberOfThreads.value(),1)
3632 p.options = dict(numberOfStreams =2,
3633 numberOfThreads =2)
3634 self.assertEqual(p.options.numberOfThreads.value(),2)
3635 self.assertEqual(p.options.numberOfStreams.value(),2)
3636 del p.options
3637 self.assertRaises(TypeError, setattr, p, 'options', untracked.PSet(numberOfThreads = int32(-1)))
3638 p.options = untracked.PSet(numberOfThreads = untracked.uint32(4))
3639 self.assertEqual(p.options.numberOfThreads.value(), 4)
3640
3641 def testMaxEvents(self):
3642 p = Process("Test")
3643 p.maxEvents.input = 10
3644 self.assertEqual(p.maxEvents.input.value(),10)
3645 p = Process("Test")
3646 p.maxEvents.output = 10
3647 self.assertEqual(p.maxEvents.output.value(),10)
3648 p = Process("Test")
3649 p.maxEvents.output = PSet(out=untracked.int32(10))
3650 self.assertEqual(p.maxEvents.output.out.value(), 10)
3651 p = Process("Test")
3652 p.maxEvents = untracked.PSet(input = untracked.int32(5))
3653 self.assertEqual(p.maxEvents.input.value(), 5)
3654 del p.maxEvents
3655 self.assertRaises(TypeError, setattr, p, 'maxEvents', untracked.PSet(input = untracked.uint32(1)))
3656 p.maxEvents = untracked.PSet(input = untracked.int32(1))
3657 self.assertEqual(p.maxEvents.input.value(), 1)
3658
3659 def testExamples(self):
3660 p = Process("Test")
3661 p.source = Source("PoolSource",fileNames = untracked(string("file:reco.root")))
3662 p.foos = EDProducer("FooProducer")
3663 p.bars = EDProducer("BarProducer", foos=InputTag("foos"))
3664 p.out = OutputModule("PoolOutputModule",fileName=untracked(string("file:foos.root")))
3665 p.bars.foos = 'Foosball'
3666 self.assertEqual(p.bars.foos, InputTag('Foosball'))
3667 p.p = Path(p.foos*p.bars)
3668 p.e = EndPath(p.out)
3669 p.add_(Service("MessageLogger"))
3670
3671 def testPrefers(self):
3672 p = Process("Test")
3673 p.add_(ESSource("ForceSource"))
3674 p.juicer = ESProducer("JuicerProducer")
3675 p.prefer("ForceSource")
3676 p.prefer("juicer")
3677 self.assertEqual(_lineDiff(p.dumpPython(), Process('Test').dumpPython()),
3678 """process.juicer = cms.ESProducer("JuicerProducer")
3679 process.ForceSource = cms.ESSource("ForceSource")
3680 process.prefer("ForceSource")
3681 process.prefer("juicer")""")
3682 p.prefer("juicer",fooRcd=vstring("Foo"))
3683 self.assertEqual(_lineDiff(p.dumpPython(), Process('Test').dumpPython()),
3684 """process.juicer = cms.ESProducer("JuicerProducer")
3685 process.ForceSource = cms.ESSource("ForceSource")
3686 process.prefer("ForceSource")
3687 process.prefer("juicer",
3688 fooRcd = cms.vstring('Foo')
3689 )""")
3690
3691 def testFreeze(self):
3692 process = Process("Freeze")
3693 m = EDProducer("M", p=PSet(i = int32(1)))
3694 m.p.i = 2
3695 process.m = m
3696
3697
3698
3699
3700
3701
3702 process.m.p.i = 4
3703 self.assertEqual(process.m.p.i.value(), 4)
3704 process.m.p = PSet(j=int32(1))
3705
3706 m2 = m.clone(p = PSet(i = int32(5)), j = int32(8))
3707 m2.p.i = 6
3708 m2.j = 8
3709 def testSubProcess(self):
3710 process = Process("Parent")
3711 subProcess = Process("Child")
3712 subProcess.a = EDProducer("A")
3713 subProcess.p = Path(subProcess.a)
3714 subProcess.add_(Service("Foo"))
3715 process.addSubProcess(SubProcess(subProcess))
3716 d = process.dumpPython()
3717 equalD ="""parentProcess = process
3718 process.a = cms.EDProducer("A")
3719 process.Foo = cms.Service("Foo")
3720 process.p = cms.Path(process.a)
3721 childProcess = process
3722 process = parentProcess
3723 process.addSubProcess(cms.SubProcess(process = childProcess, SelectEvents = cms.untracked.PSet(
3724 ), outputCommands = cms.untracked.vstring()))"""
3725 equalD = equalD.replace("parentProcess","parentProcess"+str(hash(process.subProcesses_()[0])))
3726
3727
3728
3729
3730
3731
3732
3733 messageLoggerSplit = 'process.MessageLogger = cms.Service'
3734 parentDumpSplit = Process('Parent').dumpPython().split(messageLoggerSplit)
3735 childProcess = Process('Child')
3736 del childProcess.MessageLogger
3737 combinedDump = parentDumpSplit[0] + childProcess.dumpPython() + messageLoggerSplit + parentDumpSplit[1]
3738 self.assertEqual(_lineDiff(d, combinedDump), equalD)
3739 p = TestMakePSet()
3740 process.fillProcessDesc(p)
3741 self.assertEqual((True,['a']),p.values["subProcesses"][1][0].values["process"][1].values['@all_modules'])
3742 self.assertEqual((True,['p']),p.values["subProcesses"][1][0].values["process"][1].values['@paths'])
3743 self.assertEqual({'@service_type':(True,'Foo')}, p.values["subProcesses"][1][0].values["process"][1].values["services"][1][0].values)
3744 def testRefToPSet(self):
3745 proc = Process("test")
3746 proc.top = PSet(a = int32(1))
3747 proc.ref = PSet(refToPSet_ = string("top"))
3748 proc.ref2 = PSet( a = int32(1), b = PSet( refToPSet_ = string("top")))
3749 proc.ref3 = PSet(refToPSet_ = string("ref"))
3750 proc.ref4 = VPSet(PSet(refToPSet_ = string("top")),
3751 PSet(refToPSet_ = string("ref2")))
3752 p = TestMakePSet()
3753 proc.fillProcessDesc(p)
3754 self.assertEqual((True,1),p.values["ref"][1].values["a"])
3755 self.assertEqual((True,1),p.values["ref3"][1].values["a"])
3756 self.assertEqual((True,1),p.values["ref2"][1].values["a"])
3757 self.assertEqual((True,1),p.values["ref2"][1].values["b"][1].values["a"])
3758 self.assertEqual((True,1),p.values["ref4"][1][0].values["a"])
3759 self.assertEqual((True,1),p.values["ref4"][1][1].values["a"])
3760 def testSwitchProducer(self):
3761 proc = Process("test")
3762 proc.sp = SwitchProducerTest(test2 = EDProducer("Foo",
3763 a = int32(1),
3764 b = PSet(c = int32(2))),
3765 test1 = EDProducer("Bar",
3766 aa = int32(11),
3767 bb = PSet(cc = int32(12))))
3768 self.assertEqual(proc.sp.label_(), "sp")
3769 self.assertEqual(proc.sp.test1.label_(), "sp@test1")
3770 self.assertEqual(proc.sp.test2.label_(), "sp@test2")
3771
3772 proc.a = EDProducer("A")
3773 proc.s = Sequence(proc.a + proc.sp)
3774 proc.t = Task(proc.a, proc.sp)
3775 proc.p = Path()
3776 proc.p.associate(proc.t)
3777 p = TestMakePSet()
3778 proc.fillProcessDesc(p)
3779 self.assertEqual((True,"EDProducer"), p.values["sp"][1].values["@module_edm_type"])
3780 self.assertEqual((True, "SwitchProducer"), p.values["sp"][1].values["@module_type"])
3781 self.assertEqual((True, "sp"), p.values["sp"][1].values["@module_label"])
3782 all_cases = copy.deepcopy(p.values["sp"][1].values["@all_cases"])
3783 all_cases[1].sort()
3784 self.assertEqual((True, ["sp@test1", "sp@test2"]), all_cases)
3785 self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
3786 self.assertEqual(["a", "sp", "sp@test1", "sp@test2"], p.values["@all_modules"][1])
3787 self.assertEqual((True,"EDProducer"), p.values["sp@test1"][1].values["@module_edm_type"])
3788 self.assertEqual((True,"Bar"), p.values["sp@test1"][1].values["@module_type"])
3789 self.assertEqual((True,"EDProducer"), p.values["sp@test2"][1].values["@module_edm_type"])
3790 self.assertEqual((True,"Foo"), p.values["sp@test2"][1].values["@module_type"])
3791 dump = proc.dumpPython()
3792 self.assertEqual(dump.find('@'), -1)
3793 self.assertEqual(specialImportRegistry.getSpecialImports(), ["from test import SwitchProducerTest"])
3794 self.assertTrue(dump.find("\nfrom test import SwitchProducerTest\n") != -1)
3795
3796
3797 proc = Process("test")
3798 proc.sp = SwitchProducerTest(test2 = EDProducer("Foo",
3799 a = int32(1),
3800 b = PSet(c = int32(2))),
3801 test1 = EDAlias(a = VPSet(PSet(type = string("Bar")))))
3802 proc.a = EDProducer("A")
3803 proc.s = Sequence(proc.a + proc.sp)
3804 proc.t = Task(proc.a, proc.sp)
3805 proc.p = Path()
3806 proc.p.associate(proc.t)
3807 p = TestMakePSet()
3808 proc.fillProcessDesc(p)
3809 self.assertEqual((True,"EDProducer"), p.values["sp"][1].values["@module_edm_type"])
3810 self.assertEqual((True, "SwitchProducer"), p.values["sp"][1].values["@module_type"])
3811 self.assertEqual((True, "sp"), p.values["sp"][1].values["@module_label"])
3812 all_cases = copy.deepcopy(p.values["sp"][1].values["@all_cases"])
3813 all_cases[1].sort()
3814 self.assertEqual((True, ["sp@test1", "sp@test2"]), all_cases)
3815 self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
3816 self.assertEqual(["a", "sp", "sp@test2"], p.values["@all_modules"][1])
3817 self.assertEqual(["sp@test1"], p.values["@all_aliases"][1])
3818 self.assertEqual((True,"EDProducer"), p.values["sp@test2"][1].values["@module_edm_type"])
3819 self.assertEqual((True,"Foo"), p.values["sp@test2"][1].values["@module_type"])
3820 self.assertEqual((True,"EDAlias"), p.values["sp@test1"][1].values["@module_edm_type"])
3821 self.assertEqual((True,"Bar"), p.values["sp@test1"][1].values["a"][1][0].values["type"])
3822
3823
3824 proc = Process("test")
3825 proc.sp = SwitchProducerTest(test1 = EDProducer("Foo",
3826 a = int32(1),
3827 b = PSet(c = int32(2))),
3828 test2 = EDAlias(a = VPSet(PSet(type = string("Bar")))))
3829 proc.a = EDProducer("A")
3830 proc.s = Sequence(proc.a + proc.sp)
3831 proc.t = Task(proc.a, proc.sp)
3832 proc.p = Path()
3833 proc.p.associate(proc.t)
3834 p = TestMakePSet()
3835 proc.fillProcessDesc(p)
3836 self.assertEqual((True,"EDProducer"), p.values["sp"][1].values["@module_edm_type"])
3837 self.assertEqual((True, "SwitchProducer"), p.values["sp"][1].values["@module_type"])
3838 self.assertEqual((True, "sp"), p.values["sp"][1].values["@module_label"])
3839 self.assertEqual((True, ["sp@test1", "sp@test2"]), p.values["sp"][1].values["@all_cases"])
3840 self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
3841 self.assertEqual(["a", "sp", "sp@test1"], p.values["@all_modules"][1])
3842 self.assertEqual(["sp@test2"], p.values["@all_aliases"][1])
3843 self.assertEqual((True,"EDProducer"), p.values["sp@test1"][1].values["@module_edm_type"])
3844 self.assertEqual((True,"Foo"), p.values["sp@test1"][1].values["@module_type"])
3845 self.assertEqual((True,"EDAlias"), p.values["sp@test2"][1].values["@module_edm_type"])
3846 self.assertEqual((True,"Bar"), p.values["sp@test2"][1].values["a"][1][0].values["type"])
3847
3848
3849 proc = Process("test")
3850 proc.spct = SwitchProducerTest(test2 = EDProducer("Foo",
3851 a = int32(1),
3852 b = PSet(c = int32(2))),
3853 test1 = EDProducer("Bar",
3854 aa = int32(11),
3855 bb = PSet(cc = int32(12))),
3856 test3 = EDAlias(a = VPSet(PSet(type = string("Bar")))))
3857 proc.spp = proc.spct.clone()
3858 proc.a = EDProducer("A")
3859 proc.ct = ConditionalTask(proc.spct)
3860 proc.p = Path(proc.a, proc.ct)
3861 proc.pp = Path(proc.a + proc.spp)
3862 p = TestMakePSet()
3863 proc.fillProcessDesc(p)
3864 self.assertEqual(["a", "spct", "spct@test1", "spct@test2", "spp", "spp@test1", "spp@test2"], p.values["@all_modules"][1])
3865 self.assertEqual(["a", "#", "spct", "spct@test1", "spct@test2", "@"], p.values["p"][1])
3866 self.assertEqual(["a", "spp", "#", "spp@test1", "spp@test2", "@"], p.values["pp"][1])
3867
3868 def testPrune(self):
3869 p = Process("test")
3870 p.a = EDAnalyzer("MyAnalyzer")
3871 p.b = EDAnalyzer("YourAnalyzer")
3872 p.c = EDAnalyzer("OurAnalyzer")
3873 p.d = EDAnalyzer("OurAnalyzer")
3874 p.e = EDProducer("MyProducer")
3875 p.f = EDProducer("YourProducer")
3876 p.g = EDProducer("TheirProducer")
3877 p.h = EDProducer("OnesProducer")
3878 p.s = Sequence(p.d)
3879 p.t1 = Task(p.e)
3880 p.t2 = Task(p.f)
3881 p.t3 = Task(p.g, p.t1)
3882 p.ct1 = ConditionalTask(p.h)
3883 p.ct2 = ConditionalTask(p.f)
3884 p.ct3 = ConditionalTask(p.ct1)
3885 p.path1 = Path(p.a, p.t3, p.ct3)
3886 p.path2 = Path(p.b)
3887 self.assertTrue(p.schedule is None)
3888 pths = p.paths
3889 keys = pths.keys()
3890 self.assertEqual(pths[keys[0]],p.path1)
3891 self.assertEqual(pths[keys[1]],p.path2)
3892 p.pset1 = PSet(parA = string("pset1"))
3893 p.pset2 = untracked.PSet(parA = string("pset2"))
3894 p.vpset1 = VPSet()
3895 p.vpset2 = untracked.VPSet()
3896 p.prune()
3897 self.assertTrue(hasattr(p, 'a'))
3898 self.assertTrue(hasattr(p, 'b'))
3899 self.assertTrue(not hasattr(p, 'c'))
3900 self.assertTrue(not hasattr(p, 'd'))
3901 self.assertTrue(hasattr(p, 'e'))
3902 self.assertTrue(not hasattr(p, 'f'))
3903 self.assertTrue(hasattr(p, 'g'))
3904 self.assertTrue(hasattr(p, 'h'))
3905 self.assertTrue(not hasattr(p, 's'))
3906 self.assertTrue(hasattr(p, 't1'))
3907 self.assertTrue(not hasattr(p, 't2'))
3908 self.assertTrue(hasattr(p, 't3'))
3909 self.assertTrue(hasattr(p, 'path1'))
3910 self.assertTrue(hasattr(p, 'path2'))
3911
3912
3913
3914
3915
3916 p = Process("test")
3917 p.a = EDAnalyzer("MyAnalyzer")
3918 p.b = EDAnalyzer("YourAnalyzer")
3919 p.c = EDAnalyzer("OurAnalyzer")
3920 p.d = EDAnalyzer("OurAnalyzer")
3921 p.e = EDAnalyzer("OurAnalyzer")
3922 p.f = EDProducer("MyProducer")
3923 p.g = EDProducer("YourProducer")
3924 p.h = EDProducer("TheirProducer")
3925 p.i = EDProducer("OurProducer")
3926 p.j = EDProducer("OurProducer")
3927 p.k = EDProducer("OurProducer")
3928 p.l = EDProducer("OurProducer")
3929 p.t1 = Task(p.f)
3930 p.t2 = Task(p.g)
3931 p.t3 = Task(p.h)
3932 p.t4 = Task(p.i)
3933 p.ct1 = Task(p.f)
3934 p.ct2 = Task(p.j)
3935 p.ct3 = Task(p.k)
3936 p.ct4 = Task(p.l)
3937 p.s = Sequence(p.d, p.t1, p.ct1)
3938 p.s2 = Sequence(p.b, p.t2, p.ct2)
3939 p.s3 = Sequence(p.e)
3940 p.path1 = Path(p.a, p.t3, p.ct3)
3941 p.path2 = Path(p.b)
3942 p.path3 = Path(p.b+p.s2)
3943 p.path4 = Path(p.b+p.s3)
3944 p.schedule = Schedule(p.path1,p.path2,p.path3)
3945 p.schedule.associate(p.t4)
3946 pths = p.paths
3947 keys = pths.keys()
3948 self.assertEqual(pths[keys[0]],p.path1)
3949 self.assertEqual(pths[keys[1]],p.path2)
3950 p.prune()
3951 self.assertTrue(hasattr(p, 'a'))
3952 self.assertTrue(hasattr(p, 'b'))
3953 self.assertTrue(not hasattr(p, 'c'))
3954 self.assertTrue(not hasattr(p, 'd'))
3955 self.assertTrue(not hasattr(p, 'e'))
3956 self.assertTrue(not hasattr(p, 'f'))
3957 self.assertTrue(hasattr(p, 'g'))
3958 self.assertTrue(hasattr(p, 'h'))
3959 self.assertTrue(hasattr(p, 'i'))
3960 self.assertTrue(hasattr(p, 'j'))
3961 self.assertTrue(hasattr(p, 'k'))
3962 self.assertTrue(not hasattr(p, 'l'))
3963 self.assertTrue(not hasattr(p, 't1'))
3964 self.assertTrue(hasattr(p, 't2'))
3965 self.assertTrue(hasattr(p, 't3'))
3966 self.assertTrue(hasattr(p, 't4'))
3967 self.assertTrue(not hasattr(p, 'ct1'))
3968 self.assertTrue(hasattr(p, 'ct2'))
3969 self.assertTrue(hasattr(p, 'ct3'))
3970 self.assertTrue(not hasattr(p, 'ct4'))
3971 self.assertTrue(not hasattr(p, 's'))
3972 self.assertTrue(hasattr(p, 's2'))
3973 self.assertTrue(not hasattr(p, 's3'))
3974 self.assertTrue(hasattr(p, 'path1'))
3975 self.assertTrue(hasattr(p, 'path2'))
3976 self.assertTrue(hasattr(p, 'path3'))
3977 self.assertTrue(not hasattr(p, 'path4'))
3978
3979 p = Process("test")
3980 p.a = EDAnalyzer("MyAnalyzer")
3981 p.b = EDAnalyzer("YourAnalyzer")
3982 p.s = Sequence(SequencePlaceholder("a")+p.b)
3983 p.pth = Path(p.s)
3984 p.prune()
3985 self.assertTrue(hasattr(p, 'a'))
3986 self.assertTrue(hasattr(p, 'b'))
3987 self.assertTrue(hasattr(p, 's'))
3988 self.assertTrue(hasattr(p, 'pth'))
3989
3990 p = Process("test")
3991 p.b = EDAnalyzer("YourAnalyzer")
3992 p.s = Sequence(SequencePlaceholder("a")+p.b)
3993 p.pth = Path(p.s)
3994 p.prune(keepUnresolvedSequencePlaceholders=True)
3995 self.assertTrue(hasattr(p, 'b'))
3996 self.assertTrue(hasattr(p, 's'))
3997 self.assertTrue(hasattr(p, 'pth'))
3998 self.assertEqual(p.s.dumpPython(),'cms.Sequence(cms.SequencePlaceholder("a")+process.b)\n')
3999
4000 p = Process("test")
4001 p.a = EDProducer("MyProducer")
4002 p.b = EDProducer("YourProducer")
4003 p.s = Task(TaskPlaceholder("a"),p.b)
4004 p.pth = Path(p.s)
4005 p.prune()
4006 self.assertTrue(hasattr(p, 'a'))
4007 self.assertTrue(hasattr(p, 'b'))
4008 self.assertTrue(hasattr(p, 's'))
4009 self.assertTrue(hasattr(p, 'pth'))
4010
4011 p = Process("test")
4012 p.a = EDProducer("MyProducer")
4013 p.b = EDProducer("YourProducer")
4014 p.s = ConditionalTask(ConditionalTaskPlaceholder("a"),p.b)
4015 p.pth = Path(p.s)
4016 p.prune()
4017 self.assertTrue(hasattr(p, 'a'))
4018 self.assertTrue(hasattr(p, 'b'))
4019 self.assertTrue(hasattr(p, 's'))
4020 self.assertTrue(hasattr(p, 'pth'))
4021
4022 p = Process("test")
4023 p.b = EDProducer("YourAnalyzer")
4024 p.s = Task(TaskPlaceholder("a"),p.b)
4025 p.pth = Path(p.s)
4026 p.prune(keepUnresolvedSequencePlaceholders=True)
4027 self.assertTrue(hasattr(p, 'b'))
4028 self.assertTrue(hasattr(p, 's'))
4029 self.assertTrue(hasattr(p, 'pth'))
4030 self.assertEqual(p.s.dumpPython(),'cms.Task(cms.TaskPlaceholder("a"), process.b)\n')
4031 def testTaskPlaceholder(self):
4032 p = Process("test")
4033 p.a = EDProducer("ma")
4034 p.b = EDAnalyzer("mb")
4035 p.t1 = Task(TaskPlaceholder("c"))
4036 p.t2 = Task(p.a, TaskPlaceholder("d"), p.t1)
4037 p.t3 = Task(TaskPlaceholder("e"))
4038 p.path1 = Path(p.b, p.t2, p.t3)
4039 p.t5 = Task(p.a, TaskPlaceholder("g"), TaskPlaceholder("t4"))
4040 p.t4 = Task(TaskPlaceholder("f"))
4041 p.endpath1 = EndPath(p.b, p.t5)
4042 p.t6 = Task(TaskPlaceholder("h"))
4043 p.t7 = Task(p.a, TaskPlaceholder("i"), p.t6)
4044 p.t8 = Task(TaskPlaceholder("j"))
4045 p.schedule = Schedule(p.path1, p.endpath1,tasks=[p.t7,p.t8])
4046 p.c = EDProducer("mc")
4047 p.d = EDProducer("md")
4048 p.e = EDProducer("me")
4049 p.f = EDProducer("mf")
4050 p.g = EDProducer("mg")
4051 p.h = EDProducer("mh")
4052 p.i = EDProducer("mi")
4053 p.j = EDProducer("mj")
4054 self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4055 """process.a = cms.EDProducer("ma")
4056 process.c = cms.EDProducer("mc")
4057 process.d = cms.EDProducer("md")
4058 process.e = cms.EDProducer("me")
4059 process.f = cms.EDProducer("mf")
4060 process.g = cms.EDProducer("mg")
4061 process.h = cms.EDProducer("mh")
4062 process.i = cms.EDProducer("mi")
4063 process.j = cms.EDProducer("mj")
4064 process.b = cms.EDAnalyzer("mb")
4065 process.t1 = cms.Task(cms.TaskPlaceholder("c"))
4066 process.t2 = cms.Task(cms.TaskPlaceholder("d"), process.a, process.t1)
4067 process.t3 = cms.Task(cms.TaskPlaceholder("e"))
4068 process.t5 = cms.Task(cms.TaskPlaceholder("g"), cms.TaskPlaceholder("t4"), process.a)
4069 process.t4 = cms.Task(cms.TaskPlaceholder("f"))
4070 process.t6 = cms.Task(cms.TaskPlaceholder("h"))
4071 process.t7 = cms.Task(cms.TaskPlaceholder("i"), process.a, process.t6)
4072 process.t8 = cms.Task(cms.TaskPlaceholder("j"))
4073 process.path1 = cms.Path(process.b, process.t2, process.t3)
4074 process.endpath1 = cms.EndPath(process.b, process.t5)
4075 process.schedule = cms.Schedule(*[ process.path1, process.endpath1 ], tasks=[process.t7, process.t8])""")
4076 p.resolve()
4077 self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4078 """process.a = cms.EDProducer("ma")
4079 process.c = cms.EDProducer("mc")
4080 process.d = cms.EDProducer("md")
4081 process.e = cms.EDProducer("me")
4082 process.f = cms.EDProducer("mf")
4083 process.g = cms.EDProducer("mg")
4084 process.h = cms.EDProducer("mh")
4085 process.i = cms.EDProducer("mi")
4086 process.j = cms.EDProducer("mj")
4087 process.b = cms.EDAnalyzer("mb")
4088 process.t1 = cms.Task(process.c)
4089 process.t2 = cms.Task(process.a, process.d, process.t1)
4090 process.t3 = cms.Task(process.e)
4091 process.t4 = cms.Task(process.f)
4092 process.t6 = cms.Task(process.h)
4093 process.t7 = cms.Task(process.a, process.i, process.t6)
4094 process.t8 = cms.Task(process.j)
4095 process.t5 = cms.Task(process.a, process.g, process.t4)
4096 process.path1 = cms.Path(process.b, process.t2, process.t3)
4097 process.endpath1 = cms.EndPath(process.b, process.t5)
4098 process.schedule = cms.Schedule(*[ process.path1, process.endpath1 ], tasks=[process.t7, process.t8])""")
4099 def testConditionalTaskPlaceholder(self):
4100 p = Process("test")
4101 p.a = EDProducer("ma")
4102 p.b = EDAnalyzer("mb")
4103 p.t1 = ConditionalTask(ConditionalTaskPlaceholder("c"))
4104 p.t2 = ConditionalTask(p.a, ConditionalTaskPlaceholder("d"), p.t1)
4105 p.t3 = ConditionalTask(ConditionalTaskPlaceholder("e"))
4106 p.path1 = Path(p.b, p.t2, p.t3)
4107 p.t5 = ConditionalTask(p.a, ConditionalTaskPlaceholder("g"), ConditionalTaskPlaceholder("t4"))
4108 p.t4 = ConditionalTask(ConditionalTaskPlaceholder("f"))
4109 p.path2 = Path(p.b, p.t5)
4110 p.schedule = Schedule(p.path1, p.path2)
4111 p.c = EDProducer("mc")
4112 p.d = EDProducer("md")
4113 p.e = EDProducer("me")
4114 p.f = EDProducer("mf")
4115 p.g = EDProducer("mg")
4116 p.h = EDProducer("mh")
4117 p.i = EDProducer("mi")
4118 p.j = EDProducer("mj")
4119 self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4120 """process.a = cms.EDProducer("ma")
4121 process.c = cms.EDProducer("mc")
4122 process.d = cms.EDProducer("md")
4123 process.e = cms.EDProducer("me")
4124 process.f = cms.EDProducer("mf")
4125 process.g = cms.EDProducer("mg")
4126 process.h = cms.EDProducer("mh")
4127 process.i = cms.EDProducer("mi")
4128 process.j = cms.EDProducer("mj")
4129 process.b = cms.EDAnalyzer("mb")
4130 process.t1 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("c"))
4131 process.t2 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("d"), process.a, process.t1)
4132 process.t3 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("e"))
4133 process.t5 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("g"), cms.ConditionalTaskPlaceholder("t4"), process.a)
4134 process.t4 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("f"))
4135 process.path1 = cms.Path(process.b, process.t2, process.t3)
4136 process.path2 = cms.Path(process.b, process.t5)
4137 process.schedule = cms.Schedule(*[ process.path1, process.path2 ])""")
4138 p.resolve()
4139 self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4140 """process.a = cms.EDProducer("ma")
4141 process.c = cms.EDProducer("mc")
4142 process.d = cms.EDProducer("md")
4143 process.e = cms.EDProducer("me")
4144 process.f = cms.EDProducer("mf")
4145 process.g = cms.EDProducer("mg")
4146 process.h = cms.EDProducer("mh")
4147 process.i = cms.EDProducer("mi")
4148 process.j = cms.EDProducer("mj")
4149 process.b = cms.EDAnalyzer("mb")
4150 process.t1 = cms.ConditionalTask(process.c)
4151 process.t2 = cms.ConditionalTask(process.a, process.d, process.t1)
4152 process.t3 = cms.ConditionalTask(process.e)
4153 process.t4 = cms.ConditionalTask(process.f)
4154 process.t5 = cms.ConditionalTask(process.a, process.g, process.t4)
4155 process.path1 = cms.Path(process.b, process.t2, process.t3)
4156 process.path2 = cms.Path(process.b, process.t5)
4157 process.schedule = cms.Schedule(*[ process.path1, process.path2 ])""")
4158
4159 def testDelete(self):
4160 p = Process("test")
4161 p.a = EDAnalyzer("MyAnalyzer")
4162 p.b = EDAnalyzer("YourAnalyzer")
4163 p.c = EDAnalyzer("OurAnalyzer")
4164 p.d = EDAnalyzer("OurAnalyzer")
4165 p.e = EDAnalyzer("OurAnalyzer")
4166 p.f = EDAnalyzer("OurAnalyzer")
4167 p.g = EDProducer("OurProducer")
4168 p.h = EDProducer("YourProducer")
4169 p.i = SwitchProducerTest(
4170 test1 = EDProducer("OneProducer"),
4171 test2 = EDProducer("TwoProducer")
4172 )
4173 p.t1 = Task(p.g, p.h, p.i)
4174 t2 = Task(p.g, p.h, p.i)
4175 t3 = Task(p.g, p.h)
4176 p.t4 = Task(p.h)
4177 p.ct1 = ConditionalTask(p.g, p.h, p.i)
4178 ct2 = ConditionalTask(p.g, p.h)
4179 ct3 = ConditionalTask(p.g, p.h)
4180 p.ct4 = ConditionalTask(p.h)
4181 p.s = Sequence(p.d+p.e)
4182 p.path1 = Path(p.a+p.f+p.s,t2,ct2)
4183 p.path2 = Path(p.a)
4184 p.path3 = Path(ct3, p.ct4)
4185 p.endpath2 = EndPath(p.b)
4186 p.endpath1 = EndPath(p.b+p.f)
4187 p.schedule = Schedule(p.path2, p.path3, p.endpath2, tasks=[t3, p.t4])
4188 self.assertTrue(hasattr(p, 'f'))
4189 self.assertTrue(hasattr(p, 'g'))
4190 self.assertTrue(hasattr(p, 'i'))
4191 del p.e
4192 del p.f
4193 del p.g
4194 del p.i
4195 self.assertFalse(hasattr(p, 'f'))
4196 self.assertFalse(hasattr(p, 'g'))
4197 self.assertEqual(p.t1.dumpPython(), 'cms.Task(process.h)\n')
4198 self.assertEqual(p.ct1.dumpPython(), 'cms.ConditionalTask(process.h)\n')
4199 self.assertEqual(p.s.dumpPython(), 'cms.Sequence(process.d)\n')
4200 self.assertEqual(p.path1.dumpPython(), 'cms.Path(process.a+process.s, cms.ConditionalTask(process.h), cms.Task(process.h))\n')
4201 self.assertEqual(p.endpath1.dumpPython(), 'cms.EndPath(process.b)\n')
4202 self.assertEqual(p.path3.dumpPython(), 'cms.Path(cms.ConditionalTask(process.h), process.ct4)\n')
4203 del p.s
4204 self.assertEqual(p.path1.dumpPython(), 'cms.Path(process.a+(process.d), cms.ConditionalTask(process.h), cms.Task(process.h))\n')
4205 self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(*[ process.path2, process.path3, process.endpath2 ], tasks=[cms.Task(process.h), process.t4])\n')
4206 del p.path2
4207 self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(*[ process.path3, process.endpath2 ], tasks=[cms.Task(process.h), process.t4])\n')
4208 del p.path3
4209 self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(*[ process.endpath2 ], tasks=[cms.Task(process.h), process.t4])\n')
4210 del p.endpath2
4211 self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(tasks=[cms.Task(process.h), process.t4])\n')
4212 del p.t4
4213 self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(tasks=[cms.Task(process.h)])\n')
4214 def testModifier(self):
4215 m1 = Modifier()
4216 Process._firstProcess = True
4217 p = Process("test")
4218 self.assertRaises(RuntimeError, lambda: Process("test2", m1))
4219 m1 = Modifier()
4220 Process._firstProcess = True
4221 p = Process("test",m1)
4222 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1))
4223 def _mod_fred(obj):
4224 obj.fred = 2
4225 m1.toModify(p.a,_mod_fred)
4226 self.assertEqual(p.a.fred.value(),2)
4227 p.b = EDAnalyzer("YourAnalyzer", wilma = int32(1))
4228 m1.toModify(p.b, wilma = 2)
4229 self.assertEqual(p.b.wilma.value(),2)
4230 self.assertTrue(p.isUsingModifier(m1))
4231
4232 m1 = Modifier()
4233 Process._firstProcess = True
4234 p = Process("test")
4235 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1))
4236 m1.toModify(p.a,_mod_fred)
4237 p.b = EDAnalyzer("YourAnalyzer", wilma = int32(1))
4238 m1.toModify(p.b, wilma = 2)
4239 self.assertEqual(p.a.fred.value(),1)
4240 self.assertEqual(p.b.wilma.value(),1)
4241 self.assertEqual(p.isUsingModifier(m1),False)
4242
4243 m1 = Modifier()
4244 Process._firstProcess = True
4245 p = Process("test",m1)
4246 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4247 m1.toModify(p.a, fred = int32(2))
4248 p.b = p.a.clone(wilma = int32(3))
4249 self.assertEqual(p.a.fred.value(),2)
4250 self.assertEqual(p.a.wilma.value(),1)
4251 self.assertEqual(p.b.fred.value(),2)
4252 self.assertEqual(p.b.wilma.value(),3)
4253
4254 m1 = Modifier()
4255 Process._firstProcess = True
4256 p = Process("test",m1)
4257 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1), fintstones = PSet(fred = int32(1)))
4258 m1.toModify(p.a, fred = None, fintstones = dict(fred = None))
4259 self.assertEqual(hasattr(p.a, "fred"), False)
4260 self.assertEqual(hasattr(p.a.fintstones, "fred"), False)
4261 self.assertEqual(p.a.wilma.value(),1)
4262
4263 m1 = Modifier()
4264 Process._firstProcess = True
4265 p = Process("test",m1)
4266 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1))
4267 m1.toModify(p.a, wilma = int32(2))
4268 self.assertEqual(p.a.fred.value(), 1)
4269 self.assertEqual(p.a.wilma.value(),2)
4270
4271 m1 = Modifier()
4272 Process._firstProcess = True
4273 p = Process("test",m1)
4274 p.a = EDAnalyzer("MyAnalyzer", flintstones = PSet(fred = int32(1), wilma = int32(1)))
4275 m1.toModify(p.a, flintstones = dict(fred = int32(2)))
4276 self.assertEqual(p.a.flintstones.fred.value(),2)
4277 self.assertEqual(p.a.flintstones.wilma.value(),1)
4278
4279 m1 = Modifier()
4280 Process._firstProcess = True
4281 p = Process("test",m1)
4282 p.a = EDAnalyzer("MyAnalyzer", flintstones = PSet(fred = PSet(wilma = int32(1))))
4283 self.assertRaises(KeyError, lambda: m1.toModify(p.a, flintstones = dict(imnothere = dict(wilma=2))))
4284 self.assertRaises(KeyError, lambda: m1.toModify(p.a, foo = 1))
4285
4286 m1 = Modifier()
4287 Process._firstProcess = True
4288 p = Process("test",m1)
4289 p.a = EDAnalyzer("MyAnalyzer", flintstones = VPSet(PSet(fred = int32(1)), PSet(wilma = int32(1))))
4290 m1.toModify(p.a, flintstones = {1:dict(wilma = int32(2))})
4291 self.assertEqual(p.a.flintstones[0].fred.value(),1)
4292 self.assertEqual(p.a.flintstones[1].wilma.value(),2)
4293
4294 m1 = Modifier()
4295 Process._firstProcess = True
4296 p = Process("test",m1)
4297 p.a = EDAnalyzer("MyAnalyzer", fred = vuint32(1,2,3))
4298 m1.toModify(p.a, fred = {1:7})
4299 self.assertEqual(p.a.fred[0],1)
4300 self.assertEqual(p.a.fred[1],7)
4301 self.assertEqual(p.a.fred[2],3)
4302
4303 m1 = Modifier()
4304 Process._firstProcess = True
4305 p = Process("test",m1)
4306 p.a = EDAnalyzer("MyAnalyzer", fred = vuint32(1,2,3))
4307 raised = False
4308 try: m1.toModify(p.a, fred = {5:7})
4309 except IndexError as e: raised = True
4310 self.assertEqual(raised, True)
4311
4312 m1 = Modifier()
4313 Process._firstProcess = True
4314 p = Process("test",m1)
4315 p.a = EDAnalyzer("MyAnalyzer", flintstones = VPSet(PSet(fred = int32(1)), PSet(wilma = int32(1))))
4316 raised = False
4317 try: m1.toModify(p.a, flintstones = dict(bogus = int32(37)))
4318 except TypeError as e: raised = True
4319 self.assertEqual(raised, True)
4320
4321 def _rem_a(proc):
4322 del proc.a
4323 class ProcModifierMod(object):
4324 def __init__(self,modifier,func):
4325 self.proc_mod_ = modifier.makeProcessModifier(func)
4326 class DummyMod(object):
4327 def __init__(self):
4328 self.a = EDAnalyzer("Dummy")
4329 testMod = DummyMod()
4330 p.extend(testMod)
4331 self.assertTrue(hasattr(p,"a"))
4332 m1 = Modifier()
4333 Process._firstProcess = True
4334 p = Process("test",m1)
4335 testProcMod = ProcModifierMod(m1,_rem_a)
4336 p.extend(testMod)
4337 p.extend(testProcMod)
4338 self.assertTrue(not hasattr(p,"a"))
4339
4340 m1 = Modifier()
4341 mc = ModifierChain(m1)
4342 Process._firstProcess = True
4343 p = Process("test",mc)
4344 self.assertTrue(p.isUsingModifier(m1))
4345 self.assertTrue(p.isUsingModifier(mc))
4346 testMod = DummyMod()
4347 p.b = EDAnalyzer("Dummy2", fred = int32(1))
4348 m1.toModify(p.b, fred = int32(3))
4349 p.extend(testMod)
4350 testProcMod = ProcModifierMod(m1,_rem_a)
4351 p.extend(testProcMod)
4352 self.assertTrue(not hasattr(p,"a"))
4353 self.assertEqual(p.b.fred.value(),3)
4354
4355 m1 = Modifier()
4356 m2 = Modifier()
4357 mc = ModifierChain(m1,m2)
4358 mclone = mc.copyAndExclude([m2])
4359 self.assertTrue(not mclone._isOrContains(m2))
4360 self.assertTrue(mclone._isOrContains(m1))
4361 m3 = Modifier()
4362 mc2 = ModifierChain(mc,m3)
4363 mclone = mc2.copyAndExclude([m2])
4364 self.assertTrue(not mclone._isOrContains(m2))
4365 self.assertTrue(mclone._isOrContains(m1))
4366 self.assertTrue(mclone._isOrContains(m3))
4367
4368 m1 = Modifier()
4369 m2 = Modifier()
4370 Process._firstProcess = True
4371 p = Process("test",m1)
4372 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4373 (m1 & m2).toModify(p.a, fred = int32(2))
4374 self.assertRaises(TypeError, lambda: (m1 & m2).toModify(p.a, 1, wilma=2))
4375 self.assertEqual(p.a.fred, 1)
4376 m1 = Modifier()
4377 m2 = Modifier()
4378 Process._firstProcess = True
4379 p = Process("test",m1,m2)
4380 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4381 (m1 & m2).toModify(p.a, fred = int32(2))
4382 self.assertEqual(p.a.fred, 2)
4383 m1 = Modifier()
4384 m2 = Modifier()
4385 m3 = Modifier()
4386 Process._firstProcess = True
4387 p = Process("test",m1,m2,m3)
4388 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4389 (m1 & m2 & m3).toModify(p.a, fred = int32(2))
4390 self.assertEqual(p.a.fred, 2)
4391 (m1 & (m2 & m3)).toModify(p.a, fred = int32(3))
4392 self.assertEqual(p.a.fred, 3)
4393 ((m1 & m2) & m3).toModify(p.a, fred = int32(4))
4394 self.assertEqual(p.a.fred, 4)
4395
4396 m1 = Modifier()
4397 m2 = Modifier()
4398 Process._firstProcess = True
4399 p = Process("test", m1)
4400 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4401 (~m1).toModify(p.a, fred=2)
4402 self.assertEqual(p.a.fred, 1)
4403 (~m2).toModify(p.a, wilma=2)
4404 self.assertEqual(p.a.wilma, 2)
4405 self.assertRaises(TypeError, lambda: (~m1).toModify(p.a, 1, wilma=2))
4406 self.assertRaises(TypeError, lambda: (~m2).toModify(p.a, 1, wilma=2))
4407
4408 m1 = Modifier()
4409 m2 = Modifier()
4410 m3 = Modifier()
4411 Process._firstProcess = True
4412 p = Process("test", m1)
4413 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4414 (m1 | m2).toModify(p.a, fred=2)
4415 self.assertEqual(p.a.fred, 2)
4416 (m1 | m2 | m3).toModify(p.a, fred=3)
4417 self.assertEqual(p.a.fred, 3)
4418 (m3 | m2 | m1).toModify(p.a, fred=4)
4419 self.assertEqual(p.a.fred, 4)
4420 ((m1 | m2) | m3).toModify(p.a, fred=5)
4421 self.assertEqual(p.a.fred, 5)
4422 (m1 | (m2 | m3)).toModify(p.a, fred=6)
4423 self.assertEqual(p.a.fred, 6)
4424 (m2 | m3).toModify(p.a, fred=7)
4425 self.assertEqual(p.a.fred, 6)
4426 self.assertRaises(TypeError, lambda: (m1 | m2).toModify(p.a, 1, wilma=2))
4427 self.assertRaises(TypeError, lambda: (m2 | m3).toModify(p.a, 1, wilma=2))
4428
4429 m1 = Modifier()
4430 m2 = Modifier()
4431 m3 = Modifier()
4432 m4 = Modifier()
4433 Process._firstProcess = True
4434 p = Process("test", m1, m2)
4435 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4436 (m1 & ~m2).toModify(p.a, fred=2)
4437 self.assertEqual(p.a.fred, 1)
4438 (m1 & ~m3).toModify(p.a, fred=2)
4439 self.assertEqual(p.a.fred, 2)
4440 (m1 | ~m2).toModify(p.a, fred=3)
4441 self.assertEqual(p.a.fred, 3)
4442 (~m1 | ~m2).toModify(p.a, fred=4)
4443 self.assertEqual(p.a.fred, 3)
4444 (~m3 & ~m4).toModify(p.a, fred=4)
4445 self.assertEqual(p.a.fred, 4)
4446 ((m1 & m3) | ~m4).toModify(p.a, fred=5)
4447 self.assertEqual(p.a.fred, 5)
4448
4449 m1 = Modifier()
4450 Process._firstProcess = True
4451 p = Process("test",m1)
4452 p.a =EDAnalyzer("MyAnalyzer", fred = int32(1))
4453 m1.toReplaceWith(p.a, EDAnalyzer("YourAnalyzer", wilma = int32(3)))
4454 self.assertRaises(TypeError, lambda: m1.toReplaceWith(p.a, EDProducer("YourProducer")))
4455
4456 p.b =EDAnalyzer("BAn")
4457 p.c =EDProducer("c")
4458 p.d =EDProducer("d")
4459 p.tc = Task(p.c)
4460 p.td = Task(p.d)
4461 p.s = Sequence(p.a, p.tc)
4462 m1.toReplaceWith(p.s, Sequence(p.a+p.b, p.td))
4463 self.assertEqual(p.a.wilma.value(),3)
4464 self.assertEqual(p.a.type_(),"YourAnalyzer")
4465 self.assertEqual(hasattr(p,"fred"),False)
4466 self.assertTrue(p.s.dumpPython() == "cms.Sequence(process.a+process.b, process.td)\n")
4467 p.e =EDProducer("e")
4468 m1.toReplaceWith(p.td, Task(p.e))
4469 self.assertTrue(p.td._collection == OrderedSet([p.e]))
4470
4471 p.b =EDAnalyzer("BAn")
4472 p.c =EDProducer("c")
4473 p.d =EDProducer("d")
4474 del p.tc
4475 del p.td
4476 p.tc = ConditionalTask(p.c)
4477 p.td = ConditionalTask(p.d)
4478 p.s = Sequence(p.a, p.tc)
4479 m1.toReplaceWith(p.s, Sequence(p.a+p.b, p.td))
4480 self.assertEqual(p.a.wilma.value(),3)
4481 self.assertEqual(p.a.type_(),"YourAnalyzer")
4482 self.assertEqual(hasattr(p,"fred"),False)
4483 self.assertTrue(p.s.dumpPython() == "cms.Sequence(process.a+process.b, process.td)\n")
4484 p.e =EDProducer("e")
4485 m1.toReplaceWith(p.td, ConditionalTask(p.e))
4486 self.assertTrue(p.td._collection == OrderedSet([p.e]))
4487
4488 m1 = Modifier()
4489 Process._firstProcess = True
4490 p = Process("test")
4491 p.a =EDAnalyzer("MyAnalyzer", fred = int32(1))
4492 m1.toReplaceWith(p.a, EDAnalyzer("YourAnalyzer", wilma = int32(3)))
4493 self.assertEqual(p.a.type_(),"MyAnalyzer")
4494
4495 m1 = Modifier()
4496 m2 = Modifier()
4497 m3 = Modifier()
4498 m4 = Modifier()
4499 Process._firstProcess = True
4500 p = Process("test", m1, m2)
4501 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4502 self.assertRaises(TypeError, lambda: (m1 & m2).toReplaceWith(p.a, EDProducer("YourProducer")))
4503 self.assertRaises(TypeError, lambda: (m3 & m4).toReplaceWith(p.a, EDProducer("YourProducer")))
4504 self.assertRaises(TypeError, lambda: (~m3).toReplaceWith(p.a, EDProducer("YourProducer")))
4505 self.assertRaises(TypeError, lambda: (~m1).toReplaceWith(p.a, EDProducer("YourProducer")))
4506 self.assertRaises(TypeError, lambda: (m1 | m3).toReplaceWith(p.a, EDProducer("YourProducer")))
4507 self.assertRaises(TypeError, lambda: (m3 | m4).toReplaceWith(p.a, EDProducer("YourProducer")))
4508 (m1 & m2).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer1"))
4509 self.assertEqual(p.a.type_(), "YourAnalyzer1")
4510 (m1 & m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer2"))
4511 self.assertEqual(p.a.type_(), "YourAnalyzer1")
4512 (~m1).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer2"))
4513 self.assertEqual(p.a.type_(), "YourAnalyzer1")
4514 (~m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer2"))
4515 self.assertEqual(p.a.type_(), "YourAnalyzer2")
4516 (m1 | m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer3"))
4517 self.assertEqual(p.a.type_(), "YourAnalyzer3")
4518 (m3 | m4).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer4"))
4519 self.assertEqual(p.a.type_(), "YourAnalyzer3")
4520
4521 m1 = Modifier()
4522 m2 = Modifier()
4523 m3 = Modifier()
4524 Process._firstProcess = True
4525 p = Process("test", m1, m2)
4526 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4527 p.b = EDProducer("MyProducer", barney = int32(1), betty = int32(1))
4528 (m1 & m2).toModify(p.a, fred = 2).toModify(p.b, betty = 3)
4529 self.assertEqual(p.a.fred, 2)
4530 self.assertEqual(p.a.wilma, 1)
4531 self.assertEqual(p.b.barney, 1)
4532 self.assertEqual(p.b.betty, 3)
4533 (m1 | m3).toModify(p.a, wilma = 4).toModify(p.b, barney = 5)
4534 self.assertEqual(p.a.fred, 2)
4535 self.assertEqual(p.a.wilma, 4)
4536 self.assertEqual(p.b.barney, 5)
4537 self.assertEqual(p.b.betty, 3)
4538 (m2 & ~m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer")).toModify(p.b, barney = 6)
4539 self.assertEqual(p.a.type_(), "YourAnalyzer")
4540 self.assertEqual(p.b.barney, 6)
4541 self.assertEqual(p.b.betty, 3)
4542 (m1 & ~m3).toModify(p.a, param=int32(42)).toReplaceWith(p.b, EDProducer("YourProducer"))
4543 self.assertEqual(p.a.type_(), "YourAnalyzer")
4544 self.assertEqual(p.a.param, 42)
4545 self.assertEqual(p.b.type_(), "YourProducer")
4546
4547
4548 a = EDAlias(foo2 = VPSet(PSet(type = string("Foo2"))))
4549 m = Modifier()
4550 m._setChosen()
4551
4552 m.toModify(a, foo2 = {0: dict(type = "Foo3")})
4553 self.assertEqual(a.foo2[0].type, "Foo3")
4554
4555 m.toModify(a, foo4 = VPSet(PSet(type = string("Foo4"))))
4556 self.assertEqual(a.foo2[0].type, "Foo3")
4557 self.assertEqual(a.foo4[0].type, "Foo4")
4558
4559 m.toModify(a, foo2 = None)
4560 self.assertFalse(hasattr(a, "foo2"))
4561 self.assertEqual(a.foo4[0].type, "Foo4")
4562
4563 m.toReplaceWith(a, EDAlias(bar = VPSet(PSet(type = string("Bar")))))
4564 self.assertFalse(hasattr(a, "foo2"))
4565 self.assertFalse(hasattr(a, "foo4"))
4566 self.assertTrue(hasattr(a, "bar"))
4567 self.assertEqual(a.bar[0].type, "Bar")
4568
4569
4570 sp = SwitchProducerTest(test1 = EDProducer("Foo",
4571 a = int32(1),
4572 b = PSet(c = int32(2))),
4573 test2 = EDProducer("Bar",
4574 aa = int32(11),
4575 bb = PSet(cc = int32(12))))
4576 m = Modifier()
4577 m._setChosen()
4578
4579 m.toModify(sp,
4580 test1 = dict(a = 4, b = dict(c = None)),
4581 test2 = dict(aa = 15, bb = dict(cc = 45, dd = string("foo"))))
4582 self.assertEqual(sp.test1.a.value(), 4)
4583 self.assertEqual(sp.test1.b.hasParameter("c"), False)
4584 self.assertEqual(sp.test2.aa.value(), 15)
4585 self.assertEqual(sp.test2.bb.cc.value(), 45)
4586 self.assertEqual(sp.test2.bb.dd.value(), "foo")
4587
4588 m.toReplaceWith(sp.test1, EDProducer("Fred", x = int32(42)))
4589 self.assertEqual(sp.test1.type_(), "Fred")
4590 self.assertEqual(sp.test1.x.value(), 42)
4591 self.assertRaises(TypeError, lambda: m.toReplaceWith(sp.test1, EDAnalyzer("Foo")))
4592
4593 m.toModify(sp, test2 = EDProducer("Xyzzy", x = int32(24)))
4594 self.assertEqual(sp.test2.type_(), "Xyzzy")
4595 self.assertEqual(sp.test2.x.value(), 24)
4596 self.assertRaises(TypeError, lambda: m.toModify(sp, test2 = EDAnalyzer("Foo")))
4597
4598 m.toModify(sp, test3 = EDProducer("Wilma", y = int32(24)))
4599 self.assertEqual(sp.test3.type_(), "Wilma")
4600 self.assertEqual(sp.test3.y.value(), 24)
4601 self.assertRaises(TypeError, lambda: m.toModify(sp, test4 = EDAnalyzer("Foo")))
4602
4603 m.toModify(sp, test2 = None)
4604 self.assertEqual(hasattr(sp, "test2"), False)
4605
4606 m.toModify(sp, test2 = EDAlias(foo = VPSet(PSet(type = string("int")))))
4607 self.assertTrue(hasattr(sp.test2, "foo"))
4608
4609 m.toReplaceWith(sp.test2, EDAlias(bar = VPSet(PSet(type = string("int")))))
4610 self.assertTrue(hasattr(sp.test2, "bar"))
4611
4612 m.toModify(sp, test2 = EDAlias(xyzzy = VPSet(PSet(type = string("int")))))
4613 self.assertTrue(hasattr(sp.test2, "xyzzy"))
4614
4615 self.assertRaises(TypeError, lambda: m.toReplaceWith(sp.test2, EDProducer("Foo")))
4616 m.toModify(sp, test2 = EDProducer("Foo"))
4617 def testProcessFragment(self):
4618
4619 f = ProcessFragment('Fragment')
4620 p = Process('PROCESS')
4621 p.maxEvents.input = 10
4622 p.options.numberOfThreads = 4
4623 p.maxLuminosityBlocks.input = 2
4624 p.extend(f)
4625 self.assertEqual(p.maxEvents.input.value(),10)
4626 self.assertEqual(p.options.numberOfThreads.value(), 4)
4627 self.assertEqual(p.maxLuminosityBlocks.input.value(),2)
4628
4629 f = ProcessFragment("Fragment")
4630 f.fltr = EDFilter("Foo")
4631 p = Process('PROCESS')
4632 p.extend(f)
4633 self.assertTrue(hasattr(p,'fltr'))
4634 def testProcessForProcessAccelerator(self):
4635 proc = Process("TEST")
4636 p = ProcessForProcessAccelerator(proc)
4637 p.TestService = Service("TestService")
4638 self.assertTrue(hasattr(proc, "TestService"))
4639 self.assertEqual(proc.TestService.type_(), "TestService")
4640 self.assertRaises(TypeError, setattr, p, "a", EDProducer("Foo"))
4641 p.add_(Service("TestServiceTwo"))
4642 self.assertTrue(hasattr(proc, "TestServiceTwo"))
4643 self.assertEqual(proc.TestServiceTwo.type_(), "TestServiceTwo")
4644 p.TestService.foo = untracked.uint32(42)
4645 self.assertEqual(proc.TestService.foo.value(), 42)
4646 proc.mod = EDProducer("Producer")
4647 self.assertRaises(TypeError, getattr, p, "mod")
4648 def testProcessAccelerator(self):
4649 proc = Process("TEST")
4650 p = TestMakePSet()
4651 proc.fillProcessDesc(p)
4652 self.assertTrue(["cpu"], p.values["@available_accelerators"][1])
4653 self.assertFalse(p.values["@selected_accelerators"][0])
4654 self.assertTrue(["cpu"], p.values["@selected_accelerators"][1])
4655 self.assertFalse(p.values["@module_type_resolver"][0])
4656 self.assertEqual("", p.values["@module_type_resolver"][1])
4657
4658 proc = Process("TEST")
4659 self.assertRaises(TypeError, setattr, proc, "processAcceleratorTest", ProcessAcceleratorTest())
4660 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4661 del proc.MessageLogger
4662 self.maxDiff = None
4663 self.assertEqual(proc.dumpPython(),
4664 """import FWCore.ParameterSet.Config as cms
4665 from test import ProcessAcceleratorTest
4666
4667 process = cms.Process("TEST")
4668
4669 process.maxEvents = cms.untracked.PSet(
4670 input = cms.optional.untracked.int32,
4671 output = cms.optional.untracked.allowed(cms.int32,cms.PSet)
4672 )
4673
4674 process.maxLuminosityBlocks = cms.untracked.PSet(
4675 input = cms.untracked.int32(-1)
4676 )
4677
4678 process.options = cms.untracked.PSet(
4679 IgnoreCompletely = cms.untracked.vstring(),
4680 Rethrow = cms.untracked.vstring(),
4681 TryToContinue = cms.untracked.vstring(),
4682 accelerators = cms.untracked.vstring('*'),
4683 allowUnscheduled = cms.obsolete.untracked.bool,
4684 canDeleteEarly = cms.untracked.vstring(),
4685 deleteNonConsumedUnscheduledModules = cms.untracked.bool(True),
4686 dumpOptions = cms.untracked.bool(False),
4687 emptyRunLumiMode = cms.obsolete.untracked.string,
4688 eventSetup = cms.untracked.PSet(
4689 forceNumberOfConcurrentIOVs = cms.untracked.PSet(
4690 allowAnyLabel_=cms.required.untracked.uint32
4691 ),
4692 numberOfConcurrentIOVs = cms.untracked.uint32(0)
4693 ),
4694 fileMode = cms.untracked.string('FULLMERGE'),
4695 forceEventSetupCacheClearOnNewRun = cms.untracked.bool(False),
4696 holdsReferencesToDeleteEarly = cms.untracked.VPSet(),
4697 makeTriggerResults = cms.obsolete.untracked.bool,
4698 modulesToCallForTryToContinue = cms.untracked.vstring(),
4699 modulesToIgnoreForDeleteEarly = cms.untracked.vstring(),
4700 numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(0),
4701 numberOfConcurrentRuns = cms.untracked.uint32(1),
4702 numberOfStreams = cms.untracked.uint32(0),
4703 numberOfThreads = cms.untracked.uint32(1),
4704 printDependencies = cms.untracked.bool(False),
4705 sizeOfStackForThreadsInKB = cms.optional.untracked.uint32,
4706 throwIfIllegalParameter = cms.untracked.bool(True),
4707 wantSummary = cms.untracked.bool(False)
4708 )
4709
4710 process.ProcessAcceleratorTest = ProcessAcceleratorTest(
4711 enabled = ['test1', 'test2', 'anothertest3']
4712 )
4713
4714
4715 """)
4716 p = TestMakePSet()
4717 proc.fillProcessDesc(p)
4718 self.assertEqual(["*"], p.values["options"][1].values["accelerators"][1])
4719 self.assertFalse(p.values["options"][1].values["accelerators"][0])
4720 self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["@selected_accelerators"][1])
4721 self.assertEqual("AcceleratorTestService", p.values["services"][1][0].values["@service_type"][1])
4722 self.assertFalse(p.values["@available_accelerators"][0])
4723 self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4724 self.assertFalse(p.values["@module_type_resolver"][0])
4725 self.assertEqual("", p.values["@module_type_resolver"][1])
4726
4727 proc = Process("TEST")
4728 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4729 proc.add_(Service("AcceleratorTestServiceRemove"))
4730 p = TestMakePSet()
4731 proc.fillProcessDesc(p)
4732 services = [x.values["@service_type"][1] for x in p.values["services"][1]]
4733 self.assertTrue("AcceleratorTestService" in services)
4734 self.assertFalse("AcceleratorTestServiceRemove" in services)
4735
4736 proc = Process("TEST")
4737 proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"])
4738 p = TestMakePSet()
4739 proc.fillProcessDesc(p)
4740 self.assertEqual(["cpu", "test1"], p.values["@selected_accelerators"][1])
4741 self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
4742
4743 proc = Process("TEST")
4744 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4745 proc.options.accelerators = ["test2"]
4746 p = TestMakePSet()
4747 proc.fillProcessDesc(p)
4748 self.assertEqual(["test2"], p.values["@selected_accelerators"][1])
4749 self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4750
4751 proc = Process("TEST")
4752 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4753 proc.options.accelerators = ["test*"]
4754 proc.fillProcessDesc(p)
4755 self.assertEqual(["test1", "test2"], p.values["@selected_accelerators"][1])
4756 self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4757
4758 proc = Process("TEST")
4759 proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"])
4760 proc.options.accelerators = ["test2"]
4761 p = TestMakePSet()
4762 proc.fillProcessDesc(p)
4763 self.assertEqual([], p.values["@selected_accelerators"][1])
4764 self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
4765
4766 proc = Process("TEST")
4767 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4768 proc.options.accelerators = ["cpu*"]
4769 p = TestMakePSet()
4770 proc.fillProcessDesc(p)
4771 self.assertEqual(["cpu"], p.values["@selected_accelerators"][1])
4772 self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4773
4774 proc = Process("TEST")
4775 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4776 proc.options.accelerators = ["test3"]
4777 p = TestMakePSet()
4778 self.assertRaises(ValueError, proc.fillProcessDesc, p)
4779
4780 proc = Process("TEST")
4781 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4782 proc.options.accelerators = ["*", "test1"]
4783 p = TestMakePSet()
4784 self.assertRaises(ValueError, proc.fillProcessDesc, p)
4785
4786 proc = Process("TEST")
4787 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4788 proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2()
4789 p = TestMakePSet()
4790 proc.fillProcessDesc(p)
4791 self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@selected_accelerators"][1])
4792 self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4793
4794 proc = Process("TEST")
4795 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4796 proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2()
4797 proc.options.accelerators = ["*test3", "c*"]
4798 p = TestMakePSet()
4799 proc.fillProcessDesc(p)
4800 self.assertEqual(["anothertest3", "cpu"], p.values["@selected_accelerators"][1])
4801 self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4802
4803 proc = Process("TEST")
4804 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4805 proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4806 a = int32(1),
4807 b = PSet(c = int32(2))),
4808 test1 = EDProducer("Bar",
4809 aa = int32(11),
4810 bb = PSet(cc = int32(12))))
4811 proc.p = Path(proc.sp)
4812 p = TestMakePSet()
4813 proc.fillProcessDesc(p)
4814 self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
4815
4816 proc = Process("TEST")
4817 proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"])
4818 proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4819 a = int32(1),
4820 b = PSet(c = int32(2))),
4821 test1 = EDProducer("Bar",
4822 aa = int32(11),
4823 bb = PSet(cc = int32(12))))
4824 proc.p = Path(proc.sp)
4825 p = TestMakePSet()
4826 proc.fillProcessDesc(p)
4827 self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
4828
4829 proc = Process("TEST")
4830 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4831 proc.options.accelerators = ["test1"]
4832 proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4833 a = int32(1),
4834 b = PSet(c = int32(2))),
4835 test1 = EDProducer("Bar",
4836 aa = int32(11),
4837 bb = PSet(cc = int32(12))))
4838 proc.p = Path(proc.sp)
4839 p = TestMakePSet()
4840 proc.fillProcessDesc(p)
4841 self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
4842
4843 proc = Process("TEST")
4844 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4845 proc.options.accelerators = ["test*"]
4846 proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4847 a = int32(1),
4848 b = PSet(c = int32(2))),
4849 test1 = EDProducer("Bar",
4850 aa = int32(11),
4851 bb = PSet(cc = int32(12))))
4852 proc.p = Path(proc.sp)
4853 p = TestMakePSet()
4854 proc.fillProcessDesc(p)
4855 self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
4856
4857 proc = Process("TEST")
4858 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4859 proc.options.accelerators = ["anothertest3"]
4860 proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4861 a = int32(1),
4862 b = PSet(c = int32(2))),
4863 test1 = EDProducer("Bar",
4864 aa = int32(11),
4865 bb = PSet(cc = int32(12))))
4866 proc.p = Path(proc.sp)
4867 p = TestMakePSet()
4868 self.assertRaises(RuntimeError, proc.fillProcessDesc, p)
4869
4870 proc = Process("TEST")
4871 proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4872 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4873 proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2()
4874 proc.normalProducer = EDProducer("FooProducer")
4875 proc.testProducer = EDProducer("BarProducer@test")
4876 proc.test2Producer = EDProducer("BarProducer@test", test=untracked.PSet(backend=untracked.string("test2_backend")))
4877 proc.testAnalyzer = EDAnalyzer("Analyzer@test")
4878 proc.testFilter = EDAnalyzer("Filter@test")
4879 proc.testESProducer = ESProducer("ESProducer@test")
4880 proc.testESSource = ESSource("ESSource@test")
4881 proc.p = Path(proc.normalProducer+proc.testProducer+proc.test2Producer+proc.testAnalyzer+proc.testFilter)
4882 p = TestMakePSet()
4883 proc.fillProcessDesc(p)
4884 self.assertEqual("TestModuleTypeResolver", p.values["@module_type_resolver"][1])
4885 self.assertEqual("FooProducer", p.values["normalProducer"][1].values["@module_type"][1])
4886 self.assertEqual(len(list(filter(lambda x: not "@" in x, p.values["normalProducer"][1].values.keys()))), 0)
4887 self.assertEqual("BarProducer@test", p.values["testProducer"][1].values["@module_type"][1])
4888 self.assertEqual(False, p.values["testProducer"][1].values["test"][0])
4889 self.assertEqual(False, p.values["testProducer"][1].values["test"][1].values["backend"][0])
4890 self.assertEqual("test1_backend", p.values["testProducer"][1].values["test"][1].values["backend"][1])
4891 self.assertEqual("BarProducer@test", p.values["test2Producer"][1].values["@module_type"][1])
4892 self.assertEqual("test2_backend", p.values["test2Producer"][1].values["test"][1].values["backend"][1])
4893 self.assertEqual("Analyzer@test", p.values["testAnalyzer"][1].values["@module_type"][1])
4894 self.assertEqual("test1_backend", p.values["testAnalyzer"][1].values["test"][1].values["backend"][1])
4895 self.assertEqual("Filter@test", p.values["testFilter"][1].values["@module_type"][1])
4896 self.assertEqual("test1_backend", p.values["testFilter"][1].values["test"][1].values["backend"][1])
4897 self.assertEqual("ESProducer@test", p.values["ESProducer@test@testESProducer"][1].values["@module_type"][1])
4898 self.assertEqual("test1_backend", p.values["ESProducer@test@testESProducer"][1].values["test"][1].values["backend"][1])
4899 self.assertEqual("ESSource@test", p.values["ESSource@test@testESSource"][1].values["@module_type"][1])
4900 self.assertEqual("test1_backend", p.values["ESSource@test@testESSource"][1].values["test"][1].values["backend"][1])
4901
4902
4903 proc = Process("Test")
4904 proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4905 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4906 proc.options.accelerators = []
4907 self.assertRaises(EDMException, proc.fillProcessDesc, p)
4908
4909 proc = Process("Test")
4910 proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4911 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4912 proc.options.accelerators = ["test1"]
4913 proc.test2Producer = EDProducer("BarProducer@test", test=untracked.PSet(backend=untracked.string("test2_backend")))
4914 proc.p = Path(proc.test2Producer)
4915 self.assertRaises(EDMException, proc.fillProcessDesc, p)
4916
4917
4918 proc = Process("TEST")
4919 proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4920 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4921 proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2(
4922 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4923 p = TestMakePSet()
4924 self.assertRaises(RuntimeError, proc.fillProcessDesc, p)
4925
4926 import pickle
4927 proc = Process("TEST")
4928 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4929 proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4930 a = int32(1),
4931 b = PSet(c = int32(2))),
4932 test1 = EDProducer("Bar",
4933 aa = int32(11),
4934 bb = PSet(cc = int32(12))))
4935 proc.p = Path(proc.sp)
4936 pkl = pickle.dumps(proc)
4937 unpkl = pickle.loads(pkl)
4938 p = TestMakePSet()
4939 unpkl.fillProcessDesc(p)
4940 self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
4941 self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4942 unpkl = pickle.loads(pkl)
4943 unpkl.ProcessAcceleratorTest.setEnabled(["test1"])
4944 p = TestMakePSet()
4945 unpkl.fillProcessDesc(p)
4946 self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
4947 self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
4948
4949 unittest.main()