File indexing completed on 2023-01-11 16:27:23
0001
0002
0003
0004 from __future__ import print_function
0005 from __future__ import absolute_import
0006 import os
0007 from .Options import Options
0008 options = Options()
0009
0010
0011
0012 import sys
0013 from .Mixins import PrintOptions,_ParameterTypeBase,_SimpleParameterTypeBase, _Parameterizable, _ConfigureComponent, _TypedParameterizable, _Labelable, _Unlabelable, _ValidatingListBase, _modifyParametersFromDict
0014 from .Mixins import *
0015 from .Types import *
0016 from .Modules import *
0017 from .Modules import _Module
0018 from .SequenceTypes import *
0019 from .SequenceTypes import _ModuleSequenceType, _Sequenceable
0020 from .SequenceVisitors import PathValidator, EndPathValidator, FinalPathValidator, ScheduleTaskValidator, NodeVisitor, CompositeVisitor, ModuleNamesFromGlobalsVisitor
0021 from .MessageLogger import MessageLogger
0022 from . import DictTypes
0023
0024 from .ExceptionHandling import *
0025
0026
0027 if sys.getrecursionlimit()<5000:
0028 sys.setrecursionlimit(5000)
0029
0030 class edm(object):
0031 class errors(object):
0032
0033 Configuration = "{Configuration}"
0034 UnavailableAccelerator = "{UnavailableAccelerator}"
0035
0036 class EDMException(Exception):
0037 def __init__(self, error, message):
0038 super().__init__(error+"\n"+message)
0039
0040
0041 def checkImportPermission(minLevel = 2, allowedPatterns = []):
0042 """
0043 Raise an exception if called by special config files. This checks
0044 the call or import stack for the importing file. An exception is raised if
0045 the importing module is not in allowedPatterns and if it is called too deeply:
0046 minLevel = 2: inclusion by top lvel cfg only
0047 minLevel = 1: No inclusion allowed
0048 allowedPatterns = ['Module1','Module2/SubModule1'] allows import
0049 by any module in Module1 or Submodule1
0050 """
0051
0052 import inspect
0053 import os
0054
0055 ignorePatterns = ['FWCore/ParameterSet/Config.py', 'FWCore/ParameterSet/python/Config.py','<string>','<frozen ']
0056 CMSSWPath = [os.environ['CMSSW_BASE'],os.environ['CMSSW_RELEASE_BASE']]
0057
0058
0059 trueStack = []
0060 for item in inspect.stack():
0061 inPath = False
0062 ignore = False
0063
0064 for pattern in CMSSWPath:
0065 if item[1].find(pattern) != -1:
0066 inPath = True
0067 break
0068 if item[1].find('/') == -1:
0069 inPath = True
0070
0071 for pattern in ignorePatterns:
0072 if item[1].find(pattern) != -1:
0073 ignore = True
0074 break
0075
0076 if inPath and not ignore:
0077 trueStack.append(item[1])
0078
0079 importedFile = trueStack[0]
0080 importedBy = ''
0081 if len(trueStack) > 1:
0082 importedBy = trueStack[1]
0083
0084 for pattern in allowedPatterns:
0085 if importedBy.find(pattern) > -1:
0086 return True
0087
0088 if len(trueStack) <= minLevel:
0089 return True
0090
0091 raise ImportError("Inclusion of %s is allowed only by cfg or specified cfi files."
0092 % importedFile)
0093
0094 def findProcess(module):
0095 """Look inside the module and find the Processes it contains"""
0096 class Temp(object):
0097 pass
0098 process = None
0099 if isinstance(module,dict):
0100 if 'process' in module:
0101 p = module['process']
0102 module = Temp()
0103 module.process = p
0104 if hasattr(module,'process'):
0105 if isinstance(module.process,Process):
0106 process = module.process
0107 else:
0108 raise RuntimeError("The attribute named 'process' does not inherit from the Process class")
0109 else:
0110 raise RuntimeError("no 'process' attribute found in the module, please add one")
0111 return process
0112
0113 class Process(object):
0114 """Root class for a CMS configuration process"""
0115 _firstProcess = True
0116 def __init__(self,name,*Mods):
0117 """The argument 'name' will be the name applied to this Process
0118 Can optionally pass as additional arguments cms.Modifier instances
0119 that will be used to modify the Process as it is built
0120 """
0121 self.__dict__['_Process__name'] = name
0122 if not name.isalnum():
0123 raise RuntimeError("Error: The process name is an empty string or contains non-alphanumeric characters")
0124 self.__dict__['_Process__filters'] = {}
0125 self.__dict__['_Process__producers'] = {}
0126 self.__dict__['_Process__switchproducers'] = {}
0127 self.__dict__['_Process__source'] = None
0128 self.__dict__['_Process__looper'] = None
0129 self.__dict__['_Process__subProcesses'] = []
0130 self.__dict__['_Process__schedule'] = None
0131 self.__dict__['_Process__analyzers'] = {}
0132 self.__dict__['_Process__outputmodules'] = {}
0133 self.__dict__['_Process__paths'] = DictTypes.SortedKeysDict()
0134 self.__dict__['_Process__endpaths'] = DictTypes.SortedKeysDict()
0135 self.__dict__['_Process__finalpaths'] = DictTypes.SortedKeysDict()
0136 self.__dict__['_Process__sequences'] = {}
0137 self.__dict__['_Process__tasks'] = {}
0138 self.__dict__['_Process__conditionaltasks'] = {}
0139 self.__dict__['_Process__services'] = {}
0140 self.__dict__['_Process__essources'] = {}
0141 self.__dict__['_Process__esproducers'] = {}
0142 self.__dict__['_Process__esprefers'] = {}
0143 self.__dict__['_Process__aliases'] = {}
0144 self.__dict__['_Process__psets']={}
0145 self.__dict__['_Process__vpsets']={}
0146 self.__dict__['_cloneToObjectDict'] = {}
0147
0148 self.__dict__['_Process__InExtendCall'] = False
0149 self.__dict__['_Process__partialschedules'] = {}
0150 self.__isStrict = False
0151 self.__dict__['_Process__modifiers'] = Mods
0152 self.__dict__['_Process__accelerators'] = {}
0153 self.options = Process.defaultOptions_()
0154 self.maxEvents = Process.defaultMaxEvents_()
0155 self.maxLuminosityBlocks = Process.defaultMaxLuminosityBlocks_()
0156
0157
0158
0159
0160 self.MessageLogger = MessageLogger
0161 if Process._firstProcess:
0162 Process._firstProcess = False
0163 else:
0164 if len(Mods) > 0:
0165 for m in self.__modifiers:
0166 if not m._isChosen():
0167 raise RuntimeError("The Process {} tried to redefine which Modifiers to use after another Process was already started".format(name))
0168 for m in self.__modifiers:
0169 m._setChosen()
0170
0171 def setStrict(self, value):
0172 self.__isStrict = value
0173 _Module.__isStrict__ = True
0174
0175
0176 def producerNames(self):
0177 """Returns a string containing all the EDProducer labels separated by a blank"""
0178 return ' '.join(self.producers_().keys())
0179 def switchProducerNames(self):
0180 """Returns a string containing all the SwitchProducer labels separated by a blank"""
0181 return ' '.join(self.switchProducers_().keys())
0182 def analyzerNames(self):
0183 """Returns a string containing all the EDAnalyzer labels separated by a blank"""
0184 return ' '.join(self.analyzers_().keys())
0185 def filterNames(self):
0186 """Returns a string containing all the EDFilter labels separated by a blank"""
0187 return ' '.join(self.filters_().keys())
0188 def pathNames(self):
0189 """Returns a string containing all the Path names separated by a blank"""
0190 return ' '.join(self.paths_().keys())
0191
0192 def __setstate__(self, pkldict):
0193 """
0194 Unpickling hook.
0195
0196 Since cloneToObjectDict stores a hash of objects by their
0197 id() it needs to be updated when unpickling to use the
0198 new object id values instantiated during the unpickle.
0199
0200 """
0201 self.__dict__.update(pkldict)
0202 tmpDict = {}
0203 for value in self._cloneToObjectDict.values():
0204 tmpDict[id(value)] = value
0205 self.__dict__['_cloneToObjectDict'] = tmpDict
0206
0207
0208
0209 def filters_(self):
0210 """returns a dict of the filters that have been added to the Process"""
0211 return DictTypes.FixedKeysDict(self.__filters)
0212 filters = property(filters_, doc="dictionary containing the filters for the process")
0213 def name_(self):
0214 return self.__name
0215 def setName_(self,name):
0216 if not name.isalnum():
0217 raise RuntimeError("Error: The process name is an empty string or contains non-alphanumeric characters")
0218 self.__dict__['_Process__name'] = name
0219 process = property(name_,setName_, doc="name of the process")
0220 def producers_(self):
0221 """returns a dict of the producers that have been added to the Process"""
0222 return DictTypes.FixedKeysDict(self.__producers)
0223 producers = property(producers_,doc="dictionary containing the producers for the process")
0224 def switchProducers_(self):
0225 """returns a dict of the SwitchProducers that have been added to the Process"""
0226 return DictTypes.FixedKeysDict(self.__switchproducers)
0227 switchProducers = property(switchProducers_,doc="dictionary containing the SwitchProducers for the process")
0228 def source_(self):
0229 """returns the source that has been added to the Process or None if none have been added"""
0230 return self.__source
0231 def setSource_(self,src):
0232 self._placeSource('source',src)
0233 source = property(source_,setSource_,doc='the main source or None if not set')
0234 def looper_(self):
0235 """returns the looper that has been added to the Process or None if none have been added"""
0236 return self.__looper
0237 def setLooper_(self,lpr):
0238 self._placeLooper('looper',lpr)
0239 looper = property(looper_,setLooper_,doc='the main looper or None if not set')
0240 @staticmethod
0241 def defaultOptions_():
0242 return untracked.PSet(numberOfThreads = untracked.uint32(1),
0243 numberOfStreams = untracked.uint32(0),
0244 numberOfConcurrentRuns = untracked.uint32(1),
0245 numberOfConcurrentLuminosityBlocks = untracked.uint32(0),
0246 eventSetup = untracked.PSet(
0247 numberOfConcurrentIOVs = untracked.uint32(0),
0248 forceNumberOfConcurrentIOVs = untracked.PSet(
0249 allowAnyLabel_ = required.untracked.uint32
0250 )
0251 ),
0252 accelerators = untracked.vstring('*'),
0253 wantSummary = untracked.bool(False),
0254 fileMode = untracked.string('FULLMERGE'),
0255 forceEventSetupCacheClearOnNewRun = untracked.bool(False),
0256 throwIfIllegalParameter = untracked.bool(True),
0257 printDependencies = untracked.bool(False),
0258 deleteNonConsumedUnscheduledModules = untracked.bool(True),
0259 sizeOfStackForThreadsInKB = optional.untracked.uint32,
0260 Rethrow = untracked.vstring(),
0261 SkipEvent = untracked.vstring(),
0262 FailPath = untracked.vstring(),
0263 IgnoreCompletely = untracked.vstring(),
0264 canDeleteEarly = untracked.vstring(),
0265 holdsReferencesToDeleteEarly = untracked.VPSet(),
0266 modulesToIgnoreForDeleteEarly = untracked.vstring(),
0267 dumpOptions = untracked.bool(False),
0268 allowUnscheduled = obsolete.untracked.bool,
0269 emptyRunLumiMode = obsolete.untracked.string,
0270 makeTriggerResults = obsolete.untracked.bool,
0271 )
0272 def __updateOptions(self,opt):
0273 newOpts = self.defaultOptions_()
0274 if isinstance(opt,dict):
0275 for k,v in opt.items():
0276 setattr(newOpts,k,v)
0277 else:
0278 for p in opt.parameters_():
0279 setattr(newOpts, p, getattr(opt,p))
0280 return newOpts
0281 @staticmethod
0282 def defaultMaxEvents_():
0283 return untracked.PSet(input=optional.untracked.int32,
0284 output=optional.untracked.allowed(int32,PSet))
0285 def __updateMaxEvents(self,ps):
0286 newMax = self.defaultMaxEvents_()
0287 if isinstance(ps,dict):
0288 for k,v in ps.items():
0289 setattr(newMax,k,v)
0290 else:
0291 for p in ps.parameters_():
0292 setattr(newMax, p, getattr(ps,p))
0293 return newMax
0294 @staticmethod
0295 def defaultMaxLuminosityBlocks_():
0296 return untracked.PSet(input=untracked.int32(-1))
0297 def subProcesses_(self):
0298 """returns a list of the subProcesses that have been added to the Process"""
0299 return self.__subProcesses
0300 subProcesses = property(subProcesses_,doc='the SubProcesses that have been added to the Process')
0301 def analyzers_(self):
0302 """returns a dict of the analyzers that have been added to the Process"""
0303 return DictTypes.FixedKeysDict(self.__analyzers)
0304 analyzers = property(analyzers_,doc="dictionary containing the analyzers for the process")
0305 def outputModules_(self):
0306 """returns a dict of the output modules that have been added to the Process"""
0307 return DictTypes.FixedKeysDict(self.__outputmodules)
0308 outputModules = property(outputModules_,doc="dictionary containing the output_modules for the process")
0309 def paths_(self):
0310 """returns a dict of the paths that have been added to the Process"""
0311 return DictTypes.SortedAndFixedKeysDict(self.__paths)
0312 paths = property(paths_,doc="dictionary containing the paths for the process")
0313 def endpaths_(self):
0314 """returns a dict of the endpaths that have been added to the Process"""
0315 return DictTypes.SortedAndFixedKeysDict(self.__endpaths)
0316 endpaths = property(endpaths_,doc="dictionary containing the endpaths for the process")
0317 def finalpaths_(self):
0318 """returns a dict of the finalpaths that have been added to the Process"""
0319 return DictTypes.SortedAndFixedKeysDict(self.__finalpaths)
0320 finalpaths = property(finalpaths_,doc="dictionary containing the finalpaths for the process")
0321 def sequences_(self):
0322 """returns a dict of the sequences that have been added to the Process"""
0323 return DictTypes.FixedKeysDict(self.__sequences)
0324 sequences = property(sequences_,doc="dictionary containing the sequences for the process")
0325 def tasks_(self):
0326 """returns a dict of the tasks that have been added to the Process"""
0327 return DictTypes.FixedKeysDict(self.__tasks)
0328 tasks = property(tasks_,doc="dictionary containing the tasks for the process")
0329 def conditionaltasks_(self):
0330 """returns a dict of the conditionaltasks that have been added to the Process"""
0331 return DictTypes.FixedKeysDict(self.__conditionaltasks)
0332 conditionaltasks = property(conditionaltasks_,doc="dictionary containing the conditionatasks for the process")
0333 def schedule_(self):
0334 """returns the schedule that has been added to the Process or None if none have been added"""
0335 return self.__schedule
0336 def setPartialSchedule_(self,sch,label):
0337 if label == "schedule":
0338 self.setSchedule_(sch)
0339 else:
0340 self._place(label, sch, self.__partialschedules)
0341 def setSchedule_(self,sch):
0342
0343 index = 0
0344 try:
0345 for p in sch:
0346 p.label_()
0347 index +=1
0348 except:
0349 raise RuntimeError("The path at index "+str(index)+" in the Schedule was not attached to the process.")
0350 self.__dict__['_Process__schedule'] = sch
0351 schedule = property(schedule_,setSchedule_,doc='the schedule or None if not set')
0352 def services_(self):
0353 """returns a dict of the services that have been added to the Process"""
0354 return DictTypes.FixedKeysDict(self.__services)
0355 services = property(services_,doc="dictionary containing the services for the process")
0356 def processAccelerators_(self):
0357 """returns a dict of the ProcessAccelerators that have been added to the Process"""
0358 return DictTypes.FixedKeysDict(self.__accelerators)
0359 processAccelerators = property(processAccelerators_,doc="dictionary containing the ProcessAccelerators for the process")
0360 def es_producers_(self):
0361 """returns a dict of the esproducers that have been added to the Process"""
0362 return DictTypes.FixedKeysDict(self.__esproducers)
0363 es_producers = property(es_producers_,doc="dictionary containing the es_producers for the process")
0364 def es_sources_(self):
0365 """returns a the es_sources that have been added to the Process"""
0366 return DictTypes.FixedKeysDict(self.__essources)
0367 es_sources = property(es_sources_,doc="dictionary containing the es_sources for the process")
0368 def es_prefers_(self):
0369 """returns a dict of the es_prefers that have been added to the Process"""
0370 return DictTypes.FixedKeysDict(self.__esprefers)
0371 es_prefers = property(es_prefers_,doc="dictionary containing the es_prefers for the process")
0372 def aliases_(self):
0373 """returns a dict of the aliases that have been added to the Process"""
0374 return DictTypes.FixedKeysDict(self.__aliases)
0375 aliases = property(aliases_,doc="dictionary containing the aliases for the process")
0376 def psets_(self):
0377 """returns a dict of the PSets that have been added to the Process"""
0378 return DictTypes.FixedKeysDict(self.__psets)
0379 psets = property(psets_,doc="dictionary containing the PSets for the process")
0380 def vpsets_(self):
0381 """returns a dict of the VPSets that have been added to the Process"""
0382 return DictTypes.FixedKeysDict(self.__vpsets)
0383 vpsets = property(vpsets_,doc="dictionary containing the PSets for the process")
0384
0385 def isUsingModifier(self,mod):
0386 """returns True if the Modifier is in used by this Process"""
0387 if mod._isChosen():
0388 for m in self.__modifiers:
0389 if m._isOrContains(mod):
0390 return True
0391 return False
0392
0393 def __setObjectLabel(self, object, newLabel) :
0394 if not object.hasLabel_() :
0395 object.setLabel(newLabel)
0396 return
0397 if newLabel == object.label_() :
0398 return
0399 if newLabel is None :
0400 object.setLabel(None)
0401 return
0402 if (hasattr(self, object.label_()) and id(getattr(self, object.label_())) == id(object)) :
0403 msg100 = "Attempting to change the label of an attribute of the Process\n"
0404 msg101 = "Old label = "+object.label_()+" New label = "+newLabel+"\n"
0405 msg102 = "Type = "+str(type(object))+"\n"
0406 msg103 = "Some possible solutions:\n"
0407 msg104 = " 1. Clone modules instead of using simple assignment. Cloning is\n"
0408 msg105 = " also preferred for other types when possible.\n"
0409 msg106 = " 2. Declare new names starting with an underscore if they are\n"
0410 msg107 = " for temporaries you do not want propagated into the Process. The\n"
0411 msg108 = " underscore tells \"from x import *\" and process.load not to import\n"
0412 msg109 = " the name.\n"
0413 msg110 = " 3. Reorganize so the assigment is not necessary. Giving a second\n"
0414 msg111 = " name to the same object usually causes confusion and problems.\n"
0415 msg112 = " 4. Compose Sequences: newName = cms.Sequence(oldName)\n"
0416 raise ValueError(msg100+msg101+msg102+msg103+msg104+msg105+msg106+msg107+msg108+msg109+msg110+msg111+msg112)
0417 object.setLabel(None)
0418 object.setLabel(newLabel)
0419
0420 def __setattr__(self,name,value):
0421
0422 if not name.replace('_','').isalnum():
0423 raise ValueError('The label '+name+' contains forbiden characters')
0424
0425 if name == 'options':
0426 value = self.__updateOptions(value)
0427 if name == 'maxEvents':
0428 value = self.__updateMaxEvents(value)
0429
0430
0431 if name.startswith('_Process__'):
0432 self.__dict__[name]=value
0433 return
0434 if not isinstance(value,_ConfigureComponent):
0435 raise TypeError("can only assign labels to an object that inherits from '_ConfigureComponent'\n"
0436 +"an instance of "+str(type(value))+" will not work - requested label is "+name)
0437 if not isinstance(value,_Labelable) and not isinstance(value,Source) and not isinstance(value,Looper) and not isinstance(value,Schedule):
0438 if name == value.type_():
0439 if hasattr(self,name) and (getattr(self,name)!=value):
0440 self._replaceInTasks(name, value)
0441 self._replaceInConditionalTasks(name, value)
0442
0443 self.add_(value)
0444 return
0445 else:
0446 raise TypeError("an instance of "+str(type(value))+" can not be assigned the label '"+name+"'.\n"+
0447 "Please either use the label '"+value.type_()+" or use the 'add_' method instead.")
0448
0449 if self.__isStrict:
0450 newValue =value.copy()
0451 try:
0452 newValue._filename = value._filename
0453 except:
0454 pass
0455 value.setIsFrozen()
0456 else:
0457 newValue =value
0458 if not self._okToPlace(name, value, self.__dict__):
0459 newFile='top level config'
0460 if hasattr(value,'_filename'):
0461 newFile = value._filename
0462 oldFile='top level config'
0463 oldValue = getattr(self,name)
0464 if hasattr(oldValue,'_filename'):
0465 oldFile = oldValue._filename
0466 msg = "Trying to override definition of process."+name
0467 msg += "\n new object defined in: "+newFile
0468 msg += "\n existing object defined in: "+oldFile
0469 raise ValueError(msg)
0470
0471 if hasattr(self,name) and not (getattr(self,name)==newValue):
0472
0473
0474
0475 if newValue._isTaskComponent():
0476 if not self.__InExtendCall:
0477 self._replaceInTasks(name, newValue)
0478 self._replaceInConditionalTasks(name, newValue)
0479 self._replaceInSchedule(name, newValue)
0480 else:
0481 if not isinstance(newValue, Task):
0482
0483 newFile='top level config'
0484 if hasattr(value,'_filename'):
0485 newFile = value._filename
0486 oldFile='top level config'
0487 oldValue = getattr(self,name)
0488 if hasattr(oldValue,'_filename'):
0489 oldFile = oldValue._filename
0490 msg1 = "Trying to override definition of "+name+" while it is used by the task "
0491 msg2 = "\n new object defined in: "+newFile
0492 msg2 += "\n existing object defined in: "+oldFile
0493 s = self.__findFirstUsingModule(self.tasks,oldValue)
0494 if s is not None:
0495 raise ValueError(msg1+s.label_()+msg2)
0496
0497 if isinstance(newValue, _Sequenceable) or newValue._isTaskComponent() or isinstance(newValue, ConditionalTask):
0498 if not self.__InExtendCall:
0499 if isinstance(newValue, ConditionalTask):
0500 self._replaceInConditionalTasks(name, newValue)
0501 self._replaceInSequences(name, newValue)
0502 else:
0503
0504 newFile='top level config'
0505 if hasattr(value,'_filename'):
0506 newFile = value._filename
0507 oldFile='top level config'
0508 oldValue = getattr(self,name)
0509 if hasattr(oldValue,'_filename'):
0510 oldFile = oldValue._filename
0511 msg1 = "Trying to override definition of "+name+" while it is used by the "
0512 msg2 = "\n new object defined in: "+newFile
0513 msg2 += "\n existing object defined in: "+oldFile
0514 s = self.__findFirstUsingModule(self.sequences,oldValue)
0515 if s is not None:
0516 raise ValueError(msg1+"sequence "+s.label_()+msg2)
0517 s = self.__findFirstUsingModule(self.paths,oldValue)
0518 if s is not None:
0519 raise ValueError(msg1+"path "+s.label_()+msg2)
0520 s = self.__findFirstUsingModule(self.endpaths,oldValue)
0521 if s is not None:
0522 raise ValueError(msg1+"endpath "+s.label_()+msg2)
0523 s = self.__findFirstUsingModule(self.finalpaths,oldValue)
0524 if s is not None:
0525 raise ValueError(msg1+"finalpath "+s.label_()+msg2)
0526
0527
0528 if isinstance(newValue, EDAlias):
0529 oldValue = getattr(self, name)
0530
0531 newFile='top level config'
0532 if hasattr(value,'_filename'):
0533 newFile = value._filename
0534 oldFile='top level config'
0535 if hasattr(oldValue,'_filename'):
0536 oldFile = oldValue._filename
0537 msg1 = "Trying to override definition of "+name+" with an EDAlias while it is used by the "
0538 msg2 = "\n new object defined in: "+newFile
0539 msg2 += "\n existing object defined in: "+oldFile
0540 s = self.__findFirstUsingModule(self.tasks,oldValue)
0541 if s is not None:
0542 raise ValueError(msg1+"task "+s.label_()+msg2)
0543 s = self.__findFirstUsingModule(self.sequences,oldValue)
0544 if s is not None:
0545 raise ValueError(msg1+"sequence "+s.label_()+msg2)
0546 s = self.__findFirstUsingModule(self.paths,oldValue)
0547 if s is not None:
0548 raise ValueError(msg1+"path "+s.label_()+msg2)
0549 s = self.__findFirstUsingModule(self.endpaths,oldValue)
0550 if s is not None:
0551 raise ValueError(msg1+"endpath "+s.label_()+msg2)
0552 s = self.__findFirstUsingModule(self.finalpaths,oldValue)
0553 if s is not None:
0554 raise ValueError(msg1+"finalpath "+s.label_()+msg2)
0555
0556 if not self.__InExtendCall and (Schedule._itemIsValid(newValue) or isinstance(newValue, Task)):
0557 self._replaceInScheduleDirectly(name, newValue)
0558
0559 self._delattrFromSetattr(name)
0560 self.__dict__[name]=newValue
0561 if isinstance(newValue,_Labelable):
0562 self.__setObjectLabel(newValue, name)
0563 self._cloneToObjectDict[id(value)] = newValue
0564 self._cloneToObjectDict[id(newValue)] = newValue
0565
0566 newValue._place(name,self)
0567 def __findFirstUsingModule(self, seqsOrTasks, mod):
0568 """Given a container of sequences or tasks, find the first sequence or task
0569 containing mod and return it. If none is found, return None"""
0570 from FWCore.ParameterSet.SequenceTypes import ModuleNodeVisitor
0571 l = list()
0572 for seqOrTask in seqsOrTasks.values():
0573 l[:] = []
0574 v = ModuleNodeVisitor(l)
0575 seqOrTask.visit(v)
0576 if mod in l:
0577 return seqOrTask
0578 return None
0579
0580 def _delHelper(self,name):
0581 if not hasattr(self,name):
0582 raise KeyError('process does not know about '+name)
0583 elif name.startswith('_Process__'):
0584 raise ValueError('this attribute cannot be deleted')
0585
0586
0587 dicts = [item for item in self.__dict__.values() if (isinstance(item, dict) or isinstance(item, DictTypes.SortedKeysDict))]
0588 for reg in dicts:
0589 if name in reg: del reg[name]
0590
0591 obj = getattr(self,name)
0592 if isinstance(obj,_Labelable):
0593 obj.setLabel(None)
0594 if isinstance(obj,Service):
0595 obj._inProcess = False
0596
0597 def __delattr__(self,name):
0598 self._delHelper(name)
0599 obj = getattr(self,name)
0600 if not obj is None:
0601 if not isinstance(obj, Sequence) and not isinstance(obj, Task) and not isinstance(obj,ConditionalTask):
0602
0603
0604
0605
0606
0607
0608
0609
0610
0611 if obj._isTaskComponent():
0612 self._replaceInTasks(name, None)
0613 self._replaceInConditionalTasks(name, None)
0614 self._replaceInSchedule(name, None)
0615 if isinstance(obj, _Sequenceable) or obj._isTaskComponent():
0616 self._replaceInSequences(name, None)
0617 if Schedule._itemIsValid(obj) or isinstance(obj, Task):
0618 self._replaceInScheduleDirectly(name, None)
0619
0620 try:
0621 del self.__dict__[name]
0622 except:
0623 pass
0624
0625 def _delattrFromSetattr(self,name):
0626 """Similar to __delattr__ but we need different behavior when called from __setattr__"""
0627 self._delHelper(name)
0628
0629 try:
0630 del self.__dict__[name]
0631 except:
0632 pass
0633
0634 def add_(self,value):
0635 """Allows addition of components that do not have to have a label, e.g. Services"""
0636 if not isinstance(value,_ConfigureComponent):
0637 raise TypeError
0638 if not isinstance(value,_Unlabelable):
0639 raise TypeError
0640
0641 if self.__isStrict:
0642 newValue =value.copy()
0643 value.setIsFrozen()
0644 else:
0645 newValue =value
0646 newValue._place('',self)
0647
0648 def _okToPlace(self, name, mod, d):
0649 if not self.__InExtendCall:
0650
0651 return True
0652 elif not self.__isStrict:
0653 return True
0654 elif name in d:
0655
0656
0657
0658
0659 if mod._isModified:
0660 if d[name]._isModified:
0661 return False
0662 else:
0663 return True
0664 else:
0665 return True
0666 else:
0667 return True
0668
0669 def _place(self, name, mod, d):
0670 if self._okToPlace(name, mod, d):
0671 if self.__isStrict and isinstance(mod, _ModuleSequenceType):
0672 d[name] = mod._postProcessFixup(self._cloneToObjectDict)
0673 else:
0674 d[name] = mod
0675 if isinstance(mod,_Labelable):
0676 self.__setObjectLabel(mod, name)
0677 def _placeOutputModule(self,name,mod):
0678 self._place(name, mod, self.__outputmodules)
0679 def _placeProducer(self,name,mod):
0680 self._place(name, mod, self.__producers)
0681 def _placeSwitchProducer(self,name,mod):
0682 self._place(name, mod, self.__switchproducers)
0683 def _placeFilter(self,name,mod):
0684 self._place(name, mod, self.__filters)
0685 def _placeAnalyzer(self,name,mod):
0686 self._place(name, mod, self.__analyzers)
0687 def _placePath(self,name,mod):
0688 self._validateSequence(mod, name)
0689 try:
0690 self._place(name, mod, self.__paths)
0691 except ModuleCloneError as msg:
0692 context = format_outerframe(4)
0693 raise Exception("%sThe module %s in path %s is unknown to the process %s." %(context, msg, name, self._Process__name))
0694 def _placeEndPath(self,name,mod):
0695 self._validateSequence(mod, name)
0696 try:
0697 self._place(name, mod, self.__endpaths)
0698 except ModuleCloneError as msg:
0699 context = format_outerframe(4)
0700 raise Exception("%sThe module %s in endpath %s is unknown to the process %s." %(context, msg, name, self._Process__name))
0701 def _placeFinalPath(self,name,mod):
0702 self._validateSequence(mod, name)
0703 try:
0704 self._place(name, mod, self.__finalpaths)
0705 except ModuleCloneError as msg:
0706 context = format_outerframe(4)
0707 raise Exception("%sThe module %s in finalpath %s is unknown to the process %s." %(context, msg, name, self._Process__name))
0708 def _placeSequence(self,name,mod):
0709 self._validateSequence(mod, name)
0710 self._place(name, mod, self.__sequences)
0711 def _placeESProducer(self,name,mod):
0712 self._place(name, mod, self.__esproducers)
0713 def _placeESPrefer(self,name,mod):
0714 self._place(name, mod, self.__esprefers)
0715 def _placeESSource(self,name,mod):
0716 self._place(name, mod, self.__essources)
0717 def _placeTask(self,name,task):
0718 self._validateTask(task, name)
0719 self._place(name, task, self.__tasks)
0720 def _placeConditionalTask(self,name,task):
0721 self._validateConditionalTask(task, name)
0722 self._place(name, task, self.__conditionaltasks)
0723 def _placeAlias(self,name,mod):
0724 self._place(name, mod, self.__aliases)
0725 def _placePSet(self,name,mod):
0726 self._place(name, mod, self.__psets)
0727 def _placeVPSet(self,name,mod):
0728 self._place(name, mod, self.__vpsets)
0729 def _placeSource(self,name,mod):
0730 """Allow the source to be referenced by 'source' or by type name"""
0731 if name != 'source':
0732 raise ValueError("The label '"+name+"' can not be used for a Source. Only 'source' is allowed.")
0733 if self.__dict__['_Process__source'] is not None :
0734 del self.__dict__[self.__dict__['_Process__source'].type_()]
0735 self.__dict__['_Process__source'] = mod
0736 self.__dict__[mod.type_()] = mod
0737 def _placeLooper(self,name,mod):
0738 if name != 'looper':
0739 raise ValueError("The label '"+name+"' can not be used for a Looper. Only 'looper' is allowed.")
0740 self.__dict__['_Process__looper'] = mod
0741 self.__dict__[mod.type_()] = mod
0742 def _placeSubProcess(self,name,mod):
0743 self.__dict__['_Process__subProcess'] = mod
0744 self.__dict__[mod.type_()] = mod
0745 def addSubProcess(self,mod):
0746 self.__subProcesses.append(mod)
0747 def _placeService(self,typeName,mod):
0748 self._place(typeName, mod, self.__services)
0749 if typeName in self.__dict__:
0750 self.__dict__[typeName]._inProcess = False
0751 self.__dict__[typeName]=mod
0752 def _placeAccelerator(self,typeName,mod):
0753 self._place(typeName, mod, self.__accelerators)
0754 self.__dict__[typeName]=mod
0755 def load(self, moduleName):
0756 moduleName = moduleName.replace("/",".")
0757 module = __import__(moduleName)
0758 self.extend(sys.modules[moduleName])
0759 def extend(self,other,items=()):
0760 """Look in other and find types that we can use"""
0761
0762 self.__dict__['_Process__InExtendCall'] = True
0763
0764 seqs = dict()
0765 tasksToAttach = dict()
0766 mods = []
0767 for name in dir(other):
0768
0769 if name.startswith('_'):
0770 continue
0771 item = getattr(other,name)
0772 if name == "source" or name == "looper":
0773
0774 if item is not None:
0775 self.__setattr__(name,item)
0776 elif isinstance(item,_ModuleSequenceType):
0777 seqs[name]=item
0778 elif isinstance(item,Task) or isinstance(item, ConditionalTask):
0779 tasksToAttach[name] = item
0780 elif isinstance(item,_Labelable):
0781 self.__setattr__(name,item)
0782 if not item.hasLabel_() :
0783 item.setLabel(name)
0784 elif isinstance(item,Schedule):
0785 self.__setattr__(name,item)
0786 elif isinstance(item,_Unlabelable):
0787 self.add_(item)
0788 elif isinstance(item,ProcessModifier):
0789 mods.append(item)
0790 elif isinstance(item,ProcessFragment):
0791 self.extend(item)
0792
0793
0794 for name,seq in seqs.items():
0795 if id(seq) not in self._cloneToObjectDict:
0796 self.__setattr__(name,seq)
0797 else:
0798 newSeq = self._cloneToObjectDict[id(seq)]
0799 self.__dict__[name]=newSeq
0800 self.__setObjectLabel(newSeq, name)
0801
0802 newSeq._place(name,self)
0803
0804 for name, task in tasksToAttach.items():
0805 self.__setattr__(name, task)
0806
0807
0808 for item in mods:
0809 item.apply(self)
0810
0811 self.__dict__['_Process__InExtendCall'] = False
0812
0813 def _dumpConfigNamedList(self,items,typeName,options):
0814 returnValue = ''
0815 for name,item in items:
0816 returnValue +=options.indentation()+typeName+' '+name+' = '+item.dumpConfig(options)
0817 return returnValue
0818
0819 def _dumpConfigUnnamedList(self,items,typeName,options):
0820 returnValue = ''
0821 for name,item in items:
0822 returnValue +=options.indentation()+typeName+' = '+item.dumpConfig(options)
0823 return returnValue
0824
0825 def _dumpConfigOptionallyNamedList(self,items,typeName,options):
0826 returnValue = ''
0827 for name,item in items:
0828 if name == item.type_():
0829 name = ''
0830 returnValue +=options.indentation()+typeName+' '+name+' = '+item.dumpConfig(options)
0831 return returnValue
0832
0833 def dumpConfig(self, options=PrintOptions()):
0834 """return a string containing the equivalent process defined using the old configuration language"""
0835 config = "process "+self.__name+" = {\n"
0836 options.indent()
0837 if self.source_():
0838 config += options.indentation()+"source = "+self.source_().dumpConfig(options)
0839 if self.looper_():
0840 config += options.indentation()+"looper = "+self.looper_().dumpConfig(options)
0841
0842 config+=self._dumpConfigNamedList(self.subProcesses_(),
0843 'subProcess',
0844 options)
0845 config+=self._dumpConfigNamedList(self.producers_().items(),
0846 'module',
0847 options)
0848 config+=self._dumpConfigNamedList(self.switchProducers_().items(),
0849 'module',
0850 options)
0851 config+=self._dumpConfigNamedList(self.filters_().items(),
0852 'module',
0853 options)
0854 config+=self._dumpConfigNamedList(self.analyzers_().items(),
0855 'module',
0856 options)
0857 config+=self._dumpConfigNamedList(self.outputModules_().items(),
0858 'module',
0859 options)
0860 config+=self._dumpConfigNamedList(self.sequences_().items(),
0861 'sequence',
0862 options)
0863 config+=self._dumpConfigNamedList(self.paths_().items(),
0864 'path',
0865 options)
0866 config+=self._dumpConfigNamedList(self.endpaths_().items(),
0867 'endpath',
0868 options)
0869 config+=self._dumpConfigNamedList(self.finalpaths_().items(),
0870 'finalpath',
0871 options)
0872 config+=self._dumpConfigUnnamedList(self.services_().items(),
0873 'service',
0874 options)
0875 config+=self._dumpConfigNamedList(self.aliases_().items(),
0876 'alias',
0877 options)
0878 config+=self._dumpConfigOptionallyNamedList(
0879 self.es_producers_().items(),
0880 'es_module',
0881 options)
0882 config+=self._dumpConfigOptionallyNamedList(
0883 self.es_sources_().items(),
0884 'es_source',
0885 options)
0886 config += self._dumpConfigESPrefers(options)
0887 for name,item in self.psets.items():
0888 config +=options.indentation()+item.configTypeName()+' '+name+' = '+item.configValue(options)
0889 for name,item in self.vpsets.items():
0890 config +=options.indentation()+'VPSet '+name+' = '+item.configValue(options)
0891 if self.schedule:
0892 pathNames = [p.label_() for p in self.schedule]
0893 config +=options.indentation()+'schedule = {'+','.join(pathNames)+'}\n'
0894
0895
0896
0897
0898 config += "}\n"
0899 options.unindent()
0900 return config
0901
0902 def _dumpConfigESPrefers(self, options):
0903 result = ''
0904 for item in self.es_prefers_().values():
0905 result +=options.indentation()+'es_prefer '+item.targetLabel_()+' = '+item.dumpConfig(options)
0906 return result
0907
0908 def _dumpPythonSubProcesses(self, l, options):
0909 returnValue = ''
0910 for item in l:
0911 returnValue += item.dumpPython(options)+'\n\n'
0912 return returnValue
0913
0914 def _dumpPythonList(self, d, options):
0915 returnValue = ''
0916 if isinstance(d, DictTypes.SortedKeysDict):
0917 for name,item in d.items():
0918 returnValue +='process.'+name+' = '+item.dumpPython(options)+'\n\n'
0919 else:
0920 for name,item in sorted(d.items()):
0921 returnValue +='process.'+name+' = '+item.dumpPython(options)+'\n\n'
0922 return returnValue
0923
0924 def _splitPythonList(self, subfolder, d, options):
0925 parts = DictTypes.SortedKeysDict()
0926 for name, item in d.items() if isinstance(d, DictTypes.SortedKeysDict) else sorted(d.items()):
0927 code = ''
0928 dependencies = item.directDependencies()
0929 for module_subfolder, module in dependencies:
0930 module = module + '_cfi'
0931 if options.useSubdirectories and module_subfolder:
0932 module = module_subfolder + '.' + module
0933 if options.targetDirectory is not None:
0934 if options.useSubdirectories and subfolder:
0935 module = '..' + module
0936 else:
0937 module = '.' + module
0938 code += 'from ' + module + ' import *\n'
0939 if dependencies:
0940 code += '\n'
0941 code += name + ' = ' + item.dumpPython(options)
0942 parts[name] = subfolder, code
0943 return parts
0944
0945 def _validateSequence(self, sequence, label):
0946
0947 try:
0948 l = set()
0949 visitor = NodeNameVisitor(l)
0950 sequence.visit(visitor)
0951 except Exception as e:
0952 raise RuntimeError("An entry in sequence {} has no label\n Seen entries: {}\n Error: {}".format(label, l, e))
0953
0954 def _validateTask(self, task, label):
0955
0956 try:
0957 l = set()
0958 visitor = NodeNameVisitor(l)
0959 task.visit(visitor)
0960 except:
0961 raise RuntimeError("An entry in task " + label + ' has not been attached to the process')
0962 def _validateConditionalTask(self, task, label):
0963
0964 try:
0965 l = set()
0966 visitor = NodeNameVisitor(l)
0967 task.visit(visitor)
0968 except:
0969 raise RuntimeError("An entry in task " + label + ' has not been attached to the process')
0970
0971 def _itemsInDependencyOrder(self, processDictionaryOfItems):
0972
0973
0974
0975
0976 returnValue=DictTypes.SortedKeysDict()
0977
0978
0979
0980 dependencies = {}
0981 for label,item in processDictionaryOfItems.items():
0982 containedItems = []
0983 if isinstance(item, Task):
0984 v = TaskVisitor(containedItems)
0985 elif isinstance(item, ConditionalTask):
0986 v = ConditionalTaskVisitor(containedItems)
0987 else:
0988 v = SequenceVisitor(containedItems)
0989 try:
0990 item.visit(v)
0991 except RuntimeError:
0992 if isinstance(item, Task):
0993 raise RuntimeError("Failed in a Task visitor. Probably " \
0994 "a circular dependency discovered in Task with label " + label)
0995 elif isinstance(item, ConditionalTask):
0996 raise RuntimeError("Failed in a ConditionalTask visitor. Probably " \
0997 "a circular dependency discovered in ConditionalTask with label " + label)
0998 else:
0999 raise RuntimeError("Failed in a Sequence visitor. Probably a " \
1000 "circular dependency discovered in Sequence with label " + label)
1001 for containedItem in containedItems:
1002
1003
1004
1005
1006
1007 if containedItem.hasLabel_():
1008 testItem = processDictionaryOfItems.get(containedItem.label_())
1009 if testItem is None or containedItem != testItem:
1010 if isinstance(item, Task):
1011 raise RuntimeError("Task has a label, but using its label to get an attribute" \
1012 " from the process yields a different object or None\n"+
1013 "label = " + containedItem.label_())
1014 if isinstance(item, ConditionalTask):
1015 raise RuntimeError("ConditionalTask has a label, but using its label to get an attribute" \
1016 " from the process yields a different object or None\n"+
1017 "label = " + containedItem.label_())
1018 else:
1019 raise RuntimeError("Sequence has a label, but using its label to get an attribute" \
1020 " from the process yields a different object or None\n"+
1021 "label = " + containedItem.label_())
1022 dependencies[label]=[dep.label_() for dep in containedItems if dep.hasLabel_()]
1023
1024
1025 while dependencies:
1026 oldDeps = dict(dependencies)
1027 for label,deps in oldDeps.items():
1028 if len(deps)==0:
1029 returnValue[label]=processDictionaryOfItems[label]
1030
1031 del dependencies[label]
1032 for lb2,deps2 in dependencies.items():
1033 while deps2.count(label):
1034 deps2.remove(label)
1035 return returnValue
1036
1037 def _dumpPython(self, d, options):
1038 result = ''
1039 for name, value in sorted(d.items()):
1040 result += value.dumpPythonAs(name,options)+'\n'
1041 return result
1042
1043 def _splitPython(self, subfolder, d, options):
1044 result = {}
1045 for name, value in sorted(d.items()):
1046 result[name] = subfolder, value.dumpPythonAs(name, options) + '\n'
1047 return result
1048
1049 def dumpPython(self, options=PrintOptions()):
1050 """return a string containing the equivalent process defined using python"""
1051 specialImportRegistry._reset()
1052 header = "import FWCore.ParameterSet.Config as cms"
1053 result = "process = cms.Process(\""+self.__name+"\")\n\n"
1054 if self.source_():
1055 result += "process.source = "+self.source_().dumpPython(options)
1056 if self.looper_():
1057 result += "process.looper = "+self.looper_().dumpPython()
1058 result+=self._dumpPythonList(self.psets, options)
1059 result+=self._dumpPythonList(self.vpsets, options)
1060 result+=self._dumpPythonSubProcesses(self.subProcesses_(), options)
1061 result+=self._dumpPythonList(self.producers_(), options)
1062 result+=self._dumpPythonList(self.switchProducers_(), options)
1063 result+=self._dumpPythonList(self.filters_() , options)
1064 result+=self._dumpPythonList(self.analyzers_(), options)
1065 result+=self._dumpPythonList(self.outputModules_(), options)
1066 result+=self._dumpPythonList(self.services_(), options)
1067 result+=self._dumpPythonList(self.processAccelerators_(), options)
1068 result+=self._dumpPythonList(self.es_producers_(), options)
1069 result+=self._dumpPythonList(self.es_sources_(), options)
1070 result+=self._dumpPython(self.es_prefers_(), options)
1071 result+=self._dumpPythonList(self._itemsInDependencyOrder(self.tasks), options)
1072 result+=self._dumpPythonList(self._itemsInDependencyOrder(self.conditionaltasks), options)
1073 result+=self._dumpPythonList(self._itemsInDependencyOrder(self.sequences), options)
1074 result+=self._dumpPythonList(self.paths_(), options)
1075 result+=self._dumpPythonList(self.endpaths_(), options)
1076 result+=self._dumpPythonList(self.finalpaths_(), options)
1077 result+=self._dumpPythonList(self.aliases_(), options)
1078 if not self.schedule_() == None:
1079 result += 'process.schedule = ' + self.schedule.dumpPython(options)
1080 imports = specialImportRegistry.getSpecialImports()
1081 if len(imports) > 0:
1082 header += "\n" + "\n".join(imports)
1083 header += "\n\n"
1084 return header+result
1085
1086 def splitPython(self, options = PrintOptions()):
1087 """return a map of file names to python configuration fragments"""
1088 specialImportRegistry._reset()
1089
1090 options.isCfg = False
1091 header = "import FWCore.ParameterSet.Config as cms"
1092 result = ''
1093 parts = {}
1094 files = {}
1095
1096 result = 'process = cms.Process("' + self.__name + '")\n\n'
1097
1098 if self.source_():
1099 parts['source'] = (None, 'source = ' + self.source_().dumpPython(options))
1100
1101 if self.looper_():
1102 parts['looper'] = (None, 'looper = ' + self.looper_().dumpPython())
1103
1104 parts.update(self._splitPythonList('psets', self.psets, options))
1105 parts.update(self._splitPythonList('psets', self.vpsets, options))
1106
1107
1108 if len(self.subProcesses_()):
1109 sys.stderr.write("error: subprocesses are not supported yet\n\n")
1110 parts.update(self._splitPythonList('modules', self.producers_(), options))
1111 parts.update(self._splitPythonList('modules', self.switchProducers_(), options))
1112 parts.update(self._splitPythonList('modules', self.filters_() , options))
1113 parts.update(self._splitPythonList('modules', self.analyzers_(), options))
1114 parts.update(self._splitPythonList('modules', self.outputModules_(), options))
1115 parts.update(self._splitPythonList('services', self.services_(), options))
1116 parts.update(self._splitPythonList('eventsetup', self.es_producers_(), options))
1117 parts.update(self._splitPythonList('eventsetup', self.es_sources_(), options))
1118 parts.update(self._splitPython('eventsetup', self.es_prefers_(), options))
1119 parts.update(self._splitPythonList('tasks', self._itemsInDependencyOrder(self.tasks), options))
1120 parts.update(self._splitPythonList('sequences', self._itemsInDependencyOrder(self.sequences), options))
1121 parts.update(self._splitPythonList('paths', self.paths_(), options))
1122 parts.update(self._splitPythonList('paths', self.endpaths_(), options))
1123 parts.update(self._splitPythonList('paths', self.finalpaths_(), options))
1124 parts.update(self._splitPythonList('modules', self.aliases_(), options))
1125
1126 if options.targetDirectory is not None:
1127 files[options.targetDirectory + '/__init__.py'] = ''
1128
1129 if options.useSubdirectories:
1130 for sub in 'psets', 'modules', 'services', 'eventsetup', 'tasks', 'sequences', 'paths':
1131 if options.targetDirectory is not None:
1132 sub = options.targetDirectory + '/' + sub
1133 files[sub + '/__init__.py'] = ''
1134
1135
1136 parts = sorted(parts.items(), key = lambda nsc: (nsc[1][0].lower() if nsc[1][0] else '', nsc[0].lower()))
1137
1138 for (name, (subfolder, code)) in parts:
1139 filename = name + '_cfi'
1140 if options.useSubdirectories and subfolder:
1141 filename = subfolder + '/' + filename
1142 if options.targetDirectory is not None:
1143 filename = options.targetDirectory + '/' + filename
1144 result += 'process.load("%s")\n' % filename
1145 files[filename + '.py'] = header + '\n\n' + code
1146
1147 if self.schedule_() is not None:
1148 options.isCfg = True
1149 result += '\nprocess.schedule = ' + self.schedule.dumpPython(options)
1150
1151 imports = specialImportRegistry.getSpecialImports()
1152 if len(imports) > 0:
1153 header += '\n' + '\n'.join(imports)
1154 files['-'] = header + '\n\n' + result
1155 return files
1156
1157 def _replaceInSequences(self, label, new):
1158 old = getattr(self,label)
1159
1160
1161
1162
1163
1164 for sequenceable in self.sequences.values():
1165 sequenceable._replaceIfHeldDirectly(old,new)
1166 for sequenceable in self.sequences.values():
1167 sequenceable.replace(old,new)
1168 for sequenceable in self.paths.values():
1169 sequenceable.replace(old,new)
1170 for sequenceable in self.endpaths.values():
1171 sequenceable.replace(old,new)
1172 for sequenceable in self.finalpaths.values():
1173 sequenceable.replace(old,new)
1174 def _replaceInTasks(self, label, new):
1175 old = getattr(self,label)
1176 for task in self.tasks.values():
1177 task.replace(old, new)
1178 def _replaceInConditionalTasks(self, label, new):
1179 old = getattr(self,label)
1180 for task in self.conditionaltasks.values():
1181 task.replace(old, new)
1182 def _replaceInSchedule(self, label, new):
1183 if self.schedule_() == None:
1184 return
1185 old = getattr(self,label)
1186 for task in self.schedule_()._tasks:
1187 task.replace(old, new)
1188 def _replaceInScheduleDirectly(self, label, new):
1189 if self.schedule_() == None:
1190 return
1191 old = getattr(self,label)
1192 self.schedule_()._replaceIfHeldDirectly(old, new)
1193 def globalReplace(self,label,new):
1194 """ Replace the item with label 'label' by object 'new' in the process and all sequences/paths/tasks"""
1195 if not hasattr(self,label):
1196 raise LookupError("process has no item of label "+label)
1197 setattr(self,label,new)
1198 def _insertInto(self, parameterSet, itemDict):
1199 for name,value in itemDict.items():
1200 value.insertInto(parameterSet, name)
1201 def _insertOneInto(self, parameterSet, label, item, tracked):
1202 vitems = []
1203 if not item == None:
1204 newlabel = item.nameInProcessDesc_(label)
1205 vitems = [newlabel]
1206 item.insertInto(parameterSet, newlabel)
1207 parameterSet.addVString(tracked, label, vitems)
1208 def _insertManyInto(self, parameterSet, label, itemDict, tracked):
1209 l = []
1210 for name,value in itemDict.items():
1211 value.appendToProcessDescList_(l, name)
1212 value.insertInto(parameterSet, name)
1213
1214 l.sort()
1215 parameterSet.addVString(tracked, label, l)
1216 def _insertSwitchProducersInto(self, parameterSet, labelModules, labelAliases, itemDict, tracked):
1217 modules = parameterSet.getVString(tracked, labelModules)
1218 aliases = parameterSet.getVString(tracked, labelAliases)
1219 accelerators = parameterSet.getVString(False, "@selected_accelerators")
1220 for name,value in itemDict.items():
1221 value.appendToProcessDescLists_(modules, aliases, name)
1222 value.insertInto(parameterSet, name, accelerators)
1223 modules.sort()
1224 aliases.sort()
1225 parameterSet.addVString(tracked, labelModules, modules)
1226 parameterSet.addVString(tracked, labelAliases, aliases)
1227 def _insertSubProcessesInto(self, parameterSet, label, itemList, tracked):
1228 l = []
1229 subprocs = []
1230 for value in itemList:
1231 name = value.getProcessName()
1232 newLabel = value.nameInProcessDesc_(name)
1233 l.append(newLabel)
1234 pset = value.getSubProcessPSet(parameterSet)
1235 subprocs.append(pset)
1236
1237 l.sort()
1238 parameterSet.addVString(tracked, label, l)
1239 parameterSet.addVPSet(False,"subProcesses",subprocs)
1240 def _insertPaths(self, processPSet, nodeVisitor):
1241 scheduledPaths = []
1242 triggerPaths = []
1243 endpaths = []
1244 finalpaths = []
1245 if self.schedule_() == None:
1246
1247 for name in self.paths_():
1248 scheduledPaths.append(name)
1249 triggerPaths.append(name)
1250 for name in self.endpaths_():
1251 scheduledPaths.append(name)
1252 endpaths.append(name)
1253 for name in self.finalpaths_():
1254 finalpaths.append(name)
1255 else:
1256 for path in self.schedule_():
1257 pathname = path.label_()
1258 if pathname in self.endpaths_():
1259 endpaths.append(pathname)
1260 scheduledPaths.append(pathname)
1261 elif pathname in self.finalpaths_():
1262 finalpaths.append(pathname)
1263 else:
1264 scheduledPaths.append(pathname)
1265 triggerPaths.append(pathname)
1266 for task in self.schedule_()._tasks:
1267 task.resolve(self.__dict__)
1268 scheduleTaskValidator = ScheduleTaskValidator()
1269 task.visit(scheduleTaskValidator)
1270 task.visit(nodeVisitor)
1271
1272 endPathWithFinalPathModulesName ="@finalPath"
1273 finalPathEndPath = EndPath()
1274 if finalpaths:
1275 endpaths.append(endPathWithFinalPathModulesName)
1276 scheduledPaths.append(endPathWithFinalPathModulesName)
1277 finalpathValidator = FinalPathValidator()
1278 modulesOnFinalPath = []
1279 for finalpathname in finalpaths:
1280 iFinalPath = self.finalpaths_()[finalpathname]
1281 iFinalPath.resolve(self.__dict__)
1282 finalpathValidator.setLabel(finalpathname)
1283 iFinalPath.visit(finalpathValidator)
1284 if finalpathValidator.filtersOnFinalpaths or finalpathValidator.producersOnFinalpaths:
1285 names = [p.label_ for p in finalpathValidator.filtersOnFinalpaths]
1286 names.extend( [p.label_ for p in finalpathValidator.producersOnFinalpaths])
1287 raise RuntimeError("FinalPath %s has non OutputModules %s" % (finalpathname, ",".join(names)))
1288 modulesOnFinalPath.extend(iFinalPath.moduleNames())
1289 for m in modulesOnFinalPath:
1290 mod = getattr(self, m)
1291 setattr(mod, "@onFinalPath", untracked.bool(True))
1292 finalPathEndPath += mod
1293
1294 processPSet.addVString(True, "@end_paths", endpaths)
1295 processPSet.addVString(True, "@paths", scheduledPaths)
1296
1297 p = processPSet.newPSet()
1298 p.addVString(True, "@trigger_paths", triggerPaths)
1299 processPSet.addPSet(True, "@trigger_paths", p)
1300
1301 pathValidator = PathValidator()
1302 endpathValidator = EndPathValidator()
1303 decoratedList = []
1304 lister = DecoratedNodeNameVisitor(decoratedList)
1305 condTaskModules = []
1306 condTaskVistor = ModuleNodeOnConditionalTaskVisitor(condTaskModules)
1307 pathCompositeVisitor = CompositeVisitor(pathValidator, nodeVisitor, lister, condTaskVistor)
1308 endpathCompositeVisitor = CompositeVisitor(endpathValidator, nodeVisitor, lister)
1309 for triggername in triggerPaths:
1310 iPath = self.paths_()[triggername]
1311 iPath.resolve(self.__dict__)
1312 pathValidator.setLabel(triggername)
1313 lister.initialize()
1314 condTaskModules[:] = []
1315 iPath.visit(pathCompositeVisitor)
1316 if condTaskModules:
1317 decoratedList.append("#")
1318 l = list({x.label_() for x in condTaskModules})
1319 l.sort()
1320 decoratedList.extend(l)
1321 decoratedList.append("@")
1322 iPath.insertInto(processPSet, triggername, decoratedList[:])
1323 for endpathname in endpaths:
1324 if endpathname is not endPathWithFinalPathModulesName:
1325 iEndPath = self.endpaths_()[endpathname]
1326 else:
1327 iEndPath = finalPathEndPath
1328 iEndPath.resolve(self.__dict__)
1329 endpathValidator.setLabel(endpathname)
1330 lister.initialize()
1331 iEndPath.visit(endpathCompositeVisitor)
1332 iEndPath.insertInto(processPSet, endpathname, decoratedList[:])
1333 processPSet.addVString(False, "@filters_on_endpaths", endpathValidator.filtersOnEndpaths)
1334
1335
1336 def resolve(self,keepUnresolvedSequencePlaceholders=False):
1337 for x in self.paths.values():
1338 x.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1339 for x in self.endpaths.values():
1340 x.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1341 for x in self.finalpaths.values():
1342 x.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1343 if not self.schedule_() == None:
1344 for task in self.schedule_()._tasks:
1345 task.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1346
1347 def prune(self,verbose=False,keepUnresolvedSequencePlaceholders=False):
1348 """ Remove clutter from the process that we think is unnecessary:
1349 tracked PSets, VPSets and unused modules and sequences. If a Schedule has been set, then Paths and EndPaths
1350 not in the schedule will also be removed, along with an modules and sequences used only by
1351 those removed Paths and EndPaths. The keepUnresolvedSequencePlaceholders keeps also unresolved TaskPlaceholders."""
1352
1353
1354
1355
1356
1357 for name in self.vpsets_():
1358 delattr(self, name)
1359
1360 self.resolve(keepUnresolvedSequencePlaceholders)
1361 usedModules = set()
1362 unneededPaths = set()
1363 tasks = list()
1364 tv = TaskVisitor(tasks)
1365 if self.schedule_():
1366 usedModules=set(self.schedule_().moduleNames())
1367
1368 schedNames = set(( x.label_() for x in self.schedule_()))
1369 names = set(self.paths)
1370 names.update(set(self.endpaths))
1371 names.update(set(self.finalpaths))
1372 unneededPaths = names - schedNames
1373 for n in unneededPaths:
1374 delattr(self,n)
1375 for t in self.schedule_().tasks():
1376 tv.enter(t)
1377 t.visit(tv)
1378 tv.leave(t)
1379 else:
1380 pths = list(self.paths.values())
1381 pths.extend(self.endpaths.values())
1382 pths.extend(self.finalpaths.values())
1383 temp = Schedule(*pths)
1384 usedModules=set(temp.moduleNames())
1385 unneededModules = self._pruneModules(self.producers_(), usedModules)
1386 unneededModules.update(self._pruneModules(self.switchProducers_(), usedModules))
1387 unneededModules.update(self._pruneModules(self.filters_(), usedModules))
1388 unneededModules.update(self._pruneModules(self.analyzers_(), usedModules))
1389
1390 seqs = list()
1391 sv = SequenceVisitor(seqs)
1392 for p in self.paths.values():
1393 p.visit(sv)
1394 p.visit(tv)
1395 for p in self.endpaths.values():
1396 p.visit(sv)
1397 p.visit(tv)
1398 for p in self.finalpaths.values():
1399 p.visit(sv)
1400 p.visit(tv)
1401 def removeUnneeded(seqOrTasks, allSequencesOrTasks):
1402 _keepSet = set(( s for s in seqOrTasks if s.hasLabel_()))
1403 _availableSet = set(allSequencesOrTasks.values())
1404 _unneededSet = _availableSet-_keepSet
1405 _unneededLabels = []
1406 for s in _unneededSet:
1407 _unneededLabels.append(s.label_())
1408 delattr(self,s.label_())
1409 return _unneededLabels
1410 unneededSeqLabels = removeUnneeded(seqs, self.sequences)
1411 unneededTaskLabels = removeUnneeded(tasks, self.tasks)
1412 if verbose:
1413 print("prune removed the following:")
1414 print(" modules:"+",".join(unneededModules))
1415 print(" tasks:"+",".join(unneededTaskLabels))
1416 print(" sequences:"+",".join(unneededSeqLabels))
1417 print(" paths/endpaths/finalpaths:"+",".join(unneededPaths))
1418 def _pruneModules(self, d, scheduledNames):
1419 moduleNames = set(d.keys())
1420 junk = moduleNames - scheduledNames
1421 for name in junk:
1422 delattr(self, name)
1423 return junk
1424
1425 def fillProcessDesc(self, processPSet):
1426 """Used by the framework to convert python to C++ objects"""
1427 class ServiceInjectorAdaptor(object):
1428 def __init__(self,ppset,thelist):
1429 self.__thelist = thelist
1430 self.__processPSet = ppset
1431 def addService(self,pset):
1432 self.__thelist.append(pset)
1433 def newPSet(self):
1434 return self.__processPSet.newPSet()
1435
1436
1437
1438 class TopLevelPSetAcessorAdaptor(object):
1439 def __init__(self,ppset,process):
1440 self.__ppset = ppset
1441 self.__process = process
1442 def __getattr__(self,attr):
1443 return getattr(self.__ppset,attr)
1444 def getTopPSet_(self,label):
1445 return getattr(self.__process,label)
1446 def newPSet(self):
1447 return TopLevelPSetAcessorAdaptor(self.__ppset.newPSet(),self.__process)
1448 def addPSet(self,tracked,name,ppset):
1449 return self.__ppset.addPSet(tracked,name,self.__extractPSet(ppset))
1450 def addVPSet(self,tracked,name,vpset):
1451 return self.__ppset.addVPSet(tracked,name,[self.__extractPSet(x) for x in vpset])
1452 def __extractPSet(self,pset):
1453 if isinstance(pset,TopLevelPSetAcessorAdaptor):
1454 return pset.__ppset
1455 return pset
1456
1457 self.validate()
1458 processPSet.addString(True, "@process_name", self.name_())
1459 self.handleProcessAccelerators(processPSet)
1460 all_modules = self.producers_().copy()
1461 all_modules.update(self.filters_())
1462 all_modules.update(self.analyzers_())
1463 all_modules.update(self.outputModules_())
1464 adaptor = TopLevelPSetAcessorAdaptor(processPSet,self)
1465 self._insertInto(adaptor, self.psets_())
1466 self._insertInto(adaptor, self.vpsets_())
1467 self._insertOneInto(adaptor, "@all_sources", self.source_(), True)
1468 self._insertOneInto(adaptor, "@all_loopers", self.looper_(), True)
1469 self._insertSubProcessesInto(adaptor, "@all_subprocesses", self.subProcesses_(), False)
1470 self._insertManyInto(adaptor, "@all_esprefers", self.es_prefers_(), True)
1471 self._insertManyInto(adaptor, "@all_aliases", self.aliases_(), True)
1472
1473
1474
1475 nodeVisitor = NodeVisitor()
1476 self._insertPaths(adaptor, nodeVisitor)
1477 all_modules_onTasksOrScheduled = { key:value for key, value in all_modules.items() if value in nodeVisitor.modules }
1478 self._insertManyInto(adaptor, "@all_modules", all_modules_onTasksOrScheduled, True)
1479 all_switches = self.switchProducers_().copy()
1480 all_switches_onTasksOrScheduled = {key:value for key, value in all_switches.items() if value in nodeVisitor.modules }
1481 self._insertSwitchProducersInto(adaptor, "@all_modules", "@all_aliases", all_switches_onTasksOrScheduled, True)
1482
1483
1484 processNodeVisitor = NodeVisitor()
1485 for pTask in self.tasks.values():
1486 pTask.visit(processNodeVisitor)
1487 esProducersToEnable = {}
1488 for esProducerName, esProducer in self.es_producers_().items():
1489 if esProducer in nodeVisitor.esProducers or not (esProducer in processNodeVisitor.esProducers):
1490 esProducersToEnable[esProducerName] = esProducer
1491 self._insertManyInto(adaptor, "@all_esmodules", esProducersToEnable, True)
1492 esSourcesToEnable = {}
1493 for esSourceName, esSource in self.es_sources_().items():
1494 if esSource in nodeVisitor.esSources or not (esSource in processNodeVisitor.esSources):
1495 esSourcesToEnable[esSourceName] = esSource
1496 self._insertManyInto(adaptor, "@all_essources", esSourcesToEnable, True)
1497
1498 services = []
1499 for serviceName, serviceObject in self.services_().items():
1500 if serviceObject in nodeVisitor.services or not (serviceObject in processNodeVisitor.services):
1501 serviceObject.insertInto(ServiceInjectorAdaptor(adaptor,services))
1502 adaptor.addVPSet(False,"services",services)
1503 return processPSet
1504
1505 def validate(self):
1506
1507
1508
1509
1510 pass
1511
1512 def handleProcessAccelerators(self, parameterSet):
1513
1514 allAccelerators = set(["cpu"])
1515 availableAccelerators = set(["cpu"])
1516 for acc in self.__dict__['_Process__accelerators'].values():
1517 allAccelerators.update(acc.labels())
1518 availableAccelerators.update(acc.enabledLabels())
1519 availableAccelerators = sorted(list(availableAccelerators))
1520 parameterSet.addVString(False, "@available_accelerators", availableAccelerators)
1521
1522
1523 selectedAccelerators = []
1524 if "*" in self.options.accelerators:
1525 if len(self.options.accelerators) >= 2:
1526 raise ValueError("process.options.accelerators may contain '*' only as the only element, now it has {} elements".format(len(self.options.accelerators)))
1527 selectedAccelerators = availableAccelerators
1528 else:
1529 import fnmatch
1530 resolved = set()
1531 invalid = []
1532 for pattern in self.options.accelerators:
1533 acc = [a for a in availableAccelerators if fnmatch.fnmatchcase(a, pattern)]
1534 if len(acc) == 0:
1535 if not any(fnmatch.fnmatchcase(a, pattern) for a in allAccelerators):
1536 invalid.append(pattern)
1537 else:
1538 resolved.update(acc)
1539
1540 if len(invalid) != 0:
1541 raise ValueError("Invalid pattern{} of {} in process.options.accelerators, valid values are {} or a pattern matching to some of them.".format(
1542 "s" if len(invalid) > 2 else "",
1543 ",".join(invalid),
1544 ",".join(sorted(list(allAccelerators)))))
1545 selectedAccelerators = sorted(list(resolved))
1546 parameterSet.addVString(False, "@selected_accelerators", selectedAccelerators)
1547
1548
1549 moduleTypeResolver = None
1550 moduleTypeResolverPlugin = ""
1551 for acc in self.__dict__['_Process__accelerators'].values():
1552 resolver = acc.moduleTypeResolver(selectedAccelerators)
1553 if resolver is not None:
1554 if moduleTypeResolver is not None:
1555 raise RuntimeError("Module type resolver was already set to {} when {} tried to set it to {}. A job can have at most one ProcessAccelerator that sets module type resolver.".format(
1556 moduleTypeResolver.__class__.__name__,
1557 acc.__class__.__name__,
1558 resolver.__class__.__name__))
1559 moduleTypeResolver = resolver
1560 if moduleTypeResolver is not None:
1561
1562 moduleTypeResolverPlugin = moduleTypeResolver.plugin()
1563
1564
1565 for modlist in [self.producers_, self.filters_, self.analyzers_,
1566 self.es_producers_, self.es_sources_]:
1567 for module in modlist().values():
1568 moduleTypeResolver.setModuleVariant(module)
1569
1570 parameterSet.addString(False, "@module_type_resolver", moduleTypeResolverPlugin)
1571
1572
1573 wrapped = ProcessForProcessAccelerator(self)
1574 for acc in self.__dict__['_Process__accelerators'].values():
1575 acc.apply(wrapped, selectedAccelerators)
1576
1577 def prefer(self, esmodule,*args,**kargs):
1578 """Prefer this ES source or producer. The argument can
1579 either be an object label, e.g.,
1580 process.prefer(process.juicerProducer) (not supported yet)
1581 or a name of an ESSource or ESProducer
1582 process.prefer("juicer")
1583 or a type of unnamed ESSource or ESProducer
1584 process.prefer("JuicerProducer")
1585 In addition, you can pass as a labelled arguments the name of the Record you wish to
1586 prefer where the type passed is a cms.vstring and that vstring can contain the
1587 name of the C++ types in the Record that are being preferred, e.g.,
1588 #prefer all data in record 'OrangeRecord' from 'juicer'
1589 process.prefer("juicer", OrangeRecord=cms.vstring())
1590 or
1591 #prefer only "Orange" data in "OrangeRecord" from "juicer"
1592 process.prefer("juicer", OrangeRecord=cms.vstring("Orange"))
1593 or
1594 #prefer only "Orange" data with label "ExtraPulp" in "OrangeRecord" from "juicer"
1595 ESPrefer("ESJuicerProd", OrangeRecord=cms.vstring("Orange/ExtraPulp"))
1596 """
1597
1598 if isinstance(esmodule, ESSource) or isinstance(esmodule, ESProducer):
1599 raise RuntimeError("Syntax of process.prefer(process.esmodule) not supported yet")
1600 elif self._findPreferred(esmodule, self.es_producers_(),*args,**kargs) or \
1601 self._findPreferred(esmodule, self.es_sources_(),*args,**kargs):
1602 pass
1603 else:
1604 raise RuntimeError("Cannot resolve prefer for "+repr(esmodule))
1605
1606 def _findPreferred(self, esname, d,*args,**kargs):
1607
1608 if esname in d:
1609 typ = d[esname].type_()
1610 if typ == esname:
1611 self.__setattr__( esname+"_prefer", ESPrefer(typ,*args,**kargs) )
1612 else:
1613 self.__setattr__( esname+"_prefer", ESPrefer(typ, esname,*args,**kargs) )
1614 return True
1615 else:
1616
1617 found = False
1618 for name, value in d.items():
1619 if value.type_() == esname:
1620 if found:
1621 raise RuntimeError("More than one ES module for "+esname)
1622 found = True
1623 self.__setattr__(esname+"_prefer", ESPrefer(d[esname].type_()) )
1624 return found
1625
1626
1627 class ProcessFragment(object):
1628 def __init__(self, process):
1629 if isinstance(process, Process):
1630 self.__process = process
1631 elif isinstance(process, str):
1632 self.__process = Process(process)
1633
1634 del self.__process.options
1635 del self.__process.maxEvents
1636 del self.__process.maxLuminosityBlocks
1637 else:
1638 raise TypeError('a ProcessFragment can only be constructed from an existig Process or from process name')
1639 def __dir__(self):
1640 return [ x for x in dir(self.__process) if isinstance(getattr(self.__process, x), _ConfigureComponent) ]
1641 def __getattribute__(self, name):
1642 if name == '_ProcessFragment__process':
1643 return object.__getattribute__(self, '_ProcessFragment__process')
1644 else:
1645 return getattr(self.__process, name)
1646 def __setattr__(self, name, value):
1647 if name == '_ProcessFragment__process':
1648 object.__setattr__(self, name, value)
1649 else:
1650 setattr(self.__process, name, value)
1651 def __delattr__(self, name):
1652 if name == '_ProcessFragment__process':
1653 pass
1654 else:
1655 return delattr(self.__process, name)
1656
1657
1658 class FilteredStream(dict):
1659 """a dictionary with fixed keys"""
1660 def _blocked_attribute(obj):
1661 raise AttributeError("An FilteredStream defintion cannot be modified after creation.")
1662 _blocked_attribute = property(_blocked_attribute)
1663 __setattr__ = __delitem__ = __setitem__ = clear = _blocked_attribute
1664 pop = popitem = setdefault = update = _blocked_attribute
1665 def __new__(cls, *args, **kw):
1666 new = dict.__new__(cls)
1667 dict.__init__(new, *args, **kw)
1668 keys = sorted(kw.keys())
1669 if keys != ['content', 'dataTier', 'name', 'paths', 'responsible', 'selectEvents']:
1670 raise ValueError("The needed parameters are: content, dataTier, name, paths, responsible, selectEvents")
1671 if not isinstance(kw['name'],str):
1672 raise ValueError("name must be of type string")
1673 if not isinstance(kw['content'], vstring) and not isinstance(kw['content'],str):
1674 raise ValueError("content must be of type vstring or string")
1675 if not isinstance(kw['dataTier'], string):
1676 raise ValueError("dataTier must be of type string")
1677 if not isinstance(kw['selectEvents'], PSet):
1678 raise ValueError("selectEvents must be of type PSet")
1679 if not isinstance(kw['paths'],(tuple, Path)):
1680 raise ValueError("'paths' must be a tuple of paths")
1681 return new
1682 def __init__(self, *args, **kw):
1683 pass
1684 def __repr__(self):
1685 return "FilteredStream object: %s" %self["name"]
1686 def __getattr__(self,attr):
1687 return self[attr]
1688
1689 class SubProcess(_Unlabelable):
1690 """Allows embedding another process within a parent process. This allows one to
1691 chain processes together directly in one cmsRun job rather than having to run
1692 separate jobs that are connected via a temporary file.
1693 """
1694 def __init__(self,process, SelectEvents = untracked.PSet(), outputCommands = untracked.vstring()):
1695 """
1696 """
1697 if not isinstance(process, Process):
1698 raise ValueError("the 'process' argument must be of type cms.Process")
1699 if not isinstance(SelectEvents,PSet):
1700 raise ValueError("the 'SelectEvents' argument must be of type cms.untracked.PSet")
1701 if not isinstance(outputCommands,vstring):
1702 raise ValueError("the 'outputCommands' argument must be of type cms.untracked.vstring")
1703 self.__process = process
1704 self.__SelectEvents = SelectEvents
1705 self.__outputCommands = outputCommands
1706
1707 if self.__process.MessageLogger is not MessageLogger:
1708 print("""Warning: You have reconfigured service
1709 'edm::MessageLogger' in a subprocess.
1710 This service has already been configured.
1711 This particular service may not be reconfigured in a subprocess.
1712 The reconfiguration will be ignored.""")
1713 del self.__process.MessageLogger
1714 def dumpPython(self, options=PrintOptions()):
1715 out = "parentProcess"+str(hash(self))+" = process\n"
1716 out += self.__process.dumpPython()
1717 out += "childProcess = process\n"
1718 out += "process = parentProcess"+str(hash(self))+"\n"
1719 out += "process.addSubProcess(cms.SubProcess(process = childProcess, SelectEvents = "+self.__SelectEvents.dumpPython(options) +", outputCommands = "+self.__outputCommands.dumpPython(options) +"))"
1720 return out
1721 def getProcessName(self):
1722 return self.__process.name_()
1723 def process(self):
1724 return self.__process
1725 def SelectEvents(self):
1726 return self.__SelectEvents
1727 def outputCommands(self):
1728 return self.__outputCommands
1729 def type_(self):
1730 return 'subProcess'
1731 def nameInProcessDesc_(self,label):
1732 return label
1733 def _place(self,label,process):
1734 process._placeSubProcess('subProcess',self)
1735 def getSubProcessPSet(self,parameterSet):
1736 topPSet = parameterSet.newPSet()
1737 self.__process.fillProcessDesc(topPSet)
1738 subProcessPSet = parameterSet.newPSet()
1739 self.__SelectEvents.insertInto(subProcessPSet,"SelectEvents")
1740 self.__outputCommands.insertInto(subProcessPSet,"outputCommands")
1741 subProcessPSet.addPSet(False,"process",topPSet)
1742 return subProcessPSet
1743
1744 class _ParameterModifier(object):
1745 """Helper class for Modifier that takes key/value pairs and uses them to reset parameters of the object"""
1746 def __init__(self,args):
1747 self.__args = args
1748 def __call__(self,obj):
1749 params = {}
1750 for k in self.__args.keys():
1751 if hasattr(obj,k):
1752 params[k] = getattr(obj,k)
1753 _modifyParametersFromDict(params, self.__args, self._raiseUnknownKey)
1754 for k in self.__args.keys():
1755 if k in params:
1756 setattr(obj,k,params[k])
1757 else:
1758
1759 delattr(obj,k)
1760 @staticmethod
1761 def _raiseUnknownKey(key):
1762 raise KeyError("Unknown parameter name "+key+" specified while calling Modifier")
1763
1764 class _BoolModifierBase(object):
1765 """A helper base class for _AndModifier, _InvertModifier, and _OrModifier to contain the common code"""
1766 def __init__(self, lhs, rhs=None):
1767 self._lhs = lhs
1768 if rhs is not None:
1769 self._rhs = rhs
1770 def toModify(self,obj, func=None,**kw):
1771 Modifier._toModifyCheck(obj,func,**kw)
1772 if self._isChosen():
1773 Modifier._toModify(obj,func,**kw)
1774 return self
1775 def toReplaceWith(self,toObj,fromObj):
1776 Modifier._toReplaceWithCheck(toObj,fromObj)
1777 if self._isChosen():
1778 Modifier._toReplaceWith(toObj,fromObj)
1779 return self
1780 def makeProcessModifier(self,func):
1781 """This is used to create a ProcessModifer that can perform actions on the process as a whole.
1782 This takes as argument a callable object (e.g. function) that takes as its sole argument an instance of Process.
1783 In order to work, the value returned from this function must be assigned to a uniquely named variable."""
1784 return ProcessModifier(self,func)
1785 def __and__(self, other):
1786 return _AndModifier(self,other)
1787 def __invert__(self):
1788 return _InvertModifier(self)
1789 def __or__(self, other):
1790 return _OrModifier(self,other)
1791
1792 class _AndModifier(_BoolModifierBase):
1793 """A modifier which only applies if multiple Modifiers are chosen"""
1794 def __init__(self, lhs, rhs):
1795 super(_AndModifier,self).__init__(lhs, rhs)
1796 def _isChosen(self):
1797 return self._lhs._isChosen() and self._rhs._isChosen()
1798
1799 class _InvertModifier(_BoolModifierBase):
1800 """A modifier which only applies if a Modifier is not chosen"""
1801 def __init__(self, lhs):
1802 super(_InvertModifier,self).__init__(lhs)
1803 def _isChosen(self):
1804 return not self._lhs._isChosen()
1805
1806 class _OrModifier(_BoolModifierBase):
1807 """A modifier which only applies if at least one of multiple Modifiers is chosen"""
1808 def __init__(self, lhs, rhs):
1809 super(_OrModifier,self).__init__(lhs, rhs)
1810 def _isChosen(self):
1811 return self._lhs._isChosen() or self._rhs._isChosen()
1812
1813
1814 class Modifier(object):
1815 """This class is used to define standard modifications to a Process.
1816 An instance of this class is declared to denote a specific modification,e.g. era2017 could
1817 reconfigure items in a process to match our expectation of running in 2017. Once declared,
1818 these Modifier instances are imported into a configuration and items that need to be modified
1819 are then associated with the Modifier and with the action to do the modification.
1820 The registered modifications will only occur if the Modifier was passed to
1821 the cms.Process' constructor.
1822 """
1823 def __init__(self):
1824 self.__processModifiers = []
1825 self.__chosen = False
1826 def makeProcessModifier(self,func):
1827 """This is used to create a ProcessModifer that can perform actions on the process as a whole.
1828 This takes as argument a callable object (e.g. function) that takes as its sole argument an instance of Process.
1829 In order to work, the value returned from this function must be assigned to a uniquely named variable.
1830 """
1831 return ProcessModifier(self,func)
1832 @staticmethod
1833 def _toModifyCheck(obj,func,**kw):
1834 if func is not None and len(kw) != 0:
1835 raise TypeError("toModify takes either two arguments or one argument and key/value pairs")
1836 def toModify(self,obj, func=None,**kw):
1837 """This is used to register an action to be performed on the specific object. Two different forms are allowed
1838 Form 1: A callable object (e.g. function) can be passed as the second. This callable object is expected to take one argument
1839 that will be the object passed in as the first argument.
1840 Form 2: A list of parameter name, value pairs can be passed
1841 mod.toModify(foo, fred=cms.int32(7), barney = cms.double(3.14))
1842 This form can also be used to remove a parameter by passing the value of None
1843 #remove the parameter foo.fred
1844 mod.toModify(foo, fred = None)
1845 Additionally, parameters embedded within PSets can also be modified using a dictionary
1846 #change foo.fred.pebbles to 3 and foo.fred.friend to "barney"
1847 mod.toModify(foo, fred = dict(pebbles = 3, friend = "barney)) )
1848 """
1849 Modifier._toModifyCheck(obj,func,**kw)
1850 if self._isChosen():
1851 Modifier._toModify(obj,func,**kw)
1852 return self
1853 @staticmethod
1854 def _toModify(obj,func,**kw):
1855 if func is not None:
1856 func(obj)
1857 else:
1858 temp =_ParameterModifier(kw)
1859 temp(obj)
1860 @staticmethod
1861 def _toReplaceWithCheck(toObj,fromObj):
1862 if not isinstance(fromObj, type(toObj)):
1863 raise TypeError("toReplaceWith requires both arguments to be the same class type")
1864 def toReplaceWith(self,toObj,fromObj):
1865 """If the Modifier is chosen the internals of toObj will be associated with the internals of fromObj
1866 """
1867 Modifier._toReplaceWithCheck(toObj,fromObj)
1868 if self._isChosen():
1869 Modifier._toReplaceWith(toObj,fromObj)
1870 return self
1871 @staticmethod
1872 def _toReplaceWith(toObj,fromObj):
1873 if isinstance(fromObj,_ModuleSequenceType):
1874 toObj._seq = fromObj._seq
1875 toObj._tasks = fromObj._tasks
1876 elif isinstance(fromObj,Task):
1877 toObj._collection = fromObj._collection
1878 elif isinstance(fromObj,ConditionalTask):
1879 toObj._collection = fromObj._collection
1880 elif isinstance(fromObj,_Parameterizable):
1881
1882 for p in toObj.parameterNames_():
1883 delattr(toObj,p)
1884 for p in fromObj.parameterNames_():
1885 setattr(toObj,p,getattr(fromObj,p))
1886 if isinstance(fromObj,_TypedParameterizable):
1887 toObj._TypedParameterizable__type = fromObj._TypedParameterizable__type
1888
1889 else:
1890 raise TypeError("toReplaceWith does not work with type "+str(type(toObj)))
1891
1892 def _setChosen(self):
1893 """Should only be called by cms.Process instances"""
1894 self.__chosen = True
1895 def _isChosen(self):
1896 return self.__chosen
1897 def __and__(self, other):
1898 return _AndModifier(self,other)
1899 def __invert__(self):
1900 return _InvertModifier(self)
1901 def __or__(self, other):
1902 return _OrModifier(self,other)
1903 def _isOrContains(self, other):
1904 return self == other
1905
1906
1907 class ModifierChain(object):
1908 """A Modifier made up of a list of Modifiers
1909 """
1910 def __init__(self, *chainedModifiers):
1911 self.__chosen = False
1912 self.__chain = chainedModifiers
1913 def _applyNewProcessModifiers(self,process):
1914 """Should only be called by cms.Process instances
1915 applies list of accumulated changes to the process"""
1916 for m in self.__chain:
1917 m._applyNewProcessModifiers(process)
1918 def _setChosen(self):
1919 """Should only be called by cms.Process instances"""
1920 self.__chosen = True
1921 for m in self.__chain:
1922 m._setChosen()
1923 def _isChosen(self):
1924 return self.__chosen
1925 def copyAndExclude(self, toExclude):
1926 """Creates a new ModifierChain which is a copy of
1927 this ModifierChain but excludes any Modifier or
1928 ModifierChain in the list toExclude.
1929 The exclusion is done recursively down the chain.
1930 """
1931 newMods = []
1932 for m in self.__chain:
1933 if m not in toExclude:
1934 s = m
1935 if isinstance(m,ModifierChain):
1936 s = m.__copyIfExclude(toExclude)
1937 newMods.append(s)
1938 return ModifierChain(*newMods)
1939 def __copyIfExclude(self,toExclude):
1940 shouldCopy = False
1941 for m in toExclude:
1942 if self._isOrContains(m):
1943 shouldCopy = True
1944 break
1945 if shouldCopy:
1946 return self.copyAndExclude(toExclude)
1947 return self
1948 def _isOrContains(self, other):
1949 if self is other:
1950 return True
1951 for m in self.__chain:
1952 if m._isOrContains(other):
1953 return True
1954 return False
1955
1956 class ProcessModifier(object):
1957 """A class used by a Modifier to affect an entire Process instance.
1958 When a Process 'loads' a module containing a ProcessModifier, that
1959 ProcessModifier will be applied to the Process if and only if the
1960 Modifier passed to the constructor has been chosen.
1961 """
1962 def __init__(self, modifier, func):
1963 self.__modifier = modifier
1964 self.__func = func
1965 self.__seenProcesses = set()
1966 def apply(self,process):
1967 if self.__modifier._isChosen():
1968 if process not in self.__seenProcesses:
1969 self.__func(process)
1970 self.__seenProcesses.add(process)
1971
1972 class ProcessAccelerator(_ConfigureComponent,_Unlabelable):
1973 """A class used to specify possible compute accelerators in a Process
1974 instance. It is intended to be derived for any
1975 accelerator/portability technology, and provides hooks such that a
1976 specific customization can be applied to the Process on a worker
1977 node at the point where the python configuration is serialized for C++.
1978
1979 The customization must not change the configuration hash. To
1980 enforce this reuirement, the customization gets a
1981 ProcessForProcessAccelerator wrapper that gives access to only
1982 those parts of the configuration that can be changed. Nevertheless
1983 it would be good to have specific unit test for each deriving
1984 class to ensure that all combinations of the enabled accelerators
1985 give the same configuration hash.
1986 """
1987 def __init__(self):
1988 pass
1989 def _place(self, name, proc):
1990 proc._placeAccelerator(self.type_(), self)
1991 def type_(self):
1992 return type(self).__name__
1993 def dumpPython(self, options=PrintOptions()):
1994 specialImportRegistry.registerUse(self)
1995 result = self.__class__.__name__+"("
1996 options.indent()
1997 res = self.dumpPythonImpl(options)
1998 options.unindent()
1999 if len(res) > 0:
2000 result += "\n"+res+"\n"
2001 result += ")\n"
2002 return result
2003
2004
2005 def dumpPythonImpl(self, options):
2006 """Override if need to add any 'body' content to dumpPython(). Returns a string."""
2007 return ""
2008 def labels(self):
2009 """Override to return a list of strings for the accelerator labels."""
2010 return []
2011 def enabledLabels(self):
2012 """Override to return a list of strings for the accelerator labels
2013 that are enabled in the system the job is being run on."""
2014 return []
2015 def moduleTypeResolver(self, accelerators):
2016 """Override to return an object that implements "module type resolver"
2017 in python. The object should have the following methods
2018 - __init__(self, accelerators)
2019 * accelerators = list of selected accelerators
2020 - plugin(self):
2021 * should return a string for the type resolver plugin name
2022 - setModuleVariant(self, module):
2023 * Called for each ED and ES module. Should act only if
2024 module.type_() contains the magic identifier
2025
2026 At most one of the ProcessAccelerators in a job can return a
2027 non-None object
2028 """
2029 return None
2030 def apply(self, process, accelerators):
2031 """Override if need to customize the Process at worker node. The
2032 selected available accelerator labels are given in the
2033 'accelerators' argument (the patterns, e.g. '*' have been
2034 expanded to concrete labels).
2035
2036 This function may touch only untracked parameters.
2037 """
2038 pass
2039
2040 class ProcessForProcessAccelerator(object):
2041 """This class is inteded to wrap the Process object to constrain the
2042 available functionality for ProcessAccelerator.apply()"""
2043 def __init__(self, process):
2044 self.__process = process
2045 def __getattr__(self, label):
2046 value = getattr(self.__process, label)
2047 if not isinstance(value, Service):
2048 raise TypeError("ProcessAccelerator.apply() can get only Services. Tried to get {} with label {}".format(str(type(value)), label))
2049 return value
2050 def __setattr__(self, label, value):
2051 if label == "_ProcessForProcessAccelerator__process":
2052 super().__setattr__(label, value)
2053 else:
2054 if not isinstance(value, Service):
2055 raise TypeError("ProcessAccelerator.apply() can only set Services. Tried to set {} with label {}".format(str(type(value)), label))
2056 setattr(self.__process, label, value)
2057 def add_(self, value):
2058 if not isinstance(value, Service):
2059 raise TypeError("ProcessAccelerator.apply() can only add Services. Tried to set {} with label {}".format(str(type(value)), label))
2060 self.__process.add_(value)
2061
2062
2063
2064 def _switchproducer_test2_case1(accelerators):
2065 return ("test1" in accelerators, -10)
2066 def _switchproducer_test2_case2(accelerators):
2067 return ("test2" in accelerators, -9)
2068
2069 if __name__=="__main__":
2070 import unittest
2071 import copy
2072
2073 def _lineDiff(newString, oldString):
2074 newString = ( x for x in newString.split('\n') if len(x) > 0)
2075 oldString = [ x for x in oldString.split('\n') if len(x) > 0]
2076 diff = []
2077 oldStringLine = 0
2078 for l in newString:
2079 if oldStringLine >= len(oldString):
2080 diff.append(l)
2081 continue
2082 if l == oldString[oldStringLine]:
2083 oldStringLine +=1
2084 continue
2085 diff.append(l)
2086 return "\n".join( diff )
2087
2088 class TestMakePSet(object):
2089 """Has same interface as the C++ object that creates PSets
2090 """
2091 def __init__(self):
2092 self.values = dict()
2093 def __insertValue(self,tracked,label,value):
2094 self.values[label]=(tracked,value)
2095 def __getValue(self,tracked,label):
2096 pair = self.values[label]
2097 if pair[0] != tracked:
2098 raise Exception("Asked for %s parameter '%s', but it is %s" % ("tracked" if tracked else "untracked",
2099 label,
2100 "tracked" if pair[0] else "untracked"))
2101 return pair[1]
2102 def addInt32(self,tracked,label,value):
2103 self.__insertValue(tracked,label,value)
2104 def addVInt32(self,tracked,label,value):
2105 self.__insertValue(tracked,label,value)
2106 def addUInt32(self,tracked,label,value):
2107 self.__insertValue(tracked,label,value)
2108 def addVUInt32(self,tracked,label,value):
2109 self.__insertValue(tracked,label,value)
2110 def addInt64(self,tracked,label,value):
2111 self.__insertValue(tracked,label,value)
2112 def addVInt64(self,tracked,label,value):
2113 self.__insertValue(tracked,label,value)
2114 def addUInt64(self,tracked,label,value):
2115 self.__insertValue(tracked,label,value)
2116 def addVUInt64(self,tracked,label,value):
2117 self.__insertValue(tracked,label,value)
2118 def addDouble(self,tracked,label,value):
2119 self.__insertValue(tracked,label,value)
2120 def addVDouble(self,tracked,label,value):
2121 self.__insertValue(tracked,label,value)
2122 def addBool(self,tracked,label,value):
2123 self.__insertValue(tracked,label,value)
2124 def addString(self,tracked,label,value):
2125 self.__insertValue(tracked,label,value)
2126 def addVString(self,tracked,label,value):
2127 self.__insertValue(tracked,label,value)
2128 def getVString(self,tracked,label):
2129 return self.__getValue(tracked, label)
2130 def addInputTag(self,tracked,label,value):
2131 self.__insertValue(tracked,label,value)
2132 def addVInputTag(self,tracked,label,value):
2133 self.__insertValue(tracked,label,value)
2134 def addESInputTag(self,tracked,label,value):
2135 self.__insertValue(tracked,label,value)
2136 def addVESInputTag(self,tracked,label,value):
2137 self.__insertValue(tracked,label,value)
2138 def addEventID(self,tracked,label,value):
2139 self.__insertValue(tracked,label,value)
2140 def addVEventID(self,tracked,label,value):
2141 self.__insertValue(tracked,label,value)
2142 def addLuminosityBlockID(self,tracked,label,value):
2143 self.__insertValue(tracked,label,value)
2144 def addLuminosityBlockID(self,tracked,label,value):
2145 self.__insertValue(tracked,label,value)
2146 def addEventRange(self,tracked,label,value):
2147 self.__insertValue(tracked,label,value)
2148 def addVEventRange(self,tracked,label,value):
2149 self.__insertValue(tracked,label,value)
2150 def addPSet(self,tracked,label,value):
2151 self.__insertValue(tracked,label,value)
2152 def addVPSet(self,tracked,label,value):
2153 self.__insertValue(tracked,label,value)
2154 def addFileInPath(self,tracked,label,value):
2155 self.__insertValue(tracked,label,value)
2156 def newPSet(self):
2157 return TestMakePSet()
2158
2159 class SwitchProducerTest(SwitchProducer):
2160 def __init__(self, **kargs):
2161 super(SwitchProducerTest,self).__init__(
2162 dict(
2163 test1 = lambda accelerators: (True, -10),
2164 test2 = lambda accelerators: (True, -9),
2165 test3 = lambda accelerators: (True, -8),
2166 test4 = lambda accelerators: (True, -7)
2167 ), **kargs)
2168 specialImportRegistry.registerSpecialImportForType(SwitchProducerTest, "from test import SwitchProducerTest")
2169
2170 class SwitchProducerTest2(SwitchProducer):
2171 def __init__(self, **kargs):
2172 super(SwitchProducerTest2,self).__init__(
2173 dict(
2174 test1 = _switchproducer_test2_case1,
2175 test2 = _switchproducer_test2_case2,
2176 ), **kargs)
2177 specialImportRegistry.registerSpecialImportForType(SwitchProducerTest2, "from test import SwitchProducerTest2")
2178
2179 class TestModuleTypeResolver:
2180 def __init__(self, accelerators):
2181
2182 self._valid_backends = []
2183 if "test1" in accelerators:
2184 self._valid_backends.append("test1_backend")
2185 if "test2" in accelerators:
2186 self._valid_backends.append("test2_backend")
2187 if len(self._valid_backends) == 0:
2188 raise EDMException(edm.errors.UnavailableAccelerator, "Machine has no accelerators that Test supports (has {})".format(", ".join(accelerators)))
2189
2190 def plugin(self):
2191 return "TestModuleTypeResolver"
2192
2193 def setModuleVariant(self, module):
2194 if "@test" in module.type_():
2195 defaultBackend = self._valid_backends[0]
2196 if hasattr(module, "test"):
2197 if hasattr(module.test, "backend"):
2198 if module.test.backend.value() not in self._valid_backends:
2199 raise EDMException(edm.errors.UnavailableAccelerator, "Module {} has the Test backend set explicitly, but its accelerator is not available for the job".format(module.label_()))
2200 else:
2201 module.test.backend = untracked.string(defaultBackend)
2202 else:
2203 module.test = untracked.PSet(
2204 backend = untracked.string(defaultBackend)
2205 )
2206
2207 class ProcessAcceleratorTest(ProcessAccelerator):
2208 def __init__(self, enabled=["test1", "test2", "anothertest3"], moduleTypeResolverMaker=None):
2209 super().__init__()
2210 self._labels = ["test1", "test2", "anothertest3"]
2211 self.setEnabled(enabled)
2212 self._moduleTypeResolverMaker = moduleTypeResolverMaker
2213 def setEnabled(self, enabled):
2214 invalid = set(enabled).difference(set(self._labels))
2215 if len(invalid) > 0:
2216 raise Exception("Tried to enabled nonexistent test accelerators {}".format(",".join(invalid)))
2217 self._enabled = enabled[:]
2218 def dumpPythonImpl(self,options):
2219 result = "{}enabled = [{}]".format(options.indentation(),
2220 ", ".join(["'{}'".format(e) for e in self._enabled]))
2221 return result
2222 def labels(self):
2223 return self._labels
2224 def enabledLabels(self):
2225 return self._enabled
2226 def moduleTypeResolver(self, accelerators):
2227 if not self._moduleTypeResolverMaker:
2228 return super().moduleTypeResolver(accelerators)
2229 return self._moduleTypeResolverMaker(accelerators)
2230 def apply(self, process, accelerators):
2231 process.AcceleratorTestService = Service("AcceleratorTestService")
2232 specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorTest, "from test import ProcessAcceleratorTest")
2233
2234 class ProcessAcceleratorTest2(ProcessAccelerator):
2235 def __init__(self, enabled=["anothertest3", "anothertest4"], moduleTypeResolverMaker=None):
2236 super().__init__()
2237 self._labels = ["anothertest3", "anothertest4"]
2238 self.setEnabled(enabled)
2239 self._moduleTypeResolverMaker = moduleTypeResolverMaker
2240 def setEnabled(self, enabled):
2241 invalid = set(enabled).difference(set(self._labels))
2242 if len(invalid) > 0:
2243 raise Exception("Tried to enabled nonexistent test accelerators {}".format(",".join(invalid)))
2244 self._enabled = enabled[:]
2245 def dumpPythonImpl(self,options):
2246 result = "{}enabled = [{}]".format(options.indentation(),
2247 ", ".join(["'{}'".format(e) for e in self._enabled]))
2248 return result
2249 def labels(self):
2250 return self._labels
2251 def enabledLabels(self):
2252 return self._enabled
2253 def moduleTypeResolver(self, accelerators):
2254 if not self._moduleTypeResolverMaker:
2255 return super().moduleTypeResolver(accelerators)
2256 return self._moduleTypeResolverMaker(accelerators)
2257 def apply(self, process, accelerators):
2258 pass
2259 specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorTest2, "from test import ProcessAcceleratorTest2")
2260
2261 class TestModuleCommand(unittest.TestCase):
2262 def setUp(self):
2263 """Nothing to do """
2264 None
2265 def testParameterizable(self):
2266 p = _Parameterizable()
2267 self.assertEqual(len(p.parameterNames_()),0)
2268 p.a = int32(1)
2269 self.assertTrue('a' in p.parameterNames_())
2270 self.assertEqual(p.a.value(), 1)
2271 p.a = 10
2272 self.assertEqual(p.a.value(), 10)
2273 p.a = untracked(int32(1))
2274 self.assertEqual(p.a.value(), 1)
2275 self.assertFalse(p.a.isTracked())
2276 p.a = untracked.int32(1)
2277 self.assertEqual(p.a.value(), 1)
2278 self.assertFalse(p.a.isTracked())
2279 p = _Parameterizable(foo=int32(10), bar = untracked(double(1.0)))
2280 self.assertEqual(p.foo.value(), 10)
2281 self.assertEqual(p.bar.value(),1.0)
2282 self.assertFalse(p.bar.isTracked())
2283 self.assertRaises(TypeError,setattr,(p,'c',1))
2284 p = _Parameterizable(a=PSet(foo=int32(10), bar = untracked(double(1.0))))
2285 self.assertEqual(p.a.foo.value(),10)
2286 self.assertEqual(p.a.bar.value(),1.0)
2287 p.b = untracked(PSet(fii = int32(1)))
2288 self.assertEqual(p.b.fii.value(),1)
2289 self.assertFalse(p.b.isTracked())
2290
2291 v = int32(10)
2292 p=_Parameterizable(a=v)
2293 v.setValue(11)
2294 self.assertEqual(p.a.value(),11)
2295 p.a = 12
2296 self.assertEqual(p.a.value(),12)
2297 self.assertEqual(v.value(),12)
2298 def testTypedParameterizable(self):
2299 p = _TypedParameterizable("blah", b=int32(1))
2300
2301 other = p.copy()
2302 other.b = 2
2303 self.assertNotEqual(p.b,other.b)
2304
2305 def testProcessInsertion(self):
2306 p = Process("test")
2307 p.a = EDAnalyzer("MyAnalyzer")
2308 self.assertTrue( 'a' in p.analyzers_() )
2309 self.assertTrue( 'a' in p.analyzers)
2310 p.add_(Service("SomeService"))
2311 self.assertTrue('SomeService' in p.services_())
2312 self.assertEqual(p.SomeService.type_(), "SomeService")
2313 p.Tracer = Service("Tracer")
2314 self.assertTrue('Tracer' in p.services_())
2315 self.assertRaises(TypeError, setattr, *(p,'b',"this should fail"))
2316 self.assertRaises(TypeError, setattr, *(p,'bad',Service("MessageLogger")))
2317 self.assertRaises(ValueError, setattr, *(p,'bad',Source("PoolSource")))
2318 p.out = OutputModule("Outer")
2319 self.assertEqual(p.out.type_(), 'Outer')
2320 self.assertTrue( 'out' in p.outputModules_() )
2321
2322 p.geom = ESSource("GeomProd")
2323 self.assertTrue('geom' in p.es_sources_())
2324 p.add_(ESSource("ConfigDB"))
2325 self.assertTrue('ConfigDB' in p.es_sources_())
2326
2327 p.aliasfoo1 = EDAlias(foo1 = VPSet(PSet(type = string("Foo1"))))
2328 self.assertTrue('aliasfoo1' in p.aliases_())
2329
2330 def testProcessExtend(self):
2331 class FromArg(object):
2332 def __init__(self,*arg,**args):
2333 for name in args.keys():
2334 self.__dict__[name]=args[name]
2335
2336 a=EDAnalyzer("MyAnalyzer")
2337 t=EDAnalyzer("MyAnalyzer")
2338 t.setLabel("foo")
2339 s1 = Sequence(a)
2340 s2 = Sequence(s1)
2341 s3 = Sequence(s2)
2342 d = FromArg(
2343 a=a,
2344 b=Service("Full"),
2345 c=Path(a),
2346 d=s2,
2347 e=s1,
2348 f=s3,
2349 g=Sequence(s1+s2+s3)
2350 )
2351 p = Process("Test")
2352 p.extend(d)
2353 self.assertEqual(p.a.type_(),"MyAnalyzer")
2354 self.assertEqual(p.a.label_(),"a")
2355 self.assertRaises(AttributeError,getattr,p,'b')
2356 self.assertEqual(p.Full.type_(),"Full")
2357 self.assertEqual(str(p.c),'a')
2358 self.assertEqual(str(p.d),'a')
2359
2360 z1 = FromArg(
2361 a=a,
2362 b=Service("Full"),
2363 c=Path(a),
2364 d=s2,
2365 e=s1,
2366 f=s3,
2367 s4=s3,
2368 g=Sequence(s1+s2+s3)
2369 )
2370
2371 p1 = Process("Test")
2372
2373 self.assertRaises(ValueError, p1.extend, z1)
2374
2375 z2 = FromArg(
2376 a=a,
2377 b=Service("Full"),
2378 c=Path(a),
2379 d=s2,
2380 e=s1,
2381 f=s3,
2382 aaa=copy.deepcopy(a),
2383 s4=copy.deepcopy(s3),
2384 g=Sequence(s1+s2+s3),
2385 t=t
2386 )
2387 p2 = Process("Test")
2388 p2.extend(z2)
2389
2390 self.assertEqual(p2.s4.label_(),"s4")
2391
2392 self.assertRaises(ValueError, p2.s4.setLabel, "foo")
2393 p2.s4.setLabel("s4")
2394 p2.s4.setLabel(None)
2395 p2.s4.setLabel("foo")
2396 p2._Process__setObjectLabel(p2.s4, "foo")
2397 p2._Process__setObjectLabel(p2.s4, None)
2398 p2._Process__setObjectLabel(p2.s4, "bar")
2399
2400
2401 p = Process('test')
2402 p.a = EDProducer("MyProducer")
2403 p.t = Task(p.a)
2404 p.p = Path(p.t)
2405 self.assertRaises(ValueError, p.extend, FromArg(a = EDProducer("YourProducer")))
2406 self.assertRaises(ValueError, p.extend, FromArg(a = EDAlias()))
2407 self.assertRaises(ValueError, p.__setattr__, "a", EDAlias())
2408
2409 p = Process('test')
2410 p.a = EDProducer("MyProducer")
2411 p.t = ConditionalTask(p.a)
2412 p.p = Path(p.t)
2413 self.assertRaises(ValueError, p.extend, FromArg(a = EDProducer("YourProducer")))
2414 self.assertRaises(ValueError, p.extend, FromArg(a = EDAlias()))
2415 self.assertRaises(ValueError, p.__setattr__, "a", EDAlias())
2416
2417 p = Process('test')
2418 p.a = EDProducer("MyProducer")
2419 p.s = Sequence(p.a)
2420 p.p = Path(p.s)
2421 self.assertRaises(ValueError, p.extend, FromArg(a = EDProducer("YourProducer")))
2422 self.assertRaises(ValueError, p.extend, FromArg(a = EDAlias()))
2423 self.assertRaises(ValueError, p.__setattr__, "a", EDAlias())
2424
2425 def testProcessDumpPython(self):
2426 self.assertEqual(Process("test").dumpPython(),
2427 """import FWCore.ParameterSet.Config as cms
2428
2429 process = cms.Process("test")
2430
2431 process.maxEvents = cms.untracked.PSet(
2432 input = cms.optional.untracked.int32,
2433 output = cms.optional.untracked.allowed(cms.int32,cms.PSet)
2434 )
2435
2436 process.maxLuminosityBlocks = cms.untracked.PSet(
2437 input = cms.untracked.int32(-1)
2438 )
2439
2440 process.options = cms.untracked.PSet(
2441 FailPath = cms.untracked.vstring(),
2442 IgnoreCompletely = cms.untracked.vstring(),
2443 Rethrow = cms.untracked.vstring(),
2444 SkipEvent = cms.untracked.vstring(),
2445 accelerators = cms.untracked.vstring('*'),
2446 allowUnscheduled = cms.obsolete.untracked.bool,
2447 canDeleteEarly = cms.untracked.vstring(),
2448 deleteNonConsumedUnscheduledModules = cms.untracked.bool(True),
2449 dumpOptions = cms.untracked.bool(False),
2450 emptyRunLumiMode = cms.obsolete.untracked.string,
2451 eventSetup = cms.untracked.PSet(
2452 forceNumberOfConcurrentIOVs = cms.untracked.PSet(
2453 allowAnyLabel_=cms.required.untracked.uint32
2454 ),
2455 numberOfConcurrentIOVs = cms.untracked.uint32(0)
2456 ),
2457 fileMode = cms.untracked.string('FULLMERGE'),
2458 forceEventSetupCacheClearOnNewRun = cms.untracked.bool(False),
2459 holdsReferencesToDeleteEarly = cms.untracked.VPSet(),
2460 makeTriggerResults = cms.obsolete.untracked.bool,
2461 modulesToIgnoreForDeleteEarly = cms.untracked.vstring(),
2462 numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(0),
2463 numberOfConcurrentRuns = cms.untracked.uint32(1),
2464 numberOfStreams = cms.untracked.uint32(0),
2465 numberOfThreads = cms.untracked.uint32(1),
2466 printDependencies = cms.untracked.bool(False),
2467 sizeOfStackForThreadsInKB = cms.optional.untracked.uint32,
2468 throwIfIllegalParameter = cms.untracked.bool(True),
2469 wantSummary = cms.untracked.bool(False)
2470 )
2471
2472 process.MessageLogger = cms.Service("MessageLogger",
2473 cerr = cms.untracked.PSet(
2474 FwkReport = cms.untracked.PSet(
2475 limit = cms.untracked.int32(10000000),
2476 reportEvery = cms.untracked.int32(1)
2477 ),
2478 FwkSummary = cms.untracked.PSet(
2479 limit = cms.untracked.int32(10000000),
2480 reportEvery = cms.untracked.int32(1)
2481 ),
2482 INFO = cms.untracked.PSet(
2483 limit = cms.untracked.int32(0)
2484 ),
2485 Root_NoDictionary = cms.untracked.PSet(
2486 limit = cms.untracked.int32(0)
2487 ),
2488 default = cms.untracked.PSet(
2489 limit = cms.untracked.int32(10000000)
2490 ),
2491 enable = cms.untracked.bool(True),
2492 enableStatistics = cms.untracked.bool(False),
2493 lineLength = cms.optional.untracked.int32,
2494 noLineBreaks = cms.optional.untracked.bool,
2495 noTimeStamps = cms.untracked.bool(False),
2496 resetStatistics = cms.untracked.bool(False),
2497 statisticsThreshold = cms.untracked.string('WARNING'),
2498 threshold = cms.untracked.string('INFO'),
2499 allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2500 limit = cms.optional.untracked.int32,
2501 reportEvery = cms.untracked.int32(1),
2502 timespan = cms.optional.untracked.int32
2503 )
2504 ),
2505 cout = cms.untracked.PSet(
2506 enable = cms.untracked.bool(False),
2507 enableStatistics = cms.untracked.bool(False),
2508 lineLength = cms.optional.untracked.int32,
2509 noLineBreaks = cms.optional.untracked.bool,
2510 noTimeStamps = cms.optional.untracked.bool,
2511 resetStatistics = cms.untracked.bool(False),
2512 statisticsThreshold = cms.optional.untracked.string,
2513 threshold = cms.optional.untracked.string,
2514 allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2515 limit = cms.optional.untracked.int32,
2516 reportEvery = cms.untracked.int32(1),
2517 timespan = cms.optional.untracked.int32
2518 )
2519 ),
2520 debugModules = cms.untracked.vstring(),
2521 default = cms.untracked.PSet(
2522 limit = cms.optional.untracked.int32,
2523 lineLength = cms.untracked.int32(80),
2524 noLineBreaks = cms.untracked.bool(False),
2525 noTimeStamps = cms.untracked.bool(False),
2526 reportEvery = cms.untracked.int32(1),
2527 statisticsThreshold = cms.untracked.string('INFO'),
2528 threshold = cms.untracked.string('INFO'),
2529 timespan = cms.optional.untracked.int32,
2530 allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2531 limit = cms.optional.untracked.int32,
2532 reportEvery = cms.untracked.int32(1),
2533 timespan = cms.optional.untracked.int32
2534 )
2535 ),
2536 files = cms.untracked.PSet(
2537 allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2538 enableStatistics = cms.untracked.bool(False),
2539 extension = cms.optional.untracked.string,
2540 filename = cms.optional.untracked.string,
2541 lineLength = cms.optional.untracked.int32,
2542 noLineBreaks = cms.optional.untracked.bool,
2543 noTimeStamps = cms.optional.untracked.bool,
2544 output = cms.optional.untracked.string,
2545 resetStatistics = cms.untracked.bool(False),
2546 statisticsThreshold = cms.optional.untracked.string,
2547 threshold = cms.optional.untracked.string,
2548 allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2549 limit = cms.optional.untracked.int32,
2550 reportEvery = cms.untracked.int32(1),
2551 timespan = cms.optional.untracked.int32
2552 )
2553 )
2554 ),
2555 suppressDebug = cms.untracked.vstring(),
2556 suppressFwkInfo = cms.untracked.vstring(),
2557 suppressInfo = cms.untracked.vstring(),
2558 suppressWarning = cms.untracked.vstring(),
2559 allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2560 limit = cms.optional.untracked.int32,
2561 reportEvery = cms.untracked.int32(1),
2562 timespan = cms.optional.untracked.int32
2563 )
2564 )
2565
2566
2567 """)
2568 p = Process("test")
2569 p.a = EDAnalyzer("MyAnalyzer")
2570 p.p = Path(p.a)
2571 p.s = Sequence(p.a)
2572 p.r = Sequence(p.s)
2573 p.p2 = Path(p.s)
2574 p.schedule = Schedule(p.p2,p.p)
2575 d=p.dumpPython()
2576 self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2577 """process.a = cms.EDAnalyzer("MyAnalyzer")
2578 process.s = cms.Sequence(process.a)
2579 process.r = cms.Sequence(process.s)
2580 process.p = cms.Path(process.a)
2581 process.p2 = cms.Path(process.s)
2582 process.schedule = cms.Schedule(*[ process.p2, process.p ])""")
2583
2584 p = Process("test")
2585 p.a = EDAnalyzer("MyAnalyzer")
2586 p.p = Path(p.a)
2587 p.r = Sequence(p.a)
2588 p.s = Sequence(p.r)
2589 p.p2 = Path(p.r)
2590 p.schedule = Schedule(p.p2,p.p)
2591 p.b = EDAnalyzer("YourAnalyzer")
2592 d=p.dumpPython()
2593 self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2594 """process.a = cms.EDAnalyzer("MyAnalyzer")
2595 process.b = cms.EDAnalyzer("YourAnalyzer")
2596 process.r = cms.Sequence(process.a)
2597 process.s = cms.Sequence(process.r)
2598 process.p = cms.Path(process.a)
2599 process.p2 = cms.Path(process.r)
2600 process.schedule = cms.Schedule(*[ process.p2, process.p ])""")
2601
2602 p = Process("test")
2603 p.a = EDAnalyzer("MyAnalyzer")
2604 p.p = Path(p.a)
2605 s = Sequence(p.a)
2606 p.r = Sequence(s)
2607 p.p2 = Path(p.r)
2608 p.schedule = Schedule(p.p2,p.p)
2609 d=p.dumpPython()
2610 self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2611 """process.a = cms.EDAnalyzer("MyAnalyzer")
2612 process.r = cms.Sequence((process.a))
2613 process.p = cms.Path(process.a)
2614 process.p2 = cms.Path(process.r)
2615 process.schedule = cms.Schedule(*[ process.p2, process.p ])""")
2616
2617
2618 p = Process("test")
2619 p.a = EDAnalyzer("MyAnalyzer")
2620 p.b = EDProducer("bProducer")
2621 p.c = EDProducer("cProducer")
2622 p.d = EDProducer("dProducer")
2623 p.e = EDProducer("eProducer")
2624 p.f = EDProducer("fProducer")
2625 p.g = EDProducer("gProducer")
2626 p.task5 = Task()
2627 p.task3 = Task()
2628 p.task2 = Task(p.c, p.task3)
2629 p.task4 = Task(p.f, p.task2)
2630 p.task1 = Task(p.task5)
2631 p.task3.add(p.task1)
2632 p.p = Path(p.a)
2633 s = Sequence(p.a)
2634 p.r = Sequence(s)
2635 p.p2 = Path(p.r, p.task1, p.task2)
2636 p.schedule = Schedule(p.p2,p.p,tasks=[p.task3,p.task4, p.task5])
2637 d=p.dumpPython()
2638 self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2639 """process.b = cms.EDProducer("bProducer")
2640 process.c = cms.EDProducer("cProducer")
2641 process.d = cms.EDProducer("dProducer")
2642 process.e = cms.EDProducer("eProducer")
2643 process.f = cms.EDProducer("fProducer")
2644 process.g = cms.EDProducer("gProducer")
2645 process.a = cms.EDAnalyzer("MyAnalyzer")
2646 process.task5 = cms.Task()
2647 process.task1 = cms.Task(process.task5)
2648 process.task3 = cms.Task(process.task1)
2649 process.task2 = cms.Task(process.c, process.task3)
2650 process.task4 = cms.Task(process.f, process.task2)
2651 process.r = cms.Sequence((process.a))
2652 process.p = cms.Path(process.a)
2653 process.p2 = cms.Path(process.r, process.task1, process.task2)
2654 process.schedule = cms.Schedule(*[ process.p2, process.p ], tasks=[process.task3, process.task4, process.task5])""")
2655
2656 p = Process("test")
2657 p.a = EDAnalyzer("MyAnalyzer")
2658 p.b = EDProducer("bProducer")
2659 p.c = EDProducer("cProducer")
2660 p.d = EDProducer("dProducer")
2661 p.e = EDProducer("eProducer")
2662 p.f = EDProducer("fProducer")
2663 p.g = EDProducer("gProducer")
2664 p.task5 = Task()
2665 p.task3 = Task()
2666 p.task2 = ConditionalTask(p.c, p.task3)
2667 p.task1 = ConditionalTask(p.task5)
2668 p.p = Path(p.a)
2669 s = Sequence(p.a)
2670 p.r = Sequence(s)
2671 p.p2 = Path(p.r, p.task1, p.task2)
2672 p.schedule = Schedule(p.p2,p.p,tasks=[p.task5])
2673 d=p.dumpPython()
2674 self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2675 """process.b = cms.EDProducer("bProducer")
2676 process.c = cms.EDProducer("cProducer")
2677 process.d = cms.EDProducer("dProducer")
2678 process.e = cms.EDProducer("eProducer")
2679 process.f = cms.EDProducer("fProducer")
2680 process.g = cms.EDProducer("gProducer")
2681 process.a = cms.EDAnalyzer("MyAnalyzer")
2682 process.task5 = cms.Task()
2683 process.task3 = cms.Task()
2684 process.task2 = cms.ConditionalTask(process.c, process.task3)
2685 process.task1 = cms.ConditionalTask(process.task5)
2686 process.r = cms.Sequence((process.a))
2687 process.p = cms.Path(process.a)
2688 process.p2 = cms.Path(process.r, process.task1, process.task2)
2689 process.schedule = cms.Schedule(*[ process.p2, process.p ], tasks=[process.task5])""")
2690
2691 p = Process("test")
2692 p.d = EDProducer("dProducer")
2693 p.e = EDProducer("eProducer")
2694 p.f = EDProducer("fProducer")
2695 p.g = EDProducer("gProducer")
2696 p.task1 = Task(p.d, p.e)
2697 task2 = Task(p.f, p.g)
2698 p.schedule = Schedule(tasks=[p.task1,task2])
2699 d=p.dumpPython()
2700 self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2701 """process.d = cms.EDProducer("dProducer")
2702 process.e = cms.EDProducer("eProducer")
2703 process.f = cms.EDProducer("fProducer")
2704 process.g = cms.EDProducer("gProducer")
2705 process.task1 = cms.Task(process.d, process.e)
2706 process.schedule = cms.Schedule(tasks=[cms.Task(process.f, process.g), process.task1])""")
2707
2708 p = Process("test")
2709 p.schedule = Schedule()
2710 d=p.dumpPython()
2711 self.assertEqual(_lineDiff(d,Process('test').dumpPython()),
2712 """process.schedule = cms.Schedule()""")
2713
2714 s = Sequence()
2715 a = EDProducer("A")
2716 s2 = Sequence(a)
2717 s2 += s
2718 process = Process("DUMP")
2719 process.a = a
2720 process.s2 = s2
2721 d=process.dumpPython()
2722 self.assertEqual(_lineDiff(d,Process('DUMP').dumpPython()),
2723 """process.a = cms.EDProducer("A")
2724 process.s2 = cms.Sequence(process.a)""")
2725 s = Sequence()
2726 s1 = Sequence(s)
2727 a = EDProducer("A")
2728 s3 = Sequence(a+a)
2729 s2 = Sequence(a+s3)
2730 s2 += s1
2731 process = Process("DUMP")
2732 process.a = a
2733 process.s2 = s2
2734 d=process.dumpPython()
2735 self.assertEqual(_lineDiff(d,Process('DUMP').dumpPython()),
2736 """process.a = cms.EDProducer("A")
2737 process.s2 = cms.Sequence(process.a+(process.a+process.a))""")
2738
2739 def testSecSource(self):
2740 p = Process('test')
2741 p.a = SecSource("MySecSource")
2742 self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),'process.a = cms.SecSource("MySecSource")')
2743
2744 def testGlobalReplace(self):
2745 p = Process('test')
2746 p.a = EDAnalyzer("MyAnalyzer")
2747 old = p.a
2748 p.b = EDAnalyzer("YourAnalyzer")
2749 p.c = EDAnalyzer("OurAnalyzer")
2750 p.d = EDProducer("MyProducer")
2751 old2 = p.d
2752 p.t1 = Task(p.d)
2753 t2 = Task(p.d)
2754 t3 = Task(p.d)
2755 t4 = Task(p.d)
2756 t5 = Task(p.d)
2757 t6 = Task(p.d)
2758 p.ct1 = ConditionalTask(p.d)
2759 s = Sequence(p.a*p.b)
2760 p.s4 = Sequence(p.a*p.b, p.ct1)
2761 s.associate(t2)
2762 p.s4.associate(t2)
2763 p.p = Path(p.c+s+p.a)
2764 p.p2 = Path(p.c+p.s4+p.a, p.ct1)
2765 p.e3 = EndPath(p.c+s+p.a)
2766 new = EDAnalyzer("NewAnalyzer")
2767 new2 = EDProducer("NewProducer")
2768 visitor1 = NodeVisitor()
2769 p.p.visit(visitor1)
2770 self.assertTrue(visitor1.modules == set([old,old2,p.b,p.c]))
2771 p.schedule = Schedule(tasks=[t6])
2772 p.globalReplace("a",new)
2773 p.globalReplace("d",new2)
2774 visitor2 = NodeVisitor()
2775 p.p.visit(visitor2)
2776 self.assertTrue(visitor2.modules == set([new,new2,p.b,p.c]))
2777 self.assertEqual(p.p.dumpPython()[:-1], "cms.Path(process.c+process.a+process.b+process.a, cms.Task(process.d))")
2778 visitor_p2 = NodeVisitor()
2779 p.p2.visit(visitor_p2)
2780 self.assertTrue(visitor_p2.modules == set([new,new2,p.b,p.c]))
2781 self.assertEqual(p.p2.dumpPython()[:-1], "cms.Path(process.c+process.s4+process.a, process.ct1)")
2782 visitor3 = NodeVisitor()
2783 p.e3.visit(visitor3)
2784 self.assertTrue(visitor3.modules == set([new,new2,p.b,p.c]))
2785 visitor4 = NodeVisitor()
2786 p.s4.visit(visitor4)
2787 self.assertTrue(visitor4.modules == set([new,new2,p.b]))
2788 self.assertEqual(p.s4.dumpPython()[:-1],"cms.Sequence(process.a+process.b, cms.Task(process.d), process.ct1)")
2789 visitor5 = NodeVisitor()
2790 p.t1.visit(visitor5)
2791 self.assertTrue(visitor5.modules == set([new2]))
2792 visitor6 = NodeVisitor()
2793 listOfTasks = list(p.schedule._tasks)
2794 listOfTasks[0].visit(visitor6)
2795 self.assertTrue(visitor6.modules == set([new2]))
2796 visitor7 = NodeVisitor()
2797 p.ct1.visit(visitor7)
2798 self.assertTrue(visitor7.modules == set([new2]))
2799 visitor8 = NodeVisitor()
2800 listOfConditionalTasks = list(p.conditionaltasks.values())
2801 listOfConditionalTasks[0].visit(visitor8)
2802 self.assertTrue(visitor8.modules == set([new2]))
2803
2804
2805 p.d2 = EDProducer("YourProducer")
2806 p.schedule = Schedule(p.p, p.p2, p.e3, tasks=[p.t1])
2807 self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2808 p.p = Path(p.c+s)
2809 self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2810 p.e3 = EndPath(p.c)
2811 self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2812 p.t1 = Task(p.d2)
2813 self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2814
2815 def testSequence(self):
2816 p = Process('test')
2817 p.a = EDAnalyzer("MyAnalyzer")
2818 p.b = EDAnalyzer("YourAnalyzer")
2819 p.c = EDAnalyzer("OurAnalyzer")
2820 p.s = Sequence(p.a*p.b)
2821 self.assertEqual(str(p.s),'a+b')
2822 self.assertEqual(p.s.label_(),'s')
2823 path = Path(p.c+p.s)
2824 self.assertEqual(str(path),'c+a+b')
2825 p._validateSequence(path, 'p1')
2826 notInProcess = EDAnalyzer('NotInProcess')
2827 p2 = Path(p.c+p.s*notInProcess)
2828 self.assertRaises(RuntimeError, p._validateSequence, p2, 'p2')
2829
2830 def testSequence2(self):
2831 p = Process('test')
2832 p.a = EDAnalyzer("MyAnalyzer")
2833 p.b = EDAnalyzer("YourAnalyzer")
2834 p.c = EDAnalyzer("OurAnalyzer")
2835 testseq = Sequence(p.a*p.b)
2836 p.s = testseq
2837
2838 self.assertRaises(ValueError, p.__setattr__, "y", testseq)
2839
2840 def testServiceInProcess(self):
2841 service = Service("d")
2842 self.assertFalse(service._inProcess)
2843 process = Process("test")
2844 process.d = service
2845 self.assertTrue(service._inProcess)
2846 service2 = Service("d")
2847 process.d = service2
2848 self.assertFalse(service._inProcess)
2849 self.assertTrue(service2._inProcess)
2850 del process.d
2851 self.assertFalse(service2._inProcess)
2852
2853 def testTask(self):
2854
2855
2856 edanalyzer = EDAnalyzer("a")
2857 edproducer = EDProducer("b")
2858 edproducer2 = EDProducer("b2")
2859 edproducer3 = EDProducer("b3")
2860 edproducer4 = EDProducer("b4")
2861 edproducer8 = EDProducer("b8")
2862 edproducer9 = EDProducer("b9")
2863 edfilter = EDFilter("c")
2864 service = Service("d")
2865 service3 = Service("d", v = untracked.uint32(3))
2866 essource = ESSource("e")
2867 esproducer = ESProducer("f")
2868 testTask2 = Task()
2869
2870
2871 testTask1 = Task(edproducer, edfilter)
2872 self.assertRaises(RuntimeError, testTask1.add, edanalyzer)
2873 testTask1.add(essource, service)
2874 testTask1.add(essource, esproducer)
2875 testTask1.add(testTask2)
2876 coll = testTask1._collection
2877 self.assertTrue(edproducer in coll)
2878 self.assertTrue(edfilter in coll)
2879 self.assertTrue(service in coll)
2880 self.assertTrue(essource in coll)
2881 self.assertTrue(esproducer in coll)
2882 self.assertTrue(testTask2 in coll)
2883 self.assertTrue(len(coll) == 6)
2884 self.assertTrue(len(testTask2._collection) == 0)
2885
2886 taskContents = []
2887 for i in testTask1:
2888 taskContents.append(i)
2889 self.assertTrue(taskContents == [edproducer, edfilter, essource, service, esproducer, testTask2])
2890
2891
2892 process = Process("test")
2893
2894 process.mproducer = edproducer
2895 process.mproducer2 = edproducer2
2896 process.mfilter = edfilter
2897 process.messource = essource
2898 process.mesproducer = esproducer
2899 process.d = service
2900
2901 testTask3 = Task(edproducer, edproducer2)
2902 testTask1.add(testTask3)
2903 process.myTask1 = testTask1
2904
2905
2906
2907
2908 l = set()
2909 visitor = NodeNameVisitor(l)
2910 testTask1.visit(visitor)
2911 self.assertTrue(l == set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
2912 l2 = testTask1.moduleNames
2913 self.assertTrue(l == set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
2914
2915 testTask4 = Task(edproducer3)
2916 l.clear()
2917 self.assertRaises(RuntimeError, testTask4.visit, visitor)
2918 try:
2919 process.myTask4 = testTask4
2920 self.assertTrue(False)
2921 except RuntimeError:
2922 pass
2923
2924 testTask5 = Task(service3)
2925 l.clear()
2926 self.assertRaises(RuntimeError, testTask5.visit, visitor)
2927 try:
2928 process.myTask5 = testTask5
2929 self.assertTrue(False)
2930 except RuntimeError:
2931 pass
2932
2933 process.d = service3
2934 process.myTask5 = testTask5
2935
2936
2937 expectedDict = { 'myTask1' : testTask1, 'myTask5' : testTask5 }
2938 expectedFixedDict = DictTypes.FixedKeysDict(expectedDict);
2939 self.assertTrue(process.tasks == expectedFixedDict)
2940 self.assertTrue(process.tasks['myTask1'] == testTask1)
2941 self.assertTrue(process.myTask1 == testTask1)
2942
2943
2944
2945 process.mproducer2 = edproducer4
2946 process.d = service
2947 l = list()
2948 visitor1 = ModuleNodeVisitor(l)
2949 testTask1.visit(visitor1)
2950 l.sort(key=lambda mod: mod.__str__())
2951 expectedList = sorted([edproducer,essource,esproducer,service,edfilter,edproducer,edproducer4],key=lambda mod: mod.__str__())
2952 self.assertTrue(expectedList == l)
2953 process.myTask6 = Task()
2954 process.myTask7 = Task()
2955 process.mproducer8 = edproducer8
2956 process.myTask8 = Task(process.mproducer8)
2957 process.myTask6.add(process.myTask7)
2958 process.myTask7.add(process.myTask8)
2959 process.myTask1.add(process.myTask6)
2960 process.myTask8.add(process.myTask5)
2961
2962 testDict = process._itemsInDependencyOrder(process.tasks)
2963 expectedLabels = ["myTask5", "myTask8", "myTask7", "myTask6", "myTask1"]
2964 expectedTasks = [process.myTask5, process.myTask8, process.myTask7, process.myTask6, process.myTask1]
2965 index = 0
2966 for testLabel, testTask in testDict.items():
2967 self.assertTrue(testLabel == expectedLabels[index])
2968 self.assertTrue(testTask == expectedTasks[index])
2969 index += 1
2970
2971 pythonDump = testTask1.dumpPython(PrintOptions())
2972
2973
2974 expectedPythonDump = 'cms.Task(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer, process.mproducer2, process.myTask6)\n'
2975 self.assertTrue(pythonDump == expectedPythonDump)
2976
2977 process.myTask5 = Task()
2978 process.myTask100 = Task()
2979 process.mproducer9 = edproducer9
2980 sequence1 = Sequence(process.mproducer8, process.myTask1, process.myTask5, testTask2, testTask3)
2981 sequence2 = Sequence(process.mproducer8 + process.mproducer9)
2982 process.sequence3 = Sequence((process.mproducer8 + process.mfilter))
2983 sequence4 = Sequence()
2984 process.path1 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+sequence4)
2985 process.path1.associate(process.myTask1, process.myTask5, testTask2, testTask3)
2986 process.path11 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+ sequence4,process.myTask1, process.myTask5, testTask2, testTask3, process.myTask100)
2987 process.path2 = Path(process.mproducer)
2988 process.path3 = Path(process.mproducer9+process.mproducer8,testTask2)
2989
2990 self.assertTrue(process.path1.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)\n')
2991
2992 self.assertTrue(process.path11.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask100, process.myTask5)\n')
2993
2994
2995 l = set()
2996 nameVisitor = NodeNameVisitor(l)
2997 process.path1.visit(nameVisitor)
2998 self.assertTrue(l == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
2999 self.assertTrue(process.path1.moduleNames() == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3000
3001
3002 process.mproducer10 = EDProducer("b10")
3003 process.path21 = process.path11.copy()
3004 process.path21.replace(process.mproducer, process.mproducer10)
3005
3006 self.assertTrue(process.path11.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask100, process.myTask5)\n')
3007
3008
3009
3010
3011
3012
3013
3014
3015 self.assertTrue(process.path21.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer10), cms.Task(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), process.myTask100, process.myTask5)\n')
3016
3017 process.path22 = process.path21.copyAndExclude([process.d, process.mesproducer, process.mfilter])
3018 self.assertTrue(process.path22.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer10), cms.Task(process.messource, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), process.myTask100, process.myTask5)\n')
3019
3020 process.path23 = process.path22.copyAndExclude([process.messource, process.mproducer10])
3021 self.assertTrue(process.path23.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.Task(), cms.Task(process.None), cms.Task(process.mproducer2, process.mproducer8, process.myTask5), process.myTask100, process.myTask5)\n')
3022
3023 process.a = EDAnalyzer("MyAnalyzer")
3024 process.b = OutputModule("MyOutputModule")
3025 process.c = EDFilter("MyFilter")
3026 process.d = EDProducer("MyProducer")
3027 process.e = ESProducer("MyESProducer")
3028 process.f = ESSource("MyESSource")
3029 process.g = ESProducer("g")
3030 process.path24 = Path(process.a+process.b+process.c+process.d)
3031 process.path25 = process.path24.copyAndExclude([process.a,process.b,process.c])
3032 self.assertTrue(process.path25.dumpPython() == 'cms.Path(process.d)\n')
3033
3034
3035
3036 process.path200 = EndPath(Sequence(process.c,Task(process.e)))
3037 process.path200.replace(process.c,process.b)
3038 process.path200.replace(process.e,process.f)
3039 self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.b, cms.Task(process.f))\n")
3040 process.path200.replace(process.b,process.c)
3041 process.path200.replace(process.f,process.e)
3042 self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.c, cms.Task(process.e))\n")
3043 process.path200.replace(process.c,process.a)
3044 process.path200.replace(process.e,process.g)
3045 self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.a, cms.Task(process.g))\n")
3046 process.path200.replace(process.a,process.c)
3047 process.path200.replace(process.g,process.e)
3048 self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.c, cms.Task(process.e))\n")
3049
3050 def testConditionalTask(self):
3051
3052
3053 edanalyzer = EDAnalyzer("a")
3054 edproducer = EDProducer("b")
3055 edproducer2 = EDProducer("b2")
3056 edproducer3 = EDProducer("b3")
3057 edproducer4 = EDProducer("b4")
3058 edproducer8 = EDProducer("b8")
3059 edproducer9 = EDProducer("b9")
3060 edfilter = EDFilter("c")
3061 service = Service("d")
3062 service3 = Service("d", v = untracked.uint32(3))
3063 essource = ESSource("e")
3064 esproducer = ESProducer("f")
3065 testTask2 = Task()
3066 testCTask2 = ConditionalTask()
3067
3068
3069 testTask1 = ConditionalTask(edproducer, edfilter)
3070 self.assertRaises(RuntimeError, testTask1.add, edanalyzer)
3071 testTask1.add(essource, service)
3072 testTask1.add(essource, esproducer)
3073 testTask1.add(testTask2)
3074 testTask1.add(testCTask2)
3075 coll = testTask1._collection
3076 self.assertTrue(edproducer in coll)
3077 self.assertTrue(edfilter in coll)
3078 self.assertTrue(service in coll)
3079 self.assertTrue(essource in coll)
3080 self.assertTrue(esproducer in coll)
3081 self.assertTrue(testTask2 in coll)
3082 self.assertTrue(testCTask2 in coll)
3083 self.assertTrue(len(coll) == 7)
3084 self.assertTrue(len(testTask2._collection) == 0)
3085
3086 taskContents = []
3087 for i in testTask1:
3088 taskContents.append(i)
3089 self.assertEqual(taskContents, [edproducer, edfilter, essource, service, esproducer, testTask2, testCTask2])
3090
3091
3092 process = Process("test")
3093
3094 process.mproducer = edproducer
3095 process.mproducer2 = edproducer2
3096 process.mfilter = edfilter
3097 process.messource = essource
3098 process.mesproducer = esproducer
3099 process.d = service
3100
3101 testTask3 = ConditionalTask(edproducer, edproducer2)
3102 testTask1.add(testTask3)
3103 process.myTask1 = testTask1
3104
3105
3106
3107
3108 l = set()
3109 visitor = NodeNameVisitor(l)
3110 testTask1.visit(visitor)
3111 self.assertEqual(l, set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
3112 l2 = testTask1.moduleNames()
3113 self.assertEqual(l2, set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
3114
3115 testTask4 = ConditionalTask(edproducer3)
3116 l.clear()
3117 self.assertRaises(RuntimeError, testTask4.visit, visitor)
3118 try:
3119 process.myTask4 = testTask4
3120 self.assertTrue(False)
3121 except RuntimeError:
3122 pass
3123
3124 testTask5 = ConditionalTask(service3)
3125 l.clear()
3126 self.assertRaises(RuntimeError, testTask5.visit, visitor)
3127 try:
3128 process.myTask5 = testTask5
3129 self.assertTrue(False)
3130 except RuntimeError:
3131 pass
3132
3133 process.d = service3
3134 process.myTask5 = testTask5
3135
3136
3137 expectedDict = { 'myTask1' : testTask1, 'myTask5' : testTask5 }
3138 expectedFixedDict = DictTypes.FixedKeysDict(expectedDict);
3139 self.assertEqual(process.conditionaltasks, expectedFixedDict)
3140 self.assertEqual(process.conditionaltasks['myTask1'], testTask1)
3141 self.assertEqual(process.myTask1, testTask1)
3142
3143
3144
3145 process.mproducer2 = edproducer4
3146 process.d = service
3147 l = list()
3148 visitor1 = ModuleNodeVisitor(l)
3149 testTask1.visit(visitor1)
3150 l.sort(key=lambda mod: mod.__str__())
3151 expectedList = sorted([edproducer,essource,esproducer,service,edfilter,edproducer,edproducer4],key=lambda mod: mod.__str__())
3152 self.assertEqual(expectedList, l)
3153 process.myTask6 = ConditionalTask()
3154 process.myTask7 = ConditionalTask()
3155 process.mproducer8 = edproducer8
3156 process.myTask8 = ConditionalTask(process.mproducer8)
3157 process.myTask6.add(process.myTask7)
3158 process.myTask7.add(process.myTask8)
3159 process.myTask1.add(process.myTask6)
3160 process.myTask8.add(process.myTask5)
3161 self.assertEqual(process.myTask8.dumpPython(), "cms.ConditionalTask(process.mproducer8, process.myTask5)\n")
3162
3163 testDict = process._itemsInDependencyOrder(process.conditionaltasks)
3164 expectedLabels = ["myTask5", "myTask8", "myTask7", "myTask6", "myTask1"]
3165 expectedTasks = [process.myTask5, process.myTask8, process.myTask7, process.myTask6, process.myTask1]
3166 index = 0
3167 for testLabel, testTask in testDict.items():
3168 self.assertEqual(testLabel, expectedLabels[index])
3169 self.assertEqual(testTask, expectedTasks[index])
3170 index += 1
3171
3172 pythonDump = testTask1.dumpPython(PrintOptions())
3173
3174
3175 expectedPythonDump = 'cms.ConditionalTask(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer, process.mproducer2, process.myTask6)\n'
3176 self.assertEqual(pythonDump, expectedPythonDump)
3177
3178 process.myTask5 = ConditionalTask()
3179 self.assertEqual(process.myTask8.dumpPython(), "cms.ConditionalTask(process.mproducer8, process.myTask5)\n")
3180 process.myTask100 = ConditionalTask()
3181 process.mproducer9 = edproducer9
3182 sequence1 = Sequence(process.mproducer8, process.myTask1, process.myTask5, testTask2, testTask3)
3183 sequence2 = Sequence(process.mproducer8 + process.mproducer9)
3184 process.sequence3 = Sequence((process.mproducer8 + process.mfilter))
3185 sequence4 = Sequence()
3186 process.path1 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+sequence4)
3187 process.path1.associate(process.myTask1, process.myTask5, testTask2, testTask3)
3188 process.path11 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+ sequence4,process.myTask1, process.myTask5, testTask2, testTask3, process.myTask100)
3189 process.path2 = Path(process.mproducer)
3190 process.path3 = Path(process.mproducer9+process.mproducer8,testTask2)
3191
3192 self.assertEqual(process.path1.dumpPython(PrintOptions()), 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)\n')
3193
3194 self.assertEqual(process.path11.dumpPython(PrintOptions()), 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask100, process.myTask5)\n')
3195
3196
3197 l = set()
3198 nameVisitor = NodeNameVisitor(l)
3199 process.path1.visit(nameVisitor)
3200 self.assertTrue(l == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3201 self.assertTrue(process.path1.moduleNames() == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3202
3203
3204 process.mproducer10 = EDProducer("b10")
3205 process.path21 = process.path11.copy()
3206 process.path21.replace(process.mproducer, process.mproducer10)
3207
3208 self.assertEqual(process.path11.dumpPython(PrintOptions()), 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask100, process.myTask5)\n')
3209
3210
3211
3212
3213
3214
3215
3216
3217 self.assertEqual(process.path21.dumpPython(PrintOptions()), 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer10), cms.ConditionalTask(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), cms.Task(), process.myTask100, process.myTask5)\n')
3218
3219 process.path22 = process.path21.copyAndExclude([process.d, process.mesproducer, process.mfilter])
3220 self.assertEqual(process.path22.dumpPython(PrintOptions()), 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.ConditionalTask(process.None, process.mproducer10), cms.ConditionalTask(process.messource, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), cms.Task(), process.myTask100, process.myTask5)\n')
3221
3222 process.path23 = process.path22.copyAndExclude([process.messource, process.mproducer10])
3223 self.assertEqual(process.path23.dumpPython(PrintOptions()), 'cms.Path(process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.ConditionalTask(process.None), cms.ConditionalTask(process.mproducer2, process.mproducer8, process.myTask5), cms.Task(), process.myTask100, process.myTask5)\n')
3224
3225 process = Process("Test")
3226
3227 process.b = EDProducer("b")
3228 process.b2 = EDProducer("b2")
3229 process.b3 = EDProducer("b3")
3230 process.p = Path(process.b, ConditionalTask(process.b3, process.b2))
3231 p = TestMakePSet()
3232 process.fillProcessDesc(p)
3233 self.assertEqual(p.values["@all_modules"], (True, ['b', 'b2', 'b3']))
3234 self.assertEqual(p.values["@paths"], (True, ['p']))
3235 self.assertEqual(p.values["p"], (True, ['b','#','b2','b3','@']))
3236
3237
3238 def testPath(self):
3239 p = Process("test")
3240 p.a = EDAnalyzer("MyAnalyzer")
3241 p.b = EDAnalyzer("YourAnalyzer")
3242 p.c = EDAnalyzer("OurAnalyzer")
3243 path = Path(p.a)
3244 path *= p.b
3245 path += p.c
3246 self.assertEqual(str(path),'a+b+c')
3247 path = Path(p.a*p.b+p.c)
3248 self.assertEqual(str(path),'a+b+c')
3249
3250
3251 path = Path(p.a+ p.b*p.c)
3252 self.assertEqual(str(path),'a+b+c')
3253 path = Path(p.a*(p.b+p.c))
3254 self.assertEqual(str(path),'a+b+c')
3255 path = Path(p.a*(p.b+~p.c))
3256 pathx = Path(p.a*(p.b+ignore(p.c)))
3257 self.assertEqual(str(path),'a+b+~c')
3258 p.es = ESProducer("AnESProducer")
3259 self.assertRaises(TypeError,Path,p.es)
3260
3261 t = Path()
3262 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path()\n')
3263
3264 t = Path(p.a)
3265 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a)\n')
3266
3267 t = Path(Task())
3268 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(cms.Task())\n')
3269
3270 t = Path(p.a, Task())
3271 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.Task())\n')
3272
3273 p.prod = EDProducer("prodName")
3274 p.t1 = Task(p.prod)
3275 t = Path(p.a, p.t1, Task(), p.t1)
3276 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.Task(), process.t1)\n')
3277
3278 t = Path(ConditionalTask())
3279 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(cms.ConditionalTask())\n')
3280
3281 t = Path(p.a, ConditionalTask())
3282 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.ConditionalTask())\n')
3283
3284 p.prod = EDProducer("prodName")
3285 p.t1 = ConditionalTask(p.prod)
3286 t = Path(p.a, p.t1, Task(), p.t1)
3287 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.Task(), process.t1)\n')
3288
3289 def testFinalPath(self):
3290 p = Process("test")
3291 p.a = OutputModule("MyOutputModule")
3292 p.b = OutputModule("YourOutputModule")
3293 p.c = OutputModule("OurOutputModule")
3294 path = FinalPath(p.a)
3295 path *= p.b
3296 path += p.c
3297 self.assertEqual(str(path),'a+b+c')
3298 path = FinalPath(p.a*p.b+p.c)
3299 self.assertEqual(str(path),'a+b+c')
3300 path = FinalPath(p.a+ p.b*p.c)
3301 self.assertEqual(str(path),'a+b+c')
3302 path = FinalPath(p.a*(p.b+p.c))
3303 self.assertEqual(str(path),'a+b+c')
3304 p.es = ESProducer("AnESProducer")
3305 self.assertRaises(TypeError,FinalPath,p.es)
3306
3307 t = FinalPath()
3308 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.FinalPath()\n')
3309
3310 t = FinalPath(p.a)
3311 self.assertEqual(t.dumpPython(PrintOptions()), 'cms.FinalPath(process.a)\n')
3312
3313 self.assertRaises(TypeError, FinalPath, Task())
3314 self.assertRaises(TypeError, FinalPath, p.a, Task())
3315
3316 p.prod = EDProducer("prodName")
3317 p.t1 = Task(p.prod)
3318 self.assertRaises(TypeError, FinalPath, p.a, p.t1, Task(), p.t1)
3319
3320 p.prod = EDProducer("prodName")
3321 p.t1 = ConditionalTask(p.prod)
3322 self.assertRaises(TypeError, FinalPath, p.a, p.t1, ConditionalTask(), p.t1)
3323
3324 p.t = FinalPath(p.a)
3325 p.a = OutputModule("ReplacedOutputModule")
3326 self.assertEqual(p.t.dumpPython(PrintOptions()), 'cms.FinalPath(process.a)\n')
3327
3328 def testCloneSequence(self):
3329 p = Process("test")
3330 a = EDAnalyzer("MyAnalyzer")
3331 p.a = a
3332 a.setLabel("a")
3333 b = EDAnalyzer("YOurAnalyzer")
3334 p.b = b
3335 b.setLabel("b")
3336 path = Path(a * b)
3337 p.path = Path(p.a*p.b)
3338 lookuptable = {id(a): p.a, id(b): p.b}
3339
3340
3341
3342 self.assertEqual(str(path),str(p.path))
3343
3344 def testContains(self):
3345
3346 a = EDProducer("a")
3347 b = EDProducer("b")
3348 c = EDProducer("c")
3349 d = EDProducer("d")
3350 e = EDProducer("e")
3351 f = EDProducer("f")
3352 g = EDProducer("g")
3353 h = EDProducer("h")
3354 i = EDProducer("i")
3355 j = EDProducer("j")
3356 k = EDProducer("k")
3357 l = EDProducer("l")
3358 m = EDProducer("m")
3359 n = EDProducer("n")
3360
3361 seq1 = Sequence(e)
3362 task1 = Task(g)
3363 ctask1 = ConditionalTask(h)
3364 path = Path(a * c * seq1, task1, ctask1)
3365
3366 self.assertTrue(path.contains(a))
3367 self.assertFalse(path.contains(b))
3368 self.assertTrue(path.contains(c))
3369 self.assertFalse(path.contains(d))
3370 self.assertTrue(path.contains(e))
3371 self.assertFalse(path.contains(f))
3372 self.assertTrue(path.contains(g))
3373 self.assertTrue(path.contains(h))
3374
3375 endpath = EndPath(h * i)
3376 self.assertFalse(endpath.contains(b))
3377 self.assertTrue(endpath.contains(i))
3378
3379 seq = Sequence(a * c)
3380 self.assertFalse(seq.contains(b))
3381 self.assertTrue(seq.contains(c))
3382
3383 task2 = Task(l)
3384 task = Task(j, k, task2)
3385 self.assertFalse(task.contains(b))
3386 self.assertTrue(task.contains(j))
3387 self.assertTrue(task.contains(k))
3388 self.assertTrue(task.contains(l))
3389
3390 task3 = Task(m)
3391 path2 = Path(n)
3392 sch = Schedule(path, path2, tasks=[task,task3])
3393 self.assertFalse(sch.contains(b))
3394 self.assertTrue(sch.contains(a))
3395 self.assertTrue(sch.contains(c))
3396 self.assertTrue(sch.contains(e))
3397 self.assertTrue(sch.contains(g))
3398 self.assertTrue(sch.contains(n))
3399 self.assertTrue(sch.contains(j))
3400 self.assertTrue(sch.contains(k))
3401 self.assertTrue(sch.contains(l))
3402 self.assertTrue(sch.contains(m))
3403
3404 ctask2 = ConditionalTask(l, task1)
3405 ctask = ConditionalTask(j, k, ctask2)
3406 self.assertFalse(ctask.contains(b))
3407 self.assertTrue(ctask.contains(j))
3408 self.assertTrue(ctask.contains(k))
3409 self.assertTrue(ctask.contains(l))
3410 self.assertTrue(ctask.contains(g))
3411
3412 def testSchedule(self):
3413 p = Process("test")
3414 p.a = EDAnalyzer("MyAnalyzer")
3415 p.b = EDAnalyzer("YourAnalyzer")
3416 p.c = EDAnalyzer("OurAnalyzer")
3417 p.d = EDAnalyzer("OurAnalyzer")
3418 p.path1 = Path(p.a)
3419 p.path2 = Path(p.b)
3420 p.path3 = Path(p.d)
3421
3422 s = Schedule(p.path1,p.path2)
3423 self.assertEqual(s[0],p.path1)
3424 self.assertEqual(s[1],p.path2)
3425 p.schedule = s
3426 self.assertTrue('b' in p.schedule.moduleNames())
3427 self.assertTrue(hasattr(p, 'b'))
3428 self.assertTrue(hasattr(p, 'c'))
3429 self.assertTrue(hasattr(p, 'd'))
3430 self.assertTrue(hasattr(p, 'path1'))
3431 self.assertTrue(hasattr(p, 'path2'))
3432 self.assertTrue(hasattr(p, 'path3'))
3433 p.prune()
3434 self.assertTrue('b' in p.schedule.moduleNames())
3435 self.assertTrue(hasattr(p, 'b'))
3436 self.assertTrue(not hasattr(p, 'c'))
3437 self.assertTrue(not hasattr(p, 'd'))
3438 self.assertTrue(hasattr(p, 'path1'))
3439 self.assertTrue(hasattr(p, 'path2'))
3440 self.assertTrue(not hasattr(p, 'path3'))
3441
3442 self.assertTrue(len(p.schedule._tasks) == 0)
3443
3444 p = Process("test")
3445 p.a = EDAnalyzer("MyAnalyzer")
3446 p.b = EDAnalyzer("YourAnalyzer")
3447 p.c = EDAnalyzer("OurAnalyzer")
3448 p.d = EDAnalyzer("dAnalyzer")
3449 p.e = EDProducer("eProducer")
3450 p.f = EDProducer("fProducer")
3451 p.Tracer = Service("Tracer")
3452 p.path1 = Path(p.a)
3453 p.path2 = Path(p.b)
3454 p.path3 = Path(p.d)
3455 p.task1 = Task(p.e)
3456 p.task2 = Task(p.f, p.Tracer)
3457 s = Schedule(p.path1,p.path2,tasks=[p.task1,p.task2,p.task1])
3458 self.assertEqual(s[0],p.path1)
3459 self.assertEqual(s[1],p.path2)
3460 self.assertTrue(len(s._tasks) == 2)
3461 self.assertTrue(p.task1 in s._tasks)
3462 self.assertTrue(p.task2 in s._tasks)
3463 listOfTasks = list(s._tasks)
3464 self.assertTrue(len(listOfTasks) == 2)
3465 self.assertTrue(p.task1 == listOfTasks[0])
3466 self.assertTrue(p.task2 == listOfTasks[1])
3467 p.schedule = s
3468 self.assertTrue('b' in p.schedule.moduleNames())
3469
3470 process2 = Process("test")
3471 process2.a = EDAnalyzer("MyAnalyzer")
3472 process2.e = EDProducer("eProducer")
3473 process2.path1 = Path(process2.a)
3474 process2.task1 = Task(process2.e)
3475 process2.schedule = Schedule(process2.path1,tasks=process2.task1)
3476 listOfTasks = list(process2.schedule._tasks)
3477 self.assertTrue(listOfTasks[0] == process2.task1)
3478
3479
3480 s2 = s.copy()
3481 self.assertEqual(s2[0],p.path1)
3482 self.assertEqual(s2[1],p.path2)
3483 self.assertTrue(len(s2._tasks) == 2)
3484 self.assertTrue(p.task1 in s2._tasks)
3485 self.assertTrue(p.task2 in s2._tasks)
3486 listOfTasks = list(s2._tasks)
3487 self.assertTrue(len(listOfTasks) == 2)
3488 self.assertTrue(p.task1 == listOfTasks[0])
3489 self.assertTrue(p.task2 == listOfTasks[1])
3490
3491 names = s.moduleNames()
3492 self.assertTrue(names == set(['a', 'b', 'e', 'Tracer', 'f']))
3493
3494 p = Process("test")
3495 p.a = EDAnalyzer("MyAnalyzer")
3496 path1 = Path(p.a)
3497 s = Schedule(path1)
3498 self.assertRaises(RuntimeError, lambda : p.setSchedule_(s) )
3499
3500
3501 p = Process("test")
3502 p.a = EDAnalyzer("MyAnalyzer")
3503 p.b = EDAnalyzer("MyOtherAnalyzer")
3504 p.c = EDProducer("MyProd")
3505 path1 = Path(p.c*Sequence(p.a+p.b))
3506 s = Schedule(path1)
3507 self.assertTrue('a' in s.moduleNames())
3508 self.assertTrue('b' in s.moduleNames())
3509 self.assertTrue('c' in s.moduleNames())
3510 p.path1 = path1
3511 p.schedule = s
3512 p.prune()
3513 self.assertTrue('a' in s.moduleNames())
3514 self.assertTrue('b' in s.moduleNames())
3515 self.assertTrue('c' in s.moduleNames())
3516
3517 def testImplicitSchedule(self):
3518 p = Process("test")
3519 p.a = EDAnalyzer("MyAnalyzer")
3520 p.b = EDAnalyzer("YourAnalyzer")
3521 p.c = EDAnalyzer("OurAnalyzer")
3522 p.path1 = Path(p.a)
3523 p.path2 = Path(p.b)
3524 self.assertTrue(p.schedule is None)
3525 pths = p.paths
3526 keys = pths.keys()
3527 self.assertEqual(pths[keys[0]],p.path1)
3528 self.assertEqual(pths[keys[1]],p.path2)
3529 p.prune()
3530 self.assertTrue(hasattr(p, 'a'))
3531 self.assertTrue(hasattr(p, 'b'))
3532 self.assertTrue(not hasattr(p, 'c'))
3533 self.assertTrue(hasattr(p, 'path1'))
3534 self.assertTrue(hasattr(p, 'path2'))
3535
3536
3537 p = Process("test")
3538 p.a = EDAnalyzer("MyAnalyzer")
3539 p.b = EDAnalyzer("YourAnalyzer")
3540 p.c = EDAnalyzer("OurAnalyzer")
3541 p.path2 = Path(p.b)
3542 p.path1 = Path(p.a)
3543 self.assertTrue(p.schedule is None)
3544 pths = p.paths
3545 keys = pths.keys()
3546 self.assertEqual(pths[keys[1]],p.path1)
3547 self.assertEqual(pths[keys[0]],p.path2)
3548
3549
3550 def testUsing(self):
3551 p = Process('test')
3552 p.block = PSet(a = int32(1))
3553 p.modu = EDAnalyzer('Analyzer', p.block, b = int32(2))
3554 self.assertEqual(p.modu.a.value(),1)
3555 self.assertEqual(p.modu.b.value(),2)
3556
3557 def testOverride(self):
3558 p = Process('test')
3559 a = EDProducer("A", a1=int32(0))
3560 self.assertTrue(not a.isModified())
3561 a.a1 = 1
3562 self.assertTrue(a.isModified())
3563 p.a = a
3564 self.assertEqual(p.a.a1.value(), 1)
3565
3566
3567 p.a = EDProducer("A", a1=int32(2))
3568 self.assertEqual(p.a.a1.value(), 2)
3569
3570
3571
3572 b = EDProducer("A", a1=int32(3))
3573 b.a1 = 4
3574
3575 ps1 = PSet(a = int32(1))
3576 ps2 = PSet(a = int32(2))
3577 self.assertRaises(ValueError, EDProducer, 'C', ps1, ps2)
3578 self.assertRaises(ValueError, EDProducer, 'C', ps1, a=int32(3))
3579
3580 def testOptions(self):
3581 p = Process('test')
3582 self.assertEqual(p.options.numberOfThreads.value(),1)
3583 p.options.numberOfThreads = 8
3584 self.assertEqual(p.options.numberOfThreads.value(),8)
3585 p.options = PSet()
3586 self.assertEqual(p.options.numberOfThreads.value(),1)
3587 p.options = dict(numberOfStreams =2,
3588 numberOfThreads =2)
3589 self.assertEqual(p.options.numberOfThreads.value(),2)
3590 self.assertEqual(p.options.numberOfStreams.value(),2)
3591
3592 def testMaxEvents(self):
3593 p = Process("Test")
3594 p.maxEvents.input = 10
3595 self.assertEqual(p.maxEvents.input.value(),10)
3596 p = Process("Test")
3597 p.maxEvents.output = 10
3598 self.assertEqual(p.maxEvents.output.value(),10)
3599 p = Process("Test")
3600 p.maxEvents.output = PSet(out=untracked.int32(10))
3601 self.assertEqual(p.maxEvents.output.out.value(), 10)
3602 p = Process("Test")
3603 p.maxEvents = untracked.PSet(input = untracked.int32(5))
3604 self.assertEqual(p.maxEvents.input.value(), 5)
3605
3606
3607 def testExamples(self):
3608 p = Process("Test")
3609 p.source = Source("PoolSource",fileNames = untracked(string("file:reco.root")))
3610 p.foos = EDProducer("FooProducer")
3611 p.bars = EDProducer("BarProducer", foos=InputTag("foos"))
3612 p.out = OutputModule("PoolOutputModule",fileName=untracked(string("file:foos.root")))
3613 p.bars.foos = 'Foosball'
3614 self.assertEqual(p.bars.foos, InputTag('Foosball'))
3615 p.p = Path(p.foos*p.bars)
3616 p.e = EndPath(p.out)
3617 p.add_(Service("MessageLogger"))
3618
3619 def testPrefers(self):
3620 p = Process("Test")
3621 p.add_(ESSource("ForceSource"))
3622 p.juicer = ESProducer("JuicerProducer")
3623 p.prefer("ForceSource")
3624 p.prefer("juicer")
3625 self.assertEqual(_lineDiff(p.dumpPython(), Process('Test').dumpPython()),
3626 """process.juicer = cms.ESProducer("JuicerProducer")
3627 process.ForceSource = cms.ESSource("ForceSource")
3628 process.prefer("ForceSource")
3629 process.prefer("juicer")""")
3630 p.prefer("juicer",fooRcd=vstring("Foo"))
3631 self.assertEqual(_lineDiff(p.dumpPython(), Process('Test').dumpPython()),
3632 """process.juicer = cms.ESProducer("JuicerProducer")
3633 process.ForceSource = cms.ESSource("ForceSource")
3634 process.prefer("ForceSource")
3635 process.prefer("juicer",
3636 fooRcd = cms.vstring('Foo')
3637 )""")
3638
3639 def testFreeze(self):
3640 process = Process("Freeze")
3641 m = EDProducer("M", p=PSet(i = int32(1)))
3642 m.p.i = 2
3643 process.m = m
3644
3645
3646
3647
3648
3649
3650 process.m.p.i = 4
3651 self.assertEqual(process.m.p.i.value(), 4)
3652 process.m.p = PSet(j=int32(1))
3653
3654 m2 = m.clone(p = PSet(i = int32(5)), j = int32(8))
3655 m2.p.i = 6
3656 m2.j = 8
3657 def testSubProcess(self):
3658 process = Process("Parent")
3659 subProcess = Process("Child")
3660 subProcess.a = EDProducer("A")
3661 subProcess.p = Path(subProcess.a)
3662 subProcess.add_(Service("Foo"))
3663 process.addSubProcess(SubProcess(subProcess))
3664 d = process.dumpPython()
3665 equalD ="""parentProcess = process
3666 process.a = cms.EDProducer("A")
3667 process.Foo = cms.Service("Foo")
3668 process.p = cms.Path(process.a)
3669 childProcess = process
3670 process = parentProcess
3671 process.addSubProcess(cms.SubProcess(process = childProcess, SelectEvents = cms.untracked.PSet(
3672 ), outputCommands = cms.untracked.vstring()))"""
3673 equalD = equalD.replace("parentProcess","parentProcess"+str(hash(process.subProcesses_()[0])))
3674
3675
3676
3677
3678
3679
3680
3681 messageLoggerSplit = 'process.MessageLogger = cms.Service'
3682 parentDumpSplit = Process('Parent').dumpPython().split(messageLoggerSplit)
3683 childProcess = Process('Child')
3684 del childProcess.MessageLogger
3685 combinedDump = parentDumpSplit[0] + childProcess.dumpPython() + messageLoggerSplit + parentDumpSplit[1]
3686 self.assertEqual(_lineDiff(d, combinedDump), equalD)
3687 p = TestMakePSet()
3688 process.fillProcessDesc(p)
3689 self.assertEqual((True,['a']),p.values["subProcesses"][1][0].values["process"][1].values['@all_modules'])
3690 self.assertEqual((True,['p']),p.values["subProcesses"][1][0].values["process"][1].values['@paths'])
3691 self.assertEqual({'@service_type':(True,'Foo')}, p.values["subProcesses"][1][0].values["process"][1].values["services"][1][0].values)
3692 def testRefToPSet(self):
3693 proc = Process("test")
3694 proc.top = PSet(a = int32(1))
3695 proc.ref = PSet(refToPSet_ = string("top"))
3696 proc.ref2 = PSet( a = int32(1), b = PSet( refToPSet_ = string("top")))
3697 proc.ref3 = PSet(refToPSet_ = string("ref"))
3698 proc.ref4 = VPSet(PSet(refToPSet_ = string("top")),
3699 PSet(refToPSet_ = string("ref2")))
3700 p = TestMakePSet()
3701 proc.fillProcessDesc(p)
3702 self.assertEqual((True,1),p.values["ref"][1].values["a"])
3703 self.assertEqual((True,1),p.values["ref3"][1].values["a"])
3704 self.assertEqual((True,1),p.values["ref2"][1].values["a"])
3705 self.assertEqual((True,1),p.values["ref2"][1].values["b"][1].values["a"])
3706 self.assertEqual((True,1),p.values["ref4"][1][0].values["a"])
3707 self.assertEqual((True,1),p.values["ref4"][1][1].values["a"])
3708 def testSwitchProducer(self):
3709 proc = Process("test")
3710 proc.sp = SwitchProducerTest(test2 = EDProducer("Foo",
3711 a = int32(1),
3712 b = PSet(c = int32(2))),
3713 test1 = EDProducer("Bar",
3714 aa = int32(11),
3715 bb = PSet(cc = int32(12))))
3716 self.assertEqual(proc.sp.label_(), "sp")
3717 self.assertEqual(proc.sp.test1.label_(), "sp@test1")
3718 self.assertEqual(proc.sp.test2.label_(), "sp@test2")
3719
3720 proc.a = EDProducer("A")
3721 proc.s = Sequence(proc.a + proc.sp)
3722 proc.t = Task(proc.a, proc.sp)
3723 proc.p = Path()
3724 proc.p.associate(proc.t)
3725 p = TestMakePSet()
3726 proc.fillProcessDesc(p)
3727 self.assertEqual((True,"EDProducer"), p.values["sp"][1].values["@module_edm_type"])
3728 self.assertEqual((True, "SwitchProducer"), p.values["sp"][1].values["@module_type"])
3729 self.assertEqual((True, "sp"), p.values["sp"][1].values["@module_label"])
3730 all_cases = copy.deepcopy(p.values["sp"][1].values["@all_cases"])
3731 all_cases[1].sort()
3732 self.assertEqual((True, ["sp@test1", "sp@test2"]), all_cases)
3733 self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
3734 self.assertEqual(["a", "sp", "sp@test1", "sp@test2"], p.values["@all_modules"][1])
3735 self.assertEqual((True,"EDProducer"), p.values["sp@test1"][1].values["@module_edm_type"])
3736 self.assertEqual((True,"Bar"), p.values["sp@test1"][1].values["@module_type"])
3737 self.assertEqual((True,"EDProducer"), p.values["sp@test2"][1].values["@module_edm_type"])
3738 self.assertEqual((True,"Foo"), p.values["sp@test2"][1].values["@module_type"])
3739 dump = proc.dumpPython()
3740 self.assertEqual(dump.find('@'), -1)
3741 self.assertEqual(specialImportRegistry.getSpecialImports(), ["from test import SwitchProducerTest"])
3742 self.assertTrue(dump.find("\nfrom test import SwitchProducerTest\n") != -1)
3743
3744
3745 proc = Process("test")
3746 proc.sp = SwitchProducerTest(test2 = EDProducer("Foo",
3747 a = int32(1),
3748 b = PSet(c = int32(2))),
3749 test1 = EDAlias(a = VPSet(PSet(type = string("Bar")))))
3750 proc.a = EDProducer("A")
3751 proc.s = Sequence(proc.a + proc.sp)
3752 proc.t = Task(proc.a, proc.sp)
3753 proc.p = Path()
3754 proc.p.associate(proc.t)
3755 p = TestMakePSet()
3756 proc.fillProcessDesc(p)
3757 self.assertEqual((True,"EDProducer"), p.values["sp"][1].values["@module_edm_type"])
3758 self.assertEqual((True, "SwitchProducer"), p.values["sp"][1].values["@module_type"])
3759 self.assertEqual((True, "sp"), p.values["sp"][1].values["@module_label"])
3760 all_cases = copy.deepcopy(p.values["sp"][1].values["@all_cases"])
3761 all_cases[1].sort()
3762 self.assertEqual((True, ["sp@test1", "sp@test2"]), all_cases)
3763 self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
3764 self.assertEqual(["a", "sp", "sp@test2"], p.values["@all_modules"][1])
3765 self.assertEqual(["sp@test1"], p.values["@all_aliases"][1])
3766 self.assertEqual((True,"EDProducer"), p.values["sp@test2"][1].values["@module_edm_type"])
3767 self.assertEqual((True,"Foo"), p.values["sp@test2"][1].values["@module_type"])
3768 self.assertEqual((True,"EDAlias"), p.values["sp@test1"][1].values["@module_edm_type"])
3769 self.assertEqual((True,"Bar"), p.values["sp@test1"][1].values["a"][1][0].values["type"])
3770
3771
3772 proc = Process("test")
3773 proc.sp = SwitchProducerTest(test1 = EDProducer("Foo",
3774 a = int32(1),
3775 b = PSet(c = int32(2))),
3776 test2 = EDAlias(a = VPSet(PSet(type = string("Bar")))))
3777 proc.a = EDProducer("A")
3778 proc.s = Sequence(proc.a + proc.sp)
3779 proc.t = Task(proc.a, proc.sp)
3780 proc.p = Path()
3781 proc.p.associate(proc.t)
3782 p = TestMakePSet()
3783 proc.fillProcessDesc(p)
3784 self.assertEqual((True,"EDProducer"), p.values["sp"][1].values["@module_edm_type"])
3785 self.assertEqual((True, "SwitchProducer"), p.values["sp"][1].values["@module_type"])
3786 self.assertEqual((True, "sp"), p.values["sp"][1].values["@module_label"])
3787 self.assertEqual((True, ["sp@test1", "sp@test2"]), p.values["sp"][1].values["@all_cases"])
3788 self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
3789 self.assertEqual(["a", "sp", "sp@test1"], p.values["@all_modules"][1])
3790 self.assertEqual(["sp@test2"], p.values["@all_aliases"][1])
3791 self.assertEqual((True,"EDProducer"), p.values["sp@test1"][1].values["@module_edm_type"])
3792 self.assertEqual((True,"Foo"), p.values["sp@test1"][1].values["@module_type"])
3793 self.assertEqual((True,"EDAlias"), p.values["sp@test2"][1].values["@module_edm_type"])
3794 self.assertEqual((True,"Bar"), p.values["sp@test2"][1].values["a"][1][0].values["type"])
3795
3796
3797 proc = Process("test")
3798 proc.spct = SwitchProducerTest(test2 = EDProducer("Foo",
3799 a = int32(1),
3800 b = PSet(c = int32(2))),
3801 test1 = EDProducer("Bar",
3802 aa = int32(11),
3803 bb = PSet(cc = int32(12))),
3804 test3 = EDAlias(a = VPSet(PSet(type = string("Bar")))))
3805 proc.spp = proc.spct.clone()
3806 proc.a = EDProducer("A")
3807 proc.ct = ConditionalTask(proc.spct)
3808 proc.p = Path(proc.a, proc.ct)
3809 proc.pp = Path(proc.a + proc.spp)
3810 p = TestMakePSet()
3811 proc.fillProcessDesc(p)
3812 self.assertEqual(["a", "spct", "spct@test1", "spct@test2", "spp", "spp@test1", "spp@test2"], p.values["@all_modules"][1])
3813 self.assertEqual(["a", "#", "spct", "spct@test1", "spct@test2", "@"], p.values["p"][1])
3814 self.assertEqual(["a", "spp", "#", "spp@test1", "spp@test2", "@"], p.values["pp"][1])
3815
3816 def testPrune(self):
3817 p = Process("test")
3818 p.a = EDAnalyzer("MyAnalyzer")
3819 p.b = EDAnalyzer("YourAnalyzer")
3820 p.c = EDAnalyzer("OurAnalyzer")
3821 p.d = EDAnalyzer("OurAnalyzer")
3822 p.e = EDProducer("MyProducer")
3823 p.f = EDProducer("YourProducer")
3824 p.g = EDProducer("TheirProducer")
3825 p.h = EDProducer("OnesProducer")
3826 p.s = Sequence(p.d)
3827 p.t1 = Task(p.e)
3828 p.t2 = Task(p.f)
3829 p.t3 = Task(p.g, p.t1)
3830 p.ct1 = ConditionalTask(p.h)
3831 p.ct2 = ConditionalTask(p.f)
3832 p.ct3 = ConditionalTask(p.ct1)
3833 p.path1 = Path(p.a, p.t3, p.ct3)
3834 p.path2 = Path(p.b)
3835 self.assertTrue(p.schedule is None)
3836 pths = p.paths
3837 keys = pths.keys()
3838 self.assertEqual(pths[keys[0]],p.path1)
3839 self.assertEqual(pths[keys[1]],p.path2)
3840 p.pset1 = PSet(parA = string("pset1"))
3841 p.pset2 = untracked.PSet(parA = string("pset2"))
3842 p.vpset1 = VPSet()
3843 p.vpset2 = untracked.VPSet()
3844 p.prune()
3845 self.assertTrue(hasattr(p, 'a'))
3846 self.assertTrue(hasattr(p, 'b'))
3847 self.assertTrue(not hasattr(p, 'c'))
3848 self.assertTrue(not hasattr(p, 'd'))
3849 self.assertTrue(hasattr(p, 'e'))
3850 self.assertTrue(not hasattr(p, 'f'))
3851 self.assertTrue(hasattr(p, 'g'))
3852 self.assertTrue(hasattr(p, 'h'))
3853 self.assertTrue(not hasattr(p, 's'))
3854 self.assertTrue(hasattr(p, 't1'))
3855 self.assertTrue(not hasattr(p, 't2'))
3856 self.assertTrue(hasattr(p, 't3'))
3857 self.assertTrue(hasattr(p, 'path1'))
3858 self.assertTrue(hasattr(p, 'path2'))
3859
3860
3861
3862
3863
3864 p = Process("test")
3865 p.a = EDAnalyzer("MyAnalyzer")
3866 p.b = EDAnalyzer("YourAnalyzer")
3867 p.c = EDAnalyzer("OurAnalyzer")
3868 p.d = EDAnalyzer("OurAnalyzer")
3869 p.e = EDAnalyzer("OurAnalyzer")
3870 p.f = EDProducer("MyProducer")
3871 p.g = EDProducer("YourProducer")
3872 p.h = EDProducer("TheirProducer")
3873 p.i = EDProducer("OurProducer")
3874 p.j = EDProducer("OurProducer")
3875 p.k = EDProducer("OurProducer")
3876 p.l = EDProducer("OurProducer")
3877 p.t1 = Task(p.f)
3878 p.t2 = Task(p.g)
3879 p.t3 = Task(p.h)
3880 p.t4 = Task(p.i)
3881 p.ct1 = Task(p.f)
3882 p.ct2 = Task(p.j)
3883 p.ct3 = Task(p.k)
3884 p.ct4 = Task(p.l)
3885 p.s = Sequence(p.d, p.t1, p.ct1)
3886 p.s2 = Sequence(p.b, p.t2, p.ct2)
3887 p.s3 = Sequence(p.e)
3888 p.path1 = Path(p.a, p.t3, p.ct3)
3889 p.path2 = Path(p.b)
3890 p.path3 = Path(p.b+p.s2)
3891 p.path4 = Path(p.b+p.s3)
3892 p.schedule = Schedule(p.path1,p.path2,p.path3)
3893 p.schedule.associate(p.t4)
3894 pths = p.paths
3895 keys = pths.keys()
3896 self.assertEqual(pths[keys[0]],p.path1)
3897 self.assertEqual(pths[keys[1]],p.path2)
3898 p.prune()
3899 self.assertTrue(hasattr(p, 'a'))
3900 self.assertTrue(hasattr(p, 'b'))
3901 self.assertTrue(not hasattr(p, 'c'))
3902 self.assertTrue(not hasattr(p, 'd'))
3903 self.assertTrue(not hasattr(p, 'e'))
3904 self.assertTrue(not hasattr(p, 'f'))
3905 self.assertTrue(hasattr(p, 'g'))
3906 self.assertTrue(hasattr(p, 'h'))
3907 self.assertTrue(hasattr(p, 'i'))
3908 self.assertTrue(hasattr(p, 'j'))
3909 self.assertTrue(hasattr(p, 'k'))
3910 self.assertTrue(not hasattr(p, 'l'))
3911 self.assertTrue(not hasattr(p, 't1'))
3912 self.assertTrue(hasattr(p, 't2'))
3913 self.assertTrue(hasattr(p, 't3'))
3914 self.assertTrue(hasattr(p, 't4'))
3915 self.assertTrue(not hasattr(p, 'ct1'))
3916 self.assertTrue(hasattr(p, 'ct2'))
3917 self.assertTrue(hasattr(p, 'ct3'))
3918 self.assertTrue(not hasattr(p, 'ct4'))
3919 self.assertTrue(not hasattr(p, 's'))
3920 self.assertTrue(hasattr(p, 's2'))
3921 self.assertTrue(not hasattr(p, 's3'))
3922 self.assertTrue(hasattr(p, 'path1'))
3923 self.assertTrue(hasattr(p, 'path2'))
3924 self.assertTrue(hasattr(p, 'path3'))
3925 self.assertTrue(not hasattr(p, 'path4'))
3926
3927 p = Process("test")
3928 p.a = EDAnalyzer("MyAnalyzer")
3929 p.b = EDAnalyzer("YourAnalyzer")
3930 p.s = Sequence(SequencePlaceholder("a")+p.b)
3931 p.pth = Path(p.s)
3932 p.prune()
3933 self.assertTrue(hasattr(p, 'a'))
3934 self.assertTrue(hasattr(p, 'b'))
3935 self.assertTrue(hasattr(p, 's'))
3936 self.assertTrue(hasattr(p, 'pth'))
3937
3938 p = Process("test")
3939 p.b = EDAnalyzer("YourAnalyzer")
3940 p.s = Sequence(SequencePlaceholder("a")+p.b)
3941 p.pth = Path(p.s)
3942 p.prune(keepUnresolvedSequencePlaceholders=True)
3943 self.assertTrue(hasattr(p, 'b'))
3944 self.assertTrue(hasattr(p, 's'))
3945 self.assertTrue(hasattr(p, 'pth'))
3946 self.assertEqual(p.s.dumpPython(),'cms.Sequence(cms.SequencePlaceholder("a")+process.b)\n')
3947
3948 p = Process("test")
3949 p.a = EDProducer("MyProducer")
3950 p.b = EDProducer("YourProducer")
3951 p.s = Task(TaskPlaceholder("a"),p.b)
3952 p.pth = Path(p.s)
3953 p.prune()
3954 self.assertTrue(hasattr(p, 'a'))
3955 self.assertTrue(hasattr(p, 'b'))
3956 self.assertTrue(hasattr(p, 's'))
3957 self.assertTrue(hasattr(p, 'pth'))
3958
3959 p = Process("test")
3960 p.a = EDProducer("MyProducer")
3961 p.b = EDProducer("YourProducer")
3962 p.s = ConditionalTask(ConditionalTaskPlaceholder("a"),p.b)
3963 p.pth = Path(p.s)
3964 p.prune()
3965 self.assertTrue(hasattr(p, 'a'))
3966 self.assertTrue(hasattr(p, 'b'))
3967 self.assertTrue(hasattr(p, 's'))
3968 self.assertTrue(hasattr(p, 'pth'))
3969
3970 p = Process("test")
3971 p.b = EDProducer("YourAnalyzer")
3972 p.s = Task(TaskPlaceholder("a"),p.b)
3973 p.pth = Path(p.s)
3974 p.prune(keepUnresolvedSequencePlaceholders=True)
3975 self.assertTrue(hasattr(p, 'b'))
3976 self.assertTrue(hasattr(p, 's'))
3977 self.assertTrue(hasattr(p, 'pth'))
3978 self.assertEqual(p.s.dumpPython(),'cms.Task(cms.TaskPlaceholder("a"), process.b)\n')
3979 def testTaskPlaceholder(self):
3980 p = Process("test")
3981 p.a = EDProducer("ma")
3982 p.b = EDAnalyzer("mb")
3983 p.t1 = Task(TaskPlaceholder("c"))
3984 p.t2 = Task(p.a, TaskPlaceholder("d"), p.t1)
3985 p.t3 = Task(TaskPlaceholder("e"))
3986 p.path1 = Path(p.b, p.t2, p.t3)
3987 p.t5 = Task(p.a, TaskPlaceholder("g"), TaskPlaceholder("t4"))
3988 p.t4 = Task(TaskPlaceholder("f"))
3989 p.endpath1 = EndPath(p.b, p.t5)
3990 p.t6 = Task(TaskPlaceholder("h"))
3991 p.t7 = Task(p.a, TaskPlaceholder("i"), p.t6)
3992 p.t8 = Task(TaskPlaceholder("j"))
3993 p.schedule = Schedule(p.path1, p.endpath1,tasks=[p.t7,p.t8])
3994 p.c = EDProducer("mc")
3995 p.d = EDProducer("md")
3996 p.e = EDProducer("me")
3997 p.f = EDProducer("mf")
3998 p.g = EDProducer("mg")
3999 p.h = EDProducer("mh")
4000 p.i = EDProducer("mi")
4001 p.j = EDProducer("mj")
4002 self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4003 """process.a = cms.EDProducer("ma")
4004 process.c = cms.EDProducer("mc")
4005 process.d = cms.EDProducer("md")
4006 process.e = cms.EDProducer("me")
4007 process.f = cms.EDProducer("mf")
4008 process.g = cms.EDProducer("mg")
4009 process.h = cms.EDProducer("mh")
4010 process.i = cms.EDProducer("mi")
4011 process.j = cms.EDProducer("mj")
4012 process.b = cms.EDAnalyzer("mb")
4013 process.t1 = cms.Task(cms.TaskPlaceholder("c"))
4014 process.t2 = cms.Task(cms.TaskPlaceholder("d"), process.a, process.t1)
4015 process.t3 = cms.Task(cms.TaskPlaceholder("e"))
4016 process.t5 = cms.Task(cms.TaskPlaceholder("g"), cms.TaskPlaceholder("t4"), process.a)
4017 process.t4 = cms.Task(cms.TaskPlaceholder("f"))
4018 process.t6 = cms.Task(cms.TaskPlaceholder("h"))
4019 process.t7 = cms.Task(cms.TaskPlaceholder("i"), process.a, process.t6)
4020 process.t8 = cms.Task(cms.TaskPlaceholder("j"))
4021 process.path1 = cms.Path(process.b, process.t2, process.t3)
4022 process.endpath1 = cms.EndPath(process.b, process.t5)
4023 process.schedule = cms.Schedule(*[ process.path1, process.endpath1 ], tasks=[process.t7, process.t8])""")
4024 p.resolve()
4025 self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4026 """process.a = cms.EDProducer("ma")
4027 process.c = cms.EDProducer("mc")
4028 process.d = cms.EDProducer("md")
4029 process.e = cms.EDProducer("me")
4030 process.f = cms.EDProducer("mf")
4031 process.g = cms.EDProducer("mg")
4032 process.h = cms.EDProducer("mh")
4033 process.i = cms.EDProducer("mi")
4034 process.j = cms.EDProducer("mj")
4035 process.b = cms.EDAnalyzer("mb")
4036 process.t1 = cms.Task(process.c)
4037 process.t2 = cms.Task(process.a, process.d, process.t1)
4038 process.t3 = cms.Task(process.e)
4039 process.t4 = cms.Task(process.f)
4040 process.t6 = cms.Task(process.h)
4041 process.t7 = cms.Task(process.a, process.i, process.t6)
4042 process.t8 = cms.Task(process.j)
4043 process.t5 = cms.Task(process.a, process.g, process.t4)
4044 process.path1 = cms.Path(process.b, process.t2, process.t3)
4045 process.endpath1 = cms.EndPath(process.b, process.t5)
4046 process.schedule = cms.Schedule(*[ process.path1, process.endpath1 ], tasks=[process.t7, process.t8])""")
4047 def testConditionalTaskPlaceholder(self):
4048 p = Process("test")
4049 p.a = EDProducer("ma")
4050 p.b = EDAnalyzer("mb")
4051 p.t1 = ConditionalTask(ConditionalTaskPlaceholder("c"))
4052 p.t2 = ConditionalTask(p.a, ConditionalTaskPlaceholder("d"), p.t1)
4053 p.t3 = ConditionalTask(ConditionalTaskPlaceholder("e"))
4054 p.path1 = Path(p.b, p.t2, p.t3)
4055 p.t5 = ConditionalTask(p.a, ConditionalTaskPlaceholder("g"), ConditionalTaskPlaceholder("t4"))
4056 p.t4 = ConditionalTask(ConditionalTaskPlaceholder("f"))
4057 p.path2 = Path(p.b, p.t5)
4058 p.schedule = Schedule(p.path1, p.path2)
4059 p.c = EDProducer("mc")
4060 p.d = EDProducer("md")
4061 p.e = EDProducer("me")
4062 p.f = EDProducer("mf")
4063 p.g = EDProducer("mg")
4064 p.h = EDProducer("mh")
4065 p.i = EDProducer("mi")
4066 p.j = EDProducer("mj")
4067 self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4068 """process.a = cms.EDProducer("ma")
4069 process.c = cms.EDProducer("mc")
4070 process.d = cms.EDProducer("md")
4071 process.e = cms.EDProducer("me")
4072 process.f = cms.EDProducer("mf")
4073 process.g = cms.EDProducer("mg")
4074 process.h = cms.EDProducer("mh")
4075 process.i = cms.EDProducer("mi")
4076 process.j = cms.EDProducer("mj")
4077 process.b = cms.EDAnalyzer("mb")
4078 process.t1 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("c"))
4079 process.t2 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("d"), process.a, process.t1)
4080 process.t3 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("e"))
4081 process.t5 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("g"), cms.ConditionalTaskPlaceholder("t4"), process.a)
4082 process.t4 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("f"))
4083 process.path1 = cms.Path(process.b, process.t2, process.t3)
4084 process.path2 = cms.Path(process.b, process.t5)
4085 process.schedule = cms.Schedule(*[ process.path1, process.path2 ])""")
4086 p.resolve()
4087 self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4088 """process.a = cms.EDProducer("ma")
4089 process.c = cms.EDProducer("mc")
4090 process.d = cms.EDProducer("md")
4091 process.e = cms.EDProducer("me")
4092 process.f = cms.EDProducer("mf")
4093 process.g = cms.EDProducer("mg")
4094 process.h = cms.EDProducer("mh")
4095 process.i = cms.EDProducer("mi")
4096 process.j = cms.EDProducer("mj")
4097 process.b = cms.EDAnalyzer("mb")
4098 process.t1 = cms.ConditionalTask(process.c)
4099 process.t2 = cms.ConditionalTask(process.a, process.d, process.t1)
4100 process.t3 = cms.ConditionalTask(process.e)
4101 process.t4 = cms.ConditionalTask(process.f)
4102 process.t5 = cms.ConditionalTask(process.a, process.g, process.t4)
4103 process.path1 = cms.Path(process.b, process.t2, process.t3)
4104 process.path2 = cms.Path(process.b, process.t5)
4105 process.schedule = cms.Schedule(*[ process.path1, process.path2 ])""")
4106
4107 def testDelete(self):
4108 p = Process("test")
4109 p.a = EDAnalyzer("MyAnalyzer")
4110 p.b = EDAnalyzer("YourAnalyzer")
4111 p.c = EDAnalyzer("OurAnalyzer")
4112 p.d = EDAnalyzer("OurAnalyzer")
4113 p.e = EDAnalyzer("OurAnalyzer")
4114 p.f = EDAnalyzer("OurAnalyzer")
4115 p.g = EDProducer("OurProducer")
4116 p.h = EDProducer("YourProducer")
4117 p.i = SwitchProducerTest(
4118 test1 = EDProducer("OneProducer"),
4119 test2 = EDProducer("TwoProducer")
4120 )
4121 p.t1 = Task(p.g, p.h, p.i)
4122 t2 = Task(p.g, p.h, p.i)
4123 t3 = Task(p.g, p.h)
4124 p.t4 = Task(p.h)
4125 p.ct1 = ConditionalTask(p.g, p.h, p.i)
4126 ct2 = ConditionalTask(p.g, p.h)
4127 ct3 = ConditionalTask(p.g, p.h)
4128 p.ct4 = ConditionalTask(p.h)
4129 p.s = Sequence(p.d+p.e)
4130 p.path1 = Path(p.a+p.f+p.s,t2,ct2)
4131 p.path2 = Path(p.a)
4132 p.path3 = Path(ct3, p.ct4)
4133 p.endpath2 = EndPath(p.b)
4134 p.endpath1 = EndPath(p.b+p.f)
4135 p.schedule = Schedule(p.path2, p.path3, p.endpath2, tasks=[t3, p.t4])
4136 self.assertTrue(hasattr(p, 'f'))
4137 self.assertTrue(hasattr(p, 'g'))
4138 self.assertTrue(hasattr(p, 'i'))
4139 del p.e
4140 del p.f
4141 del p.g
4142 del p.i
4143 self.assertFalse(hasattr(p, 'f'))
4144 self.assertFalse(hasattr(p, 'g'))
4145 self.assertEqual(p.t1.dumpPython(), 'cms.Task(process.h)\n')
4146 self.assertEqual(p.ct1.dumpPython(), 'cms.ConditionalTask(process.h)\n')
4147 self.assertEqual(p.s.dumpPython(), 'cms.Sequence(process.d)\n')
4148 self.assertEqual(p.path1.dumpPython(), 'cms.Path(process.a+process.s, cms.ConditionalTask(process.h), cms.Task(process.h))\n')
4149 self.assertEqual(p.endpath1.dumpPython(), 'cms.EndPath(process.b)\n')
4150 self.assertEqual(p.path3.dumpPython(), 'cms.Path(cms.ConditionalTask(process.h), process.ct4)\n')
4151 del p.s
4152 self.assertEqual(p.path1.dumpPython(), 'cms.Path(process.a+(process.d), cms.ConditionalTask(process.h), cms.Task(process.h))\n')
4153 self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(*[ process.path2, process.path3, process.endpath2 ], tasks=[cms.Task(process.h), process.t4])\n')
4154 del p.path2
4155 self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(*[ process.path3, process.endpath2 ], tasks=[cms.Task(process.h), process.t4])\n')
4156 del p.path3
4157 self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(*[ process.endpath2 ], tasks=[cms.Task(process.h), process.t4])\n')
4158 del p.endpath2
4159 self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(tasks=[cms.Task(process.h), process.t4])\n')
4160 del p.t4
4161 self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(tasks=[cms.Task(process.h)])\n')
4162 def testModifier(self):
4163 m1 = Modifier()
4164 Process._firstProcess = True
4165 p = Process("test")
4166 self.assertRaises(RuntimeError, lambda: Process("test2", m1))
4167 m1 = Modifier()
4168 Process._firstProcess = True
4169 p = Process("test",m1)
4170 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1))
4171 def _mod_fred(obj):
4172 obj.fred = 2
4173 m1.toModify(p.a,_mod_fred)
4174 self.assertEqual(p.a.fred.value(),2)
4175 p.b = EDAnalyzer("YourAnalyzer", wilma = int32(1))
4176 m1.toModify(p.b, wilma = 2)
4177 self.assertEqual(p.b.wilma.value(),2)
4178 self.assertTrue(p.isUsingModifier(m1))
4179
4180 m1 = Modifier()
4181 Process._firstProcess = True
4182 p = Process("test")
4183 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1))
4184 m1.toModify(p.a,_mod_fred)
4185 p.b = EDAnalyzer("YourAnalyzer", wilma = int32(1))
4186 m1.toModify(p.b, wilma = 2)
4187 self.assertEqual(p.a.fred.value(),1)
4188 self.assertEqual(p.b.wilma.value(),1)
4189 self.assertEqual(p.isUsingModifier(m1),False)
4190
4191 m1 = Modifier()
4192 Process._firstProcess = True
4193 p = Process("test",m1)
4194 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4195 m1.toModify(p.a, fred = int32(2))
4196 p.b = p.a.clone(wilma = int32(3))
4197 self.assertEqual(p.a.fred.value(),2)
4198 self.assertEqual(p.a.wilma.value(),1)
4199 self.assertEqual(p.b.fred.value(),2)
4200 self.assertEqual(p.b.wilma.value(),3)
4201
4202 m1 = Modifier()
4203 Process._firstProcess = True
4204 p = Process("test",m1)
4205 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1), fintstones = PSet(fred = int32(1)))
4206 m1.toModify(p.a, fred = None, fintstones = dict(fred = None))
4207 self.assertEqual(hasattr(p.a, "fred"), False)
4208 self.assertEqual(hasattr(p.a.fintstones, "fred"), False)
4209 self.assertEqual(p.a.wilma.value(),1)
4210
4211 m1 = Modifier()
4212 Process._firstProcess = True
4213 p = Process("test",m1)
4214 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1))
4215 m1.toModify(p.a, wilma = int32(2))
4216 self.assertEqual(p.a.fred.value(), 1)
4217 self.assertEqual(p.a.wilma.value(),2)
4218
4219 m1 = Modifier()
4220 Process._firstProcess = True
4221 p = Process("test",m1)
4222 p.a = EDAnalyzer("MyAnalyzer", flintstones = PSet(fred = int32(1), wilma = int32(1)))
4223 m1.toModify(p.a, flintstones = dict(fred = int32(2)))
4224 self.assertEqual(p.a.flintstones.fred.value(),2)
4225 self.assertEqual(p.a.flintstones.wilma.value(),1)
4226
4227 m1 = Modifier()
4228 Process._firstProcess = True
4229 p = Process("test",m1)
4230 p.a = EDAnalyzer("MyAnalyzer", flintstones = PSet(fred = PSet(wilma = int32(1))))
4231 self.assertRaises(KeyError, lambda: m1.toModify(p.a, flintstones = dict(imnothere = dict(wilma=2))))
4232 self.assertRaises(KeyError, lambda: m1.toModify(p.a, foo = 1))
4233
4234 m1 = Modifier()
4235 Process._firstProcess = True
4236 p = Process("test",m1)
4237 p.a = EDAnalyzer("MyAnalyzer", flintstones = VPSet(PSet(fred = int32(1)), PSet(wilma = int32(1))))
4238 m1.toModify(p.a, flintstones = {1:dict(wilma = int32(2))})
4239 self.assertEqual(p.a.flintstones[0].fred.value(),1)
4240 self.assertEqual(p.a.flintstones[1].wilma.value(),2)
4241
4242 m1 = Modifier()
4243 Process._firstProcess = True
4244 p = Process("test",m1)
4245 p.a = EDAnalyzer("MyAnalyzer", fred = vuint32(1,2,3))
4246 m1.toModify(p.a, fred = {1:7})
4247 self.assertEqual(p.a.fred[0],1)
4248 self.assertEqual(p.a.fred[1],7)
4249 self.assertEqual(p.a.fred[2],3)
4250
4251 m1 = Modifier()
4252 Process._firstProcess = True
4253 p = Process("test",m1)
4254 p.a = EDAnalyzer("MyAnalyzer", fred = vuint32(1,2,3))
4255 raised = False
4256 try: m1.toModify(p.a, fred = {5:7})
4257 except IndexError as e: raised = True
4258 self.assertEqual(raised, True)
4259
4260 m1 = Modifier()
4261 Process._firstProcess = True
4262 p = Process("test",m1)
4263 p.a = EDAnalyzer("MyAnalyzer", flintstones = VPSet(PSet(fred = int32(1)), PSet(wilma = int32(1))))
4264 raised = False
4265 try: m1.toModify(p.a, flintstones = dict(bogus = int32(37)))
4266 except TypeError as e: raised = True
4267 self.assertEqual(raised, True)
4268
4269 def _rem_a(proc):
4270 del proc.a
4271 class ProcModifierMod(object):
4272 def __init__(self,modifier,func):
4273 self.proc_mod_ = modifier.makeProcessModifier(func)
4274 class DummyMod(object):
4275 def __init__(self):
4276 self.a = EDAnalyzer("Dummy")
4277 testMod = DummyMod()
4278 p.extend(testMod)
4279 self.assertTrue(hasattr(p,"a"))
4280 m1 = Modifier()
4281 Process._firstProcess = True
4282 p = Process("test",m1)
4283 testProcMod = ProcModifierMod(m1,_rem_a)
4284 p.extend(testMod)
4285 p.extend(testProcMod)
4286 self.assertTrue(not hasattr(p,"a"))
4287
4288 m1 = Modifier()
4289 mc = ModifierChain(m1)
4290 Process._firstProcess = True
4291 p = Process("test",mc)
4292 self.assertTrue(p.isUsingModifier(m1))
4293 self.assertTrue(p.isUsingModifier(mc))
4294 testMod = DummyMod()
4295 p.b = EDAnalyzer("Dummy2", fred = int32(1))
4296 m1.toModify(p.b, fred = int32(3))
4297 p.extend(testMod)
4298 testProcMod = ProcModifierMod(m1,_rem_a)
4299 p.extend(testProcMod)
4300 self.assertTrue(not hasattr(p,"a"))
4301 self.assertEqual(p.b.fred.value(),3)
4302
4303 m1 = Modifier()
4304 m2 = Modifier()
4305 mc = ModifierChain(m1,m2)
4306 mclone = mc.copyAndExclude([m2])
4307 self.assertTrue(not mclone._isOrContains(m2))
4308 self.assertTrue(mclone._isOrContains(m1))
4309 m3 = Modifier()
4310 mc2 = ModifierChain(mc,m3)
4311 mclone = mc2.copyAndExclude([m2])
4312 self.assertTrue(not mclone._isOrContains(m2))
4313 self.assertTrue(mclone._isOrContains(m1))
4314 self.assertTrue(mclone._isOrContains(m3))
4315
4316 m1 = Modifier()
4317 m2 = Modifier()
4318 Process._firstProcess = True
4319 p = Process("test",m1)
4320 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4321 (m1 & m2).toModify(p.a, fred = int32(2))
4322 self.assertRaises(TypeError, lambda: (m1 & m2).toModify(p.a, 1, wilma=2))
4323 self.assertEqual(p.a.fred, 1)
4324 m1 = Modifier()
4325 m2 = Modifier()
4326 Process._firstProcess = True
4327 p = Process("test",m1,m2)
4328 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4329 (m1 & m2).toModify(p.a, fred = int32(2))
4330 self.assertEqual(p.a.fred, 2)
4331 m1 = Modifier()
4332 m2 = Modifier()
4333 m3 = Modifier()
4334 Process._firstProcess = True
4335 p = Process("test",m1,m2,m3)
4336 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4337 (m1 & m2 & m3).toModify(p.a, fred = int32(2))
4338 self.assertEqual(p.a.fred, 2)
4339 (m1 & (m2 & m3)).toModify(p.a, fred = int32(3))
4340 self.assertEqual(p.a.fred, 3)
4341 ((m1 & m2) & m3).toModify(p.a, fred = int32(4))
4342 self.assertEqual(p.a.fred, 4)
4343
4344 m1 = Modifier()
4345 m2 = Modifier()
4346 Process._firstProcess = True
4347 p = Process("test", m1)
4348 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4349 (~m1).toModify(p.a, fred=2)
4350 self.assertEqual(p.a.fred, 1)
4351 (~m2).toModify(p.a, wilma=2)
4352 self.assertEqual(p.a.wilma, 2)
4353 self.assertRaises(TypeError, lambda: (~m1).toModify(p.a, 1, wilma=2))
4354 self.assertRaises(TypeError, lambda: (~m2).toModify(p.a, 1, wilma=2))
4355
4356 m1 = Modifier()
4357 m2 = Modifier()
4358 m3 = Modifier()
4359 Process._firstProcess = True
4360 p = Process("test", m1)
4361 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4362 (m1 | m2).toModify(p.a, fred=2)
4363 self.assertEqual(p.a.fred, 2)
4364 (m1 | m2 | m3).toModify(p.a, fred=3)
4365 self.assertEqual(p.a.fred, 3)
4366 (m3 | m2 | m1).toModify(p.a, fred=4)
4367 self.assertEqual(p.a.fred, 4)
4368 ((m1 | m2) | m3).toModify(p.a, fred=5)
4369 self.assertEqual(p.a.fred, 5)
4370 (m1 | (m2 | m3)).toModify(p.a, fred=6)
4371 self.assertEqual(p.a.fred, 6)
4372 (m2 | m3).toModify(p.a, fred=7)
4373 self.assertEqual(p.a.fred, 6)
4374 self.assertRaises(TypeError, lambda: (m1 | m2).toModify(p.a, 1, wilma=2))
4375 self.assertRaises(TypeError, lambda: (m2 | m3).toModify(p.a, 1, wilma=2))
4376
4377 m1 = Modifier()
4378 m2 = Modifier()
4379 m3 = Modifier()
4380 m4 = Modifier()
4381 Process._firstProcess = True
4382 p = Process("test", m1, m2)
4383 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4384 (m1 & ~m2).toModify(p.a, fred=2)
4385 self.assertEqual(p.a.fred, 1)
4386 (m1 & ~m3).toModify(p.a, fred=2)
4387 self.assertEqual(p.a.fred, 2)
4388 (m1 | ~m2).toModify(p.a, fred=3)
4389 self.assertEqual(p.a.fred, 3)
4390 (~m1 | ~m2).toModify(p.a, fred=4)
4391 self.assertEqual(p.a.fred, 3)
4392 (~m3 & ~m4).toModify(p.a, fred=4)
4393 self.assertEqual(p.a.fred, 4)
4394 ((m1 & m3) | ~m4).toModify(p.a, fred=5)
4395 self.assertEqual(p.a.fred, 5)
4396
4397 m1 = Modifier()
4398 Process._firstProcess = True
4399 p = Process("test",m1)
4400 p.a =EDAnalyzer("MyAnalyzer", fred = int32(1))
4401 m1.toReplaceWith(p.a, EDAnalyzer("YourAnalyzer", wilma = int32(3)))
4402 self.assertRaises(TypeError, lambda: m1.toReplaceWith(p.a, EDProducer("YourProducer")))
4403
4404 p.b =EDAnalyzer("BAn")
4405 p.c =EDProducer("c")
4406 p.d =EDProducer("d")
4407 p.tc = Task(p.c)
4408 p.td = Task(p.d)
4409 p.s = Sequence(p.a, p.tc)
4410 m1.toReplaceWith(p.s, Sequence(p.a+p.b, p.td))
4411 self.assertEqual(p.a.wilma.value(),3)
4412 self.assertEqual(p.a.type_(),"YourAnalyzer")
4413 self.assertEqual(hasattr(p,"fred"),False)
4414 self.assertTrue(p.s.dumpPython() == "cms.Sequence(process.a+process.b, process.td)\n")
4415 p.e =EDProducer("e")
4416 m1.toReplaceWith(p.td, Task(p.e))
4417 self.assertTrue(p.td._collection == OrderedSet([p.e]))
4418
4419 p.b =EDAnalyzer("BAn")
4420 p.c =EDProducer("c")
4421 p.d =EDProducer("d")
4422 del p.tc
4423 del p.td
4424 p.tc = ConditionalTask(p.c)
4425 p.td = ConditionalTask(p.d)
4426 p.s = Sequence(p.a, p.tc)
4427 m1.toReplaceWith(p.s, Sequence(p.a+p.b, p.td))
4428 self.assertEqual(p.a.wilma.value(),3)
4429 self.assertEqual(p.a.type_(),"YourAnalyzer")
4430 self.assertEqual(hasattr(p,"fred"),False)
4431 self.assertTrue(p.s.dumpPython() == "cms.Sequence(process.a+process.b, process.td)\n")
4432 p.e =EDProducer("e")
4433 m1.toReplaceWith(p.td, ConditionalTask(p.e))
4434 self.assertTrue(p.td._collection == OrderedSet([p.e]))
4435
4436 m1 = Modifier()
4437 Process._firstProcess = True
4438 p = Process("test")
4439 p.a =EDAnalyzer("MyAnalyzer", fred = int32(1))
4440 m1.toReplaceWith(p.a, EDAnalyzer("YourAnalyzer", wilma = int32(3)))
4441 self.assertEqual(p.a.type_(),"MyAnalyzer")
4442
4443 m1 = Modifier()
4444 m2 = Modifier()
4445 m3 = Modifier()
4446 m4 = Modifier()
4447 Process._firstProcess = True
4448 p = Process("test", m1, m2)
4449 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4450 self.assertRaises(TypeError, lambda: (m1 & m2).toReplaceWith(p.a, EDProducer("YourProducer")))
4451 self.assertRaises(TypeError, lambda: (m3 & m4).toReplaceWith(p.a, EDProducer("YourProducer")))
4452 self.assertRaises(TypeError, lambda: (~m3).toReplaceWith(p.a, EDProducer("YourProducer")))
4453 self.assertRaises(TypeError, lambda: (~m1).toReplaceWith(p.a, EDProducer("YourProducer")))
4454 self.assertRaises(TypeError, lambda: (m1 | m3).toReplaceWith(p.a, EDProducer("YourProducer")))
4455 self.assertRaises(TypeError, lambda: (m3 | m4).toReplaceWith(p.a, EDProducer("YourProducer")))
4456 (m1 & m2).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer1"))
4457 self.assertEqual(p.a.type_(), "YourAnalyzer1")
4458 (m1 & m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer2"))
4459 self.assertEqual(p.a.type_(), "YourAnalyzer1")
4460 (~m1).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer2"))
4461 self.assertEqual(p.a.type_(), "YourAnalyzer1")
4462 (~m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer2"))
4463 self.assertEqual(p.a.type_(), "YourAnalyzer2")
4464 (m1 | m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer3"))
4465 self.assertEqual(p.a.type_(), "YourAnalyzer3")
4466 (m3 | m4).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer4"))
4467 self.assertEqual(p.a.type_(), "YourAnalyzer3")
4468
4469 m1 = Modifier()
4470 m2 = Modifier()
4471 m3 = Modifier()
4472 Process._firstProcess = True
4473 p = Process("test", m1, m2)
4474 p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4475 p.b = EDProducer("MyProducer", barney = int32(1), betty = int32(1))
4476 (m1 & m2).toModify(p.a, fred = 2).toModify(p.b, betty = 3)
4477 self.assertEqual(p.a.fred, 2)
4478 self.assertEqual(p.a.wilma, 1)
4479 self.assertEqual(p.b.barney, 1)
4480 self.assertEqual(p.b.betty, 3)
4481 (m1 | m3).toModify(p.a, wilma = 4).toModify(p.b, barney = 5)
4482 self.assertEqual(p.a.fred, 2)
4483 self.assertEqual(p.a.wilma, 4)
4484 self.assertEqual(p.b.barney, 5)
4485 self.assertEqual(p.b.betty, 3)
4486 (m2 & ~m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer")).toModify(p.b, barney = 6)
4487 self.assertEqual(p.a.type_(), "YourAnalyzer")
4488 self.assertEqual(p.b.barney, 6)
4489 self.assertEqual(p.b.betty, 3)
4490 (m1 & ~m3).toModify(p.a, param=int32(42)).toReplaceWith(p.b, EDProducer("YourProducer"))
4491 self.assertEqual(p.a.type_(), "YourAnalyzer")
4492 self.assertEqual(p.a.param, 42)
4493 self.assertEqual(p.b.type_(), "YourProducer")
4494
4495
4496 a = EDAlias(foo2 = VPSet(PSet(type = string("Foo2"))))
4497 m = Modifier()
4498 m._setChosen()
4499
4500 m.toModify(a, foo2 = {0: dict(type = "Foo3")})
4501 self.assertEqual(a.foo2[0].type, "Foo3")
4502
4503 m.toModify(a, foo4 = VPSet(PSet(type = string("Foo4"))))
4504 self.assertEqual(a.foo2[0].type, "Foo3")
4505 self.assertEqual(a.foo4[0].type, "Foo4")
4506
4507 m.toModify(a, foo2 = None)
4508 self.assertFalse(hasattr(a, "foo2"))
4509 self.assertEqual(a.foo4[0].type, "Foo4")
4510
4511 m.toReplaceWith(a, EDAlias(bar = VPSet(PSet(type = string("Bar")))))
4512 self.assertFalse(hasattr(a, "foo2"))
4513 self.assertFalse(hasattr(a, "foo4"))
4514 self.assertTrue(hasattr(a, "bar"))
4515 self.assertEqual(a.bar[0].type, "Bar")
4516
4517
4518 sp = SwitchProducerTest(test1 = EDProducer("Foo",
4519 a = int32(1),
4520 b = PSet(c = int32(2))),
4521 test2 = EDProducer("Bar",
4522 aa = int32(11),
4523 bb = PSet(cc = int32(12))))
4524 m = Modifier()
4525 m._setChosen()
4526
4527 m.toModify(sp,
4528 test1 = dict(a = 4, b = dict(c = None)),
4529 test2 = dict(aa = 15, bb = dict(cc = 45, dd = string("foo"))))
4530 self.assertEqual(sp.test1.a.value(), 4)
4531 self.assertEqual(sp.test1.b.hasParameter("c"), False)
4532 self.assertEqual(sp.test2.aa.value(), 15)
4533 self.assertEqual(sp.test2.bb.cc.value(), 45)
4534 self.assertEqual(sp.test2.bb.dd.value(), "foo")
4535
4536 m.toReplaceWith(sp.test1, EDProducer("Fred", x = int32(42)))
4537 self.assertEqual(sp.test1.type_(), "Fred")
4538 self.assertEqual(sp.test1.x.value(), 42)
4539 self.assertRaises(TypeError, lambda: m.toReplaceWith(sp.test1, EDAnalyzer("Foo")))
4540
4541 m.toModify(sp, test2 = EDProducer("Xyzzy", x = int32(24)))
4542 self.assertEqual(sp.test2.type_(), "Xyzzy")
4543 self.assertEqual(sp.test2.x.value(), 24)
4544 self.assertRaises(TypeError, lambda: m.toModify(sp, test2 = EDAnalyzer("Foo")))
4545
4546 m.toModify(sp, test3 = EDProducer("Wilma", y = int32(24)))
4547 self.assertEqual(sp.test3.type_(), "Wilma")
4548 self.assertEqual(sp.test3.y.value(), 24)
4549 self.assertRaises(TypeError, lambda: m.toModify(sp, test4 = EDAnalyzer("Foo")))
4550
4551 m.toModify(sp, test2 = None)
4552 self.assertEqual(hasattr(sp, "test2"), False)
4553
4554 m.toModify(sp, test2 = EDAlias(foo = VPSet(PSet(type = string("int")))))
4555 self.assertTrue(hasattr(sp.test2, "foo"))
4556
4557 m.toReplaceWith(sp.test2, EDAlias(bar = VPSet(PSet(type = string("int")))))
4558 self.assertTrue(hasattr(sp.test2, "bar"))
4559
4560 m.toModify(sp, test2 = EDAlias(xyzzy = VPSet(PSet(type = string("int")))))
4561 self.assertTrue(hasattr(sp.test2, "xyzzy"))
4562
4563 self.assertRaises(TypeError, lambda: m.toReplaceWith(sp.test2, EDProducer("Foo")))
4564 m.toModify(sp, test2 = EDProducer("Foo"))
4565 def testProcessFragment(self):
4566
4567 f = ProcessFragment('Fragment')
4568 p = Process('PROCESS')
4569 p.maxEvents.input = 10
4570 p.options.numberOfThreads = 4
4571 p.maxLuminosityBlocks.input = 2
4572 p.extend(f)
4573 self.assertEqual(p.maxEvents.input.value(),10)
4574 self.assertEqual(p.options.numberOfThreads.value(), 4)
4575 self.assertEqual(p.maxLuminosityBlocks.input.value(),2)
4576
4577 f = ProcessFragment("Fragment")
4578 f.fltr = EDFilter("Foo")
4579 p = Process('PROCESS')
4580 p.extend(f)
4581 self.assertTrue(hasattr(p,'fltr'))
4582 def testProcessForProcessAccelerator(self):
4583 proc = Process("TEST")
4584 p = ProcessForProcessAccelerator(proc)
4585 p.TestService = Service("TestService")
4586 self.assertTrue(hasattr(proc, "TestService"))
4587 self.assertEqual(proc.TestService.type_(), "TestService")
4588 self.assertRaises(TypeError, setattr, p, "a", EDProducer("Foo"))
4589 p.add_(Service("TestServiceTwo"))
4590 self.assertTrue(hasattr(proc, "TestServiceTwo"))
4591 self.assertEqual(proc.TestServiceTwo.type_(), "TestServiceTwo")
4592 p.TestService.foo = untracked.uint32(42)
4593 self.assertEqual(proc.TestService.foo.value(), 42)
4594 proc.mod = EDProducer("Producer")
4595 self.assertRaises(TypeError, getattr, p, "mod")
4596 def testProcessAccelerator(self):
4597 proc = Process("TEST")
4598 p = TestMakePSet()
4599 proc.fillProcessDesc(p)
4600 self.assertTrue(["cpu"], p.values["@available_accelerators"][1])
4601 self.assertFalse(p.values["@selected_accelerators"][0])
4602 self.assertTrue(["cpu"], p.values["@selected_accelerators"][1])
4603 self.assertFalse(p.values["@module_type_resolver"][0])
4604 self.assertEqual("", p.values["@module_type_resolver"][1])
4605
4606 proc = Process("TEST")
4607 self.assertRaises(TypeError, setattr, proc, "processAcceleratorTest", ProcessAcceleratorTest())
4608 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4609 del proc.MessageLogger
4610 self.maxDiff = None
4611 self.assertEqual(proc.dumpPython(),
4612 """import FWCore.ParameterSet.Config as cms
4613 from test import ProcessAcceleratorTest
4614
4615 process = cms.Process("TEST")
4616
4617 process.maxEvents = cms.untracked.PSet(
4618 input = cms.optional.untracked.int32,
4619 output = cms.optional.untracked.allowed(cms.int32,cms.PSet)
4620 )
4621
4622 process.maxLuminosityBlocks = cms.untracked.PSet(
4623 input = cms.untracked.int32(-1)
4624 )
4625
4626 process.options = cms.untracked.PSet(
4627 FailPath = cms.untracked.vstring(),
4628 IgnoreCompletely = cms.untracked.vstring(),
4629 Rethrow = cms.untracked.vstring(),
4630 SkipEvent = cms.untracked.vstring(),
4631 accelerators = cms.untracked.vstring('*'),
4632 allowUnscheduled = cms.obsolete.untracked.bool,
4633 canDeleteEarly = cms.untracked.vstring(),
4634 deleteNonConsumedUnscheduledModules = cms.untracked.bool(True),
4635 dumpOptions = cms.untracked.bool(False),
4636 emptyRunLumiMode = cms.obsolete.untracked.string,
4637 eventSetup = cms.untracked.PSet(
4638 forceNumberOfConcurrentIOVs = cms.untracked.PSet(
4639 allowAnyLabel_=cms.required.untracked.uint32
4640 ),
4641 numberOfConcurrentIOVs = cms.untracked.uint32(0)
4642 ),
4643 fileMode = cms.untracked.string('FULLMERGE'),
4644 forceEventSetupCacheClearOnNewRun = cms.untracked.bool(False),
4645 holdsReferencesToDeleteEarly = cms.untracked.VPSet(),
4646 makeTriggerResults = cms.obsolete.untracked.bool,
4647 modulesToIgnoreForDeleteEarly = cms.untracked.vstring(),
4648 numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(0),
4649 numberOfConcurrentRuns = cms.untracked.uint32(1),
4650 numberOfStreams = cms.untracked.uint32(0),
4651 numberOfThreads = cms.untracked.uint32(1),
4652 printDependencies = cms.untracked.bool(False),
4653 sizeOfStackForThreadsInKB = cms.optional.untracked.uint32,
4654 throwIfIllegalParameter = cms.untracked.bool(True),
4655 wantSummary = cms.untracked.bool(False)
4656 )
4657
4658 process.ProcessAcceleratorTest = ProcessAcceleratorTest(
4659 enabled = ['test1', 'test2', 'anothertest3']
4660 )
4661
4662
4663 """)
4664 p = TestMakePSet()
4665 proc.fillProcessDesc(p)
4666 self.assertEqual(["*"], p.values["options"][1].values["accelerators"][1])
4667 self.assertFalse(p.values["options"][1].values["accelerators"][0])
4668 self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["@selected_accelerators"][1])
4669 self.assertEqual("AcceleratorTestService", p.values["services"][1][0].values["@service_type"][1])
4670 self.assertFalse(p.values["@available_accelerators"][0])
4671 self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4672 self.assertFalse(p.values["@module_type_resolver"][0])
4673 self.assertEqual("", p.values["@module_type_resolver"][1])
4674
4675 proc = Process("TEST")
4676 proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"])
4677 p = TestMakePSet()
4678 proc.fillProcessDesc(p)
4679 self.assertEqual(["cpu", "test1"], p.values["@selected_accelerators"][1])
4680 self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
4681
4682 proc = Process("TEST")
4683 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4684 proc.options.accelerators = ["test2"]
4685 p = TestMakePSet()
4686 proc.fillProcessDesc(p)
4687 self.assertEqual(["test2"], p.values["@selected_accelerators"][1])
4688 self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4689
4690 proc = Process("TEST")
4691 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4692 proc.options.accelerators = ["test*"]
4693 proc.fillProcessDesc(p)
4694 self.assertEqual(["test1", "test2"], p.values["@selected_accelerators"][1])
4695 self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4696
4697 proc = Process("TEST")
4698 proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"])
4699 proc.options.accelerators = ["test2"]
4700 p = TestMakePSet()
4701 proc.fillProcessDesc(p)
4702 self.assertEqual([], p.values["@selected_accelerators"][1])
4703 self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
4704
4705 proc = Process("TEST")
4706 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4707 proc.options.accelerators = ["cpu*"]
4708 p = TestMakePSet()
4709 proc.fillProcessDesc(p)
4710 self.assertEqual(["cpu"], p.values["@selected_accelerators"][1])
4711 self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4712
4713 proc = Process("TEST")
4714 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4715 proc.options.accelerators = ["test3"]
4716 p = TestMakePSet()
4717 self.assertRaises(ValueError, proc.fillProcessDesc, p)
4718
4719 proc = Process("TEST")
4720 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4721 proc.options.accelerators = ["*", "test1"]
4722 p = TestMakePSet()
4723 self.assertRaises(ValueError, proc.fillProcessDesc, p)
4724
4725 proc = Process("TEST")
4726 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4727 proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2()
4728 p = TestMakePSet()
4729 proc.fillProcessDesc(p)
4730 self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@selected_accelerators"][1])
4731 self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4732
4733 proc = Process("TEST")
4734 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4735 proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2()
4736 proc.options.accelerators = ["*test3", "c*"]
4737 p = TestMakePSet()
4738 proc.fillProcessDesc(p)
4739 self.assertEqual(["anothertest3", "cpu"], p.values["@selected_accelerators"][1])
4740 self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4741
4742 proc = Process("TEST")
4743 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4744 proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4745 a = int32(1),
4746 b = PSet(c = int32(2))),
4747 test1 = EDProducer("Bar",
4748 aa = int32(11),
4749 bb = PSet(cc = int32(12))))
4750 proc.p = Path(proc.sp)
4751 p = TestMakePSet()
4752 proc.fillProcessDesc(p)
4753 self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
4754
4755 proc = Process("TEST")
4756 proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"])
4757 proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4758 a = int32(1),
4759 b = PSet(c = int32(2))),
4760 test1 = EDProducer("Bar",
4761 aa = int32(11),
4762 bb = PSet(cc = int32(12))))
4763 proc.p = Path(proc.sp)
4764 p = TestMakePSet()
4765 proc.fillProcessDesc(p)
4766 self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
4767
4768 proc = Process("TEST")
4769 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4770 proc.options.accelerators = ["test1"]
4771 proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4772 a = int32(1),
4773 b = PSet(c = int32(2))),
4774 test1 = EDProducer("Bar",
4775 aa = int32(11),
4776 bb = PSet(cc = int32(12))))
4777 proc.p = Path(proc.sp)
4778 p = TestMakePSet()
4779 proc.fillProcessDesc(p)
4780 self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
4781
4782 proc = Process("TEST")
4783 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4784 proc.options.accelerators = ["test*"]
4785 proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4786 a = int32(1),
4787 b = PSet(c = int32(2))),
4788 test1 = EDProducer("Bar",
4789 aa = int32(11),
4790 bb = PSet(cc = int32(12))))
4791 proc.p = Path(proc.sp)
4792 p = TestMakePSet()
4793 proc.fillProcessDesc(p)
4794 self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
4795
4796 proc = Process("TEST")
4797 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4798 proc.options.accelerators = ["anothertest3"]
4799 proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4800 a = int32(1),
4801 b = PSet(c = int32(2))),
4802 test1 = EDProducer("Bar",
4803 aa = int32(11),
4804 bb = PSet(cc = int32(12))))
4805 proc.p = Path(proc.sp)
4806 p = TestMakePSet()
4807 self.assertRaises(RuntimeError, proc.fillProcessDesc, p)
4808
4809 proc = Process("TEST")
4810 proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4811 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4812 proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2()
4813 proc.normalProducer = EDProducer("FooProducer")
4814 proc.testProducer = EDProducer("BarProducer@test")
4815 proc.test2Producer = EDProducer("BarProducer@test", test=untracked.PSet(backend=untracked.string("test2_backend")))
4816 proc.testAnalyzer = EDAnalyzer("Analyzer@test")
4817 proc.testFilter = EDAnalyzer("Filter@test")
4818 proc.testESProducer = ESProducer("ESProducer@test")
4819 proc.testESSource = ESSource("ESSource@test")
4820 proc.p = Path(proc.normalProducer+proc.testProducer+proc.test2Producer+proc.testAnalyzer+proc.testFilter)
4821 p = TestMakePSet()
4822 proc.fillProcessDesc(p)
4823 self.assertEqual("TestModuleTypeResolver", p.values["@module_type_resolver"][1])
4824 self.assertEqual("FooProducer", p.values["normalProducer"][1].values["@module_type"][1])
4825 self.assertEqual(len(list(filter(lambda x: not "@" in x, p.values["normalProducer"][1].values.keys()))), 0)
4826 self.assertEqual("BarProducer@test", p.values["testProducer"][1].values["@module_type"][1])
4827 self.assertEqual(False, p.values["testProducer"][1].values["test"][0])
4828 self.assertEqual(False, p.values["testProducer"][1].values["test"][1].values["backend"][0])
4829 self.assertEqual("test1_backend", p.values["testProducer"][1].values["test"][1].values["backend"][1])
4830 self.assertEqual("BarProducer@test", p.values["test2Producer"][1].values["@module_type"][1])
4831 self.assertEqual("test2_backend", p.values["test2Producer"][1].values["test"][1].values["backend"][1])
4832 self.assertEqual("Analyzer@test", p.values["testAnalyzer"][1].values["@module_type"][1])
4833 self.assertEqual("test1_backend", p.values["testAnalyzer"][1].values["test"][1].values["backend"][1])
4834 self.assertEqual("Filter@test", p.values["testFilter"][1].values["@module_type"][1])
4835 self.assertEqual("test1_backend", p.values["testFilter"][1].values["test"][1].values["backend"][1])
4836 self.assertEqual("ESProducer@test", p.values["ESProducer@test@testESProducer"][1].values["@module_type"][1])
4837 self.assertEqual("test1_backend", p.values["ESProducer@test@testESProducer"][1].values["test"][1].values["backend"][1])
4838 self.assertEqual("ESSource@test", p.values["ESSource@test@testESSource"][1].values["@module_type"][1])
4839 self.assertEqual("test1_backend", p.values["ESSource@test@testESSource"][1].values["test"][1].values["backend"][1])
4840
4841
4842 proc = Process("Test")
4843 proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4844 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4845 proc.options.accelerators = []
4846 self.assertRaises(EDMException, proc.fillProcessDesc, p)
4847
4848 proc = Process("Test")
4849 proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4850 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4851 proc.options.accelerators = ["test1"]
4852 proc.test2Producer = EDProducer("BarProducer@test", test=untracked.PSet(backend=untracked.string("test2_backend")))
4853 proc.p = Path(proc.test2Producer)
4854 self.assertRaises(EDMException, proc.fillProcessDesc, p)
4855
4856
4857 proc = Process("TEST")
4858 proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4859 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4860 proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2(
4861 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4862 p = TestMakePSet()
4863 self.assertRaises(RuntimeError, proc.fillProcessDesc, p)
4864
4865 import pickle
4866 proc = Process("TEST")
4867 proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4868 proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4869 a = int32(1),
4870 b = PSet(c = int32(2))),
4871 test1 = EDProducer("Bar",
4872 aa = int32(11),
4873 bb = PSet(cc = int32(12))))
4874 proc.p = Path(proc.sp)
4875 pkl = pickle.dumps(proc)
4876 unpkl = pickle.loads(pkl)
4877 p = TestMakePSet()
4878 unpkl.fillProcessDesc(p)
4879 self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
4880 self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4881 unpkl = pickle.loads(pkl)
4882 unpkl.ProcessAcceleratorTest.setEnabled(["test1"])
4883 p = TestMakePSet()
4884 unpkl.fillProcessDesc(p)
4885 self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
4886 self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
4887
4888 unittest.main()