Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-04-06 12:12:53

0001 #!/usr/bin/env python3
0002 
0003 ### command line options helper
0004 from __future__ import print_function
0005 from __future__ import absolute_import
0006 import os
0007 from  .Options import Options
0008 options = Options()
0009 
0010 
0011 ## imports
0012 import sys
0013 from typing import Union
0014 from .Mixins import PrintOptions,_ParameterTypeBase,_SimpleParameterTypeBase, _Parameterizable, _ConfigureComponent, _TypedParameterizable, _Labelable,  _Unlabelable,  _ValidatingListBase, _modifyParametersFromDict
0015 from .Mixins import *
0016 from .Types import *
0017 from .Modules import *
0018 from .Modules import _Module
0019 from .SequenceTypes import *
0020 from .SequenceTypes import _ModuleSequenceType, _Sequenceable  #extend needs it
0021 from .SequenceVisitors import PathValidator, EndPathValidator, FinalPathValidator, ScheduleTaskValidator, NodeVisitor, CompositeVisitor, ModuleNamesFromGlobalsVisitor
0022 from .MessageLogger import MessageLogger
0023 from . import DictTypes
0024 
0025 from .ExceptionHandling import *
0026 
0027 #when building RECO paths we have hit the default recursion limit
0028 if sys.getrecursionlimit()<5000:
0029     sys.setrecursionlimit(5000)
0030 
0031 class edm(object):
0032     class errors(object):
0033         #Allowed errors to be used within Python
0034         Configuration = "{Configuration}"
0035         UnavailableAccelerator = "{UnavailableAccelerator}"
0036 
0037 class EDMException(Exception):
0038     def __init__(self, error:str, message: str):
0039         super().__init__(error+"\n"+message)
0040 
0041 
0042 def checkImportPermission(minLevel: int = 2, allowedPatterns = []):
0043     """
0044     Raise an exception if called by special config files. This checks
0045     the call or import stack for the importing file. An exception is raised if
0046     the importing module is not in allowedPatterns and if it is called too deeply:
0047     minLevel = 2: inclusion by top lvel cfg only
0048     minLevel = 1: No inclusion allowed
0049     allowedPatterns = ['Module1','Module2/SubModule1'] allows import
0050     by any module in Module1 or Submodule1
0051     """
0052 
0053     import inspect
0054     import os
0055 
0056     ignorePatterns = ['FWCore/ParameterSet/Config.py', 'FWCore/ParameterSet/python/Config.py','<string>','<frozen ']
0057     CMSSWPath = [os.environ['CMSSW_BASE'],os.environ['CMSSW_RELEASE_BASE']]
0058 
0059     # Filter the stack to things in CMSSWPath and not in ignorePatterns
0060     trueStack = []
0061     for item in inspect.stack():
0062         inPath = False
0063         ignore = False
0064 
0065         for pattern in CMSSWPath:
0066             if item[1].find(pattern) != -1:
0067                 inPath = True
0068                 break
0069         if item[1].find('/') == -1: # The base file, no path
0070             inPath = True
0071 
0072         for pattern in ignorePatterns:
0073             if item[1].find(pattern) != -1:
0074                 ignore = True
0075                 break
0076 
0077         if inPath and not ignore:
0078             trueStack.append(item[1])
0079 
0080     importedFile = trueStack[0]
0081     importedBy   = ''
0082     if len(trueStack) > 1:
0083         importedBy = trueStack[1]
0084 
0085     for pattern in allowedPatterns:
0086         if importedBy.find(pattern) > -1:
0087             return True
0088 
0089     if len(trueStack) <= minLevel: # Imported directly
0090         return True
0091 
0092     raise ImportError("Inclusion of %s is allowed only by cfg or specified cfi files."
0093                       % importedFile)
0094 
0095 def findProcess(module):
0096     """Look inside the module and find the Processes it contains"""
0097     class Temp(object):
0098         pass
0099     process = None
0100     if isinstance(module,dict):
0101         if 'process' in module:
0102             p = module['process']
0103             module = Temp()
0104             module.process = p
0105     if hasattr(module,'process'):
0106         if isinstance(module.process,Process):
0107             process = module.process
0108         else:
0109             raise RuntimeError("The attribute named 'process' does not inherit from the Process class")
0110     else:
0111         raise RuntimeError("no 'process' attribute found in the module, please add one")
0112     return process
0113 
0114 class Process(object):
0115     """Root class for a CMS configuration process"""
0116     _firstProcess = True
0117     def __init__(self,name: str, *Mods):
0118         """The argument 'name' will be the name applied to this Process
0119             Can optionally pass as additional arguments cms.Modifier instances
0120             that will be used to modify the Process as it is built
0121             """
0122         self.__dict__['_Process__name'] = name
0123         if not name.isalnum():
0124             raise RuntimeError("Error: The process name is an empty string or contains non-alphanumeric characters")
0125         self.__dict__['_Process__filters'] = {}
0126         self.__dict__['_Process__producers'] = {}
0127         self.__dict__['_Process__switchproducers'] = {}
0128         self.__dict__['_Process__source'] = None
0129         self.__dict__['_Process__looper'] = None
0130         self.__dict__['_Process__subProcesses'] = []
0131         self.__dict__['_Process__schedule'] = None
0132         self.__dict__['_Process__analyzers'] = {}
0133         self.__dict__['_Process__outputmodules'] = {}
0134         self.__dict__['_Process__paths'] = DictTypes.SortedKeysDict()    # have to keep the order
0135         self.__dict__['_Process__endpaths'] = DictTypes.SortedKeysDict() # of definition
0136         self.__dict__['_Process__finalpaths'] = DictTypes.SortedKeysDict() # of definition
0137         self.__dict__['_Process__sequences'] = {}
0138         self.__dict__['_Process__tasks'] = {}
0139         self.__dict__['_Process__conditionaltasks'] = {}
0140         self.__dict__['_Process__services'] = {}
0141         self.__dict__['_Process__essources'] = {}
0142         self.__dict__['_Process__esproducers'] = {}
0143         self.__dict__['_Process__esprefers'] = {}
0144         self.__dict__['_Process__aliases'] = {}
0145         self.__dict__['_Process__psets']={}
0146         self.__dict__['_Process__vpsets']={}
0147         self.__dict__['_cloneToObjectDict'] = {}
0148         # policy switch to avoid object overwriting during extend/load
0149         self.__dict__['_Process__InExtendCall'] = False
0150         self.__dict__['_Process__partialschedules'] = {}
0151         self.__isStrict = False
0152         self.__dict__['_Process__modifiers'] = Mods
0153         self.__dict__['_Process__accelerators'] = {}
0154         self.__injectValidValue('options', Process.defaultOptions_())
0155         self.__injectValidValue('maxEvents', Process.defaultMaxEvents_())
0156         self.maxLuminosityBlocks = Process.defaultMaxLuminosityBlocks_()
0157         # intentionally not cloned to ensure that everyone taking
0158         # MessageLogger still via
0159         # FWCore.Message(Logger|Service).MessageLogger_cfi
0160         # use the very same MessageLogger object.
0161         self.MessageLogger = MessageLogger
0162         if Process._firstProcess:
0163             Process._firstProcess = False
0164         else:
0165             if len(Mods) > 0:
0166                 for m in self.__modifiers:
0167                     if not m._isChosen():
0168                         raise RuntimeError("The Process {} tried to redefine which Modifiers to use after another Process was already started".format(name))
0169         for m in self.__modifiers:
0170             m._setChosen()
0171 
0172     def setStrict(self, value: bool):
0173         self.__isStrict = value
0174         _Module.__isStrict__ = True
0175 
0176     # some user-friendly methods for command-line browsing
0177     def producerNames(self):
0178         """Returns a string containing all the EDProducer labels separated by a blank"""
0179         return ' '.join(self.producers_().keys())
0180     def switchProducerNames(self):
0181         """Returns a string containing all the SwitchProducer labels separated by a blank"""
0182         return ' '.join(self.switchProducers_().keys())
0183     def analyzerNames(self):
0184         """Returns a string containing all the EDAnalyzer labels separated by a blank"""
0185         return ' '.join(self.analyzers_().keys())
0186     def filterNames(self):
0187         """Returns a string containing all the EDFilter labels separated by a blank"""
0188         return ' '.join(self.filters_().keys())
0189     def pathNames(self):
0190         """Returns a string containing all the Path names separated by a blank"""
0191         return ' '.join(self.paths_().keys())
0192 
0193     def __setstate__(self, pkldict):
0194         """
0195         Unpickling hook.
0196 
0197         Since cloneToObjectDict stores a hash of objects by their
0198         id() it needs to be updated when unpickling to use the
0199         new object id values instantiated during the unpickle.
0200 
0201         """
0202         self.__dict__.update(pkldict)
0203         tmpDict = {}
0204         for value in self._cloneToObjectDict.values():
0205             tmpDict[id(value)] = value
0206         self.__dict__['_cloneToObjectDict'] = tmpDict
0207 
0208 
0209 
0210     def filters_(self):
0211         """returns a dict of the filters that have been added to the Process"""
0212         return DictTypes.FixedKeysDict(self.__filters)
0213     filters = property(filters_, doc="dictionary containing the filters for the process")
0214     def name_(self) -> str:
0215         return self.__name
0216     def setName_(self,name: str):
0217         if not name.isalnum():
0218             raise RuntimeError("Error: The process name is an empty string or contains non-alphanumeric characters")
0219         self.__dict__['_Process__name'] = name
0220     process = property(name_,setName_, doc="name of the process")
0221     def producers_(self):
0222         """returns a dict of the producers that have been added to the Process"""
0223         return DictTypes.FixedKeysDict(self.__producers)
0224     producers = property(producers_,doc="dictionary containing the producers for the process")
0225     def switchProducers_(self):
0226         """returns a dict of the SwitchProducers that have been added to the Process"""
0227         return DictTypes.FixedKeysDict(self.__switchproducers)
0228     switchProducers = property(switchProducers_,doc="dictionary containing the SwitchProducers for the process")
0229     def source_(self):
0230         """returns the source that has been added to the Process or None if none have been added"""
0231         return self.__source
0232     def setSource_(self,src):
0233         self._placeSource('source',src)
0234     source = property(source_,setSource_,doc='the main source or None if not set')
0235     def looper_(self):
0236         """returns the looper that has been added to the Process or None if none have been added"""
0237         return self.__looper
0238     def setLooper_(self,lpr):
0239         self._placeLooper('looper',lpr)
0240     looper = property(looper_,setLooper_,doc='the main looper or None if not set')
0241     @staticmethod
0242     def defaultOptions_():
0243         return untracked.PSet(numberOfThreads = untracked.uint32(1),
0244                               numberOfStreams = untracked.uint32(0),
0245                               numberOfConcurrentRuns = untracked.uint32(1),
0246                               numberOfConcurrentLuminosityBlocks = untracked.uint32(0),
0247                               eventSetup = untracked.PSet(
0248                                   numberOfConcurrentIOVs = untracked.uint32(0),
0249                                   forceNumberOfConcurrentIOVs = untracked.PSet(
0250                                       allowAnyLabel_ = required.untracked.uint32
0251                                   )
0252                               ),
0253                               accelerators = untracked.vstring('*'),
0254                               wantSummary = untracked.bool(False),
0255                               fileMode = untracked.string('FULLMERGE'),
0256                               forceEventSetupCacheClearOnNewRun = untracked.bool(False),
0257                               throwIfIllegalParameter = untracked.bool(True),
0258                               printDependencies = untracked.bool(False),
0259                               deleteNonConsumedUnscheduledModules = untracked.bool(True),
0260                               sizeOfStackForThreadsInKB = optional.untracked.uint32,
0261                               Rethrow = untracked.vstring(),
0262                               TryToContinue = untracked.vstring(),
0263                               IgnoreCompletely = untracked.vstring(),
0264                               modulesToCallForTryToContinue = untracked.vstring(),
0265                               canDeleteEarly = untracked.vstring(),
0266                               holdsReferencesToDeleteEarly = untracked.VPSet(),
0267                               modulesToIgnoreForDeleteEarly = untracked.vstring(),
0268                               dumpOptions = untracked.bool(False),
0269                               allowUnscheduled = obsolete.untracked.bool,
0270                               emptyRunLumiMode = obsolete.untracked.string,
0271                               makeTriggerResults = obsolete.untracked.bool,
0272                               )
0273     def __updateOptions(self,opt):
0274         newOpts = self.defaultOptions_()
0275         if isinstance(opt,dict):
0276             for k,v in opt.items():
0277                 setattr(newOpts,k,v)
0278         else:
0279             for p in opt.parameters_():
0280                 setattr(newOpts, p, getattr(opt,p))
0281         return newOpts
0282     @staticmethod
0283     def defaultMaxEvents_():
0284         return untracked.PSet(input=optional.untracked.int32,
0285                               output=optional.untracked.allowed(int32,PSet))
0286     def __updateMaxEvents(self,ps: Union[dict,PSet]):
0287         newMax = self.defaultMaxEvents_()
0288         if isinstance(ps,dict):
0289             for k,v in ps.items():
0290                 setattr(newMax,k,v)
0291         else:
0292             for p in ps.parameters_():
0293                 setattr(newMax, p, getattr(ps,p))
0294         return newMax
0295     @staticmethod
0296     def defaultMaxLuminosityBlocks_():
0297         return untracked.PSet(input=untracked.int32(-1))
0298     def subProcesses_(self):
0299         """returns a list of the subProcesses that have been added to the Process"""
0300         return self.__subProcesses
0301     subProcesses = property(subProcesses_,doc='the SubProcesses that have been added to the Process')
0302     def analyzers_(self):
0303         """returns a dict of the analyzers that have been added to the Process"""
0304         return DictTypes.FixedKeysDict(self.__analyzers)
0305     analyzers = property(analyzers_,doc="dictionary containing the analyzers for the process")
0306     def outputModules_(self):
0307         """returns a dict of the output modules that have been added to the Process"""
0308         return DictTypes.FixedKeysDict(self.__outputmodules)
0309     outputModules = property(outputModules_,doc="dictionary containing the output_modules for the process")
0310     def paths_(self):
0311         """returns a dict of the paths that have been added to the Process"""
0312         return DictTypes.SortedAndFixedKeysDict(self.__paths)
0313     paths = property(paths_,doc="dictionary containing the paths for the process")
0314     def endpaths_(self):
0315         """returns a dict of the endpaths that have been added to the Process"""
0316         return DictTypes.SortedAndFixedKeysDict(self.__endpaths)
0317     endpaths = property(endpaths_,doc="dictionary containing the endpaths for the process")
0318     def finalpaths_(self):
0319         """returns a dict of the finalpaths that have been added to the Process"""
0320         return DictTypes.SortedAndFixedKeysDict(self.__finalpaths)
0321     finalpaths = property(finalpaths_,doc="dictionary containing the finalpaths for the process")
0322     def sequences_(self):
0323         """returns a dict of the sequences that have been added to the Process"""
0324         return DictTypes.FixedKeysDict(self.__sequences)
0325     sequences = property(sequences_,doc="dictionary containing the sequences for the process")
0326     def tasks_(self):
0327         """returns a dict of the tasks that have been added to the Process"""
0328         return DictTypes.FixedKeysDict(self.__tasks)
0329     tasks = property(tasks_,doc="dictionary containing the tasks for the process")
0330     def conditionaltasks_(self):
0331         """returns a dict of the conditionaltasks that have been added to the Process"""
0332         return DictTypes.FixedKeysDict(self.__conditionaltasks)
0333     conditionaltasks = property(conditionaltasks_,doc="dictionary containing the conditionatasks for the process")
0334     def schedule_(self):
0335         """returns the schedule that has been added to the Process or None if none have been added"""
0336         return self.__schedule
0337     def setPartialSchedule_(self,sch: Schedule,label: str):
0338         if label == "schedule":
0339             self.setSchedule_(sch)
0340         else:
0341             self._place(label, sch, self.__partialschedules)
0342     def setSchedule_(self,sch: Schedule):
0343         # See if every path and endpath has been inserted into the process
0344         index = 0
0345         try:
0346             for p in sch:
0347                 p.label_()
0348                 index +=1
0349         except:
0350             raise RuntimeError("The path at index "+str(index)+" in the Schedule was not attached to the process.")
0351         self.__dict__['_Process__schedule'] = sch
0352     schedule = property(schedule_,setSchedule_,doc='the schedule or None if not set')
0353     def services_(self):
0354         """returns a dict of the services that have been added to the Process"""
0355         return DictTypes.FixedKeysDict(self.__services)
0356     services = property(services_,doc="dictionary containing the services for the process")
0357     def processAccelerators_(self):
0358         """returns a dict of the ProcessAccelerators that have been added to the Process"""
0359         return DictTypes.FixedKeysDict(self.__accelerators)
0360     processAccelerators = property(processAccelerators_,doc="dictionary containing the ProcessAccelerators for the process")
0361     def es_producers_(self):
0362         """returns a dict of the esproducers that have been added to the Process"""
0363         return DictTypes.FixedKeysDict(self.__esproducers)
0364     es_producers = property(es_producers_,doc="dictionary containing the es_producers for the process")
0365     def es_sources_(self):
0366         """returns a the es_sources that have been added to the Process"""
0367         return DictTypes.FixedKeysDict(self.__essources)
0368     es_sources = property(es_sources_,doc="dictionary containing the es_sources for the process")
0369     def es_prefers_(self):
0370         """returns a dict of the es_prefers that have been added to the Process"""
0371         return DictTypes.FixedKeysDict(self.__esprefers)
0372     es_prefers = property(es_prefers_,doc="dictionary containing the es_prefers for the process")
0373     def aliases_(self):
0374         """returns a dict of the aliases that have been added to the Process"""
0375         return DictTypes.FixedKeysDict(self.__aliases)
0376     aliases = property(aliases_,doc="dictionary containing the aliases for the process")
0377     def psets_(self):
0378         """returns a dict of the PSets that have been added to the Process"""
0379         return DictTypes.FixedKeysDict(self.__psets)
0380     psets = property(psets_,doc="dictionary containing the PSets for the process")
0381     def vpsets_(self):
0382         """returns a dict of the VPSets that have been added to the Process"""
0383         return DictTypes.FixedKeysDict(self.__vpsets)
0384     vpsets = property(vpsets_,doc="dictionary containing the PSets for the process")
0385 
0386     def isUsingModifier(self,mod) -> bool:
0387         """returns True if the Modifier is in used by this Process"""
0388         if mod._isChosen():
0389             for m in self.__modifiers:
0390                 if m._isOrContains(mod):
0391                     return True
0392         return False
0393 
0394     def __setObjectLabel(self, object, newLabel:str) :
0395         if not object.hasLabel_() :
0396             object.setLabel(newLabel)
0397             return
0398         if newLabel == object.label_() :
0399             return
0400         if newLabel is None :
0401             object.setLabel(None)
0402             return
0403         if (hasattr(self, object.label_()) and id(getattr(self, object.label_())) == id(object)) :
0404             msg100 = "Attempting to change the label of an attribute of the Process\n"
0405             msg101 = "Old label = "+object.label_()+"  New label = "+newLabel+"\n"
0406             msg102 = "Type = "+str(type(object))+"\n"
0407             msg103 = "Some possible solutions:\n"
0408             msg104 = "  1. Clone modules instead of using simple assignment. Cloning is\n"
0409             msg105 = "  also preferred for other types when possible.\n"
0410             msg106 = "  2. Declare new names starting with an underscore if they are\n"
0411             msg107 = "  for temporaries you do not want propagated into the Process. The\n"
0412             msg108 = "  underscore tells \"from x import *\" and process.load not to import\n"
0413             msg109 = "  the name.\n"
0414             msg110 = "  3. Reorganize so the assigment is not necessary. Giving a second\n"
0415             msg111 = "  name to the same object usually causes confusion and problems.\n"
0416             msg112 = "  4. Compose Sequences: newName = cms.Sequence(oldName)\n"
0417             raise ValueError(msg100+msg101+msg102+msg103+msg104+msg105+msg106+msg107+msg108+msg109+msg110+msg111+msg112)
0418         object.setLabel(None)
0419         object.setLabel(newLabel)
0420 
0421     def __setattr__(self,name:str,value):
0422         # check if the name is well-formed (only _ and alphanumerics are allowed)
0423         if not name.replace('_','').isalnum():
0424             raise ValueError('The label '+name+' contains forbiden characters')
0425 
0426         if name == 'options':
0427             value = self.__updateOptions(value)
0428         if name == 'maxEvents':
0429             value = self.__updateMaxEvents(value)
0430 
0431         # private variable exempt from all this
0432         if name.startswith('_Process__'):
0433             self.__dict__[name]=value
0434             return
0435         if not isinstance(value,_ConfigureComponent):
0436             raise TypeError("can only assign labels to an object that inherits from '_ConfigureComponent'\n"
0437                             +"an instance of "+str(type(value))+" will not work - requested label is "+name)
0438         if not isinstance(value,_Labelable) and not isinstance(value,Source) and not isinstance(value,Looper) and not isinstance(value,Schedule):
0439             if name == value.type_():
0440                 if hasattr(self,name) and (getattr(self,name)!=value):
0441                     self._replaceInTasks(name, value)
0442                     self._replaceInConditionalTasks(name, value)
0443                 # Only Services get handled here
0444                 self.add_(value)
0445                 return
0446             else:
0447                 raise TypeError("an instance of "+str(type(value))+" can not be assigned the label '"+name+"'.\n"+
0448                                 "Please either use the label '"+value.type_()+" or use the 'add_' method instead.")
0449         #clone the item
0450         if self.__isStrict:
0451             newValue =value.copy()
0452             try:
0453                 newValue._filename = value._filename
0454             except:
0455                 pass
0456             value.setIsFrozen()
0457         else:
0458             newValue =value
0459         if not self._okToPlace(name, value, self.__dict__):
0460             newFile='top level config'
0461             if hasattr(value,'_filename'):
0462                 newFile = value._filename
0463             oldFile='top level config'
0464             oldValue = getattr(self,name)
0465             if hasattr(oldValue,'_filename'):
0466                 oldFile = oldValue._filename
0467             msg = "Trying to override definition of process."+name
0468             msg += "\n new object defined in: "+newFile
0469             msg += "\n existing object defined in: "+oldFile
0470             raise ValueError(msg)
0471         # remove the old object of the name (if there is one)
0472         if hasattr(self,name) and not (getattr(self,name)==newValue):
0473             # Complain if items in sequences or tasks from load() statements have
0474             # degenerate names, but if the user overwrites a name in the
0475             # main config, replace it everywhere
0476             if newValue._isTaskComponent():
0477                 if not self.__InExtendCall:
0478                     self._replaceInTasks(name, newValue)
0479                     self._replaceInConditionalTasks(name, newValue)
0480                     self._replaceInSchedule(name, newValue)
0481                 else:
0482                     if not isinstance(newValue, Task):
0483                         #should check to see if used in task before complaining
0484                         newFile='top level config'
0485                         if hasattr(value,'_filename'):
0486                             newFile = value._filename
0487                         oldFile='top level config'
0488                         oldValue = getattr(self,name)
0489                         if hasattr(oldValue,'_filename'):
0490                             oldFile = oldValue._filename
0491                         msg1 = "Trying to override definition of "+name+" while it is used by the task "
0492                         msg2 = "\n new object defined in: "+newFile
0493                         msg2 += "\n existing object defined in: "+oldFile
0494                         s = self.__findFirstUsingModule(self.tasks,oldValue)
0495                         if s is not None:
0496                             raise ValueError(msg1+s.label_()+msg2)
0497 
0498             if isinstance(newValue, _Sequenceable) or newValue._isTaskComponent() or isinstance(newValue, ConditionalTask):
0499                 if not self.__InExtendCall:
0500                     if isinstance(newValue, ConditionalTask):
0501                         self._replaceInConditionalTasks(name, newValue)
0502                     self._replaceInSequences(name, newValue)
0503                 else:
0504                     #should check to see if used in sequence before complaining
0505                     newFile='top level config'
0506                     if hasattr(value,'_filename'):
0507                         newFile = value._filename
0508                     oldFile='top level config'
0509                     oldValue = getattr(self,name)
0510                     if hasattr(oldValue,'_filename'):
0511                         oldFile = oldValue._filename
0512                     msg1 = "Trying to override definition of "+name+" while it is used by the "
0513                     msg2 = "\n new object defined in: "+newFile
0514                     msg2 += "\n existing object defined in: "+oldFile
0515                     s = self.__findFirstUsingModule(self.sequences,oldValue)
0516                     if s is not None:
0517                         raise ValueError(msg1+"sequence "+s.label_()+msg2)
0518                     s = self.__findFirstUsingModule(self.paths,oldValue)
0519                     if s is not None:
0520                         raise ValueError(msg1+"path "+s.label_()+msg2)
0521                     s = self.__findFirstUsingModule(self.endpaths,oldValue)
0522                     if s is not None:
0523                         raise ValueError(msg1+"endpath "+s.label_()+msg2)
0524                     s = self.__findFirstUsingModule(self.finalpaths,oldValue)
0525                     if s is not None:
0526                         raise ValueError(msg1+"finalpath "+s.label_()+msg2)
0527 
0528             # In case of EDAlias, raise Exception always to avoid surprises
0529             if isinstance(newValue, EDAlias):
0530                 oldValue = getattr(self, name)
0531                 #should check to see if used in task/sequence before complaining
0532                 newFile='top level config'
0533                 if hasattr(value,'_filename'):
0534                     newFile = value._filename
0535                 oldFile='top level config'
0536                 if hasattr(oldValue,'_filename'):
0537                     oldFile = oldValue._filename
0538                 msg1 = "Trying to override definition of "+name+" with an EDAlias while it is used by the "
0539                 msg2 = "\n new object defined in: "+newFile
0540                 msg2 += "\n existing object defined in: "+oldFile
0541                 s = self.__findFirstUsingModule(self.tasks,oldValue)
0542                 if s is not None:
0543                     raise ValueError(msg1+"task "+s.label_()+msg2)
0544                 s = self.__findFirstUsingModule(self.sequences,oldValue)
0545                 if s is not None:
0546                     raise ValueError(msg1+"sequence "+s.label_()+msg2)
0547                 s = self.__findFirstUsingModule(self.paths,oldValue)
0548                 if s is not None:
0549                     raise ValueError(msg1+"path "+s.label_()+msg2)
0550                 s = self.__findFirstUsingModule(self.endpaths,oldValue)
0551                 if s is not None:
0552                     raise ValueError(msg1+"endpath "+s.label_()+msg2)
0553                 s = self.__findFirstUsingModule(self.finalpaths,oldValue)
0554                 if s is not None:
0555                     raise ValueError(msg1+"finalpath "+s.label_()+msg2)
0556 
0557             if not self.__InExtendCall and (Schedule._itemIsValid(newValue) or isinstance(newValue, Task)):
0558                 self._replaceInScheduleDirectly(name, newValue)
0559 
0560             self._delattrFromSetattr(name)
0561         self.__injectValidValue(name, value, newValue)
0562     def __injectValidValue(self, name:str, value, newValue = None):
0563         if newValue is None:
0564             newValue = value
0565         self.__dict__[name]=newValue
0566         if isinstance(newValue,_Labelable):
0567             self.__setObjectLabel(newValue, name)
0568             self._cloneToObjectDict[id(value)] = newValue
0569             self._cloneToObjectDict[id(newValue)] = newValue
0570         #now put in proper bucket
0571         newValue._place(name,self)
0572     def __findFirstUsingModule(self, seqsOrTasks, mod):
0573         """Given a container of sequences or tasks, find the first sequence or task
0574         containing mod and return it. If none is found, return None"""
0575         from FWCore.ParameterSet.SequenceTypes import ModuleNodeVisitor
0576         l = list()
0577         for seqOrTask in seqsOrTasks.values():
0578             l[:] = []
0579             v = ModuleNodeVisitor(l)
0580             seqOrTask.visit(v)
0581             if mod in l:
0582                 return seqOrTask
0583         return None
0584 
0585     def _delHelper(self,name:str):
0586         if not hasattr(self,name):
0587             raise KeyError('process does not know about '+name)
0588         elif name.startswith('_Process__'):
0589             raise ValueError('this attribute cannot be deleted')
0590 
0591         # we have to remove it from all dictionaries/registries
0592         dicts = [item for item in self.__dict__.values() if (isinstance(item, dict) or isinstance(item, DictTypes.SortedKeysDict))]
0593         for reg in dicts:
0594             if name in reg: del reg[name]
0595         # if it was a labelable object, the label needs to be removed
0596         obj = getattr(self,name)
0597         if isinstance(obj,_Labelable):
0598             obj.setLabel(None)
0599         if isinstance(obj,Service):
0600             obj._inProcess = False
0601 
0602     def __delattr__(self,name:str):
0603         self._delHelper(name)
0604         obj = getattr(self,name)
0605         if not obj is None:
0606             if not isinstance(obj, Sequence) and not isinstance(obj, Task) and not isinstance(obj,ConditionalTask):
0607                 # For modules, ES modules and services we can also remove
0608                 # the deleted object from Sequences, Paths, EndPaths, and
0609                 # Tasks. Note that for Sequences and Tasks that cannot be done
0610                 # reliably as the places where the Sequence or Task was used
0611                 # might have been expanded so we do not even try. We considered
0612                 # raising an exception if a Sequences or Task was explicitly
0613                 # deleted, but did not because when done carefully deletion
0614                 # is sometimes OK (for example in the prune function where it
0615                 # has been checked that the deleted Sequence is not used).
0616                 if obj._isTaskComponent():
0617                     self._replaceInTasks(name, None)
0618                     self._replaceInConditionalTasks(name, None)
0619                     self._replaceInSchedule(name, None)
0620                 if isinstance(obj, _Sequenceable) or obj._isTaskComponent():
0621                     self._replaceInSequences(name, None)
0622                 if Schedule._itemIsValid(obj) or isinstance(obj, Task):
0623                     self._replaceInScheduleDirectly(name, None)
0624         # now remove it from the process itself
0625         try:
0626             del self.__dict__[name]
0627         except:
0628             pass
0629 
0630     def _delattrFromSetattr(self,name:str):
0631         """Similar to __delattr__ but we need different behavior when called from __setattr__"""
0632         self._delHelper(name)
0633         # now remove it from the process itself
0634         try:
0635             del self.__dict__[name]
0636         except:
0637             pass
0638 
0639     def add_(self,value):
0640         """Allows addition of components that do not have to have a label, e.g. Services"""
0641         if not isinstance(value,_ConfigureComponent):
0642             raise TypeError
0643         if not isinstance(value,_Unlabelable):
0644             raise TypeError
0645         #clone the item
0646         if self.__isStrict:
0647             newValue =value.copy()
0648             value.setIsFrozen()
0649         else:
0650             newValue =value
0651         newValue._place('',self)
0652 
0653     def _okToPlace(self, name:str, mod, d) -> bool:
0654         if not self.__InExtendCall:
0655             # if going
0656             return True
0657         elif not self.__isStrict:
0658             return True
0659         elif name in d:
0660             # if there's an old copy, and the new one
0661             # hasn't been modified, we're done.  Still
0662             # not quite safe if something has been defined twice.
0663             #  Need to add checks
0664             if mod._isModified:
0665                 if d[name]._isModified:
0666                     return False
0667                 else:
0668                     return True
0669             else:
0670                 return True
0671         else:
0672             return True
0673 
0674     def _place(self, name:str, mod, d):
0675         if self._okToPlace(name, mod, d):
0676             if self.__isStrict and isinstance(mod, _ModuleSequenceType):
0677                 d[name] = mod._postProcessFixup(self._cloneToObjectDict)
0678             else:
0679                 d[name] = mod
0680             if isinstance(mod,_Labelable):
0681                 self.__setObjectLabel(mod, name)
0682     def _placeOutputModule(self,name:str,mod):
0683         self._place(name, mod, self.__outputmodules)
0684     def _placeProducer(self,name:str,mod):
0685         self._place(name, mod, self.__producers)
0686     def _placeSwitchProducer(self,name:str,mod):
0687         self._place(name, mod, self.__switchproducers)
0688     def _placeFilter(self,name:str,mod):
0689         self._place(name, mod, self.__filters)
0690     def _placeAnalyzer(self,name:str,mod):
0691         self._place(name, mod, self.__analyzers)
0692     def _placePath(self,name:str,mod):
0693         self._validateSequence(mod, name)
0694         try:
0695             self._place(name, mod, self.__paths)
0696         except ModuleCloneError as msg:
0697             context = format_outerframe(4)
0698             raise Exception("%sThe module %s in path %s is unknown to the process %s." %(context, msg, name, self._Process__name))
0699     def _placeEndPath(self,name:str,mod):
0700         self._validateSequence(mod, name)
0701         try:
0702             self._place(name, mod, self.__endpaths)
0703         except ModuleCloneError as msg:
0704             context = format_outerframe(4)
0705             raise Exception("%sThe module %s in endpath %s is unknown to the process %s." %(context, msg, name, self._Process__name))
0706     def _placeFinalPath(self,name:str,mod):
0707         self._validateSequence(mod, name)
0708         try:
0709             self._place(name, mod, self.__finalpaths)
0710         except ModuleCloneError as msg:
0711             context = format_outerframe(4)
0712             raise Exception("%sThe module %s in finalpath %s is unknown to the process %s." %(context, msg, name, self._Process__name))
0713     def _placeSequence(self,name:str,mod):
0714         self._validateSequence(mod, name)
0715         self._place(name, mod, self.__sequences)
0716     def _placeESProducer(self,name:str,mod):
0717         self._place(name, mod, self.__esproducers)
0718     def _placeESPrefer(self,name:str,mod):
0719         self._place(name, mod, self.__esprefers)
0720     def _placeESSource(self,name:str,mod):
0721         self._place(name, mod, self.__essources)
0722     def _placeTask(self,name:str,task):
0723         self._validateTask(task, name)
0724         self._place(name, task, self.__tasks)
0725     def _placeConditionalTask(self,name:str,task):
0726         self._validateConditionalTask(task, name)
0727         self._place(name, task, self.__conditionaltasks)
0728     def _placeAlias(self,name:str,mod):
0729         self._place(name, mod, self.__aliases)
0730     def _placePSet(self,name:str,mod):
0731         self._place(name, mod, self.__psets)
0732     def _placeVPSet(self,name:str,mod):
0733         self._place(name, mod, self.__vpsets)
0734     def _placeSource(self,name:str,mod):
0735         """Allow the source to be referenced by 'source' or by type name"""
0736         if name != 'source':
0737             raise ValueError("The label '"+name+"' can not be used for a Source.  Only 'source' is allowed.")
0738         if self.__dict__['_Process__source'] is not None :
0739             del self.__dict__[self.__dict__['_Process__source'].type_()]
0740         self.__dict__['_Process__source'] = mod
0741         self.__dict__[mod.type_()] = mod
0742     def _placeLooper(self,name:str,mod):
0743         if name != 'looper':
0744             raise ValueError("The label '"+name+"' can not be used for a Looper.  Only 'looper' is allowed.")
0745         self.__dict__['_Process__looper'] = mod
0746         self.__dict__[mod.type_()] = mod
0747     def _placeSubProcess(self,name:str,mod):
0748         self.__dict__['_Process__subProcess'] = mod
0749         self.__dict__[mod.type_()] = mod
0750     def addSubProcess(self,mod):
0751         self.__subProcesses.append(mod)
0752     def _placeService(self,typeName:str,mod):
0753         self._place(typeName, mod, self.__services)
0754         if typeName in self.__dict__:
0755             self.__dict__[typeName]._inProcess = False
0756         self.__dict__[typeName]=mod
0757     def _placeAccelerator(self,typeName:str,mod):
0758         self._place(typeName, mod, self.__accelerators)
0759         self.__dict__[typeName]=mod
0760     def load(self, moduleName:str):
0761         moduleName = moduleName.replace("/",".")
0762         module = __import__(moduleName)
0763         self.extend(sys.modules[moduleName])
0764     def extend(self,other,items=()):
0765         """Look in other and find types that we can use"""
0766         # enable explicit check to avoid overwriting of existing objects
0767         self.__dict__['_Process__InExtendCall'] = True
0768 
0769         seqs = dict()
0770         tasksToAttach = dict()
0771         mods = []
0772         for name in dir(other):
0773             #'from XX import *' ignores these, and so should we.
0774             if name.startswith('_'):
0775                 continue
0776             item = getattr(other,name)
0777             if name == "source" or name == "looper":
0778                 # In these cases 'item' could be None if the specific object was not defined
0779                 if item is not None:
0780                     self.__setattr__(name,item)
0781             elif isinstance(item,_ModuleSequenceType):
0782                 seqs[name]=item
0783             elif isinstance(item,Task) or isinstance(item, ConditionalTask):
0784                 tasksToAttach[name] = item
0785             elif isinstance(item,_Labelable):
0786                 self.__setattr__(name,item)
0787                 if not item.hasLabel_() :
0788                     item.setLabel(name)
0789             elif isinstance(item,Schedule):
0790                 self.__setattr__(name,item)
0791             elif isinstance(item,_Unlabelable):
0792                 self.add_(item)
0793             elif isinstance(item,ProcessModifier):
0794                 mods.append(item)
0795             elif isinstance(item,ProcessFragment):
0796                 self.extend(item)
0797 
0798         #now create a sequence that uses the newly made items
0799         for name,seq in seqs.items():
0800             if id(seq) not in self._cloneToObjectDict:
0801                 self.__setattr__(name,seq)
0802             else:
0803                 newSeq = self._cloneToObjectDict[id(seq)]
0804                 self.__dict__[name]=newSeq
0805                 self.__setObjectLabel(newSeq, name)
0806                 #now put in proper bucket
0807                 newSeq._place(name,self)
0808 
0809         for name, task in tasksToAttach.items():
0810             self.__setattr__(name, task)
0811 
0812         #apply modifiers now that all names have been added
0813         for item in mods:
0814             item.apply(self)
0815 
0816         self.__dict__['_Process__InExtendCall'] = False
0817 
0818     def _dumpConfigNamedList(self,items,typeName:str,options:PrintOptions) -> str:
0819         returnValue = ''
0820         for name,item in items:
0821             returnValue +=options.indentation()+typeName+' '+name+' = '+item.dumpConfig(options)
0822         return returnValue
0823 
0824     def _dumpConfigUnnamedList(self,items,typeName:str,options:PrintOptions) -> str:
0825         returnValue = ''
0826         for name,item in items:
0827             returnValue +=options.indentation()+typeName+' = '+item.dumpConfig(options)
0828         return returnValue
0829 
0830     def _dumpConfigOptionallyNamedList(self,items,typeName:str,options:PrintOptions) -> str:
0831         returnValue = ''
0832         for name,item in items:
0833             if name == item.type_():
0834                 name = ''
0835             returnValue +=options.indentation()+typeName+' '+name+' = '+item.dumpConfig(options)
0836         return returnValue
0837 
0838     def dumpConfig(self, options:PrintOptions=PrintOptions()) -> str:
0839         """return a string containing the equivalent process defined using the old configuration language"""
0840         config = "process "+self.__name+" = {\n"
0841         options.indent()
0842         if self.source_():
0843             config += options.indentation()+"source = "+self.source_().dumpConfig(options)
0844         if self.looper_():
0845             config += options.indentation()+"looper = "+self.looper_().dumpConfig(options)
0846 
0847         config+=self._dumpConfigNamedList(self.subProcesses_(),
0848                                   'subProcess',
0849                                   options)
0850         config+=self._dumpConfigNamedList(self.producers_().items(),
0851                                   'module',
0852                                   options)
0853         config+=self._dumpConfigNamedList(self.switchProducers_().items(),
0854                                   'module',
0855                                   options)
0856         config+=self._dumpConfigNamedList(self.filters_().items(),
0857                                   'module',
0858                                   options)
0859         config+=self._dumpConfigNamedList(self.analyzers_().items(),
0860                                   'module',
0861                                   options)
0862         config+=self._dumpConfigNamedList(self.outputModules_().items(),
0863                                   'module',
0864                                   options)
0865         config+=self._dumpConfigNamedList(self.sequences_().items(),
0866                                   'sequence',
0867                                   options)
0868         config+=self._dumpConfigNamedList(self.paths_().items(),
0869                                   'path',
0870                                   options)
0871         config+=self._dumpConfigNamedList(self.endpaths_().items(),
0872                                   'endpath',
0873                                   options)
0874         config+=self._dumpConfigNamedList(self.finalpaths_().items(),
0875                                   'finalpath',
0876                                   options)
0877         config+=self._dumpConfigUnnamedList(self.services_().items(),
0878                                   'service',
0879                                   options)
0880         config+=self._dumpConfigNamedList(self.aliases_().items(),
0881                                   'alias',
0882                                   options)
0883         config+=self._dumpConfigOptionallyNamedList(
0884             self.es_producers_().items(),
0885             'es_module',
0886             options)
0887         config+=self._dumpConfigOptionallyNamedList(
0888             self.es_sources_().items(),
0889             'es_source',
0890             options)
0891         config += self._dumpConfigESPrefers(options)
0892         for name,item in self.psets.items():
0893             config +=options.indentation()+item.configTypeName()+' '+name+' = '+item.configValue(options)
0894         for name,item in self.vpsets.items():
0895             config +=options.indentation()+'VPSet '+name+' = '+item.configValue(options)
0896         if self.schedule:
0897             pathNames = [p.label_() for p in self.schedule]
0898             config +=options.indentation()+'schedule = {'+','.join(pathNames)+'}\n'
0899 
0900 #        config+=self._dumpConfigNamedList(self.vpsets.items(),
0901 #                                  'VPSet',
0902 #                                  options)
0903         config += "}\n"
0904         options.unindent()
0905         return config
0906 
0907     def _dumpConfigESPrefers(self, options:PrintOptions) -> str:
0908         result = ''
0909         for item in self.es_prefers_().values():
0910             result +=options.indentation()+'es_prefer '+item.targetLabel_()+' = '+item.dumpConfig(options)
0911         return result
0912 
0913     def _dumpPythonSubProcesses(self, l, options:PrintOptions) -> str:
0914         returnValue = ''
0915         for item in l:
0916             returnValue += item.dumpPython(options)+'\n\n'
0917         return returnValue
0918 
0919     def _dumpPythonList(self, d, options:PrintOptions) -> str:
0920         returnValue = ''
0921         if isinstance(d, DictTypes.SortedKeysDict):
0922             for name,item in d.items():
0923                 returnValue +='process.'+name+' = '+item.dumpPython(options)+'\n\n'
0924         else:
0925             for name,item in sorted(d.items()):
0926                 returnValue +='process.'+name+' = '+item.dumpPython(options)+'\n\n'
0927         return returnValue
0928 
0929     def _splitPythonList(self, subfolder, d, options:PrintOptions) -> str:
0930         parts = DictTypes.SortedKeysDict()
0931         for name, item in d.items() if isinstance(d, DictTypes.SortedKeysDict) else sorted(d.items()):
0932             code = ''
0933             dependencies = item.directDependencies()
0934             for module_subfolder, module in dependencies:
0935                 module = module + '_cfi'
0936                 if options.useSubdirectories and module_subfolder:
0937                     module = module_subfolder + '.' + module
0938                 if options.targetDirectory is not None:
0939                     if options.useSubdirectories and subfolder:
0940                       module = '..' + module
0941                     else:
0942                       module = '.' + module
0943                 code += 'from ' + module + ' import *\n'
0944             if dependencies:
0945                 code += '\n'
0946             code += name + ' = ' + item.dumpPython(options)
0947             parts[name] = subfolder, code
0948         return parts
0949 
0950     def _validateSequence(self, sequence, label):
0951         # See if every module has been inserted into the process
0952         try:
0953             l = set()
0954             visitor = NodeNameVisitor(l)
0955             sequence.visit(visitor)
0956         except Exception as e:
0957             raise RuntimeError("An entry in sequence {} has no label\n  Seen entries: {}\n  Error: {}".format(label, l, e))
0958 
0959     def _validateTask(self, task, label:str):
0960         # See if every module and service has been inserted into the process
0961         try:
0962             l = set()
0963             visitor = NodeNameVisitor(l)
0964             task.visit(visitor)
0965         except:
0966             raise RuntimeError("An entry in task " + label + ' has not been attached to the process')
0967     def _validateConditionalTask(self, task, label:str):
0968         # See if every module and service has been inserted into the process
0969         try:
0970             l = set()
0971             visitor = NodeNameVisitor(l)
0972             task.visit(visitor)
0973         except:
0974             raise RuntimeError("An entry in task " + label + ' has not been attached to the process')
0975 
0976     def _itemsInDependencyOrder(self, processDictionaryOfItems):
0977         # The items can be Sequences or Tasks and the input
0978         # argument should either be the dictionary of sequences
0979         # or the dictionary of tasks from the process.
0980 
0981         returnValue=DictTypes.SortedKeysDict()
0982 
0983         # For each item, see what other items it depends upon
0984         # For our purpose here, an item depends on the items it contains.
0985         dependencies = {}
0986         for label,item in processDictionaryOfItems.items():
0987             containedItems = []
0988             if isinstance(item, Task):
0989                 v = TaskVisitor(containedItems)
0990             elif isinstance(item, ConditionalTask):
0991                 v = ConditionalTaskVisitor(containedItems)
0992             else:
0993                 v = SequenceVisitor(containedItems)
0994             try:
0995                 item.visit(v)
0996             except RuntimeError:
0997                 if isinstance(item, Task):
0998                     raise RuntimeError("Failed in a Task visitor. Probably " \
0999                                        "a circular dependency discovered in Task with label " + label)
1000                 elif isinstance(item, ConditionalTask):
1001                     raise RuntimeError("Failed in a ConditionalTask visitor. Probably " \
1002                                        "a circular dependency discovered in ConditionalTask with label " + label)
1003                 else:
1004                     raise RuntimeError("Failed in a Sequence visitor. Probably a " \
1005                                        "circular dependency discovered in Sequence with label " + label)
1006             for containedItem in containedItems:
1007                 # Check for items that both have labels and are not in the process.
1008                 # This should not normally occur unless someone explicitly assigns a
1009                 # label without putting the item in the process (which should not ever
1010                 # be done). We check here because this problem could cause the code
1011                 # in the 'while' loop below to go into an infinite loop.
1012                 if containedItem.hasLabel_():
1013                     testItem = processDictionaryOfItems.get(containedItem.label_())
1014                     if testItem is None or containedItem != testItem:
1015                         if isinstance(item, Task):
1016                             raise RuntimeError("Task has a label, but using its label to get an attribute" \
1017                                                " from the process yields a different object or None\n"+
1018                                                "label = " + containedItem.label_())
1019                         if isinstance(item, ConditionalTask):
1020                             raise RuntimeError("ConditionalTask has a label, but using its label to get an attribute" \
1021                                                " from the process yields a different object or None\n"+
1022                                                "label = " + containedItem.label_())
1023                         else:
1024                             raise RuntimeError("Sequence has a label, but using its label to get an attribute" \
1025                                                " from the process yields a different object or None\n"+
1026                                                "label = " + containedItem.label_())
1027             dependencies[label]=[dep.label_() for dep in containedItems if dep.hasLabel_()]
1028 
1029         # keep looping until we get rid of all dependencies
1030         while dependencies:
1031             oldDeps = dict(dependencies)
1032             for label,deps in oldDeps.items():
1033                 if len(deps)==0:
1034                     returnValue[label]=processDictionaryOfItems[label]
1035                     #remove this as a dependency for all other tasks
1036                     del dependencies[label]
1037                     for lb2,deps2 in dependencies.items():
1038                         while deps2.count(label):
1039                             deps2.remove(label)
1040         return returnValue
1041 
1042     def _dumpPython(self, d, options:PrintOptions) -> str:
1043         result = ''
1044         for name, value in sorted(d.items()):
1045             result += value.dumpPythonAs(name,options)+'\n'
1046         return result
1047 
1048     def _splitPython(self, subfolder, d, options:PrintOptions) -> dict:
1049         result = {}
1050         for name, value in sorted(d.items()):
1051             result[name] = subfolder, value.dumpPythonAs(name, options) + '\n'
1052         return result
1053 
1054     def dumpPython(self, options=PrintOptions()) -> str:
1055         """return a string containing the equivalent process defined using python"""
1056         specialImportRegistry._reset()
1057         header = "import FWCore.ParameterSet.Config as cms"
1058         result = "process = cms.Process(\""+self.__name+"\")\n\n"
1059         if self.source_():
1060             result += "process.source = "+self.source_().dumpPython(options)
1061         if self.looper_():
1062             result += "process.looper = "+self.looper_().dumpPython()
1063         result+=self._dumpPythonList(self.psets, options)
1064         result+=self._dumpPythonList(self.vpsets, options)
1065         result+=self._dumpPythonSubProcesses(self.subProcesses_(), options)
1066         result+=self._dumpPythonList(self.producers_(), options)
1067         result+=self._dumpPythonList(self.switchProducers_(), options)
1068         result+=self._dumpPythonList(self.filters_() , options)
1069         result+=self._dumpPythonList(self.analyzers_(), options)
1070         result+=self._dumpPythonList(self.outputModules_(), options)
1071         result+=self._dumpPythonList(self.services_(), options)
1072         result+=self._dumpPythonList(self.processAccelerators_(), options)
1073         result+=self._dumpPythonList(self.es_producers_(), options)
1074         result+=self._dumpPythonList(self.es_sources_(), options)
1075         result+=self._dumpPython(self.es_prefers_(), options)
1076         result+=self._dumpPythonList(self._itemsInDependencyOrder(self.tasks), options)
1077         result+=self._dumpPythonList(self._itemsInDependencyOrder(self.conditionaltasks), options)
1078         result+=self._dumpPythonList(self._itemsInDependencyOrder(self.sequences), options)
1079         result+=self._dumpPythonList(self.paths_(), options)
1080         result+=self._dumpPythonList(self.endpaths_(), options)
1081         result+=self._dumpPythonList(self.finalpaths_(), options)
1082         result+=self._dumpPythonList(self.aliases_(), options)
1083         if not self.schedule_() == None:
1084             result += 'process.schedule = ' + self.schedule.dumpPython(options)
1085         imports = specialImportRegistry.getSpecialImports()
1086         if len(imports) > 0:
1087             header += "\n" + "\n".join(imports)
1088         header += "\n\n"
1089         return header+result
1090 
1091     def splitPython(self, options:PrintOptions = PrintOptions()) -> dict:
1092         """return a map of file names to python configuration fragments"""
1093         specialImportRegistry._reset()
1094         # extract individual fragments
1095         options.isCfg = False
1096         header = "import FWCore.ParameterSet.Config as cms"
1097         result = ''
1098         parts = {}
1099         files = {}
1100 
1101         result = 'process = cms.Process("' + self.__name + '")\n\n'
1102 
1103         if self.source_():
1104             parts['source'] = (None, 'source = ' + self.source_().dumpPython(options))
1105 
1106         if self.looper_():
1107             parts['looper'] = (None, 'looper = ' + self.looper_().dumpPython())
1108 
1109         parts.update(self._splitPythonList('psets', self.psets, options))
1110         parts.update(self._splitPythonList('psets', self.vpsets, options))
1111         # FIXME
1112         #parts.update(self._splitPythonSubProcesses(self.subProcesses_(), options))
1113         if len(self.subProcesses_()):
1114           sys.stderr.write("error: subprocesses are not supported yet\n\n")
1115         parts.update(self._splitPythonList('modules', self.producers_(), options))
1116         parts.update(self._splitPythonList('modules', self.switchProducers_(), options))
1117         parts.update(self._splitPythonList('modules', self.filters_() , options))
1118         parts.update(self._splitPythonList('modules', self.analyzers_(), options))
1119         parts.update(self._splitPythonList('modules', self.outputModules_(), options))
1120         parts.update(self._splitPythonList('services', self.services_(), options))
1121         parts.update(self._splitPythonList('eventsetup', self.es_producers_(), options))
1122         parts.update(self._splitPythonList('eventsetup', self.es_sources_(), options))
1123         parts.update(self._splitPython('eventsetup', self.es_prefers_(), options))
1124         parts.update(self._splitPythonList('tasks', self._itemsInDependencyOrder(self.tasks), options))
1125         parts.update(self._splitPythonList('sequences', self._itemsInDependencyOrder(self.sequences), options))
1126         parts.update(self._splitPythonList('paths', self.paths_(), options))
1127         parts.update(self._splitPythonList('paths', self.endpaths_(), options))
1128         parts.update(self._splitPythonList('paths', self.finalpaths_(), options))
1129         parts.update(self._splitPythonList('modules', self.aliases_(), options))
1130 
1131         if options.targetDirectory is not None:
1132             files[options.targetDirectory + '/__init__.py'] = ''
1133 
1134         if options.useSubdirectories:
1135           for sub in 'psets', 'modules', 'services', 'eventsetup', 'tasks', 'sequences', 'paths':
1136             if options.targetDirectory is not None:
1137                 sub = options.targetDirectory + '/' + sub
1138             files[sub + '/__init__.py'] = ''
1139 
1140         # case insensitive sort by subfolder and module name
1141         parts = sorted(parts.items(), key = lambda nsc: (nsc[1][0].lower() if nsc[1][0] else '', nsc[0].lower()))
1142 
1143         for (name, (subfolder, code)) in parts:
1144             filename = name + '_cfi'
1145             if options.useSubdirectories and subfolder:
1146                 filename = subfolder + '/' + filename
1147             if options.targetDirectory is not None:
1148                 filename = options.targetDirectory + '/' + filename
1149             result += 'process.load("%s")\n' % filename
1150             files[filename + '.py'] = header + '\n\n' + code
1151 
1152         if self.schedule_() is not None:
1153             options.isCfg = True
1154             result += '\nprocess.schedule = ' + self.schedule.dumpPython(options)
1155 
1156         imports = specialImportRegistry.getSpecialImports()
1157         if len(imports) > 0:
1158             header += '\n' + '\n'.join(imports)
1159         files['-'] = header + '\n\n' + result
1160         return files
1161 
1162     def _replaceInSequences(self, label:str, new):
1163         old = getattr(self,label)
1164         #TODO - replace by iterator concatenation
1165         #to ovoid dependency problems between sequences, first modify
1166         # process known sequences to do a non-recursive change. Then do
1167         # a recursive change to get cases where a sub-sequence unknown to
1168         # the process has the item to be replaced
1169         for sequenceable in self.sequences.values():
1170             sequenceable._replaceIfHeldDirectly(old,new)
1171         for sequenceable in self.sequences.values():
1172             sequenceable.replace(old,new)
1173         for sequenceable in self.paths.values():
1174             sequenceable.replace(old,new)
1175         for sequenceable in self.endpaths.values():
1176             sequenceable.replace(old,new)
1177         for sequenceable in self.finalpaths.values():
1178             sequenceable.replace(old,new)
1179     def _replaceInTasks(self, label:str, new):
1180         old = getattr(self,label)
1181         for task in self.tasks.values():
1182             task.replace(old, new)
1183     def _replaceInConditionalTasks(self, label:str, new):
1184         old = getattr(self,label)
1185         for task in self.conditionaltasks.values():
1186             task.replace(old, new)
1187     def _replaceInSchedule(self, label:str, new):
1188         if self.schedule_() == None:
1189             return
1190         old = getattr(self,label)
1191         for task in self.schedule_()._tasks:
1192             task.replace(old, new)
1193     def _replaceInScheduleDirectly(self, label:str, new):
1194         if self.schedule_() == None:
1195             return
1196         old = getattr(self,label)
1197         self.schedule_()._replaceIfHeldDirectly(old, new)
1198     def globalReplace(self,label:str,new):
1199         """ Replace the item with label 'label' by object 'new' in the process and all sequences/paths/tasks"""
1200         if not hasattr(self,label):
1201             raise LookupError("process has no item of label "+label)
1202         setattr(self,label,new)
1203     def _insertInto(self, parameterSet, itemDict):
1204         for name,value in itemDict.items():
1205             value.insertInto(parameterSet, name)
1206     def _insertOneInto(self, parameterSet, label:str, item, tracked:bool):
1207         vitems = []
1208         if not item == None:
1209             newlabel = item.nameInProcessDesc_(label)
1210             vitems = [newlabel]
1211             item.insertInto(parameterSet, newlabel)
1212         parameterSet.addVString(tracked, label, vitems)
1213     def _insertManyInto(self, parameterSet, label:str, itemDict, tracked:bool):
1214         l = []
1215         for name,value in itemDict.items():
1216             value.appendToProcessDescList_(l, name)
1217             value.insertInto(parameterSet, name)
1218         # alphabetical order is easier to compare with old language
1219         l.sort()
1220         parameterSet.addVString(tracked, label, l)
1221     def _insertSwitchProducersInto(self, parameterSet, labelModules, labelAliases, itemDict, tracked:bool):
1222         modules = parameterSet.getVString(tracked, labelModules)
1223         aliases = parameterSet.getVString(tracked, labelAliases)
1224         accelerators = parameterSet.getVString(False, "@selected_accelerators")
1225         for name,value in itemDict.items():
1226             value.appendToProcessDescLists_(modules, aliases, name)
1227             value.insertInto(parameterSet, name, accelerators)
1228         modules.sort()
1229         aliases.sort()
1230         parameterSet.addVString(tracked, labelModules, modules)
1231         parameterSet.addVString(tracked, labelAliases, aliases)
1232     def _insertSubProcessesInto(self, parameterSet, label:str, itemList, tracked:bool):
1233         l = []
1234         subprocs = []
1235         for value in itemList:
1236             name = value.getProcessName()
1237             newLabel = value.nameInProcessDesc_(name)
1238             l.append(newLabel)
1239             pset = value.getSubProcessPSet(parameterSet)
1240             subprocs.append(pset)
1241         # alphabetical order is easier to compare with old language
1242         l.sort()
1243         parameterSet.addVString(tracked, label, l)
1244         parameterSet.addVPSet(False,"subProcesses",subprocs)
1245     def _insertPaths(self, processPSet, nodeVisitor):
1246         scheduledPaths = []
1247         triggerPaths = []
1248         endpaths = []
1249         finalpaths = []
1250         if self.schedule_() == None:
1251             # make one from triggerpaths & endpaths
1252             for name in self.paths_():
1253                 scheduledPaths.append(name)
1254                 triggerPaths.append(name)
1255             for name in self.endpaths_():
1256                 scheduledPaths.append(name)
1257                 endpaths.append(name)
1258             for name in self.finalpaths_():
1259                 finalpaths.append(name)
1260         else:
1261             for path in self.schedule_():
1262                 pathname = path.label_()
1263                 if pathname in self.endpaths_():
1264                     endpaths.append(pathname)
1265                     scheduledPaths.append(pathname)
1266                 elif pathname in self.finalpaths_():
1267                     finalpaths.append(pathname)
1268                 else:
1269                     scheduledPaths.append(pathname)
1270                     triggerPaths.append(pathname)
1271             for task in self.schedule_()._tasks:
1272                 task.resolve(self.__dict__)
1273                 scheduleTaskValidator = ScheduleTaskValidator()
1274                 task.visit(scheduleTaskValidator)
1275                 task.visit(nodeVisitor)
1276         # consolidate all final_paths into one EndPath
1277         endPathWithFinalPathModulesName ="@finalPath"
1278         finalPathEndPath = EndPath()
1279         if finalpaths:
1280           endpaths.append(endPathWithFinalPathModulesName)
1281           scheduledPaths.append(endPathWithFinalPathModulesName)
1282           finalpathValidator = FinalPathValidator()
1283           modulesOnFinalPath = []
1284           for finalpathname in finalpaths:
1285               iFinalPath = self.finalpaths_()[finalpathname]
1286               iFinalPath.resolve(self.__dict__)
1287               finalpathValidator.setLabel(finalpathname)
1288               iFinalPath.visit(finalpathValidator)
1289               invalidModules = finalpathValidator.invalidModulesOnFinalpaths
1290               if invalidModules:
1291                   raise RuntimeError("FinalPath %s has non OutputModules %s" % (finalpathname, ",".join(invalidModules)))
1292               modulesOnFinalPath.extend(iFinalPath.moduleNames())
1293           for m in modulesOnFinalPath:
1294             mod = getattr(self, m)
1295             setattr(mod, "@onFinalPath", untracked.bool(True))
1296             finalPathEndPath += mod
1297             
1298         processPSet.addVString(True, "@end_paths", endpaths)
1299         processPSet.addVString(True, "@paths", scheduledPaths)
1300         # trigger_paths are a little different
1301         p = processPSet.newPSet()
1302         p.addVString(True, "@trigger_paths", triggerPaths)
1303         processPSet.addPSet(True, "@trigger_paths", p)
1304         # add all these paths
1305         pathValidator = PathValidator()
1306         endpathValidator = EndPathValidator()
1307         decoratedList = []
1308         lister = DecoratedNodeNameVisitor(decoratedList)
1309         condTaskModules = []
1310         condTaskVistor = ModuleNodeOnConditionalTaskVisitor(condTaskModules)
1311         pathCompositeVisitor = CompositeVisitor(pathValidator, nodeVisitor, lister, condTaskVistor)
1312         endpathCompositeVisitor = CompositeVisitor(endpathValidator, nodeVisitor, lister)
1313         for triggername in triggerPaths:
1314             iPath = self.paths_()[triggername]
1315             iPath.resolve(self.__dict__)
1316             pathValidator.setLabel(triggername)
1317             lister.initialize()
1318             condTaskModules[:] = []
1319             iPath.visit(pathCompositeVisitor)
1320             if condTaskModules:
1321               decoratedList.append("#")
1322               l = list({x.label_() for x in condTaskModules})
1323               l.sort()
1324               decoratedList.extend(l)
1325               decoratedList.append("@")
1326             iPath.insertInto(processPSet, triggername, decoratedList[:])
1327         for endpathname in endpaths:
1328             if endpathname is not endPathWithFinalPathModulesName:
1329               iEndPath = self.endpaths_()[endpathname]
1330             else:
1331               iEndPath = finalPathEndPath
1332             iEndPath.resolve(self.__dict__)
1333             endpathValidator.setLabel(endpathname)
1334             lister.initialize()
1335             iEndPath.visit(endpathCompositeVisitor)
1336             iEndPath.insertInto(processPSet, endpathname, decoratedList[:])
1337         processPSet.addVString(False, "@filters_on_endpaths", endpathValidator.filtersOnEndpaths)
1338           
1339 
1340     def resolve(self,keepUnresolvedSequencePlaceholders:bool=False):
1341         for x in self.paths.values():
1342             x.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1343         for x in self.endpaths.values():
1344             x.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1345         for x in self.finalpaths.values():
1346             x.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1347         if not self.schedule_() == None:
1348             for task in self.schedule_()._tasks:
1349                 task.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1350 
1351     def prune(self,verbose=False,keepUnresolvedSequencePlaceholders:bool=False):
1352         """ Remove clutter from the process that we think is unnecessary:
1353         tracked PSets, VPSets and unused modules and sequences. If a Schedule has been set, then Paths and EndPaths
1354         not in the schedule will also be removed, along with an modules and sequences used only by
1355         those removed Paths and EndPaths. The keepUnresolvedSequencePlaceholders keeps also unresolved TaskPlaceholders."""
1356 # need to update this to only prune psets not on refToPSets
1357 # but for now, remove the delattr
1358 #        for name in self.psets_():
1359 #            if getattr(self,name).isTracked():
1360 #                delattr(self, name)
1361         for name in self.vpsets_():
1362             delattr(self, name)
1363         #first we need to resolve any SequencePlaceholders being used
1364         self.resolve(keepUnresolvedSequencePlaceholders)
1365         usedModules = set()
1366         unneededPaths = set()
1367         tasks = list()
1368         tv = TaskVisitor(tasks)
1369         if self.schedule_():
1370             usedModules=set(self.schedule_().moduleNames())
1371             #get rid of unused paths
1372             schedNames = set(( x.label_() for x in self.schedule_()))
1373             names = set(self.paths)
1374             names.update(set(self.endpaths))
1375             names.update(set(self.finalpaths))
1376             unneededPaths = names - schedNames
1377             for n in unneededPaths:
1378                 delattr(self,n)
1379             for t in self.schedule_().tasks():
1380                 tv.enter(t)
1381                 t.visit(tv)
1382                 tv.leave(t)
1383         else:
1384             pths = list(self.paths.values())
1385             pths.extend(self.endpaths.values())
1386             pths.extend(self.finalpaths.values())
1387             temp = Schedule(*pths)
1388             usedModules=set(temp.moduleNames())
1389         unneededModules = self._pruneModules(self.producers_(), usedModules)
1390         unneededModules.update(self._pruneModules(self.switchProducers_(), usedModules))
1391         unneededModules.update(self._pruneModules(self.filters_(), usedModules))
1392         unneededModules.update(self._pruneModules(self.analyzers_(), usedModules))
1393         #remove sequences and tasks that do not appear in remaining paths and endpaths
1394         seqs = list()
1395         sv = SequenceVisitor(seqs)
1396         for p in self.paths.values():
1397             p.visit(sv)
1398             p.visit(tv)
1399         for p in self.endpaths.values():
1400             p.visit(sv)
1401             p.visit(tv)
1402         for p in self.finalpaths.values():
1403             p.visit(sv)
1404             p.visit(tv)
1405         def removeUnneeded(seqOrTasks, allSequencesOrTasks):
1406             _keepSet = set(( s for s in seqOrTasks if s.hasLabel_()))
1407             _availableSet = set(allSequencesOrTasks.values())
1408             _unneededSet = _availableSet-_keepSet
1409             _unneededLabels = []
1410             for s in _unneededSet:
1411                 _unneededLabels.append(s.label_())
1412                 delattr(self,s.label_())
1413             return _unneededLabels
1414         unneededSeqLabels = removeUnneeded(seqs, self.sequences)
1415         unneededTaskLabels = removeUnneeded(tasks, self.tasks)
1416         if verbose:
1417             print("prune removed the following:")
1418             print("  modules:"+",".join(unneededModules))
1419             print("  tasks:"+",".join(unneededTaskLabels))
1420             print("  sequences:"+",".join(unneededSeqLabels))
1421             print("  paths/endpaths/finalpaths:"+",".join(unneededPaths))
1422     def _pruneModules(self, d, scheduledNames):
1423         moduleNames = set(d.keys())
1424         junk = moduleNames - scheduledNames
1425         for name in junk:
1426             delattr(self, name)
1427         return junk
1428 
1429     def fillProcessDesc(self, processPSet):
1430         """Used by the framework to convert python to C++ objects"""
1431         class ServiceInjectorAdaptor(object):
1432             def __init__(self,ppset,thelist):
1433                 self.__thelist = thelist
1434                 self.__processPSet = ppset
1435             def addService(self,pset):
1436                 self.__thelist.append(pset)
1437             def newPSet(self):
1438                 return self.__processPSet.newPSet()
1439         #This adaptor is used to 'add' the method 'getTopPSet_'
1440         # to the ProcessDesc and PythonParameterSet C++ classes.
1441         # This method is needed for the PSet refToPSet_ functionality.
1442         class TopLevelPSetAcessorAdaptor(object):
1443             def __init__(self,ppset,process):
1444                 self.__ppset = ppset
1445                 self.__process = process
1446             def __getattr__(self,attr):
1447                 return getattr(self.__ppset,attr)
1448             def getTopPSet_(self,label):
1449                 return getattr(self.__process,label)
1450             def newPSet(self):
1451                 return TopLevelPSetAcessorAdaptor(self.__ppset.newPSet(),self.__process)
1452             def addPSet(self,tracked,name,ppset):
1453                 return self.__ppset.addPSet(tracked,name,self.__extractPSet(ppset))
1454             def addVPSet(self,tracked,name,vpset):
1455                 return self.__ppset.addVPSet(tracked,name,[self.__extractPSet(x) for x in vpset])
1456             def __extractPSet(self,pset):
1457                 if isinstance(pset,TopLevelPSetAcessorAdaptor):
1458                     return pset.__ppset
1459                 return pset
1460 
1461         self.validate()
1462         processPSet.addString(True, "@process_name", self.name_())
1463         self.handleProcessAccelerators(processPSet)
1464         all_modules = self.producers_().copy()
1465         all_modules.update(self.filters_())
1466         all_modules.update(self.analyzers_())
1467         all_modules.update(self.outputModules_())
1468         if hasattr(self.options,"modulesToCallForTryToContinue") :
1469             shouldTryToContinue = set(self.options.modulesToCallForTryToContinue)
1470             for m in all_modules:
1471                 if m in shouldTryToContinue:
1472                     setattr(getattr(self,m),"@shouldTryToContinue",untracked.bool(True))
1473             missing = shouldTryToContinue.difference(all_modules)
1474             if missing:
1475                 print("Warning: The following modules appear in options.modulesToCallForTryToContinue but are not in the Process: {} ".format(",".join(missing)))
1476         adaptor = TopLevelPSetAcessorAdaptor(processPSet,self)
1477         self._insertInto(adaptor, self.psets_())
1478         self._insertInto(adaptor, self.vpsets_())
1479         self._insertOneInto(adaptor,  "@all_sources", self.source_(), True)
1480         self._insertOneInto(adaptor,  "@all_loopers", self.looper_(), True)
1481         self._insertSubProcessesInto(adaptor, "@all_subprocesses", self.subProcesses_(), False)
1482         self._insertManyInto(adaptor, "@all_esprefers", self.es_prefers_(), True)
1483         self._insertManyInto(adaptor, "@all_aliases", self.aliases_(), True)
1484         # This will visit all the paths and endpaths that are scheduled to run,
1485         # as well as the Tasks associated to them and the schedule. It remembers
1486         # the modules, ESSources, ESProducers, and services it visits.
1487         nodeVisitor = NodeVisitor()
1488         self._insertPaths(adaptor, nodeVisitor)
1489         all_modules_onTasksOrScheduled = { key:value for key, value in all_modules.items() if value in nodeVisitor.modules }
1490         self._insertManyInto(adaptor, "@all_modules", all_modules_onTasksOrScheduled, True)
1491         all_switches = self.switchProducers_().copy()
1492         all_switches_onTasksOrScheduled = {key:value for key, value in all_switches.items() if value in nodeVisitor.modules }
1493         self._insertSwitchProducersInto(adaptor, "@all_modules", "@all_aliases", all_switches_onTasksOrScheduled, True)
1494         # Same as nodeVisitor except this one visits all the Tasks attached
1495         # to the process.
1496         processNodeVisitor = NodeVisitor()
1497         for pTask in self.tasks.values():
1498             pTask.visit(processNodeVisitor)
1499         esProducersToEnable = {}
1500         for esProducerName, esProducer in self.es_producers_().items():
1501             if esProducer in nodeVisitor.esProducers or not (esProducer in processNodeVisitor.esProducers):
1502                 esProducersToEnable[esProducerName] = esProducer
1503         self._insertManyInto(adaptor, "@all_esmodules", esProducersToEnable, True)
1504         esSourcesToEnable = {}
1505         for esSourceName, esSource in self.es_sources_().items():
1506             if esSource in nodeVisitor.esSources or not (esSource in processNodeVisitor.esSources):
1507                 esSourcesToEnable[esSourceName] = esSource
1508         self._insertManyInto(adaptor, "@all_essources", esSourcesToEnable, True)
1509         #handle services differently
1510         services = []
1511         for serviceName, serviceObject in self.services_().items():
1512             if serviceObject in nodeVisitor.services or not (serviceObject in processNodeVisitor.services):
1513                 serviceObject.insertInto(ServiceInjectorAdaptor(adaptor,services))
1514         adaptor.addVPSet(False,"services",services)
1515         return processPSet
1516 
1517     def validate(self):
1518         # check if there's some input
1519         # Breaks too many unit tests for now
1520         #if self.source_() == None and self.looper_() == None:
1521         #    raise RuntimeError("No input source was found for this process")
1522         pass
1523 
1524     def handleProcessAccelerators(self, parameterSet):
1525         # 'cpu' accelerator is always implicitly there
1526         allAccelerators = set(["cpu"])
1527         availableAccelerators = set(["cpu"])
1528         for acc in self.__dict__['_Process__accelerators'].values():
1529             allAccelerators.update(acc.labels())
1530             availableAccelerators.update(acc.enabledLabels())
1531         availableAccelerators = sorted(list(availableAccelerators))
1532         parameterSet.addVString(False, "@available_accelerators", availableAccelerators)
1533 
1534         # Resolve wildcards
1535         selectedAccelerators = []
1536         if "*" in self.options.accelerators:
1537             if len(self.options.accelerators) >= 2:
1538                 raise ValueError("process.options.accelerators may contain '*' only as the only element, now it has {} elements".format(len(self.options.accelerators)))
1539             selectedAccelerators = availableAccelerators
1540         else:
1541             import fnmatch
1542             resolved = set()
1543             invalid = []
1544             for pattern in self.options.accelerators:
1545                 acc = [a for a in availableAccelerators if fnmatch.fnmatchcase(a, pattern)]
1546                 if len(acc) == 0:
1547                     if not any(fnmatch.fnmatchcase(a, pattern) for a in allAccelerators):
1548                         invalid.append(pattern)
1549                 else:
1550                    resolved.update(acc)
1551             # Sanity check
1552             if len(invalid) != 0:
1553                 raise ValueError("Invalid pattern{} of '{}' in process.options.accelerators, valid values are '{}' or a pattern matching some of them.".format(
1554                     "s" if len(invalid) > 2 else "",
1555                     "', '".join(invalid),
1556                     "', '".join(sorted(list(allAccelerators)))))
1557             selectedAccelerators = sorted(list(resolved))
1558         parameterSet.addVString(False, "@selected_accelerators", selectedAccelerators)
1559 
1560         # Get and apply module type resolver
1561         moduleTypeResolver = None
1562         moduleTypeResolverPlugin = ""
1563         for acc in self.__dict__['_Process__accelerators'].values():
1564             resolver = acc.moduleTypeResolver(selectedAccelerators)
1565             if resolver is not None:
1566                 if moduleTypeResolver is not None:
1567                     raise RuntimeError("Module type resolver was already set to {} when {} tried to set it to {}. A job can have at most one ProcessAccelerator that sets module type resolver.".format(
1568                         moduleTypeResolver.__class__.__name__,
1569                         acc.__class__.__name__,
1570                         resolver.__class__.__name__))
1571                 moduleTypeResolver = resolver
1572         if moduleTypeResolver is not None:
1573             # Plugin name and its configuration
1574             moduleTypeResolverPlugin = moduleTypeResolver.plugin()
1575 
1576             # Customize modules
1577             for modlist in [self.producers_, self.filters_, self.analyzers_,
1578                             self.es_producers_, self.es_sources_]:
1579                 for module in modlist().values():
1580                     moduleTypeResolver.setModuleVariant(module)
1581 
1582         parameterSet.addString(False, "@module_type_resolver", moduleTypeResolverPlugin)
1583 
1584         # Customize process
1585         wrapped = ProcessForProcessAccelerator(self)
1586         for acc in self.__dict__['_Process__accelerators'].values():
1587             acc.apply(wrapped, selectedAccelerators)
1588 
1589     def prefer(self, esmodule,*args,**kargs):
1590         """Prefer this ES source or producer.  The argument can
1591            either be an object label, e.g.,
1592              process.prefer(process.juicerProducer) (not supported yet)
1593            or a name of an ESSource or ESProducer
1594              process.prefer("juicer")
1595            or a type of unnamed ESSource or ESProducer
1596              process.prefer("JuicerProducer")
1597            In addition, you can pass as a labelled arguments the name of the Record you wish to
1598            prefer where the type passed is a cms.vstring and that vstring can contain the
1599            name of the C++ types in the Record that are being preferred, e.g.,
1600               #prefer all data in record 'OrangeRecord' from 'juicer'
1601               process.prefer("juicer", OrangeRecord=cms.vstring())
1602            or
1603               #prefer only "Orange" data in "OrangeRecord" from "juicer"
1604               process.prefer("juicer", OrangeRecord=cms.vstring("Orange"))
1605            or
1606               #prefer only "Orange" data with label "ExtraPulp" in "OrangeRecord" from "juicer"
1607               ESPrefer("ESJuicerProd", OrangeRecord=cms.vstring("Orange/ExtraPulp"))
1608         """
1609         # see if this refers to a named ESProducer
1610         if isinstance(esmodule, ESSource) or isinstance(esmodule, ESProducer):
1611             raise RuntimeError("Syntax of process.prefer(process.esmodule) not supported yet")
1612         elif self._findPreferred(esmodule, self.es_producers_(),*args,**kargs) or \
1613                 self._findPreferred(esmodule, self.es_sources_(),*args,**kargs):
1614             pass
1615         else:
1616             raise RuntimeError("Cannot resolve prefer for "+repr(esmodule))
1617 
1618     def _findPreferred(self, esname:str, d,*args,**kargs) -> bool:
1619         # is esname a name in the dictionary?
1620         if esname in d:
1621             typ = d[esname].type_()
1622             if typ == esname:
1623                 self.__setattr__( esname+"_prefer", ESPrefer(typ,*args,**kargs) )
1624             else:
1625                 self.__setattr__( esname+"_prefer", ESPrefer(typ, esname,*args,**kargs) )
1626             return True
1627         else:
1628             # maybe it's an unnamed ESModule?
1629             found = False
1630             for name, value in d.items():
1631                 if value.type_() == esname:
1632                     if found:
1633                         raise RuntimeError("More than one ES module for "+esname)
1634                     found = True
1635                     self.__setattr__(esname+"_prefer",  ESPrefer(d[esname].type_()) )
1636             return found
1637 
1638 
1639 class ProcessFragment(object):
1640     def __init__(self, process: Union[Process,str]):
1641         if isinstance(process, Process):
1642             self.__process = process
1643         elif isinstance(process, str):
1644             self.__process = Process(process)
1645             #make sure we do not override the defaults
1646             del self.__process.options
1647             del self.__process.maxEvents
1648             del self.__process.maxLuminosityBlocks
1649         else:
1650             raise TypeError('a ProcessFragment can only be constructed from an existig Process or from process name')
1651     def __dir__(self):
1652         return [ x for x in dir(self.__process) if isinstance(getattr(self.__process, x), _ConfigureComponent) ]
1653     def __getattribute__(self, name:str):
1654         if name == '_ProcessFragment__process':
1655             return object.__getattribute__(self, '_ProcessFragment__process')
1656         else:
1657             return getattr(self.__process, name)
1658     def __setattr__(self, name:str, value):
1659         if name == '_ProcessFragment__process':
1660             object.__setattr__(self, name, value)
1661         else:
1662             setattr(self.__process, name, value)
1663     def __delattr__(self, name:str):
1664         if name == '_ProcessFragment__process':
1665             pass
1666         else:
1667             return delattr(self.__process, name)
1668 
1669 
1670 class FilteredStream(dict):
1671     """a dictionary with fixed keys"""
1672     def _blocked_attribute(obj):
1673         raise AttributeError("An FilteredStream defintion cannot be modified after creation.")
1674     _blocked_attribute = property(_blocked_attribute)
1675     __setattr__ = __delitem__ = __setitem__ = clear = _blocked_attribute
1676     pop = popitem = setdefault = update = _blocked_attribute
1677     def __new__(cls, *args, **kw):
1678         new = dict.__new__(cls)
1679         dict.__init__(new, *args, **kw)
1680         keys = sorted(kw.keys())
1681         if keys != ['content', 'dataTier', 'name', 'paths', 'responsible', 'selectEvents']:
1682             raise ValueError("The needed parameters are: content, dataTier, name, paths, responsible, selectEvents")
1683         if not isinstance(kw['name'],str):
1684             raise ValueError("name must be of type string")
1685         if not isinstance(kw['content'], vstring) and not isinstance(kw['content'],str):
1686             raise ValueError("content must be of type vstring or string")
1687         if not isinstance(kw['dataTier'], string):
1688             raise ValueError("dataTier must be of type string")
1689         if not isinstance(kw['selectEvents'], PSet):
1690             raise ValueError("selectEvents must be of type PSet")
1691         if not isinstance(kw['paths'],(tuple, Path)):
1692             raise ValueError("'paths' must be a tuple of paths")
1693         return new
1694     def __init__(self, *args, **kw):
1695         pass
1696     def __repr__(self) -> str:
1697         return "FilteredStream object: %s" %self["name"]
1698     def __getattr__(self,attr):
1699         return self[attr]
1700 
1701 class SubProcess(_Unlabelable):
1702     """Allows embedding another process within a parent process. This allows one to 
1703     chain processes together directly in one cmsRun job rather than having to run
1704     separate jobs that are connected via a temporary file.
1705     """
1706     def __init__(self,process, SelectEvents = untracked.PSet(), outputCommands = untracked.vstring()):
1707         """
1708         """
1709         if not isinstance(process, Process):
1710             raise ValueError("the 'process' argument must be of type cms.Process")
1711         if not isinstance(SelectEvents,PSet):
1712             raise ValueError("the 'SelectEvents' argument must be of type cms.untracked.PSet")
1713         if not isinstance(outputCommands,vstring):
1714             raise ValueError("the 'outputCommands' argument must be of type cms.untracked.vstring")
1715         self.__process = process
1716         self.__SelectEvents = SelectEvents
1717         self.__outputCommands = outputCommands
1718         # Need to remove MessageLogger from the subprocess now that MessageLogger is always present
1719         if self.__process.MessageLogger is not MessageLogger:
1720             print("""Warning: You have reconfigured service
1721 'edm::MessageLogger' in a subprocess.
1722 This service has already been configured.
1723 This particular service may not be reconfigured in a subprocess.
1724 The reconfiguration will be ignored.""")
1725         del self.__process.MessageLogger
1726     def dumpPython(self, options:PrintOptions=PrintOptions()) -> str:
1727         out = "parentProcess"+str(hash(self))+" = process\n"
1728         out += self.__process.dumpPython()
1729         out += "childProcess = process\n"
1730         out += "process = parentProcess"+str(hash(self))+"\n"
1731         out += "process.addSubProcess(cms.SubProcess(process = childProcess, SelectEvents = "+self.__SelectEvents.dumpPython(options) +", outputCommands = "+self.__outputCommands.dumpPython(options) +"))"
1732         return out
1733     def getProcessName(self) -> str:
1734         return self.__process.name_()
1735     def process(self):
1736         return self.__process
1737     def SelectEvents(self):
1738         return self.__SelectEvents
1739     def outputCommands(self):
1740         return self.__outputCommands
1741     def type_(self):
1742         return 'subProcess'
1743     def nameInProcessDesc_(self,label:str) -> str:
1744         return label
1745     def _place(self,label,process):
1746         process._placeSubProcess('subProcess',self)
1747     def getSubProcessPSet(self,parameterSet):
1748         topPSet = parameterSet.newPSet()
1749         self.__process.fillProcessDesc(topPSet)
1750         subProcessPSet = parameterSet.newPSet()
1751         self.__SelectEvents.insertInto(subProcessPSet,"SelectEvents")
1752         self.__outputCommands.insertInto(subProcessPSet,"outputCommands")
1753         subProcessPSet.addPSet(False,"process",topPSet)
1754         return subProcessPSet
1755 
1756 class _ParameterModifier(object):
1757     """Helper class for Modifier that takes key/value pairs and uses them to reset parameters of the object"""
1758     def __init__(self,args):
1759         self.__args = args
1760     def __call__(self,obj):
1761         params = {}
1762         for k in self.__args.keys():
1763             if hasattr(obj,k):
1764                 params[k] = getattr(obj,k)
1765         _modifyParametersFromDict(params, self.__args, self._raiseUnknownKey)
1766         for k in self.__args.keys():
1767             if k in params:
1768                 setattr(obj,k,params[k])
1769             else:
1770                 #the parameter must have been removed
1771                 delattr(obj,k)
1772     @staticmethod
1773     def _raiseUnknownKey(key):
1774         raise KeyError("Unknown parameter name "+key+" specified while calling Modifier")
1775 
1776 class _BoolModifierBase(object):
1777     """A helper base class for _AndModifier, _InvertModifier, and _OrModifier to contain the common code"""
1778     def __init__(self, lhs, rhs=None):
1779         self._lhs = lhs
1780         if rhs is not None:
1781             self._rhs = rhs
1782     def toModify(self,obj, func=None,**kw):
1783         Modifier._toModifyCheck(obj,func,**kw)
1784         if self._isChosen():
1785             Modifier._toModify(obj,func,**kw)
1786         return self
1787     def toReplaceWith(self,toObj,fromObj):
1788         Modifier._toReplaceWithCheck(toObj,fromObj)
1789         if self._isChosen():
1790             Modifier._toReplaceWith(toObj,fromObj)
1791         return self
1792     def makeProcessModifier(self,func):
1793         """This is used to create a ProcessModifer that can perform actions on the process as a whole.
1794             This takes as argument a callable object (e.g. function) that takes as its sole argument an instance of Process.
1795             In order to work, the value returned from this function must be assigned to a uniquely named variable."""
1796         return ProcessModifier(self,func)
1797     def __and__(self, other):
1798         return _AndModifier(self,other)
1799     def __invert__(self):
1800         return _InvertModifier(self)
1801     def __or__(self, other):
1802         return _OrModifier(self,other)
1803 
1804 class _AndModifier(_BoolModifierBase):
1805     """A modifier which only applies if multiple Modifiers are chosen"""
1806     def __init__(self, lhs, rhs):
1807         super(_AndModifier,self).__init__(lhs, rhs)
1808     def _isChosen(self) -> bool:
1809         return self._lhs._isChosen() and self._rhs._isChosen()
1810 
1811 class _InvertModifier(_BoolModifierBase):
1812     """A modifier which only applies if a Modifier is not chosen"""
1813     def __init__(self, lhs):
1814         super(_InvertModifier,self).__init__(lhs)
1815     def _isChosen(self) -> bool:
1816         return not self._lhs._isChosen()
1817 
1818 class _OrModifier(_BoolModifierBase):
1819     """A modifier which only applies if at least one of multiple Modifiers is chosen"""
1820     def __init__(self, lhs, rhs):
1821         super(_OrModifier,self).__init__(lhs, rhs)
1822     def _isChosen(self) -> bool:
1823         return self._lhs._isChosen() or self._rhs._isChosen()
1824 
1825 
1826 class Modifier(object):
1827     """This class is used to define standard modifications to a Process.
1828     An instance of this class is declared to denote a specific modification,e.g. era2017 could
1829     reconfigure items in a process to match our expectation of running in 2017. Once declared,
1830     these Modifier instances are imported into a configuration and items that need to be modified
1831     are then associated with the Modifier and with the action to do the modification.
1832     The registered modifications will only occur if the Modifier was passed to 
1833     the cms.Process' constructor.
1834     """
1835     def __init__(self):
1836         self.__processModifiers = []
1837         self.__chosen = False
1838     def makeProcessModifier(self,func):
1839         """This is used to create a ProcessModifer that can perform actions on the process as a whole.
1840            This takes as argument a callable object (e.g. function) that takes as its sole argument an instance of Process.
1841            In order to work, the value returned from this function must be assigned to a uniquely named variable.
1842         """
1843         return ProcessModifier(self,func)
1844     @staticmethod
1845     def _toModifyCheck(obj,func,**kw):
1846         if func is not None and len(kw) != 0:
1847             raise TypeError("toModify takes either two arguments or one argument and key/value pairs")
1848     def toModify(self,obj, func=None,**kw):
1849         """This is used to register an action to be performed on the specific object. Two different forms are allowed
1850         Form 1: A callable object (e.g. function) can be passed as the second. This callable object is expected to take one argument
1851         that will be the object passed in as the first argument.
1852         Form 2: A list of parameter name, value pairs can be passed
1853            mod.toModify(foo, fred=cms.int32(7), barney = cms.double(3.14))
1854         This form can also be used to remove a parameter by passing the value of None
1855             #remove the parameter foo.fred       
1856             mod.toModify(foo, fred = None)
1857         Additionally, parameters embedded within PSets can also be modified using a dictionary
1858             #change foo.fred.pebbles to 3 and foo.fred.friend to "barney"
1859             mod.toModify(foo, fred = dict(pebbles = 3, friend = "barney)) )
1860         """
1861         Modifier._toModifyCheck(obj,func,**kw)
1862         if self._isChosen():
1863             Modifier._toModify(obj,func,**kw)
1864         return self
1865     @staticmethod
1866     def _toModify(obj,func,**kw):
1867         if func is not None:
1868             func(obj)
1869         else:
1870             temp =_ParameterModifier(kw)
1871             temp(obj)
1872     @staticmethod
1873     def _toReplaceWithCheck(toObj,fromObj):
1874         if not isinstance(fromObj, type(toObj)):
1875             raise TypeError("toReplaceWith requires both arguments to be the same class type")
1876     def toReplaceWith(self,toObj,fromObj):
1877         """If the Modifier is chosen the internals of toObj will be associated with the internals of fromObj
1878         """
1879         Modifier._toReplaceWithCheck(toObj,fromObj)
1880         if self._isChosen():
1881             Modifier._toReplaceWith(toObj,fromObj)
1882         return self
1883     @staticmethod
1884     def _toReplaceWith(toObj,fromObj):
1885         if isinstance(fromObj,_ModuleSequenceType):
1886             toObj._seq = fromObj._seq
1887             toObj._tasks = fromObj._tasks
1888         elif isinstance(fromObj,Task):
1889             toObj._collection = fromObj._collection
1890         elif isinstance(fromObj,ConditionalTask):
1891             toObj._collection = fromObj._collection
1892         elif isinstance(fromObj,_Parameterizable):
1893             #clear old items just incase fromObj is not a complete superset of toObj
1894             for p in toObj.parameterNames_():
1895                 delattr(toObj,p)
1896             for p in fromObj.parameterNames_():
1897                 setattr(toObj,p,getattr(fromObj,p))
1898             if isinstance(fromObj,_TypedParameterizable):
1899                 toObj._TypedParameterizable__type = fromObj._TypedParameterizable__type
1900 
1901         else:
1902             raise TypeError("toReplaceWith does not work with type "+str(type(toObj)))
1903 
1904     def _setChosen(self):
1905         """Should only be called by cms.Process instances"""
1906         self.__chosen = True
1907     def _isChosen(self) -> bool:
1908         return self.__chosen
1909     def __and__(self, other):
1910         return _AndModifier(self,other)
1911     def __invert__(self):
1912         return _InvertModifier(self)
1913     def __or__(self, other):
1914         return _OrModifier(self,other)
1915     def _isOrContains(self, other):
1916         return self == other
1917 
1918 
1919 class ModifierChain(object):
1920     """A Modifier made up of a list of Modifiers
1921     """
1922     def __init__(self, *chainedModifiers):
1923         self.__chosen = False
1924         self.__chain = chainedModifiers
1925     def _applyNewProcessModifiers(self,process):
1926         """Should only be called by cms.Process instances
1927         applies list of accumulated changes to the process"""
1928         for m in self.__chain:
1929             m._applyNewProcessModifiers(process)
1930     def _setChosen(self):
1931         """Should only be called by cms.Process instances"""
1932         self.__chosen = True
1933         for m in self.__chain:
1934             m._setChosen()
1935     def _isChosen(self) -> bool:
1936         return self.__chosen
1937     def copyAndExclude(self, toExclude):
1938         """Creates a new ModifierChain which is a copy of
1939           this ModifierChain but excludes any Modifier or
1940           ModifierChain in the list toExclude.
1941           The exclusion is done recursively down the chain.
1942           """
1943         newMods = []
1944         for m in self.__chain:
1945             if m not in toExclude:
1946                 s = m
1947                 if isinstance(m,ModifierChain):
1948                     s = m.__copyIfExclude(toExclude)
1949                 newMods.append(s)
1950         return ModifierChain(*newMods)
1951     def __copyIfExclude(self,toExclude):
1952         shouldCopy = False
1953         for m in toExclude:
1954             if self._isOrContains(m):
1955                 shouldCopy = True
1956                 break
1957         if shouldCopy:
1958             return self.copyAndExclude(toExclude)
1959         return self
1960     def _isOrContains(self, other):
1961         if self is other:
1962             return True
1963         for m in self.__chain:
1964             if m._isOrContains(other):
1965                 return True
1966         return False
1967 
1968 class ProcessModifier(object):
1969     """A class used by a Modifier to affect an entire Process instance.
1970     When a Process 'loads' a module containing a ProcessModifier, that
1971     ProcessModifier will be applied to the Process if and only if the 
1972     Modifier passed to the constructor has been chosen.
1973     """
1974     def __init__(self, modifier, func):
1975         self.__modifier = modifier
1976         self.__func = func
1977         self.__seenProcesses = set()
1978     def apply(self,process):
1979         if self.__modifier._isChosen():
1980             if process not in self.__seenProcesses:
1981                 self.__func(process)
1982                 self.__seenProcesses.add(process)
1983 
1984 class ProcessAccelerator(_ConfigureComponent,_Unlabelable):
1985     """A class used to specify possible compute accelerators in a Process
1986     instance. It is intended to be derived for any
1987     accelerator/portability technology, and provides hooks such that a
1988     specific customization can be applied to the Process on a worker
1989     node at the point where the python configuration is serialized for C++.
1990 
1991     The customization must not change the configuration hash. To
1992     enforce this reuirement, the customization gets a
1993     ProcessForProcessAccelerator wrapper that gives access to only
1994     those parts of the configuration that can be changed. Nevertheless
1995     it would be good to have specific unit test for each deriving
1996     class to ensure that all combinations of the enabled accelerators
1997     give the same configuration hash.
1998 
1999     The deriving class must do its checks for hardware availability
2000     only in enabledLabels(), and possibly in apply() if any further
2001     fine-tuning is needed, because those two are the only functions
2002     that are guaranteed to be called at the worker node.
2003     """
2004     def __init__(self):
2005         pass
2006     def _place(self, name, proc):
2007         proc._placeAccelerator(self.type_(), self)
2008     def type_(self):
2009         return type(self).__name__
2010     def dumpPython(self, options:PrintOptions=PrintOptions()) -> str:
2011         specialImportRegistry.registerUse(self)
2012         result = self.__class__.__name__+"(" # not including cms. since the deriving classes are not in cms "namespace"
2013         options.indent()
2014         res = self.dumpPythonImpl(options)
2015         options.unindent()
2016         if len(res) > 0:
2017             result += "\n"+res+"\n"
2018         result += ")\n"
2019         return result
2020 
2021     # The following methods are hooks to be overridden (if needed) in the deriving class
2022     def dumpPythonImpl(self, options) -> str:
2023         """Override if need to add any 'body' content to dumpPython(). Returns a string."""
2024         return ""
2025     def labels(self):
2026         """Override to return a list of strings for the accelerator labels."""
2027         return []
2028     def enabledLabels(self):
2029         """Override to return a list of strings for the accelerator labels
2030         that are enabled in the system the job is being run on."""
2031         return []
2032     def moduleTypeResolver(self, accelerators):
2033         """Override to return an object that implements "module type resolver"
2034         in python. The object should have the following methods
2035         - __init__(self, accelerators)
2036           * accelerators = list of selected accelerators
2037         - plugin(self):
2038           * should return a string for the type resolver plugin name
2039         - setModuleVariant(self, module):
2040           * Called for each ED and ES module. Should act only if
2041             module.type_() contains the magic identifier
2042 
2043         At most one of the ProcessAccelerators in a job can return a
2044 non-None object
2045         """
2046         return None
2047     def apply(self, process, accelerators):
2048         """Override if need to customize the Process at worker node. The
2049         selected available accelerator labels are given in the
2050         'accelerators' argument (the patterns, e.g. '*' have been
2051         expanded to concrete labels).
2052 
2053         This function may touch only untracked parameters.
2054         """
2055         pass
2056 
2057 class ProcessForProcessAccelerator(object):
2058     """This class is inteded to wrap the Process object to constrain the
2059     available functionality for ProcessAccelerator.apply()"""
2060     def  __init__(self, process):
2061         self.__process = process
2062     def __getattr__(self, label):
2063         value = getattr(self.__process, label)
2064         if not isinstance(value, Service):
2065             raise TypeError("ProcessAccelerator.apply() can get only Services. Tried to get {} with label {}".format(str(type(value)), label))
2066         return value
2067     def __setattr__(self, label:str, value):
2068         if label == "_ProcessForProcessAccelerator__process":
2069             super().__setattr__(label, value)
2070         else:
2071             if not isinstance(value, Service):
2072                 raise TypeError("ProcessAccelerator.apply() can only set Services. Tried to set {} with label {}".format(str(type(value)), label))
2073             setattr(self.__process, label, value)
2074     def __delattr__(self, label:str):
2075         value = getattr(self.__process, label)
2076         if not isinstance(value, Service):
2077             raise TypeError("ProcessAccelerator.apply() can delete only Services. Tried to del {} with label {}".format(str(type(value)), label))
2078         delattr(self.__process, label)
2079     def add_(self, value):
2080         if not isinstance(value, Service):
2081             raise TypeError("ProcessAccelerator.apply() can only add Services. Tried to set {} with label {}".format(str(type(value)), label))
2082         self.__process.add_(value)
2083 
2084 # Need to be a module-level function for the configuration with a
2085 # SwitchProducer to be pickleable.
2086 def _switchproducer_test2_case1(accelerators):
2087     return ("test1" in accelerators, -10)
2088 def _switchproducer_test2_case2(accelerators):
2089     return ("test2" in accelerators, -9)
2090 
2091 if __name__=="__main__":
2092     import unittest
2093     import copy
2094 
2095     def _lineDiff(newString, oldString):
2096         newString = ( x for x in newString.split('\n') if len(x) > 0)
2097         oldString = [ x for x in oldString.split('\n') if len(x) > 0]
2098         diff = []
2099         oldStringLine = 0
2100         for l in newString:
2101             if oldStringLine >= len(oldString):
2102                 diff.append(l)
2103                 continue
2104             if l == oldString[oldStringLine]:
2105                 oldStringLine +=1
2106                 continue
2107             diff.append(l)
2108         return "\n".join( diff )
2109 
2110     class TestMakePSet(object):
2111         """Has same interface as the C++ object that creates PSets
2112         """
2113         def __init__(self):
2114             self.values = dict()
2115         def __insertValue(self,tracked,label,value):
2116             self.values[label]=(tracked,value)
2117         def __getValue(self,tracked,label):
2118             pair = self.values[label]
2119             if pair[0] != tracked:
2120                raise Exception("Asked for %s parameter '%s', but it is %s" % ("tracked" if tracked else "untracked",
2121                                                                               label,
2122                                                                               "tracked" if pair[0] else "untracked"))
2123             return pair[1]
2124         def addInt32(self,tracked,label,value):
2125             self.__insertValue(tracked,label,value)
2126         def addVInt32(self,tracked,label,value):
2127             self.__insertValue(tracked,label,value)
2128         def addUInt32(self,tracked,label,value):
2129             self.__insertValue(tracked,label,value)
2130         def addVUInt32(self,tracked,label,value):
2131             self.__insertValue(tracked,label,value)
2132         def addInt64(self,tracked,label,value):
2133             self.__insertValue(tracked,label,value)
2134         def addVInt64(self,tracked,label,value):
2135             self.__insertValue(tracked,label,value)
2136         def addUInt64(self,tracked,label,value):
2137             self.__insertValue(tracked,label,value)
2138         def addVUInt64(self,tracked,label,value):
2139             self.__insertValue(tracked,label,value)
2140         def addDouble(self,tracked,label,value):
2141             self.__insertValue(tracked,label,value)
2142         def addVDouble(self,tracked,label,value):
2143             self.__insertValue(tracked,label,value)
2144         def addBool(self,tracked,label,value):
2145             self.__insertValue(tracked,label,value)
2146         def addString(self,tracked,label,value):
2147             self.__insertValue(tracked,label,value)
2148         def addVString(self,tracked,label,value):
2149             self.__insertValue(tracked,label,value)
2150         def getVString(self,tracked,label):
2151             return self.__getValue(tracked, label)
2152         def addInputTag(self,tracked,label,value):
2153             self.__insertValue(tracked,label,value)
2154         def addVInputTag(self,tracked,label,value):
2155             self.__insertValue(tracked,label,value)
2156         def addESInputTag(self,tracked,label,value):
2157             self.__insertValue(tracked,label,value)
2158         def addVESInputTag(self,tracked,label,value):
2159             self.__insertValue(tracked,label,value)
2160         def addEventID(self,tracked,label,value):
2161             self.__insertValue(tracked,label,value)
2162         def addVEventID(self,tracked,label,value):
2163             self.__insertValue(tracked,label,value)
2164         def addLuminosityBlockID(self,tracked,label,value):
2165             self.__insertValue(tracked,label,value)
2166         def addLuminosityBlockID(self,tracked,label,value):
2167             self.__insertValue(tracked,label,value)
2168         def addEventRange(self,tracked,label,value):
2169             self.__insertValue(tracked,label,value)
2170         def addVEventRange(self,tracked,label,value):
2171             self.__insertValue(tracked,label,value)
2172         def addPSet(self,tracked,label,value):
2173             self.__insertValue(tracked,label,value)
2174         def addVPSet(self,tracked,label,value):
2175             self.__insertValue(tracked,label,value)
2176         def addFileInPath(self,tracked,label,value):
2177             self.__insertValue(tracked,label,value)
2178         def newPSet(self):
2179             return TestMakePSet()
2180 
2181     class SwitchProducerTest(SwitchProducer):
2182         def __init__(self, **kargs):
2183             super(SwitchProducerTest,self).__init__(
2184                 dict(
2185                     test1 = lambda accelerators: (True, -10),
2186                     test2 = lambda accelerators: (True, -9),
2187                     test3 = lambda accelerators: (True, -8),
2188                     test4 = lambda accelerators: (True, -7)
2189                 ), **kargs)
2190     specialImportRegistry.registerSpecialImportForType(SwitchProducerTest, "from test import SwitchProducerTest")
2191 
2192     class SwitchProducerTest2(SwitchProducer):
2193         def __init__(self, **kargs):
2194             super(SwitchProducerTest2,self).__init__(
2195                 dict(
2196                     test1 = _switchproducer_test2_case1,
2197                     test2 = _switchproducer_test2_case2,
2198                 ), **kargs)
2199     specialImportRegistry.registerSpecialImportForType(SwitchProducerTest2, "from test import SwitchProducerTest2")
2200 
2201     class TestModuleTypeResolver:
2202         def __init__(self, accelerators):
2203             # first element is used as the default is nothing is set
2204             self._valid_backends = []
2205             if "test1" in accelerators:
2206                 self._valid_backends.append("test1_backend")
2207             if "test2" in accelerators:
2208                 self._valid_backends.append("test2_backend")
2209             if len(self._valid_backends) == 0:
2210                 raise EDMException(edm.errors.UnavailableAccelerator, "Machine has no accelerators that Test supports (has {})".format(", ".join(accelerators)))
2211 
2212         def plugin(self):
2213             return "TestModuleTypeResolver"
2214 
2215         def setModuleVariant(self, module):
2216             if "@test" in module.type_():
2217                 defaultBackend = self._valid_backends[0]
2218                 if hasattr(module, "test"):
2219                     if hasattr(module.test, "backend"):
2220                         if module.test.backend.value() not in self._valid_backends:
2221                             raise EDMException(edm.errors.UnavailableAccelerator, "Module {} has the Test backend set explicitly, but its accelerator is not available for the job".format(module.label_()))
2222                     else:
2223                         module.test.backend = untracked.string(defaultBackend)
2224                 else:
2225                     module.test = untracked.PSet(
2226                         backend = untracked.string(defaultBackend)
2227                     )
2228 
2229     class ProcessAcceleratorTest(ProcessAccelerator):
2230         def __init__(self, enabled=["test1", "test2", "anothertest3"], moduleTypeResolverMaker=None):
2231             super().__init__()
2232             self._labels = ["test1", "test2", "anothertest3"]
2233             self.setEnabled(enabled)
2234             self._moduleTypeResolverMaker = moduleTypeResolverMaker
2235         def setEnabled(self, enabled):
2236             invalid = set(enabled).difference(set(self._labels))
2237             if len(invalid) > 0:
2238                 raise Exception("Tried to enabled nonexistent test accelerators {}".format(",".join(invalid)))
2239             self._enabled = enabled[:]
2240         def dumpPythonImpl(self,options):
2241             result = "{}enabled = [{}]".format(options.indentation(),
2242                                                ", ".join(["'{}'".format(e) for e in self._enabled]))
2243             return result
2244         def labels(self):
2245             return self._labels
2246         def enabledLabels(self):
2247             return self._enabled
2248         def moduleTypeResolver(self, accelerators):
2249             if not self._moduleTypeResolverMaker:
2250                 return super().moduleTypeResolver(accelerators)
2251             return self._moduleTypeResolverMaker(accelerators)
2252         def apply(self, process, accelerators):
2253             process.AcceleratorTestService = Service("AcceleratorTestService")
2254             if hasattr(process, "AcceleratorTestServiceRemove"):
2255                 del process.AcceleratorTestServiceRemove
2256     specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorTest, "from test import ProcessAcceleratorTest")
2257 
2258     class ProcessAcceleratorTest2(ProcessAccelerator):
2259         def __init__(self, enabled=["anothertest3", "anothertest4"], moduleTypeResolverMaker=None):
2260             super().__init__()
2261             self._labels = ["anothertest3", "anothertest4"]
2262             self.setEnabled(enabled)
2263             self._moduleTypeResolverMaker = moduleTypeResolverMaker
2264         def setEnabled(self, enabled):
2265             invalid = set(enabled).difference(set(self._labels))
2266             if len(invalid) > 0:
2267                 raise Exception("Tried to enabled nonexistent test accelerators {}".format(",".join(invalid)))
2268             self._enabled = enabled[:]
2269         def dumpPythonImpl(self,options):
2270             result = "{}enabled = [{}]".format(options.indentation(),
2271                                                ", ".join(["'{}'".format(e) for e in self._enabled]))
2272             return result
2273         def labels(self):
2274             return self._labels
2275         def enabledLabels(self):
2276             return self._enabled
2277         def moduleTypeResolver(self, accelerators):
2278             if not self._moduleTypeResolverMaker:
2279                 return super().moduleTypeResolver(accelerators)
2280             return self._moduleTypeResolverMaker(accelerators)
2281         def apply(self, process, accelerators):
2282             pass
2283     specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorTest2, "from test import ProcessAcceleratorTest2")
2284 
2285     class TestModuleCommand(unittest.TestCase):
2286         def setUp(self):
2287             """Nothing to do """
2288             None
2289         def testParameterizable(self):
2290             p = _Parameterizable()
2291             self.assertEqual(len(p.parameterNames_()),0)
2292             p.a = int32(1)
2293             self.assertTrue('a' in p.parameterNames_())
2294             self.assertEqual(p.a.value(), 1)
2295             p.a = 10
2296             self.assertEqual(p.a.value(), 10)
2297             p.a = untracked(int32(1))
2298             self.assertEqual(p.a.value(), 1)
2299             self.assertFalse(p.a.isTracked())
2300             p.a = untracked.int32(1)
2301             self.assertEqual(p.a.value(), 1)
2302             self.assertFalse(p.a.isTracked())
2303             p = _Parameterizable(foo=int32(10), bar = untracked(double(1.0)))
2304             self.assertEqual(p.foo.value(), 10)
2305             self.assertEqual(p.bar.value(),1.0)
2306             self.assertFalse(p.bar.isTracked())
2307             self.assertRaises(TypeError,setattr,(p,'c',1))
2308             p = _Parameterizable(a=PSet(foo=int32(10), bar = untracked(double(1.0))))
2309             self.assertEqual(p.a.foo.value(),10)
2310             self.assertEqual(p.a.bar.value(),1.0)
2311             p.b = untracked(PSet(fii = int32(1)))
2312             self.assertEqual(p.b.fii.value(),1)
2313             self.assertFalse(p.b.isTracked())
2314             #test the fact that values can be shared
2315             v = int32(10)
2316             p=_Parameterizable(a=v)
2317             v.setValue(11)
2318             self.assertEqual(p.a.value(),11)
2319             p.a = 12
2320             self.assertEqual(p.a.value(),12)
2321             self.assertEqual(v.value(),12)
2322         def testTypedParameterizable(self):
2323             p = _TypedParameterizable("blah", b=int32(1))
2324             #see if copy works deeply
2325             other = p.copy()
2326             other.b = 2
2327             self.assertNotEqual(p.b,other.b)
2328 
2329         def testProcessInsertion(self):
2330             p = Process("test")
2331             p.a = EDAnalyzer("MyAnalyzer")
2332             self.assertTrue( 'a' in p.analyzers_() )
2333             self.assertTrue( 'a' in p.analyzers)
2334             p.add_(Service("SomeService"))
2335             self.assertTrue('SomeService' in p.services_())
2336             self.assertEqual(p.SomeService.type_(), "SomeService")
2337             p.Tracer = Service("Tracer")
2338             self.assertTrue('Tracer' in p.services_())
2339             self.assertRaises(TypeError, setattr, *(p,'b',"this should fail"))
2340             self.assertRaises(TypeError, setattr, *(p,'bad',Service("MessageLogger")))
2341             self.assertRaises(ValueError, setattr, *(p,'bad',Source("PoolSource")))
2342             p.out = OutputModule("Outer")
2343             self.assertEqual(p.out.type_(), 'Outer')
2344             self.assertTrue( 'out' in p.outputModules_() )
2345 
2346             p.geom = ESSource("GeomProd")
2347             self.assertTrue('geom' in p.es_sources_())
2348             p.add_(ESSource("ConfigDB"))
2349             self.assertTrue('ConfigDB' in p.es_sources_())
2350 
2351             p.aliasfoo1 = EDAlias(foo1 = VPSet(PSet(type = string("Foo1"))))
2352             self.assertTrue('aliasfoo1' in p.aliases_())
2353 
2354         def testProcessExtend(self):
2355             class FromArg(object):
2356                 def __init__(self,*arg,**args):
2357                     for name in args.keys():
2358                         self.__dict__[name]=args[name]
2359 
2360             a=EDAnalyzer("MyAnalyzer")
2361             t=EDAnalyzer("MyAnalyzer")
2362             t.setLabel("foo")
2363             s1 = Sequence(a)
2364             s2 = Sequence(s1)
2365             s3 = Sequence(s2)
2366             d = FromArg(
2367                     a=a,
2368                     b=Service("Full"),
2369                     c=Path(a),
2370                     d=s2,
2371                     e=s1,
2372                     f=s3,
2373                     g=Sequence(s1+s2+s3)
2374                 )
2375             p = Process("Test")
2376             p.extend(d)
2377             self.assertEqual(p.a.type_(),"MyAnalyzer")
2378             self.assertEqual(p.a.label_(),"a")
2379             self.assertRaises(AttributeError,getattr,p,'b')
2380             self.assertEqual(p.Full.type_(),"Full")
2381             self.assertEqual(str(p.c),'a')
2382             self.assertEqual(str(p.d),'a')
2383 
2384             z1 = FromArg(
2385                     a=a,
2386                     b=Service("Full"),
2387                     c=Path(a),
2388                     d=s2,
2389                     e=s1,
2390                     f=s3,
2391                     s4=s3,
2392                     g=Sequence(s1+s2+s3)
2393                  )
2394 
2395             p1 = Process("Test")
2396             #p1.extend(z1)
2397             self.assertRaises(ValueError, p1.extend, z1)
2398 
2399             z2 = FromArg(
2400                     a=a,
2401                     b=Service("Full"),
2402                     c=Path(a),
2403                     d=s2,
2404                     e=s1,
2405                     f=s3,
2406                     aaa=copy.deepcopy(a),
2407                     s4=copy.deepcopy(s3),
2408                     g=Sequence(s1+s2+s3),
2409                     t=t
2410                 )
2411             p2 = Process("Test")
2412             p2.extend(z2)
2413             #self.assertRaises(ValueError, p2.extend, z2)
2414             self.assertEqual(p2.s4.label_(),"s4")
2415             #p2.s4.setLabel("foo")
2416             self.assertRaises(ValueError, p2.s4.setLabel, "foo")
2417             p2.s4.setLabel("s4")
2418             p2.s4.setLabel(None)
2419             p2.s4.setLabel("foo")
2420             p2._Process__setObjectLabel(p2.s4, "foo")
2421             p2._Process__setObjectLabel(p2.s4, None)
2422             p2._Process__setObjectLabel(p2.s4, "bar")
2423 
2424 
2425             p = Process('test')
2426             p.a = EDProducer("MyProducer")
2427             p.t = Task(p.a)
2428             p.p = Path(p.t)
2429             self.assertRaises(ValueError, p.extend, FromArg(a = EDProducer("YourProducer")))
2430             self.assertRaises(ValueError, p.extend, FromArg(a = EDAlias()))
2431             self.assertRaises(ValueError, p.__setattr__, "a", EDAlias())
2432 
2433             p = Process('test')
2434             p.a = EDProducer("MyProducer")
2435             p.t = ConditionalTask(p.a)
2436             p.p = Path(p.t)
2437             self.assertRaises(ValueError, p.extend, FromArg(a = EDProducer("YourProducer")))
2438             self.assertRaises(ValueError, p.extend, FromArg(a = EDAlias()))
2439             self.assertRaises(ValueError, p.__setattr__, "a", EDAlias())
2440 
2441             p = Process('test')
2442             p.a = EDProducer("MyProducer")
2443             p.s = Sequence(p.a)
2444             p.p = Path(p.s)
2445             self.assertRaises(ValueError, p.extend, FromArg(a = EDProducer("YourProducer")))
2446             self.assertRaises(ValueError, p.extend, FromArg(a = EDAlias()))
2447             self.assertRaises(ValueError, p.__setattr__, "a", EDAlias())
2448 
2449         def testProcessDumpPython(self):
2450             self.maxDiff = None
2451             self.assertEqual(Process("test").dumpPython(),
2452 """import FWCore.ParameterSet.Config as cms
2453 
2454 process = cms.Process("test")
2455 
2456 process.maxEvents = cms.untracked.PSet(
2457     input = cms.optional.untracked.int32,
2458     output = cms.optional.untracked.allowed(cms.int32,cms.PSet)
2459 )
2460 
2461 process.maxLuminosityBlocks = cms.untracked.PSet(
2462     input = cms.untracked.int32(-1)
2463 )
2464 
2465 process.options = cms.untracked.PSet(
2466     IgnoreCompletely = cms.untracked.vstring(),
2467     Rethrow = cms.untracked.vstring(),
2468     TryToContinue = cms.untracked.vstring(),
2469     accelerators = cms.untracked.vstring('*'),
2470     allowUnscheduled = cms.obsolete.untracked.bool,
2471     canDeleteEarly = cms.untracked.vstring(),
2472     deleteNonConsumedUnscheduledModules = cms.untracked.bool(True),
2473     dumpOptions = cms.untracked.bool(False),
2474     emptyRunLumiMode = cms.obsolete.untracked.string,
2475     eventSetup = cms.untracked.PSet(
2476         forceNumberOfConcurrentIOVs = cms.untracked.PSet(
2477             allowAnyLabel_=cms.required.untracked.uint32
2478         ),
2479         numberOfConcurrentIOVs = cms.untracked.uint32(0)
2480     ),
2481     fileMode = cms.untracked.string('FULLMERGE'),
2482     forceEventSetupCacheClearOnNewRun = cms.untracked.bool(False),
2483     holdsReferencesToDeleteEarly = cms.untracked.VPSet(),
2484     makeTriggerResults = cms.obsolete.untracked.bool,
2485     modulesToCallForTryToContinue = cms.untracked.vstring(),
2486     modulesToIgnoreForDeleteEarly = cms.untracked.vstring(),
2487     numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(0),
2488     numberOfConcurrentRuns = cms.untracked.uint32(1),
2489     numberOfStreams = cms.untracked.uint32(0),
2490     numberOfThreads = cms.untracked.uint32(1),
2491     printDependencies = cms.untracked.bool(False),
2492     sizeOfStackForThreadsInKB = cms.optional.untracked.uint32,
2493     throwIfIllegalParameter = cms.untracked.bool(True),
2494     wantSummary = cms.untracked.bool(False)
2495 )
2496 
2497 process.MessageLogger = cms.Service("MessageLogger",
2498     cerr = cms.untracked.PSet(
2499         FwkReport = cms.untracked.PSet(
2500             limit = cms.untracked.int32(10000000),
2501             reportEvery = cms.untracked.int32(1)
2502         ),
2503         FwkSummary = cms.untracked.PSet(
2504             limit = cms.untracked.int32(10000000),
2505             reportEvery = cms.untracked.int32(1)
2506         ),
2507         INFO = cms.untracked.PSet(
2508             limit = cms.untracked.int32(0)
2509         ),
2510         Root_NoDictionary = cms.untracked.PSet(
2511             limit = cms.untracked.int32(0)
2512         ),
2513         default = cms.untracked.PSet(
2514             limit = cms.untracked.int32(10000000)
2515         ),
2516         enable = cms.untracked.bool(True),
2517         enableStatistics = cms.untracked.bool(False),
2518         lineLength = cms.optional.untracked.int32,
2519         noLineBreaks = cms.optional.untracked.bool,
2520         noTimeStamps = cms.untracked.bool(False),
2521         resetStatistics = cms.untracked.bool(False),
2522         statisticsThreshold = cms.untracked.string('WARNING'),
2523         threshold = cms.untracked.string('INFO'),
2524         allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2525             limit = cms.optional.untracked.int32,
2526             reportEvery = cms.untracked.int32(1),
2527             timespan = cms.optional.untracked.int32
2528         )
2529     ),
2530     cout = cms.untracked.PSet(
2531         enable = cms.untracked.bool(False),
2532         enableStatistics = cms.untracked.bool(False),
2533         lineLength = cms.optional.untracked.int32,
2534         noLineBreaks = cms.optional.untracked.bool,
2535         noTimeStamps = cms.optional.untracked.bool,
2536         resetStatistics = cms.untracked.bool(False),
2537         statisticsThreshold = cms.optional.untracked.string,
2538         threshold = cms.optional.untracked.string,
2539         allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2540             limit = cms.optional.untracked.int32,
2541             reportEvery = cms.untracked.int32(1),
2542             timespan = cms.optional.untracked.int32
2543         )
2544     ),
2545     debugModules = cms.untracked.vstring(),
2546     default = cms.untracked.PSet(
2547         limit = cms.optional.untracked.int32,
2548         lineLength = cms.untracked.int32(80),
2549         noLineBreaks = cms.untracked.bool(False),
2550         noTimeStamps = cms.untracked.bool(False),
2551         reportEvery = cms.untracked.int32(1),
2552         statisticsThreshold = cms.untracked.string('INFO'),
2553         threshold = cms.untracked.string('INFO'),
2554         timespan = cms.optional.untracked.int32,
2555         allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2556             limit = cms.optional.untracked.int32,
2557             reportEvery = cms.untracked.int32(1),
2558             timespan = cms.optional.untracked.int32
2559         )
2560     ),
2561     files = cms.untracked.PSet(
2562         allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2563             enableStatistics = cms.untracked.bool(False),
2564             extension = cms.optional.untracked.string,
2565             filename = cms.optional.untracked.string,
2566             lineLength = cms.optional.untracked.int32,
2567             noLineBreaks = cms.optional.untracked.bool,
2568             noTimeStamps = cms.optional.untracked.bool,
2569             output = cms.optional.untracked.string,
2570             resetStatistics = cms.untracked.bool(False),
2571             statisticsThreshold = cms.optional.untracked.string,
2572             threshold = cms.optional.untracked.string,
2573             allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2574                 limit = cms.optional.untracked.int32,
2575                 reportEvery = cms.untracked.int32(1),
2576                 timespan = cms.optional.untracked.int32
2577             )
2578         )
2579     ),
2580     suppressDebug = cms.untracked.vstring(),
2581     suppressFwkInfo = cms.untracked.vstring(),
2582     suppressInfo = cms.untracked.vstring(),
2583     suppressWarning = cms.untracked.vstring(),
2584     allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2585         limit = cms.optional.untracked.int32,
2586         reportEvery = cms.untracked.int32(1),
2587         timespan = cms.optional.untracked.int32
2588     )
2589 )
2590 
2591 
2592 """)
2593             p = Process("test")
2594             p.a = EDAnalyzer("MyAnalyzer")
2595             p.p = Path(p.a)
2596             p.s = Sequence(p.a)
2597             p.r = Sequence(p.s)
2598             p.p2 = Path(p.s)
2599             p.schedule = Schedule(p.p2,p.p)
2600             d=p.dumpPython()
2601             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2602 """process.a = cms.EDAnalyzer("MyAnalyzer")
2603 process.s = cms.Sequence(process.a)
2604 process.r = cms.Sequence(process.s)
2605 process.p = cms.Path(process.a)
2606 process.p2 = cms.Path(process.s)
2607 process.schedule = cms.Schedule(*[ process.p2, process.p ])""")
2608             #Reverse order of 'r' and 's'
2609             p = Process("test")
2610             p.a = EDAnalyzer("MyAnalyzer")
2611             p.p = Path(p.a)
2612             p.r = Sequence(p.a)
2613             p.s = Sequence(p.r)
2614             p.p2 = Path(p.r)
2615             p.schedule = Schedule(p.p2,p.p)
2616             p.b = EDAnalyzer("YourAnalyzer")
2617             d=p.dumpPython()
2618             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2619 """process.a = cms.EDAnalyzer("MyAnalyzer")
2620 process.b = cms.EDAnalyzer("YourAnalyzer")
2621 process.r = cms.Sequence(process.a)
2622 process.s = cms.Sequence(process.r)
2623 process.p = cms.Path(process.a)
2624 process.p2 = cms.Path(process.r)
2625 process.schedule = cms.Schedule(*[ process.p2, process.p ])""")
2626         #use an anonymous sequence
2627             p = Process("test")
2628             p.a = EDAnalyzer("MyAnalyzer")
2629             p.p = Path(p.a)
2630             s = Sequence(p.a)
2631             p.r = Sequence(s)
2632             p.p2 = Path(p.r)
2633             p.schedule = Schedule(p.p2,p.p)
2634             d=p.dumpPython()
2635             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2636 """process.a = cms.EDAnalyzer("MyAnalyzer")
2637 process.r = cms.Sequence((process.a))
2638 process.p = cms.Path(process.a)
2639 process.p2 = cms.Path(process.r)
2640 process.schedule = cms.Schedule(*[ process.p2, process.p ])""")
2641 
2642             # include some tasks
2643             p = Process("test")
2644             p.a = EDAnalyzer("MyAnalyzer")
2645             p.b = EDProducer("bProducer")
2646             p.c = EDProducer("cProducer")
2647             p.d = EDProducer("dProducer")
2648             p.e = EDProducer("eProducer")
2649             p.f = EDProducer("fProducer")
2650             p.g = EDProducer("gProducer")
2651             p.task5 = Task()
2652             p.task3 = Task()
2653             p.task2 = Task(p.c, p.task3)
2654             p.task4 = Task(p.f, p.task2)
2655             p.task1 = Task(p.task5)
2656             p.task3.add(p.task1)
2657             p.p = Path(p.a)
2658             s = Sequence(p.a)
2659             p.r = Sequence(s)
2660             p.p2 = Path(p.r, p.task1, p.task2)
2661             p.schedule = Schedule(p.p2,p.p,tasks=[p.task3,p.task4, p.task5])
2662             d=p.dumpPython()
2663             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2664 """process.b = cms.EDProducer("bProducer")
2665 process.c = cms.EDProducer("cProducer")
2666 process.d = cms.EDProducer("dProducer")
2667 process.e = cms.EDProducer("eProducer")
2668 process.f = cms.EDProducer("fProducer")
2669 process.g = cms.EDProducer("gProducer")
2670 process.a = cms.EDAnalyzer("MyAnalyzer")
2671 process.task5 = cms.Task()
2672 process.task1 = cms.Task(process.task5)
2673 process.task3 = cms.Task(process.task1)
2674 process.task2 = cms.Task(process.c, process.task3)
2675 process.task4 = cms.Task(process.f, process.task2)
2676 process.r = cms.Sequence((process.a))
2677 process.p = cms.Path(process.a)
2678 process.p2 = cms.Path(process.r, process.task1, process.task2)
2679 process.schedule = cms.Schedule(*[ process.p2, process.p ], tasks=[process.task3, process.task4, process.task5])""")
2680             # include some conditional tasks
2681             p = Process("test")
2682             p.a = EDAnalyzer("MyAnalyzer")
2683             p.b = EDProducer("bProducer")
2684             p.c = EDProducer("cProducer")
2685             p.d = EDProducer("dProducer")
2686             p.e = EDProducer("eProducer")
2687             p.f = EDProducer("fProducer")
2688             p.g = EDProducer("gProducer")
2689             p.task5 = Task()
2690             p.task3 = Task()
2691             p.task2 = ConditionalTask(p.c, p.task3)
2692             p.task1 = ConditionalTask(p.task5)
2693             p.p = Path(p.a)
2694             s = Sequence(p.a)
2695             p.r = Sequence(s)
2696             p.p2 = Path(p.r, p.task1, p.task2)
2697             p.schedule = Schedule(p.p2,p.p,tasks=[p.task5])
2698             d=p.dumpPython()
2699             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2700 """process.b = cms.EDProducer("bProducer")
2701 process.c = cms.EDProducer("cProducer")
2702 process.d = cms.EDProducer("dProducer")
2703 process.e = cms.EDProducer("eProducer")
2704 process.f = cms.EDProducer("fProducer")
2705 process.g = cms.EDProducer("gProducer")
2706 process.a = cms.EDAnalyzer("MyAnalyzer")
2707 process.task5 = cms.Task()
2708 process.task3 = cms.Task()
2709 process.task2 = cms.ConditionalTask(process.c, process.task3)
2710 process.task1 = cms.ConditionalTask(process.task5)
2711 process.r = cms.Sequence((process.a))
2712 process.p = cms.Path(process.a)
2713 process.p2 = cms.Path(process.r, process.task1, process.task2)
2714 process.schedule = cms.Schedule(*[ process.p2, process.p ], tasks=[process.task5])""")
2715             # only tasks
2716             p = Process("test")
2717             p.d = EDProducer("dProducer")
2718             p.e = EDProducer("eProducer")
2719             p.f = EDProducer("fProducer")
2720             p.g = EDProducer("gProducer")
2721             p.task1 = Task(p.d, p.e)
2722             task2 = Task(p.f, p.g)
2723             p.schedule = Schedule(tasks=[p.task1,task2])
2724             d=p.dumpPython()
2725             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2726 """process.d = cms.EDProducer("dProducer")
2727 process.e = cms.EDProducer("eProducer")
2728 process.f = cms.EDProducer("fProducer")
2729 process.g = cms.EDProducer("gProducer")
2730 process.task1 = cms.Task(process.d, process.e)
2731 process.schedule = cms.Schedule(tasks=[cms.Task(process.f, process.g), process.task1])""")
2732             # empty schedule
2733             p = Process("test")
2734             p.schedule = Schedule()
2735             d=p.dumpPython()
2736             self.assertEqual(_lineDiff(d,Process('test').dumpPython()),
2737 """process.schedule = cms.Schedule()""")
2738 
2739             s = Sequence()
2740             a = EDProducer("A")
2741             s2 = Sequence(a)
2742             s2 += s
2743             process = Process("DUMP")
2744             process.a = a
2745             process.s2 = s2
2746             d=process.dumpPython()
2747             self.assertEqual(_lineDiff(d,Process('DUMP').dumpPython()),
2748 """process.a = cms.EDProducer("A")
2749 process.s2 = cms.Sequence(process.a)""")
2750             s = Sequence()
2751             s1 = Sequence(s)
2752             a = EDProducer("A")
2753             s3 = Sequence(a+a)
2754             s2 = Sequence(a+s3)
2755             s2 += s1
2756             process = Process("DUMP")
2757             process.a = a
2758             process.s2 = s2
2759             d=process.dumpPython()
2760             self.assertEqual(_lineDiff(d,Process('DUMP').dumpPython()),
2761 """process.a = cms.EDProducer("A")
2762 process.s2 = cms.Sequence(process.a+(process.a+process.a))""")
2763 
2764         def testSecSource(self):
2765             p = Process('test')
2766             p.a = SecSource("MySecSource")
2767             self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),'process.a = cms.SecSource("MySecSource")')
2768 
2769         def testGlobalReplace(self):
2770             p = Process('test')
2771             p.a = EDAnalyzer("MyAnalyzer")
2772             old = p.a
2773             p.b = EDAnalyzer("YourAnalyzer")
2774             p.c = EDAnalyzer("OurAnalyzer")
2775             p.d = EDProducer("MyProducer")
2776             old2 = p.d
2777             p.t1 = Task(p.d)
2778             t2 = Task(p.d)
2779             t3 = Task(p.d)
2780             t4 = Task(p.d)
2781             t5 = Task(p.d)
2782             t6 = Task(p.d)
2783             p.ct1 = ConditionalTask(p.d)
2784             s = Sequence(p.a*p.b)
2785             p.s4 = Sequence(p.a*p.b, p.ct1)
2786             s.associate(t2)
2787             p.s4.associate(t2)
2788             p.p = Path(p.c+s+p.a)
2789             p.p2 = Path(p.c+p.s4+p.a, p.ct1)
2790             p.e3 = EndPath(p.c+s+p.a)
2791             new = EDAnalyzer("NewAnalyzer")
2792             new2 = EDProducer("NewProducer")
2793             visitor1 = NodeVisitor()
2794             p.p.visit(visitor1)
2795             self.assertTrue(visitor1.modules == set([old,old2,p.b,p.c]))
2796             p.schedule = Schedule(tasks=[t6])
2797             p.globalReplace("a",new)
2798             p.globalReplace("d",new2)
2799             visitor2 = NodeVisitor()
2800             p.p.visit(visitor2)
2801             self.assertTrue(visitor2.modules == set([new,new2,p.b,p.c]))
2802             self.assertEqual(p.p.dumpPython()[:-1], "cms.Path(process.c+process.a+process.b+process.a, cms.Task(process.d))")
2803             visitor_p2 = NodeVisitor()
2804             p.p2.visit(visitor_p2)
2805             self.assertTrue(visitor_p2.modules == set([new,new2,p.b,p.c]))
2806             self.assertEqual(p.p2.dumpPython()[:-1], "cms.Path(process.c+process.s4+process.a, process.ct1)")
2807             visitor3 = NodeVisitor()
2808             p.e3.visit(visitor3)
2809             self.assertTrue(visitor3.modules == set([new,new2,p.b,p.c]))
2810             visitor4 = NodeVisitor()
2811             p.s4.visit(visitor4)
2812             self.assertTrue(visitor4.modules == set([new,new2,p.b]))
2813             self.assertEqual(p.s4.dumpPython()[:-1],"cms.Sequence(process.a+process.b, cms.Task(process.d), process.ct1)")
2814             visitor5 = NodeVisitor()
2815             p.t1.visit(visitor5)
2816             self.assertTrue(visitor5.modules == set([new2]))
2817             visitor6 = NodeVisitor()
2818             listOfTasks = list(p.schedule._tasks)
2819             listOfTasks[0].visit(visitor6)
2820             self.assertTrue(visitor6.modules == set([new2]))
2821             visitor7 = NodeVisitor()
2822             p.ct1.visit(visitor7)
2823             self.assertTrue(visitor7.modules == set([new2]))
2824             visitor8 = NodeVisitor()
2825             listOfConditionalTasks = list(p.conditionaltasks.values())
2826             listOfConditionalTasks[0].visit(visitor8)
2827             self.assertTrue(visitor8.modules == set([new2]))
2828 
2829 
2830             p.d2 = EDProducer("YourProducer")
2831             p.schedule = Schedule(p.p, p.p2, p.e3, tasks=[p.t1])
2832             self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2833             p.p = Path(p.c+s)
2834             self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2835             p.e3 = EndPath(p.c)
2836             self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2837             p.t1 = Task(p.d2)
2838             self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2839 
2840         def testSequence(self):
2841             p = Process('test')
2842             p.a = EDAnalyzer("MyAnalyzer")
2843             p.b = EDAnalyzer("YourAnalyzer")
2844             p.c = EDAnalyzer("OurAnalyzer")
2845             p.s = Sequence(p.a*p.b)
2846             self.assertEqual(str(p.s),'a+b')
2847             self.assertEqual(p.s.label_(),'s')
2848             path = Path(p.c+p.s)
2849             self.assertEqual(str(path),'c+a+b')
2850             p._validateSequence(path, 'p1')
2851             notInProcess = EDAnalyzer('NotInProcess')
2852             p2 = Path(p.c+p.s*notInProcess)
2853             self.assertRaises(RuntimeError, p._validateSequence, p2, 'p2')
2854 
2855         def testSequence2(self):
2856             p = Process('test')
2857             p.a = EDAnalyzer("MyAnalyzer")
2858             p.b = EDAnalyzer("YourAnalyzer")
2859             p.c = EDAnalyzer("OurAnalyzer")
2860             testseq = Sequence(p.a*p.b)
2861             p.s = testseq
2862             #p.y = testseq
2863             self.assertRaises(ValueError, p.__setattr__, "y", testseq) 
2864 
2865         def testServiceInProcess(self):
2866             service = Service("d")
2867             self.assertFalse(service._inProcess)
2868             process = Process("test")
2869             process.d = service
2870             self.assertTrue(service._inProcess)
2871             service2 = Service("d")
2872             process.d = service2
2873             self.assertFalse(service._inProcess)
2874             self.assertTrue(service2._inProcess)
2875             del process.d
2876             self.assertFalse(service2._inProcess)
2877 
2878         def testTask(self):
2879 
2880             # create some objects to use in tests
2881             edanalyzer = EDAnalyzer("a")
2882             edproducer = EDProducer("b")
2883             edproducer2 = EDProducer("b2")
2884             edproducer3 = EDProducer("b3")
2885             edproducer4 = EDProducer("b4")
2886             edproducer8 = EDProducer("b8")
2887             edproducer9 = EDProducer("b9")
2888             edfilter = EDFilter("c")
2889             service = Service("d")
2890             service3 = Service("d", v = untracked.uint32(3))
2891             essource = ESSource("e")
2892             esproducer = ESProducer("f")
2893             testTask2 = Task()
2894 
2895             # test adding things to Tasks
2896             testTask1 = Task(edproducer, edfilter)
2897             self.assertRaises(RuntimeError, testTask1.add, edanalyzer)
2898             testTask1.add(essource, service)
2899             testTask1.add(essource, esproducer)
2900             testTask1.add(testTask2)
2901             coll = testTask1._collection
2902             self.assertTrue(edproducer in coll)
2903             self.assertTrue(edfilter in coll)
2904             self.assertTrue(service in coll)
2905             self.assertTrue(essource in coll)
2906             self.assertTrue(esproducer in coll)
2907             self.assertTrue(testTask2 in coll)
2908             self.assertTrue(len(coll) == 6)
2909             self.assertTrue(len(testTask2._collection) == 0)
2910 
2911             taskContents = []
2912             for i in testTask1:
2913                 taskContents.append(i)
2914             self.assertTrue(taskContents == [edproducer, edfilter, essource, service, esproducer, testTask2])
2915 
2916             # test attaching Task to Process
2917             process = Process("test")
2918 
2919             process.mproducer = edproducer
2920             process.mproducer2 = edproducer2
2921             process.mfilter = edfilter
2922             process.messource = essource
2923             process.mesproducer = esproducer
2924             process.d = service
2925 
2926             testTask3 = Task(edproducer, edproducer2)
2927             testTask1.add(testTask3)
2928             process.myTask1 = testTask1
2929 
2930             # test the validation that occurs when attaching a Task to a Process
2931             # first a case that passes, then one the fails on an EDProducer
2932             # then one that fails on a service
2933             l = set()
2934             visitor = NodeNameVisitor(l)
2935             testTask1.visit(visitor)
2936             self.assertTrue(l == set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
2937             l2 = testTask1.moduleNames
2938             self.assertTrue(l == set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
2939 
2940             testTask4 = Task(edproducer3)
2941             l.clear()
2942             self.assertRaises(RuntimeError, testTask4.visit, visitor)
2943             try:
2944                 process.myTask4 = testTask4
2945                 self.assertTrue(False)
2946             except RuntimeError:
2947                 pass
2948 
2949             testTask5 = Task(service3)
2950             l.clear()
2951             self.assertRaises(RuntimeError, testTask5.visit, visitor)
2952             try:
2953                 process.myTask5 = testTask5
2954                 self.assertTrue(False)
2955             except RuntimeError:
2956                 pass
2957 
2958             process.d = service3
2959             process.myTask5 = testTask5
2960 
2961             # test placement into the Process and the tasks property
2962             expectedDict = { 'myTask1' : testTask1, 'myTask5' : testTask5 }
2963             expectedFixedDict = DictTypes.FixedKeysDict(expectedDict);
2964             self.assertTrue(process.tasks == expectedFixedDict)
2965             self.assertTrue(process.tasks['myTask1'] == testTask1)
2966             self.assertTrue(process.myTask1 == testTask1)
2967 
2968             # test replacing an EDProducer in a Task when calling __settattr__
2969             # for the EDProducer on the Process.
2970             process.mproducer2 = edproducer4
2971             process.d = service
2972             l = list()
2973             visitor1 = ModuleNodeVisitor(l)
2974             testTask1.visit(visitor1)
2975             l.sort(key=lambda mod: mod.__str__())
2976             expectedList = sorted([edproducer,essource,esproducer,service,edfilter,edproducer,edproducer4],key=lambda mod: mod.__str__())
2977             self.assertTrue(expectedList == l)
2978             process.myTask6 = Task()
2979             process.myTask7 = Task()
2980             process.mproducer8 = edproducer8
2981             process.myTask8 = Task(process.mproducer8)
2982             process.myTask6.add(process.myTask7)
2983             process.myTask7.add(process.myTask8)
2984             process.myTask1.add(process.myTask6)
2985             process.myTask8.add(process.myTask5)
2986 
2987             testDict = process._itemsInDependencyOrder(process.tasks)
2988             expectedLabels = ["myTask5", "myTask8", "myTask7", "myTask6", "myTask1"]
2989             expectedTasks = [process.myTask5, process.myTask8, process.myTask7, process.myTask6, process.myTask1]
2990             index = 0
2991             for testLabel, testTask in testDict.items():
2992                 self.assertTrue(testLabel == expectedLabels[index])
2993                 self.assertTrue(testTask == expectedTasks[index])
2994                 index += 1
2995 
2996             pythonDump = testTask1.dumpPython(PrintOptions())
2997 
2998 
2999             expectedPythonDump = 'cms.Task(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer, process.mproducer2, process.myTask6)\n'
3000             self.assertTrue(pythonDump == expectedPythonDump)
3001 
3002             process.myTask5 = Task()
3003             process.myTask100 = Task()
3004             process.mproducer9 = edproducer9
3005             sequence1 = Sequence(process.mproducer8, process.myTask1, process.myTask5, testTask2, testTask3)
3006             sequence2 = Sequence(process.mproducer8 + process.mproducer9)
3007             process.sequence3 = Sequence((process.mproducer8 + process.mfilter))
3008             sequence4 = Sequence()
3009             process.path1 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+sequence4)
3010             process.path1.associate(process.myTask1, process.myTask5, testTask2, testTask3)
3011             process.path11 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+ sequence4,process.myTask1, process.myTask5, testTask2, testTask3, process.myTask100)
3012             process.path2 = Path(process.mproducer)
3013             process.path3 = Path(process.mproducer9+process.mproducer8,testTask2)
3014 
3015             self.assertTrue(process.path1.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)\n')
3016 
3017             self.assertTrue(process.path11.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask100, process.myTask5)\n')
3018 
3019             # test NodeNameVisitor and moduleNames
3020             l = set()
3021             nameVisitor = NodeNameVisitor(l)
3022             process.path1.visit(nameVisitor)
3023             self.assertTrue(l == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3024             self.assertTrue(process.path1.moduleNames() == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3025 
3026             # test copy
3027             process.mproducer10 = EDProducer("b10")
3028             process.path21 = process.path11.copy()
3029             process.path21.replace(process.mproducer, process.mproducer10)
3030 
3031             self.assertTrue(process.path11.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask100, process.myTask5)\n')
3032 
3033             # Some peculiarities of the way things work show up here. dumpPython sorts tasks and
3034             # removes duplication at the level of strings. The Task and Sequence objects themselves
3035             # remove duplicate tasks in their contents if the instances are the same (exact same python
3036             # object id which is not the same as the string representation being the same).
3037             # Also note that the mutating visitor replaces sequences and tasks that have
3038             # modified contents with their modified contents, it does not modify the sequence
3039             # or task itself.
3040             self.assertTrue(process.path21.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer10), cms.Task(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), process.myTask100, process.myTask5)\n')
3041 
3042             process.path22 = process.path21.copyAndExclude([process.d, process.mesproducer, process.mfilter])
3043             self.assertTrue(process.path22.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer10), cms.Task(process.messource, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), process.myTask100, process.myTask5)\n')
3044 
3045             process.path23 = process.path22.copyAndExclude([process.messource, process.mproducer10])
3046             self.assertTrue(process.path23.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.Task(), cms.Task(process.None), cms.Task(process.mproducer2, process.mproducer8, process.myTask5), process.myTask100, process.myTask5)\n')
3047 
3048             process.a = EDAnalyzer("MyAnalyzer")
3049             process.b = OutputModule("MyOutputModule")
3050             process.c = EDFilter("MyFilter")
3051             process.d = EDProducer("MyProducer")
3052             process.e = ESProducer("MyESProducer")
3053             process.f = ESSource("MyESSource")
3054             process.g = ESProducer("g")
3055             process.path24 = Path(process.a+process.b+process.c+process.d)
3056             process.path25 = process.path24.copyAndExclude([process.a,process.b,process.c])
3057             self.assertTrue(process.path25.dumpPython() == 'cms.Path(process.d)\n')
3058             #print process.path3
3059             #print process.dumpPython()
3060 
3061             process.path200 = EndPath(Sequence(process.c,Task(process.e)))
3062             process.path200.replace(process.c,process.b)
3063             process.path200.replace(process.e,process.f)
3064             self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.b, cms.Task(process.f))\n")
3065             process.path200.replace(process.b,process.c)
3066             process.path200.replace(process.f,process.e)
3067             self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.c, cms.Task(process.e))\n")
3068             process.path200.replace(process.c,process.a)
3069             process.path200.replace(process.e,process.g)
3070             self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.a, cms.Task(process.g))\n")
3071             process.path200.replace(process.a,process.c)
3072             process.path200.replace(process.g,process.e)
3073             self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.c, cms.Task(process.e))\n")
3074 
3075         def testConditionalTask(self):
3076 
3077             # create some objects to use in tests
3078             edanalyzer = EDAnalyzer("a")
3079             edproducer = EDProducer("b")
3080             edproducer2 = EDProducer("b2")
3081             edproducer3 = EDProducer("b3")
3082             edproducer4 = EDProducer("b4")
3083             edproducer8 = EDProducer("b8")
3084             edproducer9 = EDProducer("b9")
3085             edfilter = EDFilter("c")
3086             service = Service("d")
3087             service3 = Service("d", v = untracked.uint32(3))
3088             essource = ESSource("e")
3089             esproducer = ESProducer("f")
3090             testTask2 = Task()
3091             testCTask2 = ConditionalTask()
3092 
3093             # test adding things to Tasks
3094             testTask1 = ConditionalTask(edproducer, edfilter)
3095             self.assertRaises(RuntimeError, testTask1.add, edanalyzer)
3096             testTask1.add(essource, service)
3097             testTask1.add(essource, esproducer)
3098             testTask1.add(testTask2)
3099             testTask1.add(testCTask2)
3100             coll = testTask1._collection
3101             self.assertTrue(edproducer in coll)
3102             self.assertTrue(edfilter in coll)
3103             self.assertTrue(service in coll)
3104             self.assertTrue(essource in coll)
3105             self.assertTrue(esproducer in coll)
3106             self.assertTrue(testTask2 in coll)
3107             self.assertTrue(testCTask2 in coll)
3108             self.assertTrue(len(coll) == 7)
3109             self.assertTrue(len(testTask2._collection) == 0)
3110 
3111             taskContents = []
3112             for i in testTask1:
3113                 taskContents.append(i)
3114             self.assertEqual(taskContents, [edproducer, edfilter, essource, service, esproducer, testTask2, testCTask2])
3115 
3116             # test attaching Task to Process
3117             process = Process("test")
3118 
3119             process.mproducer = edproducer
3120             process.mproducer2 = edproducer2
3121             process.mfilter = edfilter
3122             process.messource = essource
3123             process.mesproducer = esproducer
3124             process.d = service
3125 
3126             testTask3 = ConditionalTask(edproducer, edproducer2)
3127             testTask1.add(testTask3)
3128             process.myTask1 = testTask1
3129 
3130             # test the validation that occurs when attaching a ConditionalTask to a Process
3131             # first a case that passes, then one the fails on an EDProducer
3132             # then one that fails on a service
3133             l = set()
3134             visitor = NodeNameVisitor(l)
3135             testTask1.visit(visitor)
3136             self.assertEqual(l, set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
3137             l2 = testTask1.moduleNames()
3138             self.assertEqual(l2, set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
3139 
3140             testTask4 = ConditionalTask(edproducer3)
3141             l.clear()
3142             self.assertRaises(RuntimeError, testTask4.visit, visitor)
3143             try:
3144                 process.myTask4 = testTask4
3145                 self.assertTrue(False)
3146             except RuntimeError:
3147                 pass
3148 
3149             testTask5 = ConditionalTask(service3)
3150             l.clear()
3151             self.assertRaises(RuntimeError, testTask5.visit, visitor)
3152             try:
3153                 process.myTask5 = testTask5
3154                 self.assertTrue(False)
3155             except RuntimeError:
3156                 pass
3157 
3158             process.d = service3
3159             process.myTask5 = testTask5
3160 
3161             # test placement into the Process and the tasks property
3162             expectedDict = { 'myTask1' : testTask1, 'myTask5' : testTask5 }
3163             expectedFixedDict = DictTypes.FixedKeysDict(expectedDict);
3164             self.assertEqual(process.conditionaltasks, expectedFixedDict)
3165             self.assertEqual(process.conditionaltasks['myTask1'], testTask1)
3166             self.assertEqual(process.myTask1, testTask1)
3167 
3168             # test replacing an EDProducer in a ConditionalTask when calling __settattr__
3169             # for the EDProducer on the Process.
3170             process.mproducer2 = edproducer4
3171             process.d = service
3172             l = list()
3173             visitor1 = ModuleNodeVisitor(l)
3174             testTask1.visit(visitor1)
3175             l.sort(key=lambda mod: mod.__str__())
3176             expectedList = sorted([edproducer,essource,esproducer,service,edfilter,edproducer,edproducer4],key=lambda mod: mod.__str__())
3177             self.assertEqual(expectedList, l)
3178             process.myTask6 = ConditionalTask()
3179             process.myTask7 = ConditionalTask()
3180             process.mproducer8 = edproducer8
3181             process.myTask8 = ConditionalTask(process.mproducer8)
3182             process.myTask6.add(process.myTask7)
3183             process.myTask7.add(process.myTask8)
3184             process.myTask1.add(process.myTask6)
3185             process.myTask8.add(process.myTask5)
3186             self.assertEqual(process.myTask8.dumpPython(), "cms.ConditionalTask(process.mproducer8, process.myTask5)\n")
3187 
3188             testDict = process._itemsInDependencyOrder(process.conditionaltasks)
3189             expectedLabels = ["myTask5", "myTask8", "myTask7", "myTask6", "myTask1"]
3190             expectedTasks = [process.myTask5, process.myTask8, process.myTask7, process.myTask6, process.myTask1]
3191             index = 0
3192             for testLabel, testTask in testDict.items():
3193                 self.assertEqual(testLabel, expectedLabels[index])
3194                 self.assertEqual(testTask, expectedTasks[index])
3195                 index += 1
3196 
3197             pythonDump = testTask1.dumpPython(PrintOptions())
3198 
3199 
3200             expectedPythonDump = 'cms.ConditionalTask(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer, process.mproducer2, process.myTask6)\n'
3201             self.assertEqual(pythonDump, expectedPythonDump)
3202 
3203             process.myTask5 = ConditionalTask()
3204             self.assertEqual(process.myTask8.dumpPython(), "cms.ConditionalTask(process.mproducer8, process.myTask5)\n")
3205             process.myTask100 = ConditionalTask()
3206             process.mproducer9 = edproducer9
3207             sequence1 = Sequence(process.mproducer8, process.myTask1, process.myTask5, testTask2, testTask3)
3208             sequence2 = Sequence(process.mproducer8 + process.mproducer9)
3209             process.sequence3 = Sequence((process.mproducer8 + process.mfilter))
3210             sequence4 = Sequence()
3211             process.path1 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+sequence4)
3212             process.path1.associate(process.myTask1, process.myTask5, testTask2, testTask3)
3213             process.path11 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+ sequence4,process.myTask1, process.myTask5, testTask2, testTask3, process.myTask100)
3214             process.path2 = Path(process.mproducer)
3215             process.path3 = Path(process.mproducer9+process.mproducer8,testTask2)
3216 
3217             self.assertEqual(process.path1.dumpPython(PrintOptions()), 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)\n')
3218 
3219             self.assertEqual(process.path11.dumpPython(PrintOptions()), 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask100, process.myTask5)\n')
3220 
3221             # test NodeNameVisitor and moduleNames
3222             l = set()
3223             nameVisitor = NodeNameVisitor(l)
3224             process.path1.visit(nameVisitor)
3225             self.assertTrue(l == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3226             self.assertTrue(process.path1.moduleNames() == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3227 
3228             # test copy
3229             process.mproducer10 = EDProducer("b10")
3230             process.path21 = process.path11.copy()
3231             process.path21.replace(process.mproducer, process.mproducer10)
3232 
3233             self.assertEqual(process.path11.dumpPython(PrintOptions()), 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask100, process.myTask5)\n')
3234 
3235             # Some peculiarities of the way things work show up here. dumpPython sorts tasks and
3236             # removes duplication at the level of strings. The Task and Sequence objects themselves
3237             # remove duplicate tasks in their contents if the instances are the same (exact same python
3238             # object id which is not the same as the string representation being the same).
3239             # Also note that the mutating visitor replaces sequences and tasks that have
3240             # modified contents with their modified contents, it does not modify the sequence
3241             # or task itself.
3242             self.assertEqual(process.path21.dumpPython(PrintOptions()), 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer10), cms.ConditionalTask(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), cms.Task(), process.myTask100, process.myTask5)\n')
3243 
3244             process.path22 = process.path21.copyAndExclude([process.d, process.mesproducer, process.mfilter])
3245             self.assertEqual(process.path22.dumpPython(PrintOptions()), 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.ConditionalTask(process.None, process.mproducer10), cms.ConditionalTask(process.messource, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), cms.Task(), process.myTask100, process.myTask5)\n')
3246 
3247             process.path23 = process.path22.copyAndExclude([process.messource, process.mproducer10])
3248             self.assertEqual(process.path23.dumpPython(PrintOptions()), 'cms.Path(process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.ConditionalTask(process.None), cms.ConditionalTask(process.mproducer2, process.mproducer8, process.myTask5), cms.Task(), process.myTask100, process.myTask5)\n')
3249 
3250             process = Process("Test")
3251 
3252             process.b = EDProducer("b")
3253             process.b2 = EDProducer("b2")
3254             process.b3 = EDProducer("b3")
3255             process.p = Path(process.b, ConditionalTask(process.b3, process.b2))
3256             p = TestMakePSet()
3257             process.fillProcessDesc(p)
3258             self.assertEqual(p.values["@all_modules"], (True, ['b', 'b2', 'b3']))
3259             self.assertEqual(p.values["@paths"], (True, ['p']))
3260             self.assertEqual(p.values["p"], (True, ['b','#','b2','b3','@']))
3261             
3262 
3263         def testPath(self):
3264             p = Process("test")
3265             p.a = EDAnalyzer("MyAnalyzer")
3266             p.b = EDAnalyzer("YourAnalyzer")
3267             p.c = EDAnalyzer("OurAnalyzer")
3268             path = Path(p.a)
3269             path *= p.b
3270             path += p.c
3271             self.assertEqual(str(path),'a+b+c')
3272             path = Path(p.a*p.b+p.c)
3273             self.assertEqual(str(path),'a+b+c')
3274 #            path = Path(p.a)*p.b+p.c #This leads to problems with sequences
3275 #            self.assertEqual(str(path),'((a*b)+c)')
3276             path = Path(p.a+ p.b*p.c)
3277             self.assertEqual(str(path),'a+b+c')
3278             path = Path(p.a*(p.b+p.c))
3279             self.assertEqual(str(path),'a+b+c')
3280             path = Path(p.a*(p.b+~p.c))
3281             pathx = Path(p.a*(p.b+ignore(p.c)))
3282             self.assertEqual(str(path),'a+b+~c')
3283             p.es = ESProducer("AnESProducer")
3284             self.assertRaises(TypeError,Path,p.es)
3285 
3286             t = Path()
3287             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path()\n')
3288 
3289             t = Path(p.a)
3290             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a)\n')
3291 
3292             t = Path(Task())
3293             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(cms.Task())\n')
3294 
3295             t = Path(p.a, Task())
3296             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.Task())\n')
3297 
3298             p.prod = EDProducer("prodName")
3299             p.t1 = Task(p.prod)
3300             t = Path(p.a, p.t1, Task(), p.t1)
3301             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.Task(), process.t1)\n')
3302 
3303             t = Path(ConditionalTask())
3304             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(cms.ConditionalTask())\n')
3305 
3306             t = Path(p.a, ConditionalTask())
3307             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.ConditionalTask())\n')
3308 
3309             p.prod = EDProducer("prodName")
3310             p.t1 = ConditionalTask(p.prod)
3311             t = Path(p.a, p.t1, Task(), p.t1)
3312             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.Task(), process.t1)\n')
3313 
3314         def testFinalPath(self):
3315             p = Process("test")
3316             p.a = OutputModule("MyOutputModule")
3317             p.b = OutputModule("YourOutputModule")
3318             p.c = OutputModule("OurOutputModule")
3319             path = FinalPath(p.a)
3320             path *= p.b
3321             path += p.c
3322             self.assertEqual(str(path),'a+b+c')
3323             path = FinalPath(p.a*p.b+p.c)
3324             self.assertEqual(str(path),'a+b+c')
3325             path = FinalPath(p.a+ p.b*p.c)
3326             self.assertEqual(str(path),'a+b+c')
3327             path = FinalPath(p.a*(p.b+p.c))
3328             self.assertEqual(str(path),'a+b+c')
3329             p.es = ESProducer("AnESProducer")
3330             self.assertRaises(TypeError,FinalPath, p.es)
3331 
3332             t = FinalPath()
3333             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.FinalPath()\n')
3334 
3335             t = FinalPath(p.a)
3336             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.FinalPath(process.a)\n')
3337 
3338             self.assertRaises(TypeError, FinalPath, Task())
3339             self.assertRaises(TypeError, FinalPath, p.a, Task())
3340 
3341             p.prod = EDProducer("prodName")
3342             p.t1 = Task(p.prod)
3343             self.assertRaises(TypeError, FinalPath, p.a, p.t1, Task(), p.t1)
3344 
3345             p.prod = EDProducer("prodName")
3346             p.t1 = ConditionalTask(p.prod)
3347             self.assertRaises(TypeError, FinalPath, p.a, p.t1, ConditionalTask(), p.t1)
3348 
3349             p.t = FinalPath(p.a)
3350             p.a = OutputModule("ReplacedOutputModule")
3351             self.assertEqual(p.t.dumpPython(PrintOptions()), 'cms.FinalPath(process.a)\n')
3352 
3353             p.anal = EDAnalyzer("MyAnalyzer")
3354             p.t = FinalPath(p.anal)
3355             pset = TestMakePSet()
3356             self.assertRaises(RuntimeError, p.fillProcessDesc, pset)
3357 
3358             p.prod = EDProducer("MyProducer")
3359             p.t = FinalPath(p.prod)
3360             pset = TestMakePSet()
3361             self.assertRaises(RuntimeError, p.fillProcessDesc, pset)
3362 
3363             p.filt = EDFilter("MyFilter")
3364             p.t = FinalPath(p.filt)
3365             pset = TestMakePSet()
3366             self.assertRaises(RuntimeError, p.fillProcessDesc, pset)
3367 
3368             p.outp = OutputModule("MyOutputModule")
3369             p.t = FinalPath(p.outp)
3370             pset = TestMakePSet()
3371             p.fillProcessDesc(pset)
3372 
3373         def testCloneSequence(self):
3374             p = Process("test")
3375             a = EDAnalyzer("MyAnalyzer")
3376             p.a = a
3377             a.setLabel("a")
3378             b = EDAnalyzer("YOurAnalyzer")
3379             p.b = b
3380             b.setLabel("b")
3381             path = Path(a * b)
3382             p.path = Path(p.a*p.b)
3383             lookuptable = {id(a): p.a, id(b): p.b}
3384             #self.assertEqual(str(path),str(path._postProcessFixup(lookuptable)))
3385             #lookuptable = p._cloneToObjectDict
3386             #self.assertEqual(str(path),str(path._postProcessFixup(lookuptable)))
3387             self.assertEqual(str(path),str(p.path))
3388 
3389         def testContains(self):
3390 
3391             a = EDProducer("a")
3392             b = EDProducer("b")
3393             c = EDProducer("c")
3394             d = EDProducer("d")
3395             e = EDProducer("e")
3396             f = EDProducer("f")
3397             g = EDProducer("g")
3398             h = EDProducer("h")
3399             i = EDProducer("i")
3400             j = EDProducer("j")
3401             k = EDProducer("k")
3402             l = EDProducer("l")
3403             m = EDProducer("m")
3404             n = EDProducer("n")
3405 
3406             seq1 = Sequence(e)
3407             task1 = Task(g)
3408             ctask1 = ConditionalTask(h)
3409             path = Path(a * c * seq1, task1, ctask1)
3410 
3411             self.assertTrue(path.contains(a))
3412             self.assertFalse(path.contains(b))
3413             self.assertTrue(path.contains(c))
3414             self.assertFalse(path.contains(d))
3415             self.assertTrue(path.contains(e))
3416             self.assertFalse(path.contains(f))
3417             self.assertTrue(path.contains(g))
3418             self.assertTrue(path.contains(h))
3419 
3420             endpath = EndPath(h * i)
3421             self.assertFalse(endpath.contains(b))
3422             self.assertTrue(endpath.contains(i))
3423 
3424             seq = Sequence(a * c)
3425             self.assertFalse(seq.contains(b))
3426             self.assertTrue(seq.contains(c))
3427 
3428             task2 = Task(l)
3429             task = Task(j, k, task2)
3430             self.assertFalse(task.contains(b))
3431             self.assertTrue(task.contains(j))
3432             self.assertTrue(task.contains(k))
3433             self.assertTrue(task.contains(l))
3434 
3435             task3 = Task(m)
3436             path2 = Path(n)
3437             sch = Schedule(path, path2, tasks=[task,task3])
3438             self.assertFalse(sch.contains(b))
3439             self.assertTrue(sch.contains(a))
3440             self.assertTrue(sch.contains(c))
3441             self.assertTrue(sch.contains(e))
3442             self.assertTrue(sch.contains(g))
3443             self.assertTrue(sch.contains(n))
3444             self.assertTrue(sch.contains(j))
3445             self.assertTrue(sch.contains(k))
3446             self.assertTrue(sch.contains(l))
3447             self.assertTrue(sch.contains(m))
3448 
3449             ctask2 = ConditionalTask(l, task1)
3450             ctask = ConditionalTask(j, k, ctask2)
3451             self.assertFalse(ctask.contains(b))
3452             self.assertTrue(ctask.contains(j))
3453             self.assertTrue(ctask.contains(k))
3454             self.assertTrue(ctask.contains(l))
3455             self.assertTrue(ctask.contains(g))
3456 
3457         def testSchedule(self):
3458             p = Process("test")
3459             p.a = EDAnalyzer("MyAnalyzer")
3460             p.b = EDAnalyzer("YourAnalyzer")
3461             p.c = EDAnalyzer("OurAnalyzer")
3462             p.d = EDAnalyzer("OurAnalyzer")
3463             p.path1 = Path(p.a)
3464             p.path2 = Path(p.b)
3465             p.path3 = Path(p.d)
3466 
3467             s = Schedule(p.path1,p.path2)
3468             self.assertEqual(s[0],p.path1)
3469             self.assertEqual(s[1],p.path2)
3470             p.schedule = s
3471             self.assertTrue('b' in p.schedule.moduleNames())
3472             self.assertTrue(hasattr(p, 'b'))
3473             self.assertTrue(hasattr(p, 'c'))
3474             self.assertTrue(hasattr(p, 'd'))
3475             self.assertTrue(hasattr(p, 'path1'))
3476             self.assertTrue(hasattr(p, 'path2'))
3477             self.assertTrue(hasattr(p, 'path3'))
3478             p.prune()
3479             self.assertTrue('b' in p.schedule.moduleNames())
3480             self.assertTrue(hasattr(p, 'b'))
3481             self.assertTrue(not hasattr(p, 'c'))
3482             self.assertTrue(not hasattr(p, 'd'))
3483             self.assertTrue(hasattr(p, 'path1'))
3484             self.assertTrue(hasattr(p, 'path2'))
3485             self.assertTrue(not hasattr(p, 'path3'))
3486 
3487             self.assertTrue(len(p.schedule._tasks) == 0)
3488 
3489             p = Process("test")
3490             p.a = EDAnalyzer("MyAnalyzer")
3491             p.b = EDAnalyzer("YourAnalyzer")
3492             p.c = EDAnalyzer("OurAnalyzer")
3493             p.d = EDAnalyzer("dAnalyzer")
3494             p.e = EDProducer("eProducer")
3495             p.f = EDProducer("fProducer")
3496             p.Tracer = Service("Tracer")
3497             p.path1 = Path(p.a)
3498             p.path2 = Path(p.b)
3499             p.path3 = Path(p.d)
3500             p.task1 = Task(p.e)
3501             p.task2 = Task(p.f, p.Tracer)
3502             s = Schedule(p.path1,p.path2,tasks=[p.task1,p.task2,p.task1])
3503             self.assertEqual(s[0],p.path1)
3504             self.assertEqual(s[1],p.path2)
3505             self.assertTrue(len(s._tasks) == 2)
3506             self.assertTrue(p.task1 in s._tasks)
3507             self.assertTrue(p.task2 in s._tasks)
3508             listOfTasks = list(s._tasks)
3509             self.assertTrue(len(listOfTasks) == 2)
3510             self.assertTrue(p.task1 == listOfTasks[0])
3511             self.assertTrue(p.task2 == listOfTasks[1])
3512             p.schedule = s
3513             self.assertTrue('b' in p.schedule.moduleNames())
3514 
3515             process2 = Process("test")
3516             process2.a = EDAnalyzer("MyAnalyzer")
3517             process2.e = EDProducer("eProducer")
3518             process2.path1 = Path(process2.a)
3519             process2.task1 = Task(process2.e)
3520             process2.schedule = Schedule(process2.path1,tasks=process2.task1)
3521             listOfTasks = list(process2.schedule._tasks)
3522             self.assertTrue(listOfTasks[0] == process2.task1)
3523 
3524             # test Schedule copy
3525             s2 = s.copy()
3526             self.assertEqual(s2[0],p.path1)
3527             self.assertEqual(s2[1],p.path2)
3528             self.assertTrue(len(s2._tasks) == 2)
3529             self.assertTrue(p.task1 in s2._tasks)
3530             self.assertTrue(p.task2 in s2._tasks)
3531             listOfTasks = list(s2._tasks)
3532             self.assertTrue(len(listOfTasks) == 2)
3533             self.assertTrue(p.task1 == listOfTasks[0])
3534             self.assertTrue(p.task2 == listOfTasks[1])
3535 
3536             names = s.moduleNames()
3537             self.assertTrue(names == set(['a', 'b', 'e', 'Tracer', 'f']))
3538             #adding a path not attached to the Process should cause an exception
3539             p = Process("test")
3540             p.a = EDAnalyzer("MyAnalyzer")
3541             path1 = Path(p.a)
3542             s = Schedule(path1)
3543             self.assertRaises(RuntimeError, lambda : p.setSchedule_(s) )
3544 
3545             #make sure anonymous sequences work
3546             p = Process("test")
3547             p.a = EDAnalyzer("MyAnalyzer")
3548             p.b = EDAnalyzer("MyOtherAnalyzer")
3549             p.c = EDProducer("MyProd")
3550             path1 = Path(p.c*Sequence(p.a+p.b))
3551             s = Schedule(path1)
3552             self.assertTrue('a' in s.moduleNames())
3553             self.assertTrue('b' in s.moduleNames())
3554             self.assertTrue('c' in s.moduleNames())
3555             p.path1 = path1
3556             p.schedule = s
3557             p.prune()
3558             self.assertTrue('a' in s.moduleNames())
3559             self.assertTrue('b' in s.moduleNames())
3560             self.assertTrue('c' in s.moduleNames())
3561 
3562         def testImplicitSchedule(self):
3563             p = Process("test")
3564             p.a = EDAnalyzer("MyAnalyzer")
3565             p.b = EDAnalyzer("YourAnalyzer")
3566             p.c = EDAnalyzer("OurAnalyzer")
3567             p.path1 = Path(p.a)
3568             p.path2 = Path(p.b)
3569             self.assertTrue(p.schedule is None)
3570             pths = p.paths
3571             keys = pths.keys()
3572             self.assertEqual(pths[keys[0]],p.path1)
3573             self.assertEqual(pths[keys[1]],p.path2)
3574             p.prune()
3575             self.assertTrue(hasattr(p, 'a'))
3576             self.assertTrue(hasattr(p, 'b'))
3577             self.assertTrue(not hasattr(p, 'c'))
3578             self.assertTrue(hasattr(p, 'path1'))
3579             self.assertTrue(hasattr(p, 'path2'))
3580 
3581 
3582             p = Process("test")
3583             p.a = EDAnalyzer("MyAnalyzer")
3584             p.b = EDAnalyzer("YourAnalyzer")
3585             p.c = EDAnalyzer("OurAnalyzer")
3586             p.path2 = Path(p.b)
3587             p.path1 = Path(p.a)
3588             self.assertTrue(p.schedule is None)
3589             pths = p.paths
3590             keys = pths.keys()
3591             self.assertEqual(pths[keys[1]],p.path1)
3592             self.assertEqual(pths[keys[0]],p.path2)
3593 
3594 
3595         def testUsing(self):
3596             p = Process('test')
3597             p.block = PSet(a = int32(1))
3598             p.modu = EDAnalyzer('Analyzer', p.block, b = int32(2))
3599             self.assertEqual(p.modu.a.value(),1)
3600             self.assertEqual(p.modu.b.value(),2)
3601 
3602         def testOverride(self):
3603             p = Process('test')
3604             a = EDProducer("A", a1=int32(0))
3605             self.assertTrue(not a.isModified())
3606             a.a1 = 1
3607             self.assertTrue(a.isModified())
3608             p.a = a
3609             self.assertEqual(p.a.a1.value(), 1)
3610             # try adding an unmodified module.
3611             # should accept it
3612             p.a = EDProducer("A", a1=int32(2))
3613             self.assertEqual(p.a.a1.value(), 2)
3614             # try adding a modified module.  Should throw
3615             # no longer, since the same (modified) say, geometry
3616             # could come from more than one cff
3617             b = EDProducer("A", a1=int32(3))
3618             b.a1 = 4
3619             #self.assertRaises(RuntimeError, setattr, *(p,'a',b))
3620             ps1 = PSet(a = int32(1))
3621             ps2 = PSet(a = int32(2))
3622             self.assertRaises(ValueError, EDProducer, 'C', ps1, ps2)
3623             self.assertRaises(ValueError, EDProducer, 'C', ps1, a=int32(3))
3624         
3625         def testOptions(self):
3626             p = Process('test')
3627             self.assertEqual(p.options.numberOfThreads.value(),1)
3628             p.options.numberOfThreads = 8
3629             self.assertEqual(p.options.numberOfThreads.value(),8)
3630             p.options = PSet()
3631             self.assertEqual(p.options.numberOfThreads.value(),1)
3632             p.options = dict(numberOfStreams =2,
3633                              numberOfThreads =2)
3634             self.assertEqual(p.options.numberOfThreads.value(),2)
3635             self.assertEqual(p.options.numberOfStreams.value(),2)
3636             del p.options
3637             self.assertRaises(TypeError, setattr, p, 'options', untracked.PSet(numberOfThreads = int32(-1)))
3638             p.options = untracked.PSet(numberOfThreads = untracked.uint32(4))
3639             self.assertEqual(p.options.numberOfThreads.value(), 4)
3640 
3641         def testMaxEvents(self):
3642             p = Process("Test")
3643             p.maxEvents.input = 10
3644             self.assertEqual(p.maxEvents.input.value(),10)
3645             p = Process("Test")
3646             p.maxEvents.output = 10
3647             self.assertEqual(p.maxEvents.output.value(),10)
3648             p = Process("Test")
3649             p.maxEvents.output = PSet(out=untracked.int32(10))
3650             self.assertEqual(p.maxEvents.output.out.value(), 10)
3651             p = Process("Test")
3652             p.maxEvents = untracked.PSet(input = untracked.int32(5))
3653             self.assertEqual(p.maxEvents.input.value(), 5)
3654             del p.maxEvents
3655             self.assertRaises(TypeError, setattr, p, 'maxEvents', untracked.PSet(input = untracked.uint32(1)))
3656             p.maxEvents = untracked.PSet(input = untracked.int32(1))
3657             self.assertEqual(p.maxEvents.input.value(), 1)
3658         
3659         def testExamples(self):
3660             p = Process("Test")
3661             p.source = Source("PoolSource",fileNames = untracked(string("file:reco.root")))
3662             p.foos = EDProducer("FooProducer")
3663             p.bars = EDProducer("BarProducer", foos=InputTag("foos"))
3664             p.out = OutputModule("PoolOutputModule",fileName=untracked(string("file:foos.root")))
3665             p.bars.foos = 'Foosball'
3666             self.assertEqual(p.bars.foos, InputTag('Foosball'))
3667             p.p = Path(p.foos*p.bars)
3668             p.e = EndPath(p.out)
3669             p.add_(Service("MessageLogger"))
3670 
3671         def testPrefers(self):
3672             p = Process("Test")
3673             p.add_(ESSource("ForceSource"))
3674             p.juicer = ESProducer("JuicerProducer")
3675             p.prefer("ForceSource")
3676             p.prefer("juicer")
3677             self.assertEqual(_lineDiff(p.dumpPython(), Process('Test').dumpPython()),
3678 """process.juicer = cms.ESProducer("JuicerProducer")
3679 process.ForceSource = cms.ESSource("ForceSource")
3680 process.prefer("ForceSource")
3681 process.prefer("juicer")""")
3682             p.prefer("juicer",fooRcd=vstring("Foo"))
3683             self.assertEqual(_lineDiff(p.dumpPython(), Process('Test').dumpPython()),
3684 """process.juicer = cms.ESProducer("JuicerProducer")
3685 process.ForceSource = cms.ESSource("ForceSource")
3686 process.prefer("ForceSource")
3687 process.prefer("juicer",
3688     fooRcd = cms.vstring('Foo')
3689 )""")
3690 
3691         def testFreeze(self):
3692             process = Process("Freeze")
3693             m = EDProducer("M", p=PSet(i = int32(1)))
3694             m.p.i = 2
3695             process.m = m
3696             # should be frozen
3697             #self.assertRaises(ValueError, setattr, m.p, 'i', 3)
3698             #self.assertRaises(ValueError, setattr, m, 'p', PSet(i=int32(1)))
3699             #self.assertRaises(ValueError, setattr, m.p, 'j', 1)
3700             #self.assertRaises(ValueError, setattr, m, 'j', 1)
3701             # But OK to change through the process
3702             process.m.p.i = 4
3703             self.assertEqual(process.m.p.i.value(), 4)
3704             process.m.p = PSet(j=int32(1))
3705             # should work to clone it, though
3706             m2 = m.clone(p = PSet(i = int32(5)), j = int32(8))
3707             m2.p.i = 6
3708             m2.j = 8
3709         def testSubProcess(self):
3710             process = Process("Parent")
3711             subProcess = Process("Child")
3712             subProcess.a = EDProducer("A")
3713             subProcess.p = Path(subProcess.a)
3714             subProcess.add_(Service("Foo"))
3715             process.addSubProcess(SubProcess(subProcess))
3716             d = process.dumpPython()
3717             equalD ="""parentProcess = process
3718 process.a = cms.EDProducer("A")
3719 process.Foo = cms.Service("Foo")
3720 process.p = cms.Path(process.a)
3721 childProcess = process
3722 process = parentProcess
3723 process.addSubProcess(cms.SubProcess(process = childProcess, SelectEvents = cms.untracked.PSet(
3724 ), outputCommands = cms.untracked.vstring()))"""
3725             equalD = equalD.replace("parentProcess","parentProcess"+str(hash(process.subProcesses_()[0])))
3726             # SubProcesses are dumped before Services, so in order to
3727             # craft the dump of the Parent and Child manually the dump
3728             # of the Parent needs to be split at the MessageLogger
3729             # boundary (now when it is part of Process by default),
3730             # and insert the dump of the Child between the top part of
3731             # the Parent (before MessageLogger) and the bottom part of
3732             # the Parent (after and including MessageLogger)
3733             messageLoggerSplit = 'process.MessageLogger = cms.Service'
3734             parentDumpSplit = Process('Parent').dumpPython().split(messageLoggerSplit)
3735             childProcess = Process('Child')
3736             del childProcess.MessageLogger
3737             combinedDump = parentDumpSplit[0] + childProcess.dumpPython() + messageLoggerSplit + parentDumpSplit[1]
3738             self.assertEqual(_lineDiff(d, combinedDump), equalD)
3739             p = TestMakePSet()
3740             process.fillProcessDesc(p)
3741             self.assertEqual((True,['a']),p.values["subProcesses"][1][0].values["process"][1].values['@all_modules'])
3742             self.assertEqual((True,['p']),p.values["subProcesses"][1][0].values["process"][1].values['@paths'])
3743             self.assertEqual({'@service_type':(True,'Foo')}, p.values["subProcesses"][1][0].values["process"][1].values["services"][1][0].values)
3744         def testRefToPSet(self):
3745             proc = Process("test")
3746             proc.top = PSet(a = int32(1))
3747             proc.ref = PSet(refToPSet_ = string("top"))
3748             proc.ref2 = PSet( a = int32(1), b = PSet( refToPSet_ = string("top")))
3749             proc.ref3 = PSet(refToPSet_ = string("ref"))
3750             proc.ref4 = VPSet(PSet(refToPSet_ = string("top")),
3751                               PSet(refToPSet_ = string("ref2")))
3752             p = TestMakePSet()
3753             proc.fillProcessDesc(p)
3754             self.assertEqual((True,1),p.values["ref"][1].values["a"])
3755             self.assertEqual((True,1),p.values["ref3"][1].values["a"])
3756             self.assertEqual((True,1),p.values["ref2"][1].values["a"])
3757             self.assertEqual((True,1),p.values["ref2"][1].values["b"][1].values["a"])
3758             self.assertEqual((True,1),p.values["ref4"][1][0].values["a"])
3759             self.assertEqual((True,1),p.values["ref4"][1][1].values["a"])
3760         def testSwitchProducer(self):
3761             proc = Process("test")
3762             proc.sp = SwitchProducerTest(test2 = EDProducer("Foo",
3763                                                             a = int32(1),
3764                                                             b = PSet(c = int32(2))),
3765                                          test1 = EDProducer("Bar",
3766                                                             aa = int32(11),
3767                                                             bb = PSet(cc = int32(12))))
3768             self.assertEqual(proc.sp.label_(), "sp")
3769             self.assertEqual(proc.sp.test1.label_(), "sp@test1")
3770             self.assertEqual(proc.sp.test2.label_(), "sp@test2")
3771 
3772             proc.a = EDProducer("A")
3773             proc.s = Sequence(proc.a + proc.sp)
3774             proc.t = Task(proc.a, proc.sp)
3775             proc.p = Path()
3776             proc.p.associate(proc.t)
3777             p = TestMakePSet()
3778             proc.fillProcessDesc(p)
3779             self.assertEqual((True,"EDProducer"), p.values["sp"][1].values["@module_edm_type"])
3780             self.assertEqual((True, "SwitchProducer"), p.values["sp"][1].values["@module_type"])
3781             self.assertEqual((True, "sp"), p.values["sp"][1].values["@module_label"])
3782             all_cases = copy.deepcopy(p.values["sp"][1].values["@all_cases"])
3783             all_cases[1].sort() # names of all cases come via dict, i.e. their order is undefined
3784             self.assertEqual((True, ["sp@test1", "sp@test2"]), all_cases)
3785             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
3786             self.assertEqual(["a", "sp", "sp@test1", "sp@test2"], p.values["@all_modules"][1])
3787             self.assertEqual((True,"EDProducer"), p.values["sp@test1"][1].values["@module_edm_type"])
3788             self.assertEqual((True,"Bar"), p.values["sp@test1"][1].values["@module_type"])
3789             self.assertEqual((True,"EDProducer"), p.values["sp@test2"][1].values["@module_edm_type"])
3790             self.assertEqual((True,"Foo"), p.values["sp@test2"][1].values["@module_type"])
3791             dump = proc.dumpPython()
3792             self.assertEqual(dump.find('@'), -1)
3793             self.assertEqual(specialImportRegistry.getSpecialImports(), ["from test import SwitchProducerTest"])
3794             self.assertTrue(dump.find("\nfrom test import SwitchProducerTest\n") != -1)
3795 
3796             # EDAlias as non-chosen case
3797             proc = Process("test")
3798             proc.sp = SwitchProducerTest(test2 = EDProducer("Foo",
3799                                                             a = int32(1),
3800                                                             b = PSet(c = int32(2))),
3801                                          test1 = EDAlias(a = VPSet(PSet(type = string("Bar")))))
3802             proc.a = EDProducer("A")
3803             proc.s = Sequence(proc.a + proc.sp)
3804             proc.t = Task(proc.a, proc.sp)
3805             proc.p = Path()
3806             proc.p.associate(proc.t)
3807             p = TestMakePSet()
3808             proc.fillProcessDesc(p)
3809             self.assertEqual((True,"EDProducer"), p.values["sp"][1].values["@module_edm_type"])
3810             self.assertEqual((True, "SwitchProducer"), p.values["sp"][1].values["@module_type"])
3811             self.assertEqual((True, "sp"), p.values["sp"][1].values["@module_label"])
3812             all_cases = copy.deepcopy(p.values["sp"][1].values["@all_cases"])
3813             all_cases[1].sort()
3814             self.assertEqual((True, ["sp@test1", "sp@test2"]), all_cases)
3815             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
3816             self.assertEqual(["a", "sp", "sp@test2"], p.values["@all_modules"][1])
3817             self.assertEqual(["sp@test1"], p.values["@all_aliases"][1])
3818             self.assertEqual((True,"EDProducer"), p.values["sp@test2"][1].values["@module_edm_type"])
3819             self.assertEqual((True,"Foo"), p.values["sp@test2"][1].values["@module_type"])
3820             self.assertEqual((True,"EDAlias"), p.values["sp@test1"][1].values["@module_edm_type"])
3821             self.assertEqual((True,"Bar"), p.values["sp@test1"][1].values["a"][1][0].values["type"])
3822 
3823             # EDAlias as chosen case
3824             proc = Process("test")
3825             proc.sp = SwitchProducerTest(test1 = EDProducer("Foo",
3826                                                             a = int32(1),
3827                                                             b = PSet(c = int32(2))),
3828                                          test2 = EDAlias(a = VPSet(PSet(type = string("Bar")))))
3829             proc.a = EDProducer("A")
3830             proc.s = Sequence(proc.a + proc.sp)
3831             proc.t = Task(proc.a, proc.sp)
3832             proc.p = Path()
3833             proc.p.associate(proc.t)
3834             p = TestMakePSet()
3835             proc.fillProcessDesc(p)
3836             self.assertEqual((True,"EDProducer"), p.values["sp"][1].values["@module_edm_type"])
3837             self.assertEqual((True, "SwitchProducer"), p.values["sp"][1].values["@module_type"])
3838             self.assertEqual((True, "sp"), p.values["sp"][1].values["@module_label"])
3839             self.assertEqual((True, ["sp@test1", "sp@test2"]), p.values["sp"][1].values["@all_cases"])
3840             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
3841             self.assertEqual(["a", "sp", "sp@test1"], p.values["@all_modules"][1])
3842             self.assertEqual(["sp@test2"], p.values["@all_aliases"][1])
3843             self.assertEqual((True,"EDProducer"), p.values["sp@test1"][1].values["@module_edm_type"])
3844             self.assertEqual((True,"Foo"), p.values["sp@test1"][1].values["@module_type"])
3845             self.assertEqual((True,"EDAlias"), p.values["sp@test2"][1].values["@module_edm_type"])
3846             self.assertEqual((True,"Bar"), p.values["sp@test2"][1].values["a"][1][0].values["type"])
3847 
3848             # ConditionalTask
3849             proc = Process("test")
3850             proc.spct = SwitchProducerTest(test2 = EDProducer("Foo",
3851                                                               a = int32(1),
3852                                                               b = PSet(c = int32(2))),
3853                                            test1 = EDProducer("Bar",
3854                                                               aa = int32(11),
3855                                                               bb = PSet(cc = int32(12))),
3856                                            test3 = EDAlias(a = VPSet(PSet(type = string("Bar")))))
3857             proc.spp = proc.spct.clone()
3858             proc.a = EDProducer("A")
3859             proc.ct = ConditionalTask(proc.spct)
3860             proc.p = Path(proc.a, proc.ct)
3861             proc.pp = Path(proc.a + proc.spp)
3862             p = TestMakePSet()
3863             proc.fillProcessDesc(p)
3864             self.assertEqual(["a", "spct", "spct@test1", "spct@test2", "spp", "spp@test1", "spp@test2"], p.values["@all_modules"][1])
3865             self.assertEqual(["a", "#", "spct", "spct@test1", "spct@test2", "@"], p.values["p"][1])
3866             self.assertEqual(["a", "spp", "#", "spp@test1", "spp@test2", "@"], p.values["pp"][1])
3867 
3868         def testPrune(self):
3869             p = Process("test")
3870             p.a = EDAnalyzer("MyAnalyzer")
3871             p.b = EDAnalyzer("YourAnalyzer")
3872             p.c = EDAnalyzer("OurAnalyzer")
3873             p.d = EDAnalyzer("OurAnalyzer")
3874             p.e = EDProducer("MyProducer")
3875             p.f = EDProducer("YourProducer")
3876             p.g = EDProducer("TheirProducer")
3877             p.h = EDProducer("OnesProducer")
3878             p.s = Sequence(p.d)
3879             p.t1 = Task(p.e)
3880             p.t2 = Task(p.f)
3881             p.t3 = Task(p.g, p.t1)
3882             p.ct1 = ConditionalTask(p.h)
3883             p.ct2 = ConditionalTask(p.f)
3884             p.ct3 = ConditionalTask(p.ct1)
3885             p.path1 = Path(p.a, p.t3, p.ct3)
3886             p.path2 = Path(p.b)
3887             self.assertTrue(p.schedule is None)
3888             pths = p.paths
3889             keys = pths.keys()
3890             self.assertEqual(pths[keys[0]],p.path1)
3891             self.assertEqual(pths[keys[1]],p.path2)
3892             p.pset1 = PSet(parA = string("pset1"))
3893             p.pset2 = untracked.PSet(parA = string("pset2"))
3894             p.vpset1 = VPSet()
3895             p.vpset2 = untracked.VPSet()
3896             p.prune()
3897             self.assertTrue(hasattr(p, 'a'))
3898             self.assertTrue(hasattr(p, 'b'))
3899             self.assertTrue(not hasattr(p, 'c'))
3900             self.assertTrue(not hasattr(p, 'd'))
3901             self.assertTrue(hasattr(p, 'e'))
3902             self.assertTrue(not hasattr(p, 'f'))
3903             self.assertTrue(hasattr(p, 'g'))
3904             self.assertTrue(hasattr(p, 'h'))
3905             self.assertTrue(not hasattr(p, 's'))
3906             self.assertTrue(hasattr(p, 't1'))
3907             self.assertTrue(not hasattr(p, 't2'))
3908             self.assertTrue(hasattr(p, 't3'))
3909             self.assertTrue(hasattr(p, 'path1'))
3910             self.assertTrue(hasattr(p, 'path2'))
3911 #            self.assertTrue(not hasattr(p, 'pset1'))
3912 #            self.assertTrue(hasattr(p, 'pset2'))
3913 #            self.assertTrue(not hasattr(p, 'vpset1'))
3914 #            self.assertTrue(not hasattr(p, 'vpset2'))
3915 
3916             p = Process("test")
3917             p.a = EDAnalyzer("MyAnalyzer")
3918             p.b = EDAnalyzer("YourAnalyzer")
3919             p.c = EDAnalyzer("OurAnalyzer")
3920             p.d = EDAnalyzer("OurAnalyzer")
3921             p.e = EDAnalyzer("OurAnalyzer")
3922             p.f = EDProducer("MyProducer")
3923             p.g = EDProducer("YourProducer")
3924             p.h = EDProducer("TheirProducer")
3925             p.i = EDProducer("OurProducer")
3926             p.j = EDProducer("OurProducer")
3927             p.k = EDProducer("OurProducer")
3928             p.l = EDProducer("OurProducer")
3929             p.t1 = Task(p.f)
3930             p.t2 = Task(p.g)
3931             p.t3 = Task(p.h)
3932             p.t4 = Task(p.i)
3933             p.ct1 = Task(p.f)
3934             p.ct2 = Task(p.j)
3935             p.ct3 = Task(p.k)
3936             p.ct4 = Task(p.l)
3937             p.s = Sequence(p.d, p.t1, p.ct1)
3938             p.s2 = Sequence(p.b, p.t2, p.ct2)
3939             p.s3 = Sequence(p.e)
3940             p.path1 = Path(p.a, p.t3, p.ct3)
3941             p.path2 = Path(p.b)
3942             p.path3 = Path(p.b+p.s2)
3943             p.path4 = Path(p.b+p.s3)
3944             p.schedule = Schedule(p.path1,p.path2,p.path3)
3945             p.schedule.associate(p.t4)
3946             pths = p.paths
3947             keys = pths.keys()
3948             self.assertEqual(pths[keys[0]],p.path1)
3949             self.assertEqual(pths[keys[1]],p.path2)
3950             p.prune()
3951             self.assertTrue(hasattr(p, 'a'))
3952             self.assertTrue(hasattr(p, 'b'))
3953             self.assertTrue(not hasattr(p, 'c'))
3954             self.assertTrue(not hasattr(p, 'd'))
3955             self.assertTrue(not hasattr(p, 'e'))
3956             self.assertTrue(not hasattr(p, 'f'))
3957             self.assertTrue(hasattr(p, 'g'))
3958             self.assertTrue(hasattr(p, 'h'))
3959             self.assertTrue(hasattr(p, 'i'))
3960             self.assertTrue(hasattr(p, 'j'))
3961             self.assertTrue(hasattr(p, 'k'))
3962             self.assertTrue(not hasattr(p, 'l'))
3963             self.assertTrue(not hasattr(p, 't1'))
3964             self.assertTrue(hasattr(p, 't2'))
3965             self.assertTrue(hasattr(p, 't3'))
3966             self.assertTrue(hasattr(p, 't4'))
3967             self.assertTrue(not hasattr(p, 'ct1'))
3968             self.assertTrue(hasattr(p, 'ct2'))
3969             self.assertTrue(hasattr(p, 'ct3'))
3970             self.assertTrue(not hasattr(p, 'ct4'))
3971             self.assertTrue(not hasattr(p, 's'))
3972             self.assertTrue(hasattr(p, 's2'))
3973             self.assertTrue(not hasattr(p, 's3'))
3974             self.assertTrue(hasattr(p, 'path1'))
3975             self.assertTrue(hasattr(p, 'path2'))
3976             self.assertTrue(hasattr(p, 'path3'))
3977             self.assertTrue(not hasattr(p, 'path4'))
3978             #test SequencePlaceholder
3979             p = Process("test")
3980             p.a = EDAnalyzer("MyAnalyzer")
3981             p.b = EDAnalyzer("YourAnalyzer")
3982             p.s = Sequence(SequencePlaceholder("a")+p.b)
3983             p.pth = Path(p.s)
3984             p.prune()
3985             self.assertTrue(hasattr(p, 'a'))
3986             self.assertTrue(hasattr(p, 'b'))
3987             self.assertTrue(hasattr(p, 's'))
3988             self.assertTrue(hasattr(p, 'pth'))
3989             #test unresolved SequencePlaceholder
3990             p = Process("test")
3991             p.b = EDAnalyzer("YourAnalyzer")
3992             p.s = Sequence(SequencePlaceholder("a")+p.b)
3993             p.pth = Path(p.s)
3994             p.prune(keepUnresolvedSequencePlaceholders=True)
3995             self.assertTrue(hasattr(p, 'b'))
3996             self.assertTrue(hasattr(p, 's'))
3997             self.assertTrue(hasattr(p, 'pth'))
3998             self.assertEqual(p.s.dumpPython(),'cms.Sequence(cms.SequencePlaceholder("a")+process.b)\n')
3999             #test TaskPlaceholder
4000             p = Process("test")
4001             p.a = EDProducer("MyProducer")
4002             p.b = EDProducer("YourProducer")
4003             p.s = Task(TaskPlaceholder("a"),p.b)
4004             p.pth = Path(p.s)
4005             p.prune()
4006             self.assertTrue(hasattr(p, 'a'))
4007             self.assertTrue(hasattr(p, 'b'))
4008             self.assertTrue(hasattr(p, 's'))
4009             self.assertTrue(hasattr(p, 'pth'))
4010             #test ConditionalTaskPlaceholder
4011             p = Process("test")
4012             p.a = EDProducer("MyProducer")
4013             p.b = EDProducer("YourProducer")
4014             p.s = ConditionalTask(ConditionalTaskPlaceholder("a"),p.b)
4015             p.pth = Path(p.s)
4016             p.prune()
4017             self.assertTrue(hasattr(p, 'a'))
4018             self.assertTrue(hasattr(p, 'b'))
4019             self.assertTrue(hasattr(p, 's'))
4020             self.assertTrue(hasattr(p, 'pth'))
4021             #test unresolved SequencePlaceholder
4022             p = Process("test")
4023             p.b = EDProducer("YourAnalyzer")
4024             p.s = Task(TaskPlaceholder("a"),p.b)
4025             p.pth = Path(p.s)
4026             p.prune(keepUnresolvedSequencePlaceholders=True)
4027             self.assertTrue(hasattr(p, 'b'))
4028             self.assertTrue(hasattr(p, 's'))
4029             self.assertTrue(hasattr(p, 'pth'))
4030             self.assertEqual(p.s.dumpPython(),'cms.Task(cms.TaskPlaceholder("a"), process.b)\n')
4031         def testTaskPlaceholder(self):
4032             p = Process("test")
4033             p.a = EDProducer("ma")
4034             p.b = EDAnalyzer("mb")
4035             p.t1 = Task(TaskPlaceholder("c"))
4036             p.t2 = Task(p.a, TaskPlaceholder("d"), p.t1)
4037             p.t3 = Task(TaskPlaceholder("e"))
4038             p.path1 = Path(p.b, p.t2, p.t3)
4039             p.t5 = Task(p.a, TaskPlaceholder("g"), TaskPlaceholder("t4"))
4040             p.t4 = Task(TaskPlaceholder("f"))
4041             p.endpath1 = EndPath(p.b, p.t5)
4042             p.t6 = Task(TaskPlaceholder("h"))
4043             p.t7 = Task(p.a, TaskPlaceholder("i"), p.t6)
4044             p.t8 = Task(TaskPlaceholder("j"))
4045             p.schedule = Schedule(p.path1, p.endpath1,tasks=[p.t7,p.t8])
4046             p.c = EDProducer("mc")
4047             p.d = EDProducer("md")
4048             p.e = EDProducer("me")
4049             p.f = EDProducer("mf")
4050             p.g = EDProducer("mg")
4051             p.h = EDProducer("mh")
4052             p.i = EDProducer("mi")
4053             p.j = EDProducer("mj")
4054             self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4055 """process.a = cms.EDProducer("ma")
4056 process.c = cms.EDProducer("mc")
4057 process.d = cms.EDProducer("md")
4058 process.e = cms.EDProducer("me")
4059 process.f = cms.EDProducer("mf")
4060 process.g = cms.EDProducer("mg")
4061 process.h = cms.EDProducer("mh")
4062 process.i = cms.EDProducer("mi")
4063 process.j = cms.EDProducer("mj")
4064 process.b = cms.EDAnalyzer("mb")
4065 process.t1 = cms.Task(cms.TaskPlaceholder("c"))
4066 process.t2 = cms.Task(cms.TaskPlaceholder("d"), process.a, process.t1)
4067 process.t3 = cms.Task(cms.TaskPlaceholder("e"))
4068 process.t5 = cms.Task(cms.TaskPlaceholder("g"), cms.TaskPlaceholder("t4"), process.a)
4069 process.t4 = cms.Task(cms.TaskPlaceholder("f"))
4070 process.t6 = cms.Task(cms.TaskPlaceholder("h"))
4071 process.t7 = cms.Task(cms.TaskPlaceholder("i"), process.a, process.t6)
4072 process.t8 = cms.Task(cms.TaskPlaceholder("j"))
4073 process.path1 = cms.Path(process.b, process.t2, process.t3)
4074 process.endpath1 = cms.EndPath(process.b, process.t5)
4075 process.schedule = cms.Schedule(*[ process.path1, process.endpath1 ], tasks=[process.t7, process.t8])""")
4076             p.resolve()
4077             self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4078 """process.a = cms.EDProducer("ma")
4079 process.c = cms.EDProducer("mc")
4080 process.d = cms.EDProducer("md")
4081 process.e = cms.EDProducer("me")
4082 process.f = cms.EDProducer("mf")
4083 process.g = cms.EDProducer("mg")
4084 process.h = cms.EDProducer("mh")
4085 process.i = cms.EDProducer("mi")
4086 process.j = cms.EDProducer("mj")
4087 process.b = cms.EDAnalyzer("mb")
4088 process.t1 = cms.Task(process.c)
4089 process.t2 = cms.Task(process.a, process.d, process.t1)
4090 process.t3 = cms.Task(process.e)
4091 process.t4 = cms.Task(process.f)
4092 process.t6 = cms.Task(process.h)
4093 process.t7 = cms.Task(process.a, process.i, process.t6)
4094 process.t8 = cms.Task(process.j)
4095 process.t5 = cms.Task(process.a, process.g, process.t4)
4096 process.path1 = cms.Path(process.b, process.t2, process.t3)
4097 process.endpath1 = cms.EndPath(process.b, process.t5)
4098 process.schedule = cms.Schedule(*[ process.path1, process.endpath1 ], tasks=[process.t7, process.t8])""")
4099         def testConditionalTaskPlaceholder(self):
4100             p = Process("test")
4101             p.a = EDProducer("ma")
4102             p.b = EDAnalyzer("mb")
4103             p.t1 = ConditionalTask(ConditionalTaskPlaceholder("c"))
4104             p.t2 = ConditionalTask(p.a, ConditionalTaskPlaceholder("d"), p.t1)
4105             p.t3 = ConditionalTask(ConditionalTaskPlaceholder("e"))
4106             p.path1 = Path(p.b, p.t2, p.t3)
4107             p.t5 = ConditionalTask(p.a, ConditionalTaskPlaceholder("g"), ConditionalTaskPlaceholder("t4"))
4108             p.t4 = ConditionalTask(ConditionalTaskPlaceholder("f"))
4109             p.path2 = Path(p.b, p.t5)
4110             p.schedule = Schedule(p.path1, p.path2)
4111             p.c = EDProducer("mc")
4112             p.d = EDProducer("md")
4113             p.e = EDProducer("me")
4114             p.f = EDProducer("mf")
4115             p.g = EDProducer("mg")
4116             p.h = EDProducer("mh")
4117             p.i = EDProducer("mi")
4118             p.j = EDProducer("mj")
4119             self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4120 """process.a = cms.EDProducer("ma")
4121 process.c = cms.EDProducer("mc")
4122 process.d = cms.EDProducer("md")
4123 process.e = cms.EDProducer("me")
4124 process.f = cms.EDProducer("mf")
4125 process.g = cms.EDProducer("mg")
4126 process.h = cms.EDProducer("mh")
4127 process.i = cms.EDProducer("mi")
4128 process.j = cms.EDProducer("mj")
4129 process.b = cms.EDAnalyzer("mb")
4130 process.t1 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("c"))
4131 process.t2 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("d"), process.a, process.t1)
4132 process.t3 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("e"))
4133 process.t5 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("g"), cms.ConditionalTaskPlaceholder("t4"), process.a)
4134 process.t4 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("f"))
4135 process.path1 = cms.Path(process.b, process.t2, process.t3)
4136 process.path2 = cms.Path(process.b, process.t5)
4137 process.schedule = cms.Schedule(*[ process.path1, process.path2 ])""")
4138             p.resolve()
4139             self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4140 """process.a = cms.EDProducer("ma")
4141 process.c = cms.EDProducer("mc")
4142 process.d = cms.EDProducer("md")
4143 process.e = cms.EDProducer("me")
4144 process.f = cms.EDProducer("mf")
4145 process.g = cms.EDProducer("mg")
4146 process.h = cms.EDProducer("mh")
4147 process.i = cms.EDProducer("mi")
4148 process.j = cms.EDProducer("mj")
4149 process.b = cms.EDAnalyzer("mb")
4150 process.t1 = cms.ConditionalTask(process.c)
4151 process.t2 = cms.ConditionalTask(process.a, process.d, process.t1)
4152 process.t3 = cms.ConditionalTask(process.e)
4153 process.t4 = cms.ConditionalTask(process.f)
4154 process.t5 = cms.ConditionalTask(process.a, process.g, process.t4)
4155 process.path1 = cms.Path(process.b, process.t2, process.t3)
4156 process.path2 = cms.Path(process.b, process.t5)
4157 process.schedule = cms.Schedule(*[ process.path1, process.path2 ])""")
4158 
4159         def testDelete(self):
4160             p = Process("test")
4161             p.a = EDAnalyzer("MyAnalyzer")
4162             p.b = EDAnalyzer("YourAnalyzer")
4163             p.c = EDAnalyzer("OurAnalyzer")
4164             p.d = EDAnalyzer("OurAnalyzer")
4165             p.e = EDAnalyzer("OurAnalyzer")
4166             p.f = EDAnalyzer("OurAnalyzer")
4167             p.g = EDProducer("OurProducer")
4168             p.h = EDProducer("YourProducer")
4169             p.i = SwitchProducerTest(
4170                 test1 = EDProducer("OneProducer"),
4171                 test2 = EDProducer("TwoProducer")
4172             )
4173             p.t1 = Task(p.g, p.h, p.i)
4174             t2 = Task(p.g, p.h, p.i)
4175             t3 = Task(p.g, p.h)
4176             p.t4 = Task(p.h)
4177             p.ct1 = ConditionalTask(p.g, p.h, p.i)
4178             ct2 = ConditionalTask(p.g, p.h)
4179             ct3 = ConditionalTask(p.g, p.h)
4180             p.ct4 = ConditionalTask(p.h)
4181             p.s = Sequence(p.d+p.e)
4182             p.path1 = Path(p.a+p.f+p.s,t2,ct2)
4183             p.path2 = Path(p.a)
4184             p.path3 = Path(ct3, p.ct4)
4185             p.endpath2 = EndPath(p.b)
4186             p.endpath1 = EndPath(p.b+p.f)
4187             p.schedule = Schedule(p.path2, p.path3, p.endpath2, tasks=[t3, p.t4])
4188             self.assertTrue(hasattr(p, 'f'))
4189             self.assertTrue(hasattr(p, 'g'))
4190             self.assertTrue(hasattr(p, 'i'))
4191             del p.e
4192             del p.f
4193             del p.g
4194             del p.i
4195             self.assertFalse(hasattr(p, 'f'))
4196             self.assertFalse(hasattr(p, 'g'))
4197             self.assertEqual(p.t1.dumpPython(), 'cms.Task(process.h)\n')
4198             self.assertEqual(p.ct1.dumpPython(), 'cms.ConditionalTask(process.h)\n')
4199             self.assertEqual(p.s.dumpPython(), 'cms.Sequence(process.d)\n')
4200             self.assertEqual(p.path1.dumpPython(), 'cms.Path(process.a+process.s, cms.ConditionalTask(process.h), cms.Task(process.h))\n')
4201             self.assertEqual(p.endpath1.dumpPython(), 'cms.EndPath(process.b)\n')
4202             self.assertEqual(p.path3.dumpPython(), 'cms.Path(cms.ConditionalTask(process.h), process.ct4)\n')
4203             del p.s
4204             self.assertEqual(p.path1.dumpPython(), 'cms.Path(process.a+(process.d), cms.ConditionalTask(process.h), cms.Task(process.h))\n')
4205             self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(*[ process.path2, process.path3, process.endpath2 ], tasks=[cms.Task(process.h), process.t4])\n')
4206             del p.path2
4207             self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(*[ process.path3, process.endpath2 ], tasks=[cms.Task(process.h), process.t4])\n')
4208             del p.path3
4209             self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(*[ process.endpath2 ], tasks=[cms.Task(process.h), process.t4])\n')
4210             del p.endpath2
4211             self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(tasks=[cms.Task(process.h), process.t4])\n')
4212             del p.t4
4213             self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(tasks=[cms.Task(process.h)])\n')
4214         def testModifier(self):
4215             m1 = Modifier()
4216             Process._firstProcess = True
4217             p = Process("test")
4218             self.assertRaises(RuntimeError, lambda: Process("test2", m1))
4219             m1 = Modifier()
4220             Process._firstProcess = True
4221             p = Process("test",m1)
4222             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1))
4223             def _mod_fred(obj):
4224                 obj.fred = 2
4225             m1.toModify(p.a,_mod_fred)
4226             self.assertEqual(p.a.fred.value(),2)
4227             p.b = EDAnalyzer("YourAnalyzer", wilma = int32(1))
4228             m1.toModify(p.b, wilma = 2)
4229             self.assertEqual(p.b.wilma.value(),2)
4230             self.assertTrue(p.isUsingModifier(m1))
4231             #check that Modifier not attached to a process doesn't run
4232             m1 = Modifier()
4233             Process._firstProcess = True
4234             p = Process("test")
4235             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1))
4236             m1.toModify(p.a,_mod_fred)
4237             p.b = EDAnalyzer("YourAnalyzer", wilma = int32(1))
4238             m1.toModify(p.b, wilma = 2)
4239             self.assertEqual(p.a.fred.value(),1)
4240             self.assertEqual(p.b.wilma.value(),1)
4241             self.assertEqual(p.isUsingModifier(m1),False)
4242             #make sure clones get the changes
4243             m1 = Modifier()
4244             Process._firstProcess = True
4245             p = Process("test",m1)
4246             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4247             m1.toModify(p.a, fred = int32(2))
4248             p.b = p.a.clone(wilma = int32(3))
4249             self.assertEqual(p.a.fred.value(),2)
4250             self.assertEqual(p.a.wilma.value(),1)
4251             self.assertEqual(p.b.fred.value(),2)
4252             self.assertEqual(p.b.wilma.value(),3)
4253             #test removal of parameter
4254             m1 = Modifier()
4255             Process._firstProcess = True
4256             p = Process("test",m1)
4257             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1), fintstones = PSet(fred = int32(1)))
4258             m1.toModify(p.a, fred = None, fintstones = dict(fred = None))
4259             self.assertEqual(hasattr(p.a, "fred"), False)
4260             self.assertEqual(hasattr(p.a.fintstones, "fred"), False)
4261             self.assertEqual(p.a.wilma.value(),1)
4262             #test adding a parameter
4263             m1 = Modifier()
4264             Process._firstProcess = True
4265             p = Process("test",m1)
4266             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1))
4267             m1.toModify(p.a, wilma = int32(2))
4268             self.assertEqual(p.a.fred.value(), 1)
4269             self.assertEqual(p.a.wilma.value(),2)
4270             #test setting of value in PSet
4271             m1 = Modifier()
4272             Process._firstProcess = True
4273             p = Process("test",m1)
4274             p.a = EDAnalyzer("MyAnalyzer", flintstones = PSet(fred = int32(1), wilma = int32(1)))
4275             m1.toModify(p.a, flintstones = dict(fred = int32(2)))
4276             self.assertEqual(p.a.flintstones.fred.value(),2)
4277             self.assertEqual(p.a.flintstones.wilma.value(),1)
4278             #test proper exception from nonexisting parameter name
4279             m1 = Modifier()
4280             Process._firstProcess = True
4281             p = Process("test",m1)
4282             p.a = EDAnalyzer("MyAnalyzer", flintstones = PSet(fred = PSet(wilma = int32(1))))
4283             self.assertRaises(KeyError, lambda: m1.toModify(p.a, flintstones = dict(imnothere = dict(wilma=2))))
4284             self.assertRaises(KeyError, lambda: m1.toModify(p.a, foo = 1))
4285             #test setting a value in a VPSet
4286             m1 = Modifier()
4287             Process._firstProcess = True
4288             p = Process("test",m1)
4289             p.a = EDAnalyzer("MyAnalyzer", flintstones = VPSet(PSet(fred = int32(1)), PSet(wilma = int32(1))))
4290             m1.toModify(p.a, flintstones = {1:dict(wilma = int32(2))})
4291             self.assertEqual(p.a.flintstones[0].fred.value(),1)
4292             self.assertEqual(p.a.flintstones[1].wilma.value(),2)
4293             #test setting a value in a list of values
4294             m1 = Modifier()
4295             Process._firstProcess = True
4296             p = Process("test",m1)
4297             p.a = EDAnalyzer("MyAnalyzer", fred = vuint32(1,2,3))
4298             m1.toModify(p.a, fred = {1:7})
4299             self.assertEqual(p.a.fred[0],1)
4300             self.assertEqual(p.a.fred[1],7)
4301             self.assertEqual(p.a.fred[2],3)
4302             #test IndexError setting a value in a list to an item key not in the list
4303             m1 = Modifier()
4304             Process._firstProcess = True
4305             p = Process("test",m1)
4306             p.a = EDAnalyzer("MyAnalyzer", fred = vuint32(1,2,3))
4307             raised = False
4308             try: m1.toModify(p.a, fred = {5:7})
4309             except IndexError as e: raised = True
4310             self.assertEqual(raised, True)
4311             #test TypeError setting a value in a list using a key that is not an int
4312             m1 = Modifier()
4313             Process._firstProcess = True
4314             p = Process("test",m1)
4315             p.a = EDAnalyzer("MyAnalyzer", flintstones = VPSet(PSet(fred = int32(1)), PSet(wilma = int32(1))))
4316             raised = False
4317             try: m1.toModify(p.a, flintstones = dict(bogus = int32(37)))
4318             except TypeError as e: raised = True
4319             self.assertEqual(raised, True)
4320             #test that load causes process wide methods to run
4321             def _rem_a(proc):
4322                 del proc.a
4323             class ProcModifierMod(object):
4324                 def __init__(self,modifier,func):
4325                     self.proc_mod_ = modifier.makeProcessModifier(func)
4326             class DummyMod(object):
4327                 def __init__(self):
4328                     self.a = EDAnalyzer("Dummy")
4329             testMod = DummyMod()
4330             p.extend(testMod)
4331             self.assertTrue(hasattr(p,"a"))
4332             m1 = Modifier()
4333             Process._firstProcess = True
4334             p = Process("test",m1)
4335             testProcMod = ProcModifierMod(m1,_rem_a)
4336             p.extend(testMod)
4337             p.extend(testProcMod)
4338             self.assertTrue(not hasattr(p,"a"))
4339             #test ModifierChain
4340             m1 = Modifier()
4341             mc = ModifierChain(m1)
4342             Process._firstProcess = True
4343             p = Process("test",mc)
4344             self.assertTrue(p.isUsingModifier(m1))
4345             self.assertTrue(p.isUsingModifier(mc))
4346             testMod = DummyMod()
4347             p.b = EDAnalyzer("Dummy2", fred = int32(1))
4348             m1.toModify(p.b, fred = int32(3))
4349             p.extend(testMod)
4350             testProcMod = ProcModifierMod(m1,_rem_a)
4351             p.extend(testProcMod)
4352             self.assertTrue(not hasattr(p,"a"))
4353             self.assertEqual(p.b.fred.value(),3)
4354             #check cloneAndExclude
4355             m1 = Modifier()
4356             m2 = Modifier()
4357             mc = ModifierChain(m1,m2)
4358             mclone = mc.copyAndExclude([m2])
4359             self.assertTrue(not mclone._isOrContains(m2))
4360             self.assertTrue(mclone._isOrContains(m1))
4361             m3 = Modifier()
4362             mc2 = ModifierChain(mc,m3)
4363             mclone = mc2.copyAndExclude([m2])
4364             self.assertTrue(not mclone._isOrContains(m2))
4365             self.assertTrue(mclone._isOrContains(m1))
4366             self.assertTrue(mclone._isOrContains(m3))
4367             #check combining
4368             m1 = Modifier()
4369             m2 = Modifier()
4370             Process._firstProcess = True
4371             p = Process("test",m1)
4372             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4373             (m1 & m2).toModify(p.a, fred = int32(2))
4374             self.assertRaises(TypeError, lambda: (m1 & m2).toModify(p.a, 1, wilma=2))
4375             self.assertEqual(p.a.fred, 1)
4376             m1 = Modifier()
4377             m2 = Modifier()
4378             Process._firstProcess = True
4379             p = Process("test",m1,m2)
4380             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4381             (m1 & m2).toModify(p.a, fred = int32(2))
4382             self.assertEqual(p.a.fred, 2)
4383             m1 = Modifier()
4384             m2 = Modifier()
4385             m3 = Modifier()
4386             Process._firstProcess = True
4387             p = Process("test",m1,m2,m3)
4388             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4389             (m1 & m2 & m3).toModify(p.a, fred = int32(2))
4390             self.assertEqual(p.a.fred, 2)
4391             (m1 & (m2 & m3)).toModify(p.a, fred = int32(3))
4392             self.assertEqual(p.a.fred, 3)
4393             ((m1 & m2) & m3).toModify(p.a, fred = int32(4))
4394             self.assertEqual(p.a.fred, 4)
4395             #check inverse
4396             m1 = Modifier()
4397             m2 = Modifier()
4398             Process._firstProcess = True
4399             p = Process("test", m1)
4400             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4401             (~m1).toModify(p.a, fred=2)
4402             self.assertEqual(p.a.fred, 1)
4403             (~m2).toModify(p.a, wilma=2)
4404             self.assertEqual(p.a.wilma, 2)
4405             self.assertRaises(TypeError, lambda: (~m1).toModify(p.a, 1, wilma=2))
4406             self.assertRaises(TypeError, lambda: (~m2).toModify(p.a, 1, wilma=2))
4407             # check or
4408             m1 = Modifier()
4409             m2 = Modifier()
4410             m3 = Modifier()
4411             Process._firstProcess = True
4412             p = Process("test", m1)
4413             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4414             (m1 | m2).toModify(p.a, fred=2)
4415             self.assertEqual(p.a.fred, 2)
4416             (m1 | m2 | m3).toModify(p.a, fred=3)
4417             self.assertEqual(p.a.fred, 3)
4418             (m3 | m2 | m1).toModify(p.a, fred=4)
4419             self.assertEqual(p.a.fred, 4)
4420             ((m1 | m2) | m3).toModify(p.a, fred=5)
4421             self.assertEqual(p.a.fred, 5)
4422             (m1 | (m2 | m3)).toModify(p.a, fred=6)
4423             self.assertEqual(p.a.fred, 6)
4424             (m2 | m3).toModify(p.a, fred=7)
4425             self.assertEqual(p.a.fred, 6)
4426             self.assertRaises(TypeError, lambda: (m1 | m2).toModify(p.a, 1, wilma=2))
4427             self.assertRaises(TypeError, lambda: (m2 | m3).toModify(p.a, 1, wilma=2))
4428             # check combinations
4429             m1 = Modifier()
4430             m2 = Modifier()
4431             m3 = Modifier()
4432             m4 = Modifier()
4433             Process._firstProcess = True
4434             p = Process("test", m1, m2)
4435             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4436             (m1 & ~m2).toModify(p.a, fred=2)
4437             self.assertEqual(p.a.fred, 1)
4438             (m1 & ~m3).toModify(p.a, fred=2)
4439             self.assertEqual(p.a.fred, 2)
4440             (m1 | ~m2).toModify(p.a, fred=3)
4441             self.assertEqual(p.a.fred, 3)
4442             (~m1 | ~m2).toModify(p.a, fred=4)
4443             self.assertEqual(p.a.fred, 3)
4444             (~m3 & ~m4).toModify(p.a, fred=4)
4445             self.assertEqual(p.a.fred, 4)
4446             ((m1 & m3) | ~m4).toModify(p.a, fred=5)
4447             self.assertEqual(p.a.fred, 5)
4448             #check toReplaceWith
4449             m1 = Modifier()
4450             Process._firstProcess = True
4451             p = Process("test",m1)
4452             p.a =EDAnalyzer("MyAnalyzer", fred = int32(1))
4453             m1.toReplaceWith(p.a, EDAnalyzer("YourAnalyzer", wilma = int32(3)))
4454             self.assertRaises(TypeError, lambda: m1.toReplaceWith(p.a, EDProducer("YourProducer")))
4455             #Task
4456             p.b =EDAnalyzer("BAn")
4457             p.c =EDProducer("c")
4458             p.d =EDProducer("d")
4459             p.tc = Task(p.c)
4460             p.td = Task(p.d)
4461             p.s = Sequence(p.a, p.tc)
4462             m1.toReplaceWith(p.s, Sequence(p.a+p.b, p.td))
4463             self.assertEqual(p.a.wilma.value(),3)
4464             self.assertEqual(p.a.type_(),"YourAnalyzer")
4465             self.assertEqual(hasattr(p,"fred"),False)
4466             self.assertTrue(p.s.dumpPython() == "cms.Sequence(process.a+process.b, process.td)\n")
4467             p.e =EDProducer("e")
4468             m1.toReplaceWith(p.td, Task(p.e))
4469             self.assertTrue(p.td._collection == OrderedSet([p.e]))
4470             #ConditionalTask
4471             p.b =EDAnalyzer("BAn")
4472             p.c =EDProducer("c")
4473             p.d =EDProducer("d")
4474             del p.tc
4475             del p.td
4476             p.tc = ConditionalTask(p.c)
4477             p.td = ConditionalTask(p.d)
4478             p.s = Sequence(p.a, p.tc)
4479             m1.toReplaceWith(p.s, Sequence(p.a+p.b, p.td))
4480             self.assertEqual(p.a.wilma.value(),3)
4481             self.assertEqual(p.a.type_(),"YourAnalyzer")
4482             self.assertEqual(hasattr(p,"fred"),False)
4483             self.assertTrue(p.s.dumpPython() == "cms.Sequence(process.a+process.b, process.td)\n")
4484             p.e =EDProducer("e")
4485             m1.toReplaceWith(p.td, ConditionalTask(p.e))
4486             self.assertTrue(p.td._collection == OrderedSet([p.e]))
4487             #check toReplaceWith doesn't activate not chosen
4488             m1 = Modifier()
4489             Process._firstProcess = True
4490             p = Process("test")
4491             p.a =EDAnalyzer("MyAnalyzer", fred = int32(1))
4492             m1.toReplaceWith(p.a, EDAnalyzer("YourAnalyzer", wilma = int32(3)))
4493             self.assertEqual(p.a.type_(),"MyAnalyzer")
4494             #check toReplaceWith and and/not/or combinations
4495             m1 = Modifier()
4496             m2 = Modifier()
4497             m3 = Modifier()
4498             m4 = Modifier()
4499             Process._firstProcess = True
4500             p = Process("test", m1, m2)
4501             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4502             self.assertRaises(TypeError, lambda: (m1 & m2).toReplaceWith(p.a, EDProducer("YourProducer")))
4503             self.assertRaises(TypeError, lambda: (m3 & m4).toReplaceWith(p.a, EDProducer("YourProducer")))
4504             self.assertRaises(TypeError, lambda: (~m3).toReplaceWith(p.a, EDProducer("YourProducer")))
4505             self.assertRaises(TypeError, lambda: (~m1).toReplaceWith(p.a, EDProducer("YourProducer")))
4506             self.assertRaises(TypeError, lambda: (m1 | m3).toReplaceWith(p.a, EDProducer("YourProducer")))
4507             self.assertRaises(TypeError, lambda: (m3 | m4).toReplaceWith(p.a, EDProducer("YourProducer")))
4508             (m1 & m2).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer1"))
4509             self.assertEqual(p.a.type_(), "YourAnalyzer1")
4510             (m1 & m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer2"))
4511             self.assertEqual(p.a.type_(), "YourAnalyzer1")
4512             (~m1).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer2"))
4513             self.assertEqual(p.a.type_(), "YourAnalyzer1")
4514             (~m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer2"))
4515             self.assertEqual(p.a.type_(), "YourAnalyzer2")
4516             (m1 | m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer3"))
4517             self.assertEqual(p.a.type_(), "YourAnalyzer3")
4518             (m3 | m4).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer4"))
4519             self.assertEqual(p.a.type_(), "YourAnalyzer3")
4520             #check chaining of toModify and toReplaceWith
4521             m1 = Modifier()
4522             m2 = Modifier()
4523             m3 = Modifier()
4524             Process._firstProcess = True
4525             p = Process("test", m1, m2)
4526             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4527             p.b = EDProducer("MyProducer", barney = int32(1), betty = int32(1))
4528             (m1 & m2).toModify(p.a, fred = 2).toModify(p.b, betty = 3)
4529             self.assertEqual(p.a.fred, 2)
4530             self.assertEqual(p.a.wilma, 1)
4531             self.assertEqual(p.b.barney, 1)
4532             self.assertEqual(p.b.betty, 3)
4533             (m1 | m3).toModify(p.a, wilma = 4).toModify(p.b, barney = 5)
4534             self.assertEqual(p.a.fred, 2)
4535             self.assertEqual(p.a.wilma, 4)
4536             self.assertEqual(p.b.barney, 5)
4537             self.assertEqual(p.b.betty, 3)
4538             (m2 & ~m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer")).toModify(p.b, barney = 6)
4539             self.assertEqual(p.a.type_(), "YourAnalyzer")
4540             self.assertEqual(p.b.barney, 6)
4541             self.assertEqual(p.b.betty, 3)
4542             (m1 & ~m3).toModify(p.a, param=int32(42)).toReplaceWith(p.b, EDProducer("YourProducer"))
4543             self.assertEqual(p.a.type_(), "YourAnalyzer")
4544             self.assertEqual(p.a.param, 42)
4545             self.assertEqual(p.b.type_(), "YourProducer")
4546 
4547             # EDAlias
4548             a = EDAlias(foo2 = VPSet(PSet(type = string("Foo2"))))
4549             m = Modifier()
4550             m._setChosen()
4551             # Modify parameters
4552             m.toModify(a, foo2 = {0: dict(type = "Foo3")})
4553             self.assertEqual(a.foo2[0].type, "Foo3")
4554             # Add an alias
4555             m.toModify(a, foo4 = VPSet(PSet(type = string("Foo4"))))
4556             self.assertEqual(a.foo2[0].type, "Foo3")
4557             self.assertEqual(a.foo4[0].type, "Foo4")
4558             # Remove an alias
4559             m.toModify(a, foo2 = None)
4560             self.assertFalse(hasattr(a, "foo2"))
4561             self.assertEqual(a.foo4[0].type, "Foo4")
4562             # Replace (doesn't work out of the box because EDAlias is not _Parameterizable
4563             m.toReplaceWith(a, EDAlias(bar = VPSet(PSet(type = string("Bar")))))
4564             self.assertFalse(hasattr(a, "foo2"))
4565             self.assertFalse(hasattr(a, "foo4"))
4566             self.assertTrue(hasattr(a, "bar"))
4567             self.assertEqual(a.bar[0].type, "Bar")
4568 
4569             # SwitchProducer
4570             sp = SwitchProducerTest(test1 = EDProducer("Foo",
4571                                                        a = int32(1),
4572                                                        b = PSet(c = int32(2))),
4573                                     test2 = EDProducer("Bar",
4574                                                        aa = int32(11),
4575                                                        bb = PSet(cc = int32(12))))
4576             m = Modifier()
4577             m._setChosen()
4578             # Modify parameters
4579             m.toModify(sp,
4580                        test1 = dict(a = 4, b = dict(c = None)),
4581                        test2 = dict(aa = 15, bb = dict(cc = 45, dd = string("foo"))))
4582             self.assertEqual(sp.test1.a.value(), 4)
4583             self.assertEqual(sp.test1.b.hasParameter("c"), False)
4584             self.assertEqual(sp.test2.aa.value(), 15)
4585             self.assertEqual(sp.test2.bb.cc.value(), 45)
4586             self.assertEqual(sp.test2.bb.dd.value(), "foo")
4587             # Replace a producer
4588             m.toReplaceWith(sp.test1, EDProducer("Fred", x = int32(42)))
4589             self.assertEqual(sp.test1.type_(), "Fred")
4590             self.assertEqual(sp.test1.x.value(), 42)
4591             self.assertRaises(TypeError, lambda: m.toReplaceWith(sp.test1, EDAnalyzer("Foo")))
4592             # Alternative way (only to be allow same syntax to be used as for adding)
4593             m.toModify(sp, test2 = EDProducer("Xyzzy", x = int32(24)))
4594             self.assertEqual(sp.test2.type_(), "Xyzzy")
4595             self.assertEqual(sp.test2.x.value(), 24)
4596             self.assertRaises(TypeError, lambda: m.toModify(sp, test2 = EDAnalyzer("Foo")))
4597             # Add a producer
4598             m.toModify(sp, test3 = EDProducer("Wilma", y = int32(24)))
4599             self.assertEqual(sp.test3.type_(), "Wilma")
4600             self.assertEqual(sp.test3.y.value(), 24)
4601             self.assertRaises(TypeError, lambda: m.toModify(sp, test4 = EDAnalyzer("Foo")))
4602             # Remove a producer
4603             m.toModify(sp, test2 = None)
4604             self.assertEqual(hasattr(sp, "test2"), False)
4605             # Add an alias
4606             m.toModify(sp, test2 = EDAlias(foo = VPSet(PSet(type = string("int")))))
4607             self.assertTrue(hasattr(sp.test2, "foo"))
4608             # Replace an alias
4609             m.toReplaceWith(sp.test2, EDAlias(bar = VPSet(PSet(type = string("int")))))
4610             self.assertTrue(hasattr(sp.test2, "bar"))
4611             # Alternative way
4612             m.toModify(sp, test2 = EDAlias(xyzzy = VPSet(PSet(type = string("int")))))
4613             self.assertTrue(hasattr(sp.test2, "xyzzy"))
4614             # Replace an alias with EDProducer
4615             self.assertRaises(TypeError, lambda: m.toReplaceWith(sp.test2, EDProducer("Foo")))
4616             m.toModify(sp, test2 = EDProducer("Foo"))
4617         def testProcessFragment(self):
4618             #check defaults are not overwritten
4619             f = ProcessFragment('Fragment')
4620             p = Process('PROCESS')
4621             p.maxEvents.input = 10
4622             p.options.numberOfThreads = 4
4623             p.maxLuminosityBlocks.input = 2
4624             p.extend(f)
4625             self.assertEqual(p.maxEvents.input.value(),10)
4626             self.assertEqual(p.options.numberOfThreads.value(), 4)
4627             self.assertEqual(p.maxLuminosityBlocks.input.value(),2)
4628             #general checks
4629             f = ProcessFragment("Fragment")
4630             f.fltr = EDFilter("Foo")
4631             p = Process('PROCESS')
4632             p.extend(f)
4633             self.assertTrue(hasattr(p,'fltr'))
4634         def testProcessForProcessAccelerator(self):
4635             proc = Process("TEST")
4636             p = ProcessForProcessAccelerator(proc)
4637             p.TestService = Service("TestService")
4638             self.assertTrue(hasattr(proc, "TestService"))
4639             self.assertEqual(proc.TestService.type_(), "TestService")
4640             self.assertRaises(TypeError, setattr, p, "a", EDProducer("Foo"))
4641             p.add_(Service("TestServiceTwo"))
4642             self.assertTrue(hasattr(proc, "TestServiceTwo"))
4643             self.assertEqual(proc.TestServiceTwo.type_(), "TestServiceTwo")
4644             p.TestService.foo = untracked.uint32(42)
4645             self.assertEqual(proc.TestService.foo.value(), 42)
4646             proc.mod = EDProducer("Producer")
4647             self.assertRaises(TypeError, getattr, p, "mod")
4648         def testProcessAccelerator(self):
4649             proc = Process("TEST")
4650             p = TestMakePSet()
4651             proc.fillProcessDesc(p)
4652             self.assertTrue(["cpu"], p.values["@available_accelerators"][1])
4653             self.assertFalse(p.values["@selected_accelerators"][0])
4654             self.assertTrue(["cpu"], p.values["@selected_accelerators"][1])
4655             self.assertFalse(p.values["@module_type_resolver"][0])
4656             self.assertEqual("", p.values["@module_type_resolver"][1])
4657 
4658             proc = Process("TEST")
4659             self.assertRaises(TypeError, setattr, proc, "processAcceleratorTest", ProcessAcceleratorTest())
4660             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4661             del proc.MessageLogger # remove boilerplate unnecessary for this test case
4662             self.maxDiff = None
4663             self.assertEqual(proc.dumpPython(),
4664 """import FWCore.ParameterSet.Config as cms
4665 from test import ProcessAcceleratorTest
4666 
4667 process = cms.Process("TEST")
4668 
4669 process.maxEvents = cms.untracked.PSet(
4670     input = cms.optional.untracked.int32,
4671     output = cms.optional.untracked.allowed(cms.int32,cms.PSet)
4672 )
4673 
4674 process.maxLuminosityBlocks = cms.untracked.PSet(
4675     input = cms.untracked.int32(-1)
4676 )
4677 
4678 process.options = cms.untracked.PSet(
4679     IgnoreCompletely = cms.untracked.vstring(),
4680     Rethrow = cms.untracked.vstring(),
4681     TryToContinue = cms.untracked.vstring(),
4682     accelerators = cms.untracked.vstring('*'),
4683     allowUnscheduled = cms.obsolete.untracked.bool,
4684     canDeleteEarly = cms.untracked.vstring(),
4685     deleteNonConsumedUnscheduledModules = cms.untracked.bool(True),
4686     dumpOptions = cms.untracked.bool(False),
4687     emptyRunLumiMode = cms.obsolete.untracked.string,
4688     eventSetup = cms.untracked.PSet(
4689         forceNumberOfConcurrentIOVs = cms.untracked.PSet(
4690             allowAnyLabel_=cms.required.untracked.uint32
4691         ),
4692         numberOfConcurrentIOVs = cms.untracked.uint32(0)
4693     ),
4694     fileMode = cms.untracked.string('FULLMERGE'),
4695     forceEventSetupCacheClearOnNewRun = cms.untracked.bool(False),
4696     holdsReferencesToDeleteEarly = cms.untracked.VPSet(),
4697     makeTriggerResults = cms.obsolete.untracked.bool,
4698     modulesToCallForTryToContinue = cms.untracked.vstring(),
4699     modulesToIgnoreForDeleteEarly = cms.untracked.vstring(),
4700     numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(0),
4701     numberOfConcurrentRuns = cms.untracked.uint32(1),
4702     numberOfStreams = cms.untracked.uint32(0),
4703     numberOfThreads = cms.untracked.uint32(1),
4704     printDependencies = cms.untracked.bool(False),
4705     sizeOfStackForThreadsInKB = cms.optional.untracked.uint32,
4706     throwIfIllegalParameter = cms.untracked.bool(True),
4707     wantSummary = cms.untracked.bool(False)
4708 )
4709 
4710 process.ProcessAcceleratorTest = ProcessAcceleratorTest(
4711     enabled = ['test1', 'test2', 'anothertest3']
4712 )
4713 
4714 
4715 """)
4716             p = TestMakePSet()
4717             proc.fillProcessDesc(p)
4718             self.assertEqual(["*"], p.values["options"][1].values["accelerators"][1])
4719             self.assertFalse(p.values["options"][1].values["accelerators"][0])
4720             self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["@selected_accelerators"][1])
4721             self.assertEqual("AcceleratorTestService", p.values["services"][1][0].values["@service_type"][1])
4722             self.assertFalse(p.values["@available_accelerators"][0])
4723             self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4724             self.assertFalse(p.values["@module_type_resolver"][0])
4725             self.assertEqual("", p.values["@module_type_resolver"][1])
4726 
4727             proc = Process("TEST")
4728             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4729             proc.add_(Service("AcceleratorTestServiceRemove"))
4730             p = TestMakePSet()
4731             proc.fillProcessDesc(p)
4732             services = [x.values["@service_type"][1] for x in p.values["services"][1]]
4733             self.assertTrue("AcceleratorTestService" in services)
4734             self.assertFalse("AcceleratorTestServiceRemove" in services)
4735 
4736             proc = Process("TEST")
4737             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"])
4738             p = TestMakePSet()
4739             proc.fillProcessDesc(p)
4740             self.assertEqual(["cpu", "test1"], p.values["@selected_accelerators"][1])
4741             self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
4742 
4743             proc = Process("TEST")
4744             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4745             proc.options.accelerators = ["test2"]
4746             p = TestMakePSet()
4747             proc.fillProcessDesc(p)
4748             self.assertEqual(["test2"], p.values["@selected_accelerators"][1])
4749             self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4750 
4751             proc = Process("TEST")
4752             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4753             proc.options.accelerators = ["test*"]
4754             proc.fillProcessDesc(p)
4755             self.assertEqual(["test1", "test2"], p.values["@selected_accelerators"][1])
4756             self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4757 
4758             proc = Process("TEST")
4759             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"])
4760             proc.options.accelerators = ["test2"]
4761             p = TestMakePSet()
4762             proc.fillProcessDesc(p)
4763             self.assertEqual([], p.values["@selected_accelerators"][1])
4764             self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
4765 
4766             proc = Process("TEST")
4767             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4768             proc.options.accelerators = ["cpu*"]
4769             p = TestMakePSet()
4770             proc.fillProcessDesc(p)
4771             self.assertEqual(["cpu"], p.values["@selected_accelerators"][1])
4772             self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4773 
4774             proc = Process("TEST")
4775             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4776             proc.options.accelerators = ["test3"]
4777             p = TestMakePSet()
4778             self.assertRaises(ValueError, proc.fillProcessDesc, p)
4779 
4780             proc = Process("TEST")
4781             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4782             proc.options.accelerators = ["*", "test1"]
4783             p = TestMakePSet()
4784             self.assertRaises(ValueError, proc.fillProcessDesc, p)
4785 
4786             proc = Process("TEST")
4787             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4788             proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2()
4789             p = TestMakePSet()
4790             proc.fillProcessDesc(p)
4791             self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@selected_accelerators"][1])
4792             self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4793 
4794             proc = Process("TEST")
4795             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4796             proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2()
4797             proc.options.accelerators = ["*test3", "c*"]
4798             p = TestMakePSet()
4799             proc.fillProcessDesc(p)
4800             self.assertEqual(["anothertest3", "cpu"], p.values["@selected_accelerators"][1])
4801             self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4802 
4803             proc = Process("TEST")
4804             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4805             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4806                                                              a = int32(1),
4807                                                              b = PSet(c = int32(2))),
4808                                           test1 = EDProducer("Bar",
4809                                                              aa = int32(11),
4810                                                              bb = PSet(cc = int32(12))))
4811             proc.p = Path(proc.sp)
4812             p = TestMakePSet()
4813             proc.fillProcessDesc(p)
4814             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
4815 
4816             proc = Process("TEST")
4817             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"])
4818             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4819                                                              a = int32(1),
4820                                                              b = PSet(c = int32(2))),
4821                                           test1 = EDProducer("Bar",
4822                                                              aa = int32(11),
4823                                                              bb = PSet(cc = int32(12))))
4824             proc.p = Path(proc.sp)
4825             p = TestMakePSet()
4826             proc.fillProcessDesc(p)
4827             self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
4828 
4829             proc = Process("TEST")
4830             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4831             proc.options.accelerators = ["test1"]
4832             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4833                                                              a = int32(1),
4834                                                              b = PSet(c = int32(2))),
4835                                           test1 = EDProducer("Bar",
4836                                                              aa = int32(11),
4837                                                              bb = PSet(cc = int32(12))))
4838             proc.p = Path(proc.sp)
4839             p = TestMakePSet()
4840             proc.fillProcessDesc(p)
4841             self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
4842 
4843             proc = Process("TEST")
4844             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4845             proc.options.accelerators = ["test*"]
4846             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4847                                                              a = int32(1),
4848                                                              b = PSet(c = int32(2))),
4849                                           test1 = EDProducer("Bar",
4850                                                              aa = int32(11),
4851                                                              bb = PSet(cc = int32(12))))
4852             proc.p = Path(proc.sp)
4853             p = TestMakePSet()
4854             proc.fillProcessDesc(p)
4855             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
4856 
4857             proc = Process("TEST")
4858             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4859             proc.options.accelerators = ["anothertest3"]
4860             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4861                                                              a = int32(1),
4862                                                              b = PSet(c = int32(2))),
4863                                           test1 = EDProducer("Bar",
4864                                                              aa = int32(11),
4865                                                              bb = PSet(cc = int32(12))))
4866             proc.p = Path(proc.sp)
4867             p = TestMakePSet()
4868             self.assertRaises(RuntimeError, proc.fillProcessDesc, p)
4869 
4870             proc = Process("TEST")
4871             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4872                 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4873             proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2()
4874             proc.normalProducer = EDProducer("FooProducer")
4875             proc.testProducer = EDProducer("BarProducer@test")
4876             proc.test2Producer = EDProducer("BarProducer@test", test=untracked.PSet(backend=untracked.string("test2_backend")))
4877             proc.testAnalyzer = EDAnalyzer("Analyzer@test")
4878             proc.testFilter = EDAnalyzer("Filter@test")
4879             proc.testESProducer = ESProducer("ESProducer@test")
4880             proc.testESSource = ESSource("ESSource@test")
4881             proc.p = Path(proc.normalProducer+proc.testProducer+proc.test2Producer+proc.testAnalyzer+proc.testFilter)
4882             p = TestMakePSet()
4883             proc.fillProcessDesc(p)
4884             self.assertEqual("TestModuleTypeResolver", p.values["@module_type_resolver"][1])
4885             self.assertEqual("FooProducer", p.values["normalProducer"][1].values["@module_type"][1])
4886             self.assertEqual(len(list(filter(lambda x: not "@" in x, p.values["normalProducer"][1].values.keys()))), 0)
4887             self.assertEqual("BarProducer@test", p.values["testProducer"][1].values["@module_type"][1])
4888             self.assertEqual(False, p.values["testProducer"][1].values["test"][0])
4889             self.assertEqual(False, p.values["testProducer"][1].values["test"][1].values["backend"][0])
4890             self.assertEqual("test1_backend", p.values["testProducer"][1].values["test"][1].values["backend"][1])
4891             self.assertEqual("BarProducer@test", p.values["test2Producer"][1].values["@module_type"][1])
4892             self.assertEqual("test2_backend", p.values["test2Producer"][1].values["test"][1].values["backend"][1])
4893             self.assertEqual("Analyzer@test", p.values["testAnalyzer"][1].values["@module_type"][1])
4894             self.assertEqual("test1_backend", p.values["testAnalyzer"][1].values["test"][1].values["backend"][1])
4895             self.assertEqual("Filter@test", p.values["testFilter"][1].values["@module_type"][1])
4896             self.assertEqual("test1_backend", p.values["testFilter"][1].values["test"][1].values["backend"][1])
4897             self.assertEqual("ESProducer@test", p.values["ESProducer@test@testESProducer"][1].values["@module_type"][1])
4898             self.assertEqual("test1_backend", p.values["ESProducer@test@testESProducer"][1].values["test"][1].values["backend"][1])
4899             self.assertEqual("ESSource@test", p.values["ESSource@test@testESSource"][1].values["@module_type"][1])
4900             self.assertEqual("test1_backend", p.values["ESSource@test@testESSource"][1].values["test"][1].values["backend"][1])
4901 
4902             # No required accelerators available
4903             proc = Process("Test")
4904             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4905                 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4906             proc.options.accelerators = []
4907             self.assertRaises(EDMException, proc.fillProcessDesc, p)
4908 
4909             proc = Process("Test")
4910             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4911                 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4912             proc.options.accelerators = ["test1"]
4913             proc.test2Producer = EDProducer("BarProducer@test", test=untracked.PSet(backend=untracked.string("test2_backend")))
4914             proc.p = Path(proc.test2Producer)
4915             self.assertRaises(EDMException, proc.fillProcessDesc, p)
4916 
4917             # Two ProcessAccelerators return type resolver
4918             proc = Process("TEST")
4919             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4920                 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4921             proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2(
4922                 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4923             p = TestMakePSet()
4924             self.assertRaises(RuntimeError, proc.fillProcessDesc, p)
4925 
4926             import pickle
4927             proc = Process("TEST")
4928             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4929             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4930                                                              a = int32(1),
4931                                                              b = PSet(c = int32(2))),
4932                                           test1 = EDProducer("Bar",
4933                                                              aa = int32(11),
4934                                                              bb = PSet(cc = int32(12))))
4935             proc.p = Path(proc.sp)
4936             pkl = pickle.dumps(proc)
4937             unpkl = pickle.loads(pkl)
4938             p = TestMakePSet()
4939             unpkl.fillProcessDesc(p)
4940             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
4941             self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4942             unpkl = pickle.loads(pkl)
4943             unpkl.ProcessAcceleratorTest.setEnabled(["test1"])
4944             p = TestMakePSet()
4945             unpkl.fillProcessDesc(p)
4946             self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
4947             self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
4948 
4949     unittest.main()