Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2023-01-11 16:27:23

0001 #!/usr/bin/env python3
0002 
0003 ### command line options helper
0004 from __future__ import print_function
0005 from __future__ import absolute_import
0006 import os
0007 from  .Options import Options
0008 options = Options()
0009 
0010 
0011 ## imports
0012 import sys
0013 from .Mixins import PrintOptions,_ParameterTypeBase,_SimpleParameterTypeBase, _Parameterizable, _ConfigureComponent, _TypedParameterizable, _Labelable,  _Unlabelable,  _ValidatingListBase, _modifyParametersFromDict
0014 from .Mixins import *
0015 from .Types import *
0016 from .Modules import *
0017 from .Modules import _Module
0018 from .SequenceTypes import *
0019 from .SequenceTypes import _ModuleSequenceType, _Sequenceable  #extend needs it
0020 from .SequenceVisitors import PathValidator, EndPathValidator, FinalPathValidator, ScheduleTaskValidator, NodeVisitor, CompositeVisitor, ModuleNamesFromGlobalsVisitor
0021 from .MessageLogger import MessageLogger
0022 from . import DictTypes
0023 
0024 from .ExceptionHandling import *
0025 
0026 #when building RECO paths we have hit the default recursion limit
0027 if sys.getrecursionlimit()<5000:
0028     sys.setrecursionlimit(5000)
0029 
0030 class edm(object):
0031     class errors(object):
0032         #Allowed errors to be used within Python
0033         Configuration = "{Configuration}"
0034         UnavailableAccelerator = "{UnavailableAccelerator}"
0035 
0036 class EDMException(Exception):
0037     def __init__(self, error, message):
0038         super().__init__(error+"\n"+message)
0039 
0040 
0041 def checkImportPermission(minLevel = 2, allowedPatterns = []):
0042     """
0043     Raise an exception if called by special config files. This checks
0044     the call or import stack for the importing file. An exception is raised if
0045     the importing module is not in allowedPatterns and if it is called too deeply:
0046     minLevel = 2: inclusion by top lvel cfg only
0047     minLevel = 1: No inclusion allowed
0048     allowedPatterns = ['Module1','Module2/SubModule1'] allows import
0049     by any module in Module1 or Submodule1
0050     """
0051 
0052     import inspect
0053     import os
0054 
0055     ignorePatterns = ['FWCore/ParameterSet/Config.py', 'FWCore/ParameterSet/python/Config.py','<string>','<frozen ']
0056     CMSSWPath = [os.environ['CMSSW_BASE'],os.environ['CMSSW_RELEASE_BASE']]
0057 
0058     # Filter the stack to things in CMSSWPath and not in ignorePatterns
0059     trueStack = []
0060     for item in inspect.stack():
0061         inPath = False
0062         ignore = False
0063 
0064         for pattern in CMSSWPath:
0065             if item[1].find(pattern) != -1:
0066                 inPath = True
0067                 break
0068         if item[1].find('/') == -1: # The base file, no path
0069             inPath = True
0070 
0071         for pattern in ignorePatterns:
0072             if item[1].find(pattern) != -1:
0073                 ignore = True
0074                 break
0075 
0076         if inPath and not ignore:
0077             trueStack.append(item[1])
0078 
0079     importedFile = trueStack[0]
0080     importedBy   = ''
0081     if len(trueStack) > 1:
0082         importedBy = trueStack[1]
0083 
0084     for pattern in allowedPatterns:
0085         if importedBy.find(pattern) > -1:
0086             return True
0087 
0088     if len(trueStack) <= minLevel: # Imported directly
0089         return True
0090 
0091     raise ImportError("Inclusion of %s is allowed only by cfg or specified cfi files."
0092                       % importedFile)
0093 
0094 def findProcess(module):
0095     """Look inside the module and find the Processes it contains"""
0096     class Temp(object):
0097         pass
0098     process = None
0099     if isinstance(module,dict):
0100         if 'process' in module:
0101             p = module['process']
0102             module = Temp()
0103             module.process = p
0104     if hasattr(module,'process'):
0105         if isinstance(module.process,Process):
0106             process = module.process
0107         else:
0108             raise RuntimeError("The attribute named 'process' does not inherit from the Process class")
0109     else:
0110         raise RuntimeError("no 'process' attribute found in the module, please add one")
0111     return process
0112 
0113 class Process(object):
0114     """Root class for a CMS configuration process"""
0115     _firstProcess = True
0116     def __init__(self,name,*Mods):
0117         """The argument 'name' will be the name applied to this Process
0118             Can optionally pass as additional arguments cms.Modifier instances
0119             that will be used to modify the Process as it is built
0120             """
0121         self.__dict__['_Process__name'] = name
0122         if not name.isalnum():
0123             raise RuntimeError("Error: The process name is an empty string or contains non-alphanumeric characters")
0124         self.__dict__['_Process__filters'] = {}
0125         self.__dict__['_Process__producers'] = {}
0126         self.__dict__['_Process__switchproducers'] = {}
0127         self.__dict__['_Process__source'] = None
0128         self.__dict__['_Process__looper'] = None
0129         self.__dict__['_Process__subProcesses'] = []
0130         self.__dict__['_Process__schedule'] = None
0131         self.__dict__['_Process__analyzers'] = {}
0132         self.__dict__['_Process__outputmodules'] = {}
0133         self.__dict__['_Process__paths'] = DictTypes.SortedKeysDict()    # have to keep the order
0134         self.__dict__['_Process__endpaths'] = DictTypes.SortedKeysDict() # of definition
0135         self.__dict__['_Process__finalpaths'] = DictTypes.SortedKeysDict() # of definition
0136         self.__dict__['_Process__sequences'] = {}
0137         self.__dict__['_Process__tasks'] = {}
0138         self.__dict__['_Process__conditionaltasks'] = {}
0139         self.__dict__['_Process__services'] = {}
0140         self.__dict__['_Process__essources'] = {}
0141         self.__dict__['_Process__esproducers'] = {}
0142         self.__dict__['_Process__esprefers'] = {}
0143         self.__dict__['_Process__aliases'] = {}
0144         self.__dict__['_Process__psets']={}
0145         self.__dict__['_Process__vpsets']={}
0146         self.__dict__['_cloneToObjectDict'] = {}
0147         # policy switch to avoid object overwriting during extend/load
0148         self.__dict__['_Process__InExtendCall'] = False
0149         self.__dict__['_Process__partialschedules'] = {}
0150         self.__isStrict = False
0151         self.__dict__['_Process__modifiers'] = Mods
0152         self.__dict__['_Process__accelerators'] = {}
0153         self.options = Process.defaultOptions_()
0154         self.maxEvents = Process.defaultMaxEvents_()
0155         self.maxLuminosityBlocks = Process.defaultMaxLuminosityBlocks_()
0156         # intentionally not cloned to ensure that everyone taking
0157         # MessageLogger still via
0158         # FWCore.Message(Logger|Service).MessageLogger_cfi
0159         # use the very same MessageLogger object.
0160         self.MessageLogger = MessageLogger
0161         if Process._firstProcess:
0162             Process._firstProcess = False
0163         else:
0164             if len(Mods) > 0:
0165                 for m in self.__modifiers:
0166                     if not m._isChosen():
0167                         raise RuntimeError("The Process {} tried to redefine which Modifiers to use after another Process was already started".format(name))
0168         for m in self.__modifiers:
0169             m._setChosen()
0170 
0171     def setStrict(self, value):
0172         self.__isStrict = value
0173         _Module.__isStrict__ = True
0174 
0175     # some user-friendly methods for command-line browsing
0176     def producerNames(self):
0177         """Returns a string containing all the EDProducer labels separated by a blank"""
0178         return ' '.join(self.producers_().keys())
0179     def switchProducerNames(self):
0180         """Returns a string containing all the SwitchProducer labels separated by a blank"""
0181         return ' '.join(self.switchProducers_().keys())
0182     def analyzerNames(self):
0183         """Returns a string containing all the EDAnalyzer labels separated by a blank"""
0184         return ' '.join(self.analyzers_().keys())
0185     def filterNames(self):
0186         """Returns a string containing all the EDFilter labels separated by a blank"""
0187         return ' '.join(self.filters_().keys())
0188     def pathNames(self):
0189         """Returns a string containing all the Path names separated by a blank"""
0190         return ' '.join(self.paths_().keys())
0191 
0192     def __setstate__(self, pkldict):
0193         """
0194         Unpickling hook.
0195 
0196         Since cloneToObjectDict stores a hash of objects by their
0197         id() it needs to be updated when unpickling to use the
0198         new object id values instantiated during the unpickle.
0199 
0200         """
0201         self.__dict__.update(pkldict)
0202         tmpDict = {}
0203         for value in self._cloneToObjectDict.values():
0204             tmpDict[id(value)] = value
0205         self.__dict__['_cloneToObjectDict'] = tmpDict
0206 
0207 
0208 
0209     def filters_(self):
0210         """returns a dict of the filters that have been added to the Process"""
0211         return DictTypes.FixedKeysDict(self.__filters)
0212     filters = property(filters_, doc="dictionary containing the filters for the process")
0213     def name_(self):
0214         return self.__name
0215     def setName_(self,name):
0216         if not name.isalnum():
0217             raise RuntimeError("Error: The process name is an empty string or contains non-alphanumeric characters")
0218         self.__dict__['_Process__name'] = name
0219     process = property(name_,setName_, doc="name of the process")
0220     def producers_(self):
0221         """returns a dict of the producers that have been added to the Process"""
0222         return DictTypes.FixedKeysDict(self.__producers)
0223     producers = property(producers_,doc="dictionary containing the producers for the process")
0224     def switchProducers_(self):
0225         """returns a dict of the SwitchProducers that have been added to the Process"""
0226         return DictTypes.FixedKeysDict(self.__switchproducers)
0227     switchProducers = property(switchProducers_,doc="dictionary containing the SwitchProducers for the process")
0228     def source_(self):
0229         """returns the source that has been added to the Process or None if none have been added"""
0230         return self.__source
0231     def setSource_(self,src):
0232         self._placeSource('source',src)
0233     source = property(source_,setSource_,doc='the main source or None if not set')
0234     def looper_(self):
0235         """returns the looper that has been added to the Process or None if none have been added"""
0236         return self.__looper
0237     def setLooper_(self,lpr):
0238         self._placeLooper('looper',lpr)
0239     looper = property(looper_,setLooper_,doc='the main looper or None if not set')
0240     @staticmethod
0241     def defaultOptions_():
0242         return untracked.PSet(numberOfThreads = untracked.uint32(1),
0243                               numberOfStreams = untracked.uint32(0),
0244                               numberOfConcurrentRuns = untracked.uint32(1),
0245                               numberOfConcurrentLuminosityBlocks = untracked.uint32(0),
0246                               eventSetup = untracked.PSet(
0247                                   numberOfConcurrentIOVs = untracked.uint32(0),
0248                                   forceNumberOfConcurrentIOVs = untracked.PSet(
0249                                       allowAnyLabel_ = required.untracked.uint32
0250                                   )
0251                               ),
0252                               accelerators = untracked.vstring('*'),
0253                               wantSummary = untracked.bool(False),
0254                               fileMode = untracked.string('FULLMERGE'),
0255                               forceEventSetupCacheClearOnNewRun = untracked.bool(False),
0256                               throwIfIllegalParameter = untracked.bool(True),
0257                               printDependencies = untracked.bool(False),
0258                               deleteNonConsumedUnscheduledModules = untracked.bool(True),
0259                               sizeOfStackForThreadsInKB = optional.untracked.uint32,
0260                               Rethrow = untracked.vstring(),
0261                               SkipEvent = untracked.vstring(),
0262                               FailPath = untracked.vstring(),
0263                               IgnoreCompletely = untracked.vstring(),
0264                               canDeleteEarly = untracked.vstring(),
0265                               holdsReferencesToDeleteEarly = untracked.VPSet(),
0266                               modulesToIgnoreForDeleteEarly = untracked.vstring(),
0267                               dumpOptions = untracked.bool(False),
0268                               allowUnscheduled = obsolete.untracked.bool,
0269                               emptyRunLumiMode = obsolete.untracked.string,
0270                               makeTriggerResults = obsolete.untracked.bool,
0271                               )
0272     def __updateOptions(self,opt):
0273         newOpts = self.defaultOptions_()
0274         if isinstance(opt,dict):
0275             for k,v in opt.items():
0276                 setattr(newOpts,k,v)
0277         else:
0278             for p in opt.parameters_():
0279                 setattr(newOpts, p, getattr(opt,p))
0280         return newOpts
0281     @staticmethod
0282     def defaultMaxEvents_():
0283         return untracked.PSet(input=optional.untracked.int32,
0284                               output=optional.untracked.allowed(int32,PSet))
0285     def __updateMaxEvents(self,ps):
0286         newMax = self.defaultMaxEvents_()
0287         if isinstance(ps,dict):
0288             for k,v in ps.items():
0289                 setattr(newMax,k,v)
0290         else:
0291             for p in ps.parameters_():
0292                 setattr(newMax, p, getattr(ps,p))
0293         return newMax
0294     @staticmethod
0295     def defaultMaxLuminosityBlocks_():
0296         return untracked.PSet(input=untracked.int32(-1))
0297     def subProcesses_(self):
0298         """returns a list of the subProcesses that have been added to the Process"""
0299         return self.__subProcesses
0300     subProcesses = property(subProcesses_,doc='the SubProcesses that have been added to the Process')
0301     def analyzers_(self):
0302         """returns a dict of the analyzers that have been added to the Process"""
0303         return DictTypes.FixedKeysDict(self.__analyzers)
0304     analyzers = property(analyzers_,doc="dictionary containing the analyzers for the process")
0305     def outputModules_(self):
0306         """returns a dict of the output modules that have been added to the Process"""
0307         return DictTypes.FixedKeysDict(self.__outputmodules)
0308     outputModules = property(outputModules_,doc="dictionary containing the output_modules for the process")
0309     def paths_(self):
0310         """returns a dict of the paths that have been added to the Process"""
0311         return DictTypes.SortedAndFixedKeysDict(self.__paths)
0312     paths = property(paths_,doc="dictionary containing the paths for the process")
0313     def endpaths_(self):
0314         """returns a dict of the endpaths that have been added to the Process"""
0315         return DictTypes.SortedAndFixedKeysDict(self.__endpaths)
0316     endpaths = property(endpaths_,doc="dictionary containing the endpaths for the process")
0317     def finalpaths_(self):
0318         """returns a dict of the finalpaths that have been added to the Process"""
0319         return DictTypes.SortedAndFixedKeysDict(self.__finalpaths)
0320     finalpaths = property(finalpaths_,doc="dictionary containing the finalpaths for the process")
0321     def sequences_(self):
0322         """returns a dict of the sequences that have been added to the Process"""
0323         return DictTypes.FixedKeysDict(self.__sequences)
0324     sequences = property(sequences_,doc="dictionary containing the sequences for the process")
0325     def tasks_(self):
0326         """returns a dict of the tasks that have been added to the Process"""
0327         return DictTypes.FixedKeysDict(self.__tasks)
0328     tasks = property(tasks_,doc="dictionary containing the tasks for the process")
0329     def conditionaltasks_(self):
0330         """returns a dict of the conditionaltasks that have been added to the Process"""
0331         return DictTypes.FixedKeysDict(self.__conditionaltasks)
0332     conditionaltasks = property(conditionaltasks_,doc="dictionary containing the conditionatasks for the process")
0333     def schedule_(self):
0334         """returns the schedule that has been added to the Process or None if none have been added"""
0335         return self.__schedule
0336     def setPartialSchedule_(self,sch,label):
0337         if label == "schedule":
0338             self.setSchedule_(sch)
0339         else:
0340             self._place(label, sch, self.__partialschedules)
0341     def setSchedule_(self,sch):
0342         # See if every path and endpath has been inserted into the process
0343         index = 0
0344         try:
0345             for p in sch:
0346                 p.label_()
0347                 index +=1
0348         except:
0349             raise RuntimeError("The path at index "+str(index)+" in the Schedule was not attached to the process.")
0350         self.__dict__['_Process__schedule'] = sch
0351     schedule = property(schedule_,setSchedule_,doc='the schedule or None if not set')
0352     def services_(self):
0353         """returns a dict of the services that have been added to the Process"""
0354         return DictTypes.FixedKeysDict(self.__services)
0355     services = property(services_,doc="dictionary containing the services for the process")
0356     def processAccelerators_(self):
0357         """returns a dict of the ProcessAccelerators that have been added to the Process"""
0358         return DictTypes.FixedKeysDict(self.__accelerators)
0359     processAccelerators = property(processAccelerators_,doc="dictionary containing the ProcessAccelerators for the process")
0360     def es_producers_(self):
0361         """returns a dict of the esproducers that have been added to the Process"""
0362         return DictTypes.FixedKeysDict(self.__esproducers)
0363     es_producers = property(es_producers_,doc="dictionary containing the es_producers for the process")
0364     def es_sources_(self):
0365         """returns a the es_sources that have been added to the Process"""
0366         return DictTypes.FixedKeysDict(self.__essources)
0367     es_sources = property(es_sources_,doc="dictionary containing the es_sources for the process")
0368     def es_prefers_(self):
0369         """returns a dict of the es_prefers that have been added to the Process"""
0370         return DictTypes.FixedKeysDict(self.__esprefers)
0371     es_prefers = property(es_prefers_,doc="dictionary containing the es_prefers for the process")
0372     def aliases_(self):
0373         """returns a dict of the aliases that have been added to the Process"""
0374         return DictTypes.FixedKeysDict(self.__aliases)
0375     aliases = property(aliases_,doc="dictionary containing the aliases for the process")
0376     def psets_(self):
0377         """returns a dict of the PSets that have been added to the Process"""
0378         return DictTypes.FixedKeysDict(self.__psets)
0379     psets = property(psets_,doc="dictionary containing the PSets for the process")
0380     def vpsets_(self):
0381         """returns a dict of the VPSets that have been added to the Process"""
0382         return DictTypes.FixedKeysDict(self.__vpsets)
0383     vpsets = property(vpsets_,doc="dictionary containing the PSets for the process")
0384 
0385     def isUsingModifier(self,mod):
0386         """returns True if the Modifier is in used by this Process"""
0387         if mod._isChosen():
0388             for m in self.__modifiers:
0389                 if m._isOrContains(mod):
0390                     return True
0391         return False
0392 
0393     def __setObjectLabel(self, object, newLabel) :
0394         if not object.hasLabel_() :
0395             object.setLabel(newLabel)
0396             return
0397         if newLabel == object.label_() :
0398             return
0399         if newLabel is None :
0400             object.setLabel(None)
0401             return
0402         if (hasattr(self, object.label_()) and id(getattr(self, object.label_())) == id(object)) :
0403             msg100 = "Attempting to change the label of an attribute of the Process\n"
0404             msg101 = "Old label = "+object.label_()+"  New label = "+newLabel+"\n"
0405             msg102 = "Type = "+str(type(object))+"\n"
0406             msg103 = "Some possible solutions:\n"
0407             msg104 = "  1. Clone modules instead of using simple assignment. Cloning is\n"
0408             msg105 = "  also preferred for other types when possible.\n"
0409             msg106 = "  2. Declare new names starting with an underscore if they are\n"
0410             msg107 = "  for temporaries you do not want propagated into the Process. The\n"
0411             msg108 = "  underscore tells \"from x import *\" and process.load not to import\n"
0412             msg109 = "  the name.\n"
0413             msg110 = "  3. Reorganize so the assigment is not necessary. Giving a second\n"
0414             msg111 = "  name to the same object usually causes confusion and problems.\n"
0415             msg112 = "  4. Compose Sequences: newName = cms.Sequence(oldName)\n"
0416             raise ValueError(msg100+msg101+msg102+msg103+msg104+msg105+msg106+msg107+msg108+msg109+msg110+msg111+msg112)
0417         object.setLabel(None)
0418         object.setLabel(newLabel)
0419 
0420     def __setattr__(self,name,value):
0421         # check if the name is well-formed (only _ and alphanumerics are allowed)
0422         if not name.replace('_','').isalnum():
0423             raise ValueError('The label '+name+' contains forbiden characters')
0424 
0425         if name == 'options':
0426             value = self.__updateOptions(value)
0427         if name == 'maxEvents':
0428             value = self.__updateMaxEvents(value)
0429 
0430         # private variable exempt from all this
0431         if name.startswith('_Process__'):
0432             self.__dict__[name]=value
0433             return
0434         if not isinstance(value,_ConfigureComponent):
0435             raise TypeError("can only assign labels to an object that inherits from '_ConfigureComponent'\n"
0436                             +"an instance of "+str(type(value))+" will not work - requested label is "+name)
0437         if not isinstance(value,_Labelable) and not isinstance(value,Source) and not isinstance(value,Looper) and not isinstance(value,Schedule):
0438             if name == value.type_():
0439                 if hasattr(self,name) and (getattr(self,name)!=value):
0440                     self._replaceInTasks(name, value)
0441                     self._replaceInConditionalTasks(name, value)
0442                 # Only Services get handled here
0443                 self.add_(value)
0444                 return
0445             else:
0446                 raise TypeError("an instance of "+str(type(value))+" can not be assigned the label '"+name+"'.\n"+
0447                                 "Please either use the label '"+value.type_()+" or use the 'add_' method instead.")
0448         #clone the item
0449         if self.__isStrict:
0450             newValue =value.copy()
0451             try:
0452                 newValue._filename = value._filename
0453             except:
0454                 pass
0455             value.setIsFrozen()
0456         else:
0457             newValue =value
0458         if not self._okToPlace(name, value, self.__dict__):
0459             newFile='top level config'
0460             if hasattr(value,'_filename'):
0461                 newFile = value._filename
0462             oldFile='top level config'
0463             oldValue = getattr(self,name)
0464             if hasattr(oldValue,'_filename'):
0465                 oldFile = oldValue._filename
0466             msg = "Trying to override definition of process."+name
0467             msg += "\n new object defined in: "+newFile
0468             msg += "\n existing object defined in: "+oldFile
0469             raise ValueError(msg)
0470         # remove the old object of the name (if there is one)
0471         if hasattr(self,name) and not (getattr(self,name)==newValue):
0472             # Complain if items in sequences or tasks from load() statements have
0473             # degenerate names, but if the user overwrites a name in the
0474             # main config, replace it everywhere
0475             if newValue._isTaskComponent():
0476                 if not self.__InExtendCall:
0477                     self._replaceInTasks(name, newValue)
0478                     self._replaceInConditionalTasks(name, newValue)
0479                     self._replaceInSchedule(name, newValue)
0480                 else:
0481                     if not isinstance(newValue, Task):
0482                         #should check to see if used in task before complaining
0483                         newFile='top level config'
0484                         if hasattr(value,'_filename'):
0485                             newFile = value._filename
0486                         oldFile='top level config'
0487                         oldValue = getattr(self,name)
0488                         if hasattr(oldValue,'_filename'):
0489                             oldFile = oldValue._filename
0490                         msg1 = "Trying to override definition of "+name+" while it is used by the task "
0491                         msg2 = "\n new object defined in: "+newFile
0492                         msg2 += "\n existing object defined in: "+oldFile
0493                         s = self.__findFirstUsingModule(self.tasks,oldValue)
0494                         if s is not None:
0495                             raise ValueError(msg1+s.label_()+msg2)
0496 
0497             if isinstance(newValue, _Sequenceable) or newValue._isTaskComponent() or isinstance(newValue, ConditionalTask):
0498                 if not self.__InExtendCall:
0499                     if isinstance(newValue, ConditionalTask):
0500                         self._replaceInConditionalTasks(name, newValue)
0501                     self._replaceInSequences(name, newValue)
0502                 else:
0503                     #should check to see if used in sequence before complaining
0504                     newFile='top level config'
0505                     if hasattr(value,'_filename'):
0506                         newFile = value._filename
0507                     oldFile='top level config'
0508                     oldValue = getattr(self,name)
0509                     if hasattr(oldValue,'_filename'):
0510                         oldFile = oldValue._filename
0511                     msg1 = "Trying to override definition of "+name+" while it is used by the "
0512                     msg2 = "\n new object defined in: "+newFile
0513                     msg2 += "\n existing object defined in: "+oldFile
0514                     s = self.__findFirstUsingModule(self.sequences,oldValue)
0515                     if s is not None:
0516                         raise ValueError(msg1+"sequence "+s.label_()+msg2)
0517                     s = self.__findFirstUsingModule(self.paths,oldValue)
0518                     if s is not None:
0519                         raise ValueError(msg1+"path "+s.label_()+msg2)
0520                     s = self.__findFirstUsingModule(self.endpaths,oldValue)
0521                     if s is not None:
0522                         raise ValueError(msg1+"endpath "+s.label_()+msg2)
0523                     s = self.__findFirstUsingModule(self.finalpaths,oldValue)
0524                     if s is not None:
0525                         raise ValueError(msg1+"finalpath "+s.label_()+msg2)
0526 
0527             # In case of EDAlias, raise Exception always to avoid surprises
0528             if isinstance(newValue, EDAlias):
0529                 oldValue = getattr(self, name)
0530                 #should check to see if used in task/sequence before complaining
0531                 newFile='top level config'
0532                 if hasattr(value,'_filename'):
0533                     newFile = value._filename
0534                 oldFile='top level config'
0535                 if hasattr(oldValue,'_filename'):
0536                     oldFile = oldValue._filename
0537                 msg1 = "Trying to override definition of "+name+" with an EDAlias while it is used by the "
0538                 msg2 = "\n new object defined in: "+newFile
0539                 msg2 += "\n existing object defined in: "+oldFile
0540                 s = self.__findFirstUsingModule(self.tasks,oldValue)
0541                 if s is not None:
0542                     raise ValueError(msg1+"task "+s.label_()+msg2)
0543                 s = self.__findFirstUsingModule(self.sequences,oldValue)
0544                 if s is not None:
0545                     raise ValueError(msg1+"sequence "+s.label_()+msg2)
0546                 s = self.__findFirstUsingModule(self.paths,oldValue)
0547                 if s is not None:
0548                     raise ValueError(msg1+"path "+s.label_()+msg2)
0549                 s = self.__findFirstUsingModule(self.endpaths,oldValue)
0550                 if s is not None:
0551                     raise ValueError(msg1+"endpath "+s.label_()+msg2)
0552                 s = self.__findFirstUsingModule(self.finalpaths,oldValue)
0553                 if s is not None:
0554                     raise ValueError(msg1+"finalpath "+s.label_()+msg2)
0555 
0556             if not self.__InExtendCall and (Schedule._itemIsValid(newValue) or isinstance(newValue, Task)):
0557                 self._replaceInScheduleDirectly(name, newValue)
0558 
0559             self._delattrFromSetattr(name)
0560         self.__dict__[name]=newValue
0561         if isinstance(newValue,_Labelable):
0562             self.__setObjectLabel(newValue, name)
0563             self._cloneToObjectDict[id(value)] = newValue
0564             self._cloneToObjectDict[id(newValue)] = newValue
0565         #now put in proper bucket
0566         newValue._place(name,self)
0567     def __findFirstUsingModule(self, seqsOrTasks, mod):
0568         """Given a container of sequences or tasks, find the first sequence or task
0569         containing mod and return it. If none is found, return None"""
0570         from FWCore.ParameterSet.SequenceTypes import ModuleNodeVisitor
0571         l = list()
0572         for seqOrTask in seqsOrTasks.values():
0573             l[:] = []
0574             v = ModuleNodeVisitor(l)
0575             seqOrTask.visit(v)
0576             if mod in l:
0577                 return seqOrTask
0578         return None
0579 
0580     def _delHelper(self,name):
0581         if not hasattr(self,name):
0582             raise KeyError('process does not know about '+name)
0583         elif name.startswith('_Process__'):
0584             raise ValueError('this attribute cannot be deleted')
0585 
0586         # we have to remove it from all dictionaries/registries
0587         dicts = [item for item in self.__dict__.values() if (isinstance(item, dict) or isinstance(item, DictTypes.SortedKeysDict))]
0588         for reg in dicts:
0589             if name in reg: del reg[name]
0590         # if it was a labelable object, the label needs to be removed
0591         obj = getattr(self,name)
0592         if isinstance(obj,_Labelable):
0593             obj.setLabel(None)
0594         if isinstance(obj,Service):
0595             obj._inProcess = False
0596 
0597     def __delattr__(self,name):
0598         self._delHelper(name)
0599         obj = getattr(self,name)
0600         if not obj is None:
0601             if not isinstance(obj, Sequence) and not isinstance(obj, Task) and not isinstance(obj,ConditionalTask):
0602                 # For modules, ES modules and services we can also remove
0603                 # the deleted object from Sequences, Paths, EndPaths, and
0604                 # Tasks. Note that for Sequences and Tasks that cannot be done
0605                 # reliably as the places where the Sequence or Task was used
0606                 # might have been expanded so we do not even try. We considered
0607                 # raising an exception if a Sequences or Task was explicitly
0608                 # deleted, but did not because when done carefully deletion
0609                 # is sometimes OK (for example in the prune function where it
0610                 # has been checked that the deleted Sequence is not used).
0611                 if obj._isTaskComponent():
0612                     self._replaceInTasks(name, None)
0613                     self._replaceInConditionalTasks(name, None)
0614                     self._replaceInSchedule(name, None)
0615                 if isinstance(obj, _Sequenceable) or obj._isTaskComponent():
0616                     self._replaceInSequences(name, None)
0617                 if Schedule._itemIsValid(obj) or isinstance(obj, Task):
0618                     self._replaceInScheduleDirectly(name, None)
0619         # now remove it from the process itself
0620         try:
0621             del self.__dict__[name]
0622         except:
0623             pass
0624 
0625     def _delattrFromSetattr(self,name):
0626         """Similar to __delattr__ but we need different behavior when called from __setattr__"""
0627         self._delHelper(name)
0628         # now remove it from the process itself
0629         try:
0630             del self.__dict__[name]
0631         except:
0632             pass
0633 
0634     def add_(self,value):
0635         """Allows addition of components that do not have to have a label, e.g. Services"""
0636         if not isinstance(value,_ConfigureComponent):
0637             raise TypeError
0638         if not isinstance(value,_Unlabelable):
0639             raise TypeError
0640         #clone the item
0641         if self.__isStrict:
0642             newValue =value.copy()
0643             value.setIsFrozen()
0644         else:
0645             newValue =value
0646         newValue._place('',self)
0647 
0648     def _okToPlace(self, name, mod, d):
0649         if not self.__InExtendCall:
0650             # if going
0651             return True
0652         elif not self.__isStrict:
0653             return True
0654         elif name in d:
0655             # if there's an old copy, and the new one
0656             # hasn't been modified, we're done.  Still
0657             # not quite safe if something has been defined twice.
0658             #  Need to add checks
0659             if mod._isModified:
0660                 if d[name]._isModified:
0661                     return False
0662                 else:
0663                     return True
0664             else:
0665                 return True
0666         else:
0667             return True
0668 
0669     def _place(self, name, mod, d):
0670         if self._okToPlace(name, mod, d):
0671             if self.__isStrict and isinstance(mod, _ModuleSequenceType):
0672                 d[name] = mod._postProcessFixup(self._cloneToObjectDict)
0673             else:
0674                 d[name] = mod
0675             if isinstance(mod,_Labelable):
0676                 self.__setObjectLabel(mod, name)
0677     def _placeOutputModule(self,name,mod):
0678         self._place(name, mod, self.__outputmodules)
0679     def _placeProducer(self,name,mod):
0680         self._place(name, mod, self.__producers)
0681     def _placeSwitchProducer(self,name,mod):
0682         self._place(name, mod, self.__switchproducers)
0683     def _placeFilter(self,name,mod):
0684         self._place(name, mod, self.__filters)
0685     def _placeAnalyzer(self,name,mod):
0686         self._place(name, mod, self.__analyzers)
0687     def _placePath(self,name,mod):
0688         self._validateSequence(mod, name)
0689         try:
0690             self._place(name, mod, self.__paths)
0691         except ModuleCloneError as msg:
0692             context = format_outerframe(4)
0693             raise Exception("%sThe module %s in path %s is unknown to the process %s." %(context, msg, name, self._Process__name))
0694     def _placeEndPath(self,name,mod):
0695         self._validateSequence(mod, name)
0696         try:
0697             self._place(name, mod, self.__endpaths)
0698         except ModuleCloneError as msg:
0699             context = format_outerframe(4)
0700             raise Exception("%sThe module %s in endpath %s is unknown to the process %s." %(context, msg, name, self._Process__name))
0701     def _placeFinalPath(self,name,mod):
0702         self._validateSequence(mod, name)
0703         try:
0704             self._place(name, mod, self.__finalpaths)
0705         except ModuleCloneError as msg:
0706             context = format_outerframe(4)
0707             raise Exception("%sThe module %s in finalpath %s is unknown to the process %s." %(context, msg, name, self._Process__name))
0708     def _placeSequence(self,name,mod):
0709         self._validateSequence(mod, name)
0710         self._place(name, mod, self.__sequences)
0711     def _placeESProducer(self,name,mod):
0712         self._place(name, mod, self.__esproducers)
0713     def _placeESPrefer(self,name,mod):
0714         self._place(name, mod, self.__esprefers)
0715     def _placeESSource(self,name,mod):
0716         self._place(name, mod, self.__essources)
0717     def _placeTask(self,name,task):
0718         self._validateTask(task, name)
0719         self._place(name, task, self.__tasks)
0720     def _placeConditionalTask(self,name,task):
0721         self._validateConditionalTask(task, name)
0722         self._place(name, task, self.__conditionaltasks)
0723     def _placeAlias(self,name,mod):
0724         self._place(name, mod, self.__aliases)
0725     def _placePSet(self,name,mod):
0726         self._place(name, mod, self.__psets)
0727     def _placeVPSet(self,name,mod):
0728         self._place(name, mod, self.__vpsets)
0729     def _placeSource(self,name,mod):
0730         """Allow the source to be referenced by 'source' or by type name"""
0731         if name != 'source':
0732             raise ValueError("The label '"+name+"' can not be used for a Source.  Only 'source' is allowed.")
0733         if self.__dict__['_Process__source'] is not None :
0734             del self.__dict__[self.__dict__['_Process__source'].type_()]
0735         self.__dict__['_Process__source'] = mod
0736         self.__dict__[mod.type_()] = mod
0737     def _placeLooper(self,name,mod):
0738         if name != 'looper':
0739             raise ValueError("The label '"+name+"' can not be used for a Looper.  Only 'looper' is allowed.")
0740         self.__dict__['_Process__looper'] = mod
0741         self.__dict__[mod.type_()] = mod
0742     def _placeSubProcess(self,name,mod):
0743         self.__dict__['_Process__subProcess'] = mod
0744         self.__dict__[mod.type_()] = mod
0745     def addSubProcess(self,mod):
0746         self.__subProcesses.append(mod)
0747     def _placeService(self,typeName,mod):
0748         self._place(typeName, mod, self.__services)
0749         if typeName in self.__dict__:
0750             self.__dict__[typeName]._inProcess = False
0751         self.__dict__[typeName]=mod
0752     def _placeAccelerator(self,typeName,mod):
0753         self._place(typeName, mod, self.__accelerators)
0754         self.__dict__[typeName]=mod
0755     def load(self, moduleName):
0756         moduleName = moduleName.replace("/",".")
0757         module = __import__(moduleName)
0758         self.extend(sys.modules[moduleName])
0759     def extend(self,other,items=()):
0760         """Look in other and find types that we can use"""
0761         # enable explicit check to avoid overwriting of existing objects
0762         self.__dict__['_Process__InExtendCall'] = True
0763 
0764         seqs = dict()
0765         tasksToAttach = dict()
0766         mods = []
0767         for name in dir(other):
0768             #'from XX import *' ignores these, and so should we.
0769             if name.startswith('_'):
0770                 continue
0771             item = getattr(other,name)
0772             if name == "source" or name == "looper":
0773                 # In these cases 'item' could be None if the specific object was not defined
0774                 if item is not None:
0775                     self.__setattr__(name,item)
0776             elif isinstance(item,_ModuleSequenceType):
0777                 seqs[name]=item
0778             elif isinstance(item,Task) or isinstance(item, ConditionalTask):
0779                 tasksToAttach[name] = item
0780             elif isinstance(item,_Labelable):
0781                 self.__setattr__(name,item)
0782                 if not item.hasLabel_() :
0783                     item.setLabel(name)
0784             elif isinstance(item,Schedule):
0785                 self.__setattr__(name,item)
0786             elif isinstance(item,_Unlabelable):
0787                 self.add_(item)
0788             elif isinstance(item,ProcessModifier):
0789                 mods.append(item)
0790             elif isinstance(item,ProcessFragment):
0791                 self.extend(item)
0792 
0793         #now create a sequence that uses the newly made items
0794         for name,seq in seqs.items():
0795             if id(seq) not in self._cloneToObjectDict:
0796                 self.__setattr__(name,seq)
0797             else:
0798                 newSeq = self._cloneToObjectDict[id(seq)]
0799                 self.__dict__[name]=newSeq
0800                 self.__setObjectLabel(newSeq, name)
0801                 #now put in proper bucket
0802                 newSeq._place(name,self)
0803 
0804         for name, task in tasksToAttach.items():
0805             self.__setattr__(name, task)
0806 
0807         #apply modifiers now that all names have been added
0808         for item in mods:
0809             item.apply(self)
0810 
0811         self.__dict__['_Process__InExtendCall'] = False
0812 
0813     def _dumpConfigNamedList(self,items,typeName,options):
0814         returnValue = ''
0815         for name,item in items:
0816             returnValue +=options.indentation()+typeName+' '+name+' = '+item.dumpConfig(options)
0817         return returnValue
0818 
0819     def _dumpConfigUnnamedList(self,items,typeName,options):
0820         returnValue = ''
0821         for name,item in items:
0822             returnValue +=options.indentation()+typeName+' = '+item.dumpConfig(options)
0823         return returnValue
0824 
0825     def _dumpConfigOptionallyNamedList(self,items,typeName,options):
0826         returnValue = ''
0827         for name,item in items:
0828             if name == item.type_():
0829                 name = ''
0830             returnValue +=options.indentation()+typeName+' '+name+' = '+item.dumpConfig(options)
0831         return returnValue
0832 
0833     def dumpConfig(self, options=PrintOptions()):
0834         """return a string containing the equivalent process defined using the old configuration language"""
0835         config = "process "+self.__name+" = {\n"
0836         options.indent()
0837         if self.source_():
0838             config += options.indentation()+"source = "+self.source_().dumpConfig(options)
0839         if self.looper_():
0840             config += options.indentation()+"looper = "+self.looper_().dumpConfig(options)
0841 
0842         config+=self._dumpConfigNamedList(self.subProcesses_(),
0843                                   'subProcess',
0844                                   options)
0845         config+=self._dumpConfigNamedList(self.producers_().items(),
0846                                   'module',
0847                                   options)
0848         config+=self._dumpConfigNamedList(self.switchProducers_().items(),
0849                                   'module',
0850                                   options)
0851         config+=self._dumpConfigNamedList(self.filters_().items(),
0852                                   'module',
0853                                   options)
0854         config+=self._dumpConfigNamedList(self.analyzers_().items(),
0855                                   'module',
0856                                   options)
0857         config+=self._dumpConfigNamedList(self.outputModules_().items(),
0858                                   'module',
0859                                   options)
0860         config+=self._dumpConfigNamedList(self.sequences_().items(),
0861                                   'sequence',
0862                                   options)
0863         config+=self._dumpConfigNamedList(self.paths_().items(),
0864                                   'path',
0865                                   options)
0866         config+=self._dumpConfigNamedList(self.endpaths_().items(),
0867                                   'endpath',
0868                                   options)
0869         config+=self._dumpConfigNamedList(self.finalpaths_().items(),
0870                                   'finalpath',
0871                                   options)
0872         config+=self._dumpConfigUnnamedList(self.services_().items(),
0873                                   'service',
0874                                   options)
0875         config+=self._dumpConfigNamedList(self.aliases_().items(),
0876                                   'alias',
0877                                   options)
0878         config+=self._dumpConfigOptionallyNamedList(
0879             self.es_producers_().items(),
0880             'es_module',
0881             options)
0882         config+=self._dumpConfigOptionallyNamedList(
0883             self.es_sources_().items(),
0884             'es_source',
0885             options)
0886         config += self._dumpConfigESPrefers(options)
0887         for name,item in self.psets.items():
0888             config +=options.indentation()+item.configTypeName()+' '+name+' = '+item.configValue(options)
0889         for name,item in self.vpsets.items():
0890             config +=options.indentation()+'VPSet '+name+' = '+item.configValue(options)
0891         if self.schedule:
0892             pathNames = [p.label_() for p in self.schedule]
0893             config +=options.indentation()+'schedule = {'+','.join(pathNames)+'}\n'
0894 
0895 #        config+=self._dumpConfigNamedList(self.vpsets.items(),
0896 #                                  'VPSet',
0897 #                                  options)
0898         config += "}\n"
0899         options.unindent()
0900         return config
0901 
0902     def _dumpConfigESPrefers(self, options):
0903         result = ''
0904         for item in self.es_prefers_().values():
0905             result +=options.indentation()+'es_prefer '+item.targetLabel_()+' = '+item.dumpConfig(options)
0906         return result
0907 
0908     def _dumpPythonSubProcesses(self, l, options):
0909         returnValue = ''
0910         for item in l:
0911             returnValue += item.dumpPython(options)+'\n\n'
0912         return returnValue
0913 
0914     def _dumpPythonList(self, d, options):
0915         returnValue = ''
0916         if isinstance(d, DictTypes.SortedKeysDict):
0917             for name,item in d.items():
0918                 returnValue +='process.'+name+' = '+item.dumpPython(options)+'\n\n'
0919         else:
0920             for name,item in sorted(d.items()):
0921                 returnValue +='process.'+name+' = '+item.dumpPython(options)+'\n\n'
0922         return returnValue
0923 
0924     def _splitPythonList(self, subfolder, d, options):
0925         parts = DictTypes.SortedKeysDict()
0926         for name, item in d.items() if isinstance(d, DictTypes.SortedKeysDict) else sorted(d.items()):
0927             code = ''
0928             dependencies = item.directDependencies()
0929             for module_subfolder, module in dependencies:
0930                 module = module + '_cfi'
0931                 if options.useSubdirectories and module_subfolder:
0932                     module = module_subfolder + '.' + module
0933                 if options.targetDirectory is not None:
0934                     if options.useSubdirectories and subfolder:
0935                       module = '..' + module
0936                     else:
0937                       module = '.' + module
0938                 code += 'from ' + module + ' import *\n'
0939             if dependencies:
0940                 code += '\n'
0941             code += name + ' = ' + item.dumpPython(options)
0942             parts[name] = subfolder, code
0943         return parts
0944 
0945     def _validateSequence(self, sequence, label):
0946         # See if every module has been inserted into the process
0947         try:
0948             l = set()
0949             visitor = NodeNameVisitor(l)
0950             sequence.visit(visitor)
0951         except Exception as e:
0952             raise RuntimeError("An entry in sequence {} has no label\n  Seen entries: {}\n  Error: {}".format(label, l, e))
0953 
0954     def _validateTask(self, task, label):
0955         # See if every module and service has been inserted into the process
0956         try:
0957             l = set()
0958             visitor = NodeNameVisitor(l)
0959             task.visit(visitor)
0960         except:
0961             raise RuntimeError("An entry in task " + label + ' has not been attached to the process')
0962     def _validateConditionalTask(self, task, label):
0963         # See if every module and service has been inserted into the process
0964         try:
0965             l = set()
0966             visitor = NodeNameVisitor(l)
0967             task.visit(visitor)
0968         except:
0969             raise RuntimeError("An entry in task " + label + ' has not been attached to the process')
0970 
0971     def _itemsInDependencyOrder(self, processDictionaryOfItems):
0972         # The items can be Sequences or Tasks and the input
0973         # argument should either be the dictionary of sequences
0974         # or the dictionary of tasks from the process.
0975 
0976         returnValue=DictTypes.SortedKeysDict()
0977 
0978         # For each item, see what other items it depends upon
0979         # For our purpose here, an item depends on the items it contains.
0980         dependencies = {}
0981         for label,item in processDictionaryOfItems.items():
0982             containedItems = []
0983             if isinstance(item, Task):
0984                 v = TaskVisitor(containedItems)
0985             elif isinstance(item, ConditionalTask):
0986                 v = ConditionalTaskVisitor(containedItems)
0987             else:
0988                 v = SequenceVisitor(containedItems)
0989             try:
0990                 item.visit(v)
0991             except RuntimeError:
0992                 if isinstance(item, Task):
0993                     raise RuntimeError("Failed in a Task visitor. Probably " \
0994                                        "a circular dependency discovered in Task with label " + label)
0995                 elif isinstance(item, ConditionalTask):
0996                     raise RuntimeError("Failed in a ConditionalTask visitor. Probably " \
0997                                        "a circular dependency discovered in ConditionalTask with label " + label)
0998                 else:
0999                     raise RuntimeError("Failed in a Sequence visitor. Probably a " \
1000                                        "circular dependency discovered in Sequence with label " + label)
1001             for containedItem in containedItems:
1002                 # Check for items that both have labels and are not in the process.
1003                 # This should not normally occur unless someone explicitly assigns a
1004                 # label without putting the item in the process (which should not ever
1005                 # be done). We check here because this problem could cause the code
1006                 # in the 'while' loop below to go into an infinite loop.
1007                 if containedItem.hasLabel_():
1008                     testItem = processDictionaryOfItems.get(containedItem.label_())
1009                     if testItem is None or containedItem != testItem:
1010                         if isinstance(item, Task):
1011                             raise RuntimeError("Task has a label, but using its label to get an attribute" \
1012                                                " from the process yields a different object or None\n"+
1013                                                "label = " + containedItem.label_())
1014                         if isinstance(item, ConditionalTask):
1015                             raise RuntimeError("ConditionalTask has a label, but using its label to get an attribute" \
1016                                                " from the process yields a different object or None\n"+
1017                                                "label = " + containedItem.label_())
1018                         else:
1019                             raise RuntimeError("Sequence has a label, but using its label to get an attribute" \
1020                                                " from the process yields a different object or None\n"+
1021                                                "label = " + containedItem.label_())
1022             dependencies[label]=[dep.label_() for dep in containedItems if dep.hasLabel_()]
1023 
1024         # keep looping until we get rid of all dependencies
1025         while dependencies:
1026             oldDeps = dict(dependencies)
1027             for label,deps in oldDeps.items():
1028                 if len(deps)==0:
1029                     returnValue[label]=processDictionaryOfItems[label]
1030                     #remove this as a dependency for all other tasks
1031                     del dependencies[label]
1032                     for lb2,deps2 in dependencies.items():
1033                         while deps2.count(label):
1034                             deps2.remove(label)
1035         return returnValue
1036 
1037     def _dumpPython(self, d, options):
1038         result = ''
1039         for name, value in sorted(d.items()):
1040             result += value.dumpPythonAs(name,options)+'\n'
1041         return result
1042 
1043     def _splitPython(self, subfolder, d, options):
1044         result = {}
1045         for name, value in sorted(d.items()):
1046             result[name] = subfolder, value.dumpPythonAs(name, options) + '\n'
1047         return result
1048 
1049     def dumpPython(self, options=PrintOptions()):
1050         """return a string containing the equivalent process defined using python"""
1051         specialImportRegistry._reset()
1052         header = "import FWCore.ParameterSet.Config as cms"
1053         result = "process = cms.Process(\""+self.__name+"\")\n\n"
1054         if self.source_():
1055             result += "process.source = "+self.source_().dumpPython(options)
1056         if self.looper_():
1057             result += "process.looper = "+self.looper_().dumpPython()
1058         result+=self._dumpPythonList(self.psets, options)
1059         result+=self._dumpPythonList(self.vpsets, options)
1060         result+=self._dumpPythonSubProcesses(self.subProcesses_(), options)
1061         result+=self._dumpPythonList(self.producers_(), options)
1062         result+=self._dumpPythonList(self.switchProducers_(), options)
1063         result+=self._dumpPythonList(self.filters_() , options)
1064         result+=self._dumpPythonList(self.analyzers_(), options)
1065         result+=self._dumpPythonList(self.outputModules_(), options)
1066         result+=self._dumpPythonList(self.services_(), options)
1067         result+=self._dumpPythonList(self.processAccelerators_(), options)
1068         result+=self._dumpPythonList(self.es_producers_(), options)
1069         result+=self._dumpPythonList(self.es_sources_(), options)
1070         result+=self._dumpPython(self.es_prefers_(), options)
1071         result+=self._dumpPythonList(self._itemsInDependencyOrder(self.tasks), options)
1072         result+=self._dumpPythonList(self._itemsInDependencyOrder(self.conditionaltasks), options)
1073         result+=self._dumpPythonList(self._itemsInDependencyOrder(self.sequences), options)
1074         result+=self._dumpPythonList(self.paths_(), options)
1075         result+=self._dumpPythonList(self.endpaths_(), options)
1076         result+=self._dumpPythonList(self.finalpaths_(), options)
1077         result+=self._dumpPythonList(self.aliases_(), options)
1078         if not self.schedule_() == None:
1079             result += 'process.schedule = ' + self.schedule.dumpPython(options)
1080         imports = specialImportRegistry.getSpecialImports()
1081         if len(imports) > 0:
1082             header += "\n" + "\n".join(imports)
1083         header += "\n\n"
1084         return header+result
1085 
1086     def splitPython(self, options = PrintOptions()):
1087         """return a map of file names to python configuration fragments"""
1088         specialImportRegistry._reset()
1089         # extract individual fragments
1090         options.isCfg = False
1091         header = "import FWCore.ParameterSet.Config as cms"
1092         result = ''
1093         parts = {}
1094         files = {}
1095 
1096         result = 'process = cms.Process("' + self.__name + '")\n\n'
1097 
1098         if self.source_():
1099             parts['source'] = (None, 'source = ' + self.source_().dumpPython(options))
1100 
1101         if self.looper_():
1102             parts['looper'] = (None, 'looper = ' + self.looper_().dumpPython())
1103 
1104         parts.update(self._splitPythonList('psets', self.psets, options))
1105         parts.update(self._splitPythonList('psets', self.vpsets, options))
1106         # FIXME
1107         #parts.update(self._splitPythonSubProcesses(self.subProcesses_(), options))
1108         if len(self.subProcesses_()):
1109           sys.stderr.write("error: subprocesses are not supported yet\n\n")
1110         parts.update(self._splitPythonList('modules', self.producers_(), options))
1111         parts.update(self._splitPythonList('modules', self.switchProducers_(), options))
1112         parts.update(self._splitPythonList('modules', self.filters_() , options))
1113         parts.update(self._splitPythonList('modules', self.analyzers_(), options))
1114         parts.update(self._splitPythonList('modules', self.outputModules_(), options))
1115         parts.update(self._splitPythonList('services', self.services_(), options))
1116         parts.update(self._splitPythonList('eventsetup', self.es_producers_(), options))
1117         parts.update(self._splitPythonList('eventsetup', self.es_sources_(), options))
1118         parts.update(self._splitPython('eventsetup', self.es_prefers_(), options))
1119         parts.update(self._splitPythonList('tasks', self._itemsInDependencyOrder(self.tasks), options))
1120         parts.update(self._splitPythonList('sequences', self._itemsInDependencyOrder(self.sequences), options))
1121         parts.update(self._splitPythonList('paths', self.paths_(), options))
1122         parts.update(self._splitPythonList('paths', self.endpaths_(), options))
1123         parts.update(self._splitPythonList('paths', self.finalpaths_(), options))
1124         parts.update(self._splitPythonList('modules', self.aliases_(), options))
1125 
1126         if options.targetDirectory is not None:
1127             files[options.targetDirectory + '/__init__.py'] = ''
1128 
1129         if options.useSubdirectories:
1130           for sub in 'psets', 'modules', 'services', 'eventsetup', 'tasks', 'sequences', 'paths':
1131             if options.targetDirectory is not None:
1132                 sub = options.targetDirectory + '/' + sub
1133             files[sub + '/__init__.py'] = ''
1134 
1135         # case insensitive sort by subfolder and module name
1136         parts = sorted(parts.items(), key = lambda nsc: (nsc[1][0].lower() if nsc[1][0] else '', nsc[0].lower()))
1137 
1138         for (name, (subfolder, code)) in parts:
1139             filename = name + '_cfi'
1140             if options.useSubdirectories and subfolder:
1141                 filename = subfolder + '/' + filename
1142             if options.targetDirectory is not None:
1143                 filename = options.targetDirectory + '/' + filename
1144             result += 'process.load("%s")\n' % filename
1145             files[filename + '.py'] = header + '\n\n' + code
1146 
1147         if self.schedule_() is not None:
1148             options.isCfg = True
1149             result += '\nprocess.schedule = ' + self.schedule.dumpPython(options)
1150 
1151         imports = specialImportRegistry.getSpecialImports()
1152         if len(imports) > 0:
1153             header += '\n' + '\n'.join(imports)
1154         files['-'] = header + '\n\n' + result
1155         return files
1156 
1157     def _replaceInSequences(self, label, new):
1158         old = getattr(self,label)
1159         #TODO - replace by iterator concatenation
1160         #to ovoid dependency problems between sequences, first modify
1161         # process known sequences to do a non-recursive change. Then do
1162         # a recursive change to get cases where a sub-sequence unknown to
1163         # the process has the item to be replaced
1164         for sequenceable in self.sequences.values():
1165             sequenceable._replaceIfHeldDirectly(old,new)
1166         for sequenceable in self.sequences.values():
1167             sequenceable.replace(old,new)
1168         for sequenceable in self.paths.values():
1169             sequenceable.replace(old,new)
1170         for sequenceable in self.endpaths.values():
1171             sequenceable.replace(old,new)
1172         for sequenceable in self.finalpaths.values():
1173             sequenceable.replace(old,new)
1174     def _replaceInTasks(self, label, new):
1175         old = getattr(self,label)
1176         for task in self.tasks.values():
1177             task.replace(old, new)
1178     def _replaceInConditionalTasks(self, label, new):
1179         old = getattr(self,label)
1180         for task in self.conditionaltasks.values():
1181             task.replace(old, new)
1182     def _replaceInSchedule(self, label, new):
1183         if self.schedule_() == None:
1184             return
1185         old = getattr(self,label)
1186         for task in self.schedule_()._tasks:
1187             task.replace(old, new)
1188     def _replaceInScheduleDirectly(self, label, new):
1189         if self.schedule_() == None:
1190             return
1191         old = getattr(self,label)
1192         self.schedule_()._replaceIfHeldDirectly(old, new)
1193     def globalReplace(self,label,new):
1194         """ Replace the item with label 'label' by object 'new' in the process and all sequences/paths/tasks"""
1195         if not hasattr(self,label):
1196             raise LookupError("process has no item of label "+label)
1197         setattr(self,label,new)
1198     def _insertInto(self, parameterSet, itemDict):
1199         for name,value in itemDict.items():
1200             value.insertInto(parameterSet, name)
1201     def _insertOneInto(self, parameterSet, label, item, tracked):
1202         vitems = []
1203         if not item == None:
1204             newlabel = item.nameInProcessDesc_(label)
1205             vitems = [newlabel]
1206             item.insertInto(parameterSet, newlabel)
1207         parameterSet.addVString(tracked, label, vitems)
1208     def _insertManyInto(self, parameterSet, label, itemDict, tracked):
1209         l = []
1210         for name,value in itemDict.items():
1211             value.appendToProcessDescList_(l, name)
1212             value.insertInto(parameterSet, name)
1213         # alphabetical order is easier to compare with old language
1214         l.sort()
1215         parameterSet.addVString(tracked, label, l)
1216     def _insertSwitchProducersInto(self, parameterSet, labelModules, labelAliases, itemDict, tracked):
1217         modules = parameterSet.getVString(tracked, labelModules)
1218         aliases = parameterSet.getVString(tracked, labelAliases)
1219         accelerators = parameterSet.getVString(False, "@selected_accelerators")
1220         for name,value in itemDict.items():
1221             value.appendToProcessDescLists_(modules, aliases, name)
1222             value.insertInto(parameterSet, name, accelerators)
1223         modules.sort()
1224         aliases.sort()
1225         parameterSet.addVString(tracked, labelModules, modules)
1226         parameterSet.addVString(tracked, labelAliases, aliases)
1227     def _insertSubProcessesInto(self, parameterSet, label, itemList, tracked):
1228         l = []
1229         subprocs = []
1230         for value in itemList:
1231             name = value.getProcessName()
1232             newLabel = value.nameInProcessDesc_(name)
1233             l.append(newLabel)
1234             pset = value.getSubProcessPSet(parameterSet)
1235             subprocs.append(pset)
1236         # alphabetical order is easier to compare with old language
1237         l.sort()
1238         parameterSet.addVString(tracked, label, l)
1239         parameterSet.addVPSet(False,"subProcesses",subprocs)
1240     def _insertPaths(self, processPSet, nodeVisitor):
1241         scheduledPaths = []
1242         triggerPaths = []
1243         endpaths = []
1244         finalpaths = []
1245         if self.schedule_() == None:
1246             # make one from triggerpaths & endpaths
1247             for name in self.paths_():
1248                 scheduledPaths.append(name)
1249                 triggerPaths.append(name)
1250             for name in self.endpaths_():
1251                 scheduledPaths.append(name)
1252                 endpaths.append(name)
1253             for name in self.finalpaths_():
1254                 finalpaths.append(name)
1255         else:
1256             for path in self.schedule_():
1257                 pathname = path.label_()
1258                 if pathname in self.endpaths_():
1259                     endpaths.append(pathname)
1260                     scheduledPaths.append(pathname)
1261                 elif pathname in self.finalpaths_():
1262                     finalpaths.append(pathname)
1263                 else:
1264                     scheduledPaths.append(pathname)
1265                     triggerPaths.append(pathname)
1266             for task in self.schedule_()._tasks:
1267                 task.resolve(self.__dict__)
1268                 scheduleTaskValidator = ScheduleTaskValidator()
1269                 task.visit(scheduleTaskValidator)
1270                 task.visit(nodeVisitor)
1271         # consolidate all final_paths into one EndPath
1272         endPathWithFinalPathModulesName ="@finalPath"
1273         finalPathEndPath = EndPath()
1274         if finalpaths:
1275           endpaths.append(endPathWithFinalPathModulesName)
1276           scheduledPaths.append(endPathWithFinalPathModulesName)
1277           finalpathValidator = FinalPathValidator()
1278           modulesOnFinalPath = []
1279           for finalpathname in finalpaths:
1280               iFinalPath = self.finalpaths_()[finalpathname]
1281               iFinalPath.resolve(self.__dict__)
1282               finalpathValidator.setLabel(finalpathname)
1283               iFinalPath.visit(finalpathValidator)
1284               if finalpathValidator.filtersOnFinalpaths or finalpathValidator.producersOnFinalpaths:
1285                   names = [p.label_ for p in finalpathValidator.filtersOnFinalpaths]
1286                   names.extend( [p.label_ for p in finalpathValidator.producersOnFinalpaths])
1287                   raise RuntimeError("FinalPath %s has non OutputModules %s" % (finalpathname, ",".join(names)))
1288               modulesOnFinalPath.extend(iFinalPath.moduleNames())
1289           for m in modulesOnFinalPath:
1290             mod = getattr(self, m)
1291             setattr(mod, "@onFinalPath", untracked.bool(True))
1292             finalPathEndPath += mod
1293             
1294         processPSet.addVString(True, "@end_paths", endpaths)
1295         processPSet.addVString(True, "@paths", scheduledPaths)
1296         # trigger_paths are a little different
1297         p = processPSet.newPSet()
1298         p.addVString(True, "@trigger_paths", triggerPaths)
1299         processPSet.addPSet(True, "@trigger_paths", p)
1300         # add all these paths
1301         pathValidator = PathValidator()
1302         endpathValidator = EndPathValidator()
1303         decoratedList = []
1304         lister = DecoratedNodeNameVisitor(decoratedList)
1305         condTaskModules = []
1306         condTaskVistor = ModuleNodeOnConditionalTaskVisitor(condTaskModules)
1307         pathCompositeVisitor = CompositeVisitor(pathValidator, nodeVisitor, lister, condTaskVistor)
1308         endpathCompositeVisitor = CompositeVisitor(endpathValidator, nodeVisitor, lister)
1309         for triggername in triggerPaths:
1310             iPath = self.paths_()[triggername]
1311             iPath.resolve(self.__dict__)
1312             pathValidator.setLabel(triggername)
1313             lister.initialize()
1314             condTaskModules[:] = []
1315             iPath.visit(pathCompositeVisitor)
1316             if condTaskModules:
1317               decoratedList.append("#")
1318               l = list({x.label_() for x in condTaskModules})
1319               l.sort()
1320               decoratedList.extend(l)
1321               decoratedList.append("@")
1322             iPath.insertInto(processPSet, triggername, decoratedList[:])
1323         for endpathname in endpaths:
1324             if endpathname is not endPathWithFinalPathModulesName:
1325               iEndPath = self.endpaths_()[endpathname]
1326             else:
1327               iEndPath = finalPathEndPath
1328             iEndPath.resolve(self.__dict__)
1329             endpathValidator.setLabel(endpathname)
1330             lister.initialize()
1331             iEndPath.visit(endpathCompositeVisitor)
1332             iEndPath.insertInto(processPSet, endpathname, decoratedList[:])
1333         processPSet.addVString(False, "@filters_on_endpaths", endpathValidator.filtersOnEndpaths)
1334           
1335 
1336     def resolve(self,keepUnresolvedSequencePlaceholders=False):
1337         for x in self.paths.values():
1338             x.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1339         for x in self.endpaths.values():
1340             x.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1341         for x in self.finalpaths.values():
1342             x.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1343         if not self.schedule_() == None:
1344             for task in self.schedule_()._tasks:
1345                 task.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1346 
1347     def prune(self,verbose=False,keepUnresolvedSequencePlaceholders=False):
1348         """ Remove clutter from the process that we think is unnecessary:
1349         tracked PSets, VPSets and unused modules and sequences. If a Schedule has been set, then Paths and EndPaths
1350         not in the schedule will also be removed, along with an modules and sequences used only by
1351         those removed Paths and EndPaths. The keepUnresolvedSequencePlaceholders keeps also unresolved TaskPlaceholders."""
1352 # need to update this to only prune psets not on refToPSets
1353 # but for now, remove the delattr
1354 #        for name in self.psets_():
1355 #            if getattr(self,name).isTracked():
1356 #                delattr(self, name)
1357         for name in self.vpsets_():
1358             delattr(self, name)
1359         #first we need to resolve any SequencePlaceholders being used
1360         self.resolve(keepUnresolvedSequencePlaceholders)
1361         usedModules = set()
1362         unneededPaths = set()
1363         tasks = list()
1364         tv = TaskVisitor(tasks)
1365         if self.schedule_():
1366             usedModules=set(self.schedule_().moduleNames())
1367             #get rid of unused paths
1368             schedNames = set(( x.label_() for x in self.schedule_()))
1369             names = set(self.paths)
1370             names.update(set(self.endpaths))
1371             names.update(set(self.finalpaths))
1372             unneededPaths = names - schedNames
1373             for n in unneededPaths:
1374                 delattr(self,n)
1375             for t in self.schedule_().tasks():
1376                 tv.enter(t)
1377                 t.visit(tv)
1378                 tv.leave(t)
1379         else:
1380             pths = list(self.paths.values())
1381             pths.extend(self.endpaths.values())
1382             pths.extend(self.finalpaths.values())
1383             temp = Schedule(*pths)
1384             usedModules=set(temp.moduleNames())
1385         unneededModules = self._pruneModules(self.producers_(), usedModules)
1386         unneededModules.update(self._pruneModules(self.switchProducers_(), usedModules))
1387         unneededModules.update(self._pruneModules(self.filters_(), usedModules))
1388         unneededModules.update(self._pruneModules(self.analyzers_(), usedModules))
1389         #remove sequences and tasks that do not appear in remaining paths and endpaths
1390         seqs = list()
1391         sv = SequenceVisitor(seqs)
1392         for p in self.paths.values():
1393             p.visit(sv)
1394             p.visit(tv)
1395         for p in self.endpaths.values():
1396             p.visit(sv)
1397             p.visit(tv)
1398         for p in self.finalpaths.values():
1399             p.visit(sv)
1400             p.visit(tv)
1401         def removeUnneeded(seqOrTasks, allSequencesOrTasks):
1402             _keepSet = set(( s for s in seqOrTasks if s.hasLabel_()))
1403             _availableSet = set(allSequencesOrTasks.values())
1404             _unneededSet = _availableSet-_keepSet
1405             _unneededLabels = []
1406             for s in _unneededSet:
1407                 _unneededLabels.append(s.label_())
1408                 delattr(self,s.label_())
1409             return _unneededLabels
1410         unneededSeqLabels = removeUnneeded(seqs, self.sequences)
1411         unneededTaskLabels = removeUnneeded(tasks, self.tasks)
1412         if verbose:
1413             print("prune removed the following:")
1414             print("  modules:"+",".join(unneededModules))
1415             print("  tasks:"+",".join(unneededTaskLabels))
1416             print("  sequences:"+",".join(unneededSeqLabels))
1417             print("  paths/endpaths/finalpaths:"+",".join(unneededPaths))
1418     def _pruneModules(self, d, scheduledNames):
1419         moduleNames = set(d.keys())
1420         junk = moduleNames - scheduledNames
1421         for name in junk:
1422             delattr(self, name)
1423         return junk
1424 
1425     def fillProcessDesc(self, processPSet):
1426         """Used by the framework to convert python to C++ objects"""
1427         class ServiceInjectorAdaptor(object):
1428             def __init__(self,ppset,thelist):
1429                 self.__thelist = thelist
1430                 self.__processPSet = ppset
1431             def addService(self,pset):
1432                 self.__thelist.append(pset)
1433             def newPSet(self):
1434                 return self.__processPSet.newPSet()
1435         #This adaptor is used to 'add' the method 'getTopPSet_'
1436         # to the ProcessDesc and PythonParameterSet C++ classes.
1437         # This method is needed for the PSet refToPSet_ functionality.
1438         class TopLevelPSetAcessorAdaptor(object):
1439             def __init__(self,ppset,process):
1440                 self.__ppset = ppset
1441                 self.__process = process
1442             def __getattr__(self,attr):
1443                 return getattr(self.__ppset,attr)
1444             def getTopPSet_(self,label):
1445                 return getattr(self.__process,label)
1446             def newPSet(self):
1447                 return TopLevelPSetAcessorAdaptor(self.__ppset.newPSet(),self.__process)
1448             def addPSet(self,tracked,name,ppset):
1449                 return self.__ppset.addPSet(tracked,name,self.__extractPSet(ppset))
1450             def addVPSet(self,tracked,name,vpset):
1451                 return self.__ppset.addVPSet(tracked,name,[self.__extractPSet(x) for x in vpset])
1452             def __extractPSet(self,pset):
1453                 if isinstance(pset,TopLevelPSetAcessorAdaptor):
1454                     return pset.__ppset
1455                 return pset
1456 
1457         self.validate()
1458         processPSet.addString(True, "@process_name", self.name_())
1459         self.handleProcessAccelerators(processPSet)
1460         all_modules = self.producers_().copy()
1461         all_modules.update(self.filters_())
1462         all_modules.update(self.analyzers_())
1463         all_modules.update(self.outputModules_())
1464         adaptor = TopLevelPSetAcessorAdaptor(processPSet,self)
1465         self._insertInto(adaptor, self.psets_())
1466         self._insertInto(adaptor, self.vpsets_())
1467         self._insertOneInto(adaptor,  "@all_sources", self.source_(), True)
1468         self._insertOneInto(adaptor,  "@all_loopers", self.looper_(), True)
1469         self._insertSubProcessesInto(adaptor, "@all_subprocesses", self.subProcesses_(), False)
1470         self._insertManyInto(adaptor, "@all_esprefers", self.es_prefers_(), True)
1471         self._insertManyInto(adaptor, "@all_aliases", self.aliases_(), True)
1472         # This will visit all the paths and endpaths that are scheduled to run,
1473         # as well as the Tasks associated to them and the schedule. It remembers
1474         # the modules, ESSources, ESProducers, and services it visits.
1475         nodeVisitor = NodeVisitor()
1476         self._insertPaths(adaptor, nodeVisitor)
1477         all_modules_onTasksOrScheduled = { key:value for key, value in all_modules.items() if value in nodeVisitor.modules }
1478         self._insertManyInto(adaptor, "@all_modules", all_modules_onTasksOrScheduled, True)
1479         all_switches = self.switchProducers_().copy()
1480         all_switches_onTasksOrScheduled = {key:value for key, value in all_switches.items() if value in nodeVisitor.modules }
1481         self._insertSwitchProducersInto(adaptor, "@all_modules", "@all_aliases", all_switches_onTasksOrScheduled, True)
1482         # Same as nodeVisitor except this one visits all the Tasks attached
1483         # to the process.
1484         processNodeVisitor = NodeVisitor()
1485         for pTask in self.tasks.values():
1486             pTask.visit(processNodeVisitor)
1487         esProducersToEnable = {}
1488         for esProducerName, esProducer in self.es_producers_().items():
1489             if esProducer in nodeVisitor.esProducers or not (esProducer in processNodeVisitor.esProducers):
1490                 esProducersToEnable[esProducerName] = esProducer
1491         self._insertManyInto(adaptor, "@all_esmodules", esProducersToEnable, True)
1492         esSourcesToEnable = {}
1493         for esSourceName, esSource in self.es_sources_().items():
1494             if esSource in nodeVisitor.esSources or not (esSource in processNodeVisitor.esSources):
1495                 esSourcesToEnable[esSourceName] = esSource
1496         self._insertManyInto(adaptor, "@all_essources", esSourcesToEnable, True)
1497         #handle services differently
1498         services = []
1499         for serviceName, serviceObject in self.services_().items():
1500             if serviceObject in nodeVisitor.services or not (serviceObject in processNodeVisitor.services):
1501                 serviceObject.insertInto(ServiceInjectorAdaptor(adaptor,services))
1502         adaptor.addVPSet(False,"services",services)
1503         return processPSet
1504 
1505     def validate(self):
1506         # check if there's some input
1507         # Breaks too many unit tests for now
1508         #if self.source_() == None and self.looper_() == None:
1509         #    raise RuntimeError("No input source was found for this process")
1510         pass
1511 
1512     def handleProcessAccelerators(self, parameterSet):
1513         # 'cpu' accelerator is always implicitly there
1514         allAccelerators = set(["cpu"])
1515         availableAccelerators = set(["cpu"])
1516         for acc in self.__dict__['_Process__accelerators'].values():
1517             allAccelerators.update(acc.labels())
1518             availableAccelerators.update(acc.enabledLabels())
1519         availableAccelerators = sorted(list(availableAccelerators))
1520         parameterSet.addVString(False, "@available_accelerators", availableAccelerators)
1521 
1522         # Resolve wildcards
1523         selectedAccelerators = []
1524         if "*" in self.options.accelerators:
1525             if len(self.options.accelerators) >= 2:
1526                 raise ValueError("process.options.accelerators may contain '*' only as the only element, now it has {} elements".format(len(self.options.accelerators)))
1527             selectedAccelerators = availableAccelerators
1528         else:
1529             import fnmatch
1530             resolved = set()
1531             invalid = []
1532             for pattern in self.options.accelerators:
1533                 acc = [a for a in availableAccelerators if fnmatch.fnmatchcase(a, pattern)]
1534                 if len(acc) == 0:
1535                     if not any(fnmatch.fnmatchcase(a, pattern) for a in allAccelerators):
1536                         invalid.append(pattern)
1537                 else:
1538                    resolved.update(acc)
1539             # Sanity check
1540             if len(invalid) != 0:
1541                 raise ValueError("Invalid pattern{} of {} in process.options.accelerators, valid values are {} or a pattern matching to some of them.".format(
1542                     "s" if len(invalid) > 2 else "",
1543                     ",".join(invalid),
1544                     ",".join(sorted(list(allAccelerators)))))
1545             selectedAccelerators = sorted(list(resolved))
1546         parameterSet.addVString(False, "@selected_accelerators", selectedAccelerators)
1547 
1548         # Get and apply module type resolver
1549         moduleTypeResolver = None
1550         moduleTypeResolverPlugin = ""
1551         for acc in self.__dict__['_Process__accelerators'].values():
1552             resolver = acc.moduleTypeResolver(selectedAccelerators)
1553             if resolver is not None:
1554                 if moduleTypeResolver is not None:
1555                     raise RuntimeError("Module type resolver was already set to {} when {} tried to set it to {}. A job can have at most one ProcessAccelerator that sets module type resolver.".format(
1556                         moduleTypeResolver.__class__.__name__,
1557                         acc.__class__.__name__,
1558                         resolver.__class__.__name__))
1559                 moduleTypeResolver = resolver
1560         if moduleTypeResolver is not None:
1561             # Plugin name and its configuration
1562             moduleTypeResolverPlugin = moduleTypeResolver.plugin()
1563 
1564             # Customize modules
1565             for modlist in [self.producers_, self.filters_, self.analyzers_,
1566                             self.es_producers_, self.es_sources_]:
1567                 for module in modlist().values():
1568                     moduleTypeResolver.setModuleVariant(module)
1569 
1570         parameterSet.addString(False, "@module_type_resolver", moduleTypeResolverPlugin)
1571 
1572         # Customize process
1573         wrapped = ProcessForProcessAccelerator(self)
1574         for acc in self.__dict__['_Process__accelerators'].values():
1575             acc.apply(wrapped, selectedAccelerators)
1576 
1577     def prefer(self, esmodule,*args,**kargs):
1578         """Prefer this ES source or producer.  The argument can
1579            either be an object label, e.g.,
1580              process.prefer(process.juicerProducer) (not supported yet)
1581            or a name of an ESSource or ESProducer
1582              process.prefer("juicer")
1583            or a type of unnamed ESSource or ESProducer
1584              process.prefer("JuicerProducer")
1585            In addition, you can pass as a labelled arguments the name of the Record you wish to
1586            prefer where the type passed is a cms.vstring and that vstring can contain the
1587            name of the C++ types in the Record that are being preferred, e.g.,
1588               #prefer all data in record 'OrangeRecord' from 'juicer'
1589               process.prefer("juicer", OrangeRecord=cms.vstring())
1590            or
1591               #prefer only "Orange" data in "OrangeRecord" from "juicer"
1592               process.prefer("juicer", OrangeRecord=cms.vstring("Orange"))
1593            or
1594               #prefer only "Orange" data with label "ExtraPulp" in "OrangeRecord" from "juicer"
1595               ESPrefer("ESJuicerProd", OrangeRecord=cms.vstring("Orange/ExtraPulp"))
1596         """
1597         # see if this refers to a named ESProducer
1598         if isinstance(esmodule, ESSource) or isinstance(esmodule, ESProducer):
1599             raise RuntimeError("Syntax of process.prefer(process.esmodule) not supported yet")
1600         elif self._findPreferred(esmodule, self.es_producers_(),*args,**kargs) or \
1601                 self._findPreferred(esmodule, self.es_sources_(),*args,**kargs):
1602             pass
1603         else:
1604             raise RuntimeError("Cannot resolve prefer for "+repr(esmodule))
1605 
1606     def _findPreferred(self, esname, d,*args,**kargs):
1607         # is esname a name in the dictionary?
1608         if esname in d:
1609             typ = d[esname].type_()
1610             if typ == esname:
1611                 self.__setattr__( esname+"_prefer", ESPrefer(typ,*args,**kargs) )
1612             else:
1613                 self.__setattr__( esname+"_prefer", ESPrefer(typ, esname,*args,**kargs) )
1614             return True
1615         else:
1616             # maybe it's an unnamed ESModule?
1617             found = False
1618             for name, value in d.items():
1619                 if value.type_() == esname:
1620                     if found:
1621                         raise RuntimeError("More than one ES module for "+esname)
1622                     found = True
1623                     self.__setattr__(esname+"_prefer",  ESPrefer(d[esname].type_()) )
1624             return found
1625 
1626 
1627 class ProcessFragment(object):
1628     def __init__(self, process):
1629         if isinstance(process, Process):
1630             self.__process = process
1631         elif isinstance(process, str):
1632             self.__process = Process(process)
1633             #make sure we do not override the defaults
1634             del self.__process.options
1635             del self.__process.maxEvents
1636             del self.__process.maxLuminosityBlocks
1637         else:
1638             raise TypeError('a ProcessFragment can only be constructed from an existig Process or from process name')
1639     def __dir__(self):
1640         return [ x for x in dir(self.__process) if isinstance(getattr(self.__process, x), _ConfigureComponent) ]
1641     def __getattribute__(self, name):
1642         if name == '_ProcessFragment__process':
1643             return object.__getattribute__(self, '_ProcessFragment__process')
1644         else:
1645             return getattr(self.__process, name)
1646     def __setattr__(self, name, value):
1647         if name == '_ProcessFragment__process':
1648             object.__setattr__(self, name, value)
1649         else:
1650             setattr(self.__process, name, value)
1651     def __delattr__(self, name):
1652         if name == '_ProcessFragment__process':
1653             pass
1654         else:
1655             return delattr(self.__process, name)
1656 
1657 
1658 class FilteredStream(dict):
1659     """a dictionary with fixed keys"""
1660     def _blocked_attribute(obj):
1661         raise AttributeError("An FilteredStream defintion cannot be modified after creation.")
1662     _blocked_attribute = property(_blocked_attribute)
1663     __setattr__ = __delitem__ = __setitem__ = clear = _blocked_attribute
1664     pop = popitem = setdefault = update = _blocked_attribute
1665     def __new__(cls, *args, **kw):
1666         new = dict.__new__(cls)
1667         dict.__init__(new, *args, **kw)
1668         keys = sorted(kw.keys())
1669         if keys != ['content', 'dataTier', 'name', 'paths', 'responsible', 'selectEvents']:
1670             raise ValueError("The needed parameters are: content, dataTier, name, paths, responsible, selectEvents")
1671         if not isinstance(kw['name'],str):
1672             raise ValueError("name must be of type string")
1673         if not isinstance(kw['content'], vstring) and not isinstance(kw['content'],str):
1674             raise ValueError("content must be of type vstring or string")
1675         if not isinstance(kw['dataTier'], string):
1676             raise ValueError("dataTier must be of type string")
1677         if not isinstance(kw['selectEvents'], PSet):
1678             raise ValueError("selectEvents must be of type PSet")
1679         if not isinstance(kw['paths'],(tuple, Path)):
1680             raise ValueError("'paths' must be a tuple of paths")
1681         return new
1682     def __init__(self, *args, **kw):
1683         pass
1684     def __repr__(self):
1685         return "FilteredStream object: %s" %self["name"]
1686     def __getattr__(self,attr):
1687         return self[attr]
1688 
1689 class SubProcess(_Unlabelable):
1690     """Allows embedding another process within a parent process. This allows one to 
1691     chain processes together directly in one cmsRun job rather than having to run
1692     separate jobs that are connected via a temporary file.
1693     """
1694     def __init__(self,process, SelectEvents = untracked.PSet(), outputCommands = untracked.vstring()):
1695         """
1696         """
1697         if not isinstance(process, Process):
1698             raise ValueError("the 'process' argument must be of type cms.Process")
1699         if not isinstance(SelectEvents,PSet):
1700             raise ValueError("the 'SelectEvents' argument must be of type cms.untracked.PSet")
1701         if not isinstance(outputCommands,vstring):
1702             raise ValueError("the 'outputCommands' argument must be of type cms.untracked.vstring")
1703         self.__process = process
1704         self.__SelectEvents = SelectEvents
1705         self.__outputCommands = outputCommands
1706         # Need to remove MessageLogger from the subprocess now that MessageLogger is always present
1707         if self.__process.MessageLogger is not MessageLogger:
1708             print("""Warning: You have reconfigured service
1709 'edm::MessageLogger' in a subprocess.
1710 This service has already been configured.
1711 This particular service may not be reconfigured in a subprocess.
1712 The reconfiguration will be ignored.""")
1713         del self.__process.MessageLogger
1714     def dumpPython(self, options=PrintOptions()):
1715         out = "parentProcess"+str(hash(self))+" = process\n"
1716         out += self.__process.dumpPython()
1717         out += "childProcess = process\n"
1718         out += "process = parentProcess"+str(hash(self))+"\n"
1719         out += "process.addSubProcess(cms.SubProcess(process = childProcess, SelectEvents = "+self.__SelectEvents.dumpPython(options) +", outputCommands = "+self.__outputCommands.dumpPython(options) +"))"
1720         return out
1721     def getProcessName(self):
1722         return self.__process.name_()
1723     def process(self):
1724         return self.__process
1725     def SelectEvents(self):
1726         return self.__SelectEvents
1727     def outputCommands(self):
1728         return self.__outputCommands
1729     def type_(self):
1730         return 'subProcess'
1731     def nameInProcessDesc_(self,label):
1732         return label
1733     def _place(self,label,process):
1734         process._placeSubProcess('subProcess',self)
1735     def getSubProcessPSet(self,parameterSet):
1736         topPSet = parameterSet.newPSet()
1737         self.__process.fillProcessDesc(topPSet)
1738         subProcessPSet = parameterSet.newPSet()
1739         self.__SelectEvents.insertInto(subProcessPSet,"SelectEvents")
1740         self.__outputCommands.insertInto(subProcessPSet,"outputCommands")
1741         subProcessPSet.addPSet(False,"process",topPSet)
1742         return subProcessPSet
1743 
1744 class _ParameterModifier(object):
1745     """Helper class for Modifier that takes key/value pairs and uses them to reset parameters of the object"""
1746     def __init__(self,args):
1747         self.__args = args
1748     def __call__(self,obj):
1749         params = {}
1750         for k in self.__args.keys():
1751             if hasattr(obj,k):
1752                 params[k] = getattr(obj,k)
1753         _modifyParametersFromDict(params, self.__args, self._raiseUnknownKey)
1754         for k in self.__args.keys():
1755             if k in params:
1756                 setattr(obj,k,params[k])
1757             else:
1758                 #the parameter must have been removed
1759                 delattr(obj,k)
1760     @staticmethod
1761     def _raiseUnknownKey(key):
1762         raise KeyError("Unknown parameter name "+key+" specified while calling Modifier")
1763 
1764 class _BoolModifierBase(object):
1765     """A helper base class for _AndModifier, _InvertModifier, and _OrModifier to contain the common code"""
1766     def __init__(self, lhs, rhs=None):
1767         self._lhs = lhs
1768         if rhs is not None:
1769             self._rhs = rhs
1770     def toModify(self,obj, func=None,**kw):
1771         Modifier._toModifyCheck(obj,func,**kw)
1772         if self._isChosen():
1773             Modifier._toModify(obj,func,**kw)
1774         return self
1775     def toReplaceWith(self,toObj,fromObj):
1776         Modifier._toReplaceWithCheck(toObj,fromObj)
1777         if self._isChosen():
1778             Modifier._toReplaceWith(toObj,fromObj)
1779         return self
1780     def makeProcessModifier(self,func):
1781         """This is used to create a ProcessModifer that can perform actions on the process as a whole.
1782             This takes as argument a callable object (e.g. function) that takes as its sole argument an instance of Process.
1783             In order to work, the value returned from this function must be assigned to a uniquely named variable."""
1784         return ProcessModifier(self,func)
1785     def __and__(self, other):
1786         return _AndModifier(self,other)
1787     def __invert__(self):
1788         return _InvertModifier(self)
1789     def __or__(self, other):
1790         return _OrModifier(self,other)
1791 
1792 class _AndModifier(_BoolModifierBase):
1793     """A modifier which only applies if multiple Modifiers are chosen"""
1794     def __init__(self, lhs, rhs):
1795         super(_AndModifier,self).__init__(lhs, rhs)
1796     def _isChosen(self):
1797         return self._lhs._isChosen() and self._rhs._isChosen()
1798 
1799 class _InvertModifier(_BoolModifierBase):
1800     """A modifier which only applies if a Modifier is not chosen"""
1801     def __init__(self, lhs):
1802         super(_InvertModifier,self).__init__(lhs)
1803     def _isChosen(self):
1804         return not self._lhs._isChosen()
1805 
1806 class _OrModifier(_BoolModifierBase):
1807     """A modifier which only applies if at least one of multiple Modifiers is chosen"""
1808     def __init__(self, lhs, rhs):
1809         super(_OrModifier,self).__init__(lhs, rhs)
1810     def _isChosen(self):
1811         return self._lhs._isChosen() or self._rhs._isChosen()
1812 
1813 
1814 class Modifier(object):
1815     """This class is used to define standard modifications to a Process.
1816     An instance of this class is declared to denote a specific modification,e.g. era2017 could
1817     reconfigure items in a process to match our expectation of running in 2017. Once declared,
1818     these Modifier instances are imported into a configuration and items that need to be modified
1819     are then associated with the Modifier and with the action to do the modification.
1820     The registered modifications will only occur if the Modifier was passed to 
1821     the cms.Process' constructor.
1822     """
1823     def __init__(self):
1824         self.__processModifiers = []
1825         self.__chosen = False
1826     def makeProcessModifier(self,func):
1827         """This is used to create a ProcessModifer that can perform actions on the process as a whole.
1828            This takes as argument a callable object (e.g. function) that takes as its sole argument an instance of Process.
1829            In order to work, the value returned from this function must be assigned to a uniquely named variable.
1830         """
1831         return ProcessModifier(self,func)
1832     @staticmethod
1833     def _toModifyCheck(obj,func,**kw):
1834         if func is not None and len(kw) != 0:
1835             raise TypeError("toModify takes either two arguments or one argument and key/value pairs")
1836     def toModify(self,obj, func=None,**kw):
1837         """This is used to register an action to be performed on the specific object. Two different forms are allowed
1838         Form 1: A callable object (e.g. function) can be passed as the second. This callable object is expected to take one argument
1839         that will be the object passed in as the first argument.
1840         Form 2: A list of parameter name, value pairs can be passed
1841            mod.toModify(foo, fred=cms.int32(7), barney = cms.double(3.14))
1842         This form can also be used to remove a parameter by passing the value of None
1843             #remove the parameter foo.fred       
1844             mod.toModify(foo, fred = None)
1845         Additionally, parameters embedded within PSets can also be modified using a dictionary
1846             #change foo.fred.pebbles to 3 and foo.fred.friend to "barney"
1847             mod.toModify(foo, fred = dict(pebbles = 3, friend = "barney)) )
1848         """
1849         Modifier._toModifyCheck(obj,func,**kw)
1850         if self._isChosen():
1851             Modifier._toModify(obj,func,**kw)
1852         return self
1853     @staticmethod
1854     def _toModify(obj,func,**kw):
1855         if func is not None:
1856             func(obj)
1857         else:
1858             temp =_ParameterModifier(kw)
1859             temp(obj)
1860     @staticmethod
1861     def _toReplaceWithCheck(toObj,fromObj):
1862         if not isinstance(fromObj, type(toObj)):
1863             raise TypeError("toReplaceWith requires both arguments to be the same class type")
1864     def toReplaceWith(self,toObj,fromObj):
1865         """If the Modifier is chosen the internals of toObj will be associated with the internals of fromObj
1866         """
1867         Modifier._toReplaceWithCheck(toObj,fromObj)
1868         if self._isChosen():
1869             Modifier._toReplaceWith(toObj,fromObj)
1870         return self
1871     @staticmethod
1872     def _toReplaceWith(toObj,fromObj):
1873         if isinstance(fromObj,_ModuleSequenceType):
1874             toObj._seq = fromObj._seq
1875             toObj._tasks = fromObj._tasks
1876         elif isinstance(fromObj,Task):
1877             toObj._collection = fromObj._collection
1878         elif isinstance(fromObj,ConditionalTask):
1879             toObj._collection = fromObj._collection
1880         elif isinstance(fromObj,_Parameterizable):
1881             #clear old items just incase fromObj is not a complete superset of toObj
1882             for p in toObj.parameterNames_():
1883                 delattr(toObj,p)
1884             for p in fromObj.parameterNames_():
1885                 setattr(toObj,p,getattr(fromObj,p))
1886             if isinstance(fromObj,_TypedParameterizable):
1887                 toObj._TypedParameterizable__type = fromObj._TypedParameterizable__type
1888 
1889         else:
1890             raise TypeError("toReplaceWith does not work with type "+str(type(toObj)))
1891 
1892     def _setChosen(self):
1893         """Should only be called by cms.Process instances"""
1894         self.__chosen = True
1895     def _isChosen(self):
1896         return self.__chosen
1897     def __and__(self, other):
1898         return _AndModifier(self,other)
1899     def __invert__(self):
1900         return _InvertModifier(self)
1901     def __or__(self, other):
1902         return _OrModifier(self,other)
1903     def _isOrContains(self, other):
1904         return self == other
1905 
1906 
1907 class ModifierChain(object):
1908     """A Modifier made up of a list of Modifiers
1909     """
1910     def __init__(self, *chainedModifiers):
1911         self.__chosen = False
1912         self.__chain = chainedModifiers
1913     def _applyNewProcessModifiers(self,process):
1914         """Should only be called by cms.Process instances
1915         applies list of accumulated changes to the process"""
1916         for m in self.__chain:
1917             m._applyNewProcessModifiers(process)
1918     def _setChosen(self):
1919         """Should only be called by cms.Process instances"""
1920         self.__chosen = True
1921         for m in self.__chain:
1922             m._setChosen()
1923     def _isChosen(self):
1924         return self.__chosen
1925     def copyAndExclude(self, toExclude):
1926         """Creates a new ModifierChain which is a copy of
1927           this ModifierChain but excludes any Modifier or
1928           ModifierChain in the list toExclude.
1929           The exclusion is done recursively down the chain.
1930           """
1931         newMods = []
1932         for m in self.__chain:
1933             if m not in toExclude:
1934                 s = m
1935                 if isinstance(m,ModifierChain):
1936                     s = m.__copyIfExclude(toExclude)
1937                 newMods.append(s)
1938         return ModifierChain(*newMods)
1939     def __copyIfExclude(self,toExclude):
1940         shouldCopy = False
1941         for m in toExclude:
1942             if self._isOrContains(m):
1943                 shouldCopy = True
1944                 break
1945         if shouldCopy:
1946             return self.copyAndExclude(toExclude)
1947         return self
1948     def _isOrContains(self, other):
1949         if self is other:
1950             return True
1951         for m in self.__chain:
1952             if m._isOrContains(other):
1953                 return True
1954         return False
1955 
1956 class ProcessModifier(object):
1957     """A class used by a Modifier to affect an entire Process instance.
1958     When a Process 'loads' a module containing a ProcessModifier, that
1959     ProcessModifier will be applied to the Process if and only if the 
1960     Modifier passed to the constructor has been chosen.
1961     """
1962     def __init__(self, modifier, func):
1963         self.__modifier = modifier
1964         self.__func = func
1965         self.__seenProcesses = set()
1966     def apply(self,process):
1967         if self.__modifier._isChosen():
1968             if process not in self.__seenProcesses:
1969                 self.__func(process)
1970                 self.__seenProcesses.add(process)
1971 
1972 class ProcessAccelerator(_ConfigureComponent,_Unlabelable):
1973     """A class used to specify possible compute accelerators in a Process
1974     instance. It is intended to be derived for any
1975     accelerator/portability technology, and provides hooks such that a
1976     specific customization can be applied to the Process on a worker
1977     node at the point where the python configuration is serialized for C++.
1978 
1979     The customization must not change the configuration hash. To
1980     enforce this reuirement, the customization gets a
1981     ProcessForProcessAccelerator wrapper that gives access to only
1982     those parts of the configuration that can be changed. Nevertheless
1983     it would be good to have specific unit test for each deriving
1984     class to ensure that all combinations of the enabled accelerators
1985     give the same configuration hash.
1986     """
1987     def __init__(self):
1988         pass
1989     def _place(self, name, proc):
1990         proc._placeAccelerator(self.type_(), self)
1991     def type_(self):
1992         return type(self).__name__
1993     def dumpPython(self, options=PrintOptions()):
1994         specialImportRegistry.registerUse(self)
1995         result = self.__class__.__name__+"(" # not including cms. since the deriving classes are not in cms "namespace"
1996         options.indent()
1997         res = self.dumpPythonImpl(options)
1998         options.unindent()
1999         if len(res) > 0:
2000             result += "\n"+res+"\n"
2001         result += ")\n"
2002         return result
2003 
2004     # The following methods are hooks to be overridden (if needed) in the deriving class
2005     def dumpPythonImpl(self, options):
2006         """Override if need to add any 'body' content to dumpPython(). Returns a string."""
2007         return ""
2008     def labels(self):
2009         """Override to return a list of strings for the accelerator labels."""
2010         return []
2011     def enabledLabels(self):
2012         """Override to return a list of strings for the accelerator labels
2013         that are enabled in the system the job is being run on."""
2014         return []
2015     def moduleTypeResolver(self, accelerators):
2016         """Override to return an object that implements "module type resolver"
2017         in python. The object should have the following methods
2018         - __init__(self, accelerators)
2019           * accelerators = list of selected accelerators
2020         - plugin(self):
2021           * should return a string for the type resolver plugin name
2022         - setModuleVariant(self, module):
2023           * Called for each ED and ES module. Should act only if
2024             module.type_() contains the magic identifier
2025 
2026         At most one of the ProcessAccelerators in a job can return a
2027 non-None object
2028         """
2029         return None
2030     def apply(self, process, accelerators):
2031         """Override if need to customize the Process at worker node. The
2032         selected available accelerator labels are given in the
2033         'accelerators' argument (the patterns, e.g. '*' have been
2034         expanded to concrete labels).
2035 
2036         This function may touch only untracked parameters.
2037         """
2038         pass
2039 
2040 class ProcessForProcessAccelerator(object):
2041     """This class is inteded to wrap the Process object to constrain the
2042     available functionality for ProcessAccelerator.apply()"""
2043     def  __init__(self, process):
2044         self.__process = process
2045     def __getattr__(self, label):
2046         value = getattr(self.__process, label)
2047         if not isinstance(value, Service):
2048             raise TypeError("ProcessAccelerator.apply() can get only Services. Tried to get {} with label {}".format(str(type(value)), label))
2049         return value
2050     def __setattr__(self, label, value):
2051         if label == "_ProcessForProcessAccelerator__process":
2052             super().__setattr__(label, value)
2053         else:
2054             if not isinstance(value, Service):
2055                 raise TypeError("ProcessAccelerator.apply() can only set Services. Tried to set {} with label {}".format(str(type(value)), label))
2056             setattr(self.__process, label, value)
2057     def add_(self, value):
2058         if not isinstance(value, Service):
2059             raise TypeError("ProcessAccelerator.apply() can only add Services. Tried to set {} with label {}".format(str(type(value)), label))
2060         self.__process.add_(value)
2061 
2062 # Need to be a module-level function for the configuration with a
2063 # SwitchProducer to be pickleable.
2064 def _switchproducer_test2_case1(accelerators):
2065     return ("test1" in accelerators, -10)
2066 def _switchproducer_test2_case2(accelerators):
2067     return ("test2" in accelerators, -9)
2068 
2069 if __name__=="__main__":
2070     import unittest
2071     import copy
2072 
2073     def _lineDiff(newString, oldString):
2074         newString = ( x for x in newString.split('\n') if len(x) > 0)
2075         oldString = [ x for x in oldString.split('\n') if len(x) > 0]
2076         diff = []
2077         oldStringLine = 0
2078         for l in newString:
2079             if oldStringLine >= len(oldString):
2080                 diff.append(l)
2081                 continue
2082             if l == oldString[oldStringLine]:
2083                 oldStringLine +=1
2084                 continue
2085             diff.append(l)
2086         return "\n".join( diff )
2087 
2088     class TestMakePSet(object):
2089         """Has same interface as the C++ object that creates PSets
2090         """
2091         def __init__(self):
2092             self.values = dict()
2093         def __insertValue(self,tracked,label,value):
2094             self.values[label]=(tracked,value)
2095         def __getValue(self,tracked,label):
2096             pair = self.values[label]
2097             if pair[0] != tracked:
2098                raise Exception("Asked for %s parameter '%s', but it is %s" % ("tracked" if tracked else "untracked",
2099                                                                               label,
2100                                                                               "tracked" if pair[0] else "untracked"))
2101             return pair[1]
2102         def addInt32(self,tracked,label,value):
2103             self.__insertValue(tracked,label,value)
2104         def addVInt32(self,tracked,label,value):
2105             self.__insertValue(tracked,label,value)
2106         def addUInt32(self,tracked,label,value):
2107             self.__insertValue(tracked,label,value)
2108         def addVUInt32(self,tracked,label,value):
2109             self.__insertValue(tracked,label,value)
2110         def addInt64(self,tracked,label,value):
2111             self.__insertValue(tracked,label,value)
2112         def addVInt64(self,tracked,label,value):
2113             self.__insertValue(tracked,label,value)
2114         def addUInt64(self,tracked,label,value):
2115             self.__insertValue(tracked,label,value)
2116         def addVUInt64(self,tracked,label,value):
2117             self.__insertValue(tracked,label,value)
2118         def addDouble(self,tracked,label,value):
2119             self.__insertValue(tracked,label,value)
2120         def addVDouble(self,tracked,label,value):
2121             self.__insertValue(tracked,label,value)
2122         def addBool(self,tracked,label,value):
2123             self.__insertValue(tracked,label,value)
2124         def addString(self,tracked,label,value):
2125             self.__insertValue(tracked,label,value)
2126         def addVString(self,tracked,label,value):
2127             self.__insertValue(tracked,label,value)
2128         def getVString(self,tracked,label):
2129             return self.__getValue(tracked, label)
2130         def addInputTag(self,tracked,label,value):
2131             self.__insertValue(tracked,label,value)
2132         def addVInputTag(self,tracked,label,value):
2133             self.__insertValue(tracked,label,value)
2134         def addESInputTag(self,tracked,label,value):
2135             self.__insertValue(tracked,label,value)
2136         def addVESInputTag(self,tracked,label,value):
2137             self.__insertValue(tracked,label,value)
2138         def addEventID(self,tracked,label,value):
2139             self.__insertValue(tracked,label,value)
2140         def addVEventID(self,tracked,label,value):
2141             self.__insertValue(tracked,label,value)
2142         def addLuminosityBlockID(self,tracked,label,value):
2143             self.__insertValue(tracked,label,value)
2144         def addLuminosityBlockID(self,tracked,label,value):
2145             self.__insertValue(tracked,label,value)
2146         def addEventRange(self,tracked,label,value):
2147             self.__insertValue(tracked,label,value)
2148         def addVEventRange(self,tracked,label,value):
2149             self.__insertValue(tracked,label,value)
2150         def addPSet(self,tracked,label,value):
2151             self.__insertValue(tracked,label,value)
2152         def addVPSet(self,tracked,label,value):
2153             self.__insertValue(tracked,label,value)
2154         def addFileInPath(self,tracked,label,value):
2155             self.__insertValue(tracked,label,value)
2156         def newPSet(self):
2157             return TestMakePSet()
2158 
2159     class SwitchProducerTest(SwitchProducer):
2160         def __init__(self, **kargs):
2161             super(SwitchProducerTest,self).__init__(
2162                 dict(
2163                     test1 = lambda accelerators: (True, -10),
2164                     test2 = lambda accelerators: (True, -9),
2165                     test3 = lambda accelerators: (True, -8),
2166                     test4 = lambda accelerators: (True, -7)
2167                 ), **kargs)
2168     specialImportRegistry.registerSpecialImportForType(SwitchProducerTest, "from test import SwitchProducerTest")
2169 
2170     class SwitchProducerTest2(SwitchProducer):
2171         def __init__(self, **kargs):
2172             super(SwitchProducerTest2,self).__init__(
2173                 dict(
2174                     test1 = _switchproducer_test2_case1,
2175                     test2 = _switchproducer_test2_case2,
2176                 ), **kargs)
2177     specialImportRegistry.registerSpecialImportForType(SwitchProducerTest2, "from test import SwitchProducerTest2")
2178 
2179     class TestModuleTypeResolver:
2180         def __init__(self, accelerators):
2181             # first element is used as the default is nothing is set
2182             self._valid_backends = []
2183             if "test1" in accelerators:
2184                 self._valid_backends.append("test1_backend")
2185             if "test2" in accelerators:
2186                 self._valid_backends.append("test2_backend")
2187             if len(self._valid_backends) == 0:
2188                 raise EDMException(edm.errors.UnavailableAccelerator, "Machine has no accelerators that Test supports (has {})".format(", ".join(accelerators)))
2189 
2190         def plugin(self):
2191             return "TestModuleTypeResolver"
2192 
2193         def setModuleVariant(self, module):
2194             if "@test" in module.type_():
2195                 defaultBackend = self._valid_backends[0]
2196                 if hasattr(module, "test"):
2197                     if hasattr(module.test, "backend"):
2198                         if module.test.backend.value() not in self._valid_backends:
2199                             raise EDMException(edm.errors.UnavailableAccelerator, "Module {} has the Test backend set explicitly, but its accelerator is not available for the job".format(module.label_()))
2200                     else:
2201                         module.test.backend = untracked.string(defaultBackend)
2202                 else:
2203                     module.test = untracked.PSet(
2204                         backend = untracked.string(defaultBackend)
2205                     )
2206 
2207     class ProcessAcceleratorTest(ProcessAccelerator):
2208         def __init__(self, enabled=["test1", "test2", "anothertest3"], moduleTypeResolverMaker=None):
2209             super().__init__()
2210             self._labels = ["test1", "test2", "anothertest3"]
2211             self.setEnabled(enabled)
2212             self._moduleTypeResolverMaker = moduleTypeResolverMaker
2213         def setEnabled(self, enabled):
2214             invalid = set(enabled).difference(set(self._labels))
2215             if len(invalid) > 0:
2216                 raise Exception("Tried to enabled nonexistent test accelerators {}".format(",".join(invalid)))
2217             self._enabled = enabled[:]
2218         def dumpPythonImpl(self,options):
2219             result = "{}enabled = [{}]".format(options.indentation(),
2220                                                ", ".join(["'{}'".format(e) for e in self._enabled]))
2221             return result
2222         def labels(self):
2223             return self._labels
2224         def enabledLabels(self):
2225             return self._enabled
2226         def moduleTypeResolver(self, accelerators):
2227             if not self._moduleTypeResolverMaker:
2228                 return super().moduleTypeResolver(accelerators)
2229             return self._moduleTypeResolverMaker(accelerators)
2230         def apply(self, process, accelerators):
2231             process.AcceleratorTestService = Service("AcceleratorTestService")
2232     specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorTest, "from test import ProcessAcceleratorTest")
2233 
2234     class ProcessAcceleratorTest2(ProcessAccelerator):
2235         def __init__(self, enabled=["anothertest3", "anothertest4"], moduleTypeResolverMaker=None):
2236             super().__init__()
2237             self._labels = ["anothertest3", "anothertest4"]
2238             self.setEnabled(enabled)
2239             self._moduleTypeResolverMaker = moduleTypeResolverMaker
2240         def setEnabled(self, enabled):
2241             invalid = set(enabled).difference(set(self._labels))
2242             if len(invalid) > 0:
2243                 raise Exception("Tried to enabled nonexistent test accelerators {}".format(",".join(invalid)))
2244             self._enabled = enabled[:]
2245         def dumpPythonImpl(self,options):
2246             result = "{}enabled = [{}]".format(options.indentation(),
2247                                                ", ".join(["'{}'".format(e) for e in self._enabled]))
2248             return result
2249         def labels(self):
2250             return self._labels
2251         def enabledLabels(self):
2252             return self._enabled
2253         def moduleTypeResolver(self, accelerators):
2254             if not self._moduleTypeResolverMaker:
2255                 return super().moduleTypeResolver(accelerators)
2256             return self._moduleTypeResolverMaker(accelerators)
2257         def apply(self, process, accelerators):
2258             pass
2259     specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorTest2, "from test import ProcessAcceleratorTest2")
2260 
2261     class TestModuleCommand(unittest.TestCase):
2262         def setUp(self):
2263             """Nothing to do """
2264             None
2265         def testParameterizable(self):
2266             p = _Parameterizable()
2267             self.assertEqual(len(p.parameterNames_()),0)
2268             p.a = int32(1)
2269             self.assertTrue('a' in p.parameterNames_())
2270             self.assertEqual(p.a.value(), 1)
2271             p.a = 10
2272             self.assertEqual(p.a.value(), 10)
2273             p.a = untracked(int32(1))
2274             self.assertEqual(p.a.value(), 1)
2275             self.assertFalse(p.a.isTracked())
2276             p.a = untracked.int32(1)
2277             self.assertEqual(p.a.value(), 1)
2278             self.assertFalse(p.a.isTracked())
2279             p = _Parameterizable(foo=int32(10), bar = untracked(double(1.0)))
2280             self.assertEqual(p.foo.value(), 10)
2281             self.assertEqual(p.bar.value(),1.0)
2282             self.assertFalse(p.bar.isTracked())
2283             self.assertRaises(TypeError,setattr,(p,'c',1))
2284             p = _Parameterizable(a=PSet(foo=int32(10), bar = untracked(double(1.0))))
2285             self.assertEqual(p.a.foo.value(),10)
2286             self.assertEqual(p.a.bar.value(),1.0)
2287             p.b = untracked(PSet(fii = int32(1)))
2288             self.assertEqual(p.b.fii.value(),1)
2289             self.assertFalse(p.b.isTracked())
2290             #test the fact that values can be shared
2291             v = int32(10)
2292             p=_Parameterizable(a=v)
2293             v.setValue(11)
2294             self.assertEqual(p.a.value(),11)
2295             p.a = 12
2296             self.assertEqual(p.a.value(),12)
2297             self.assertEqual(v.value(),12)
2298         def testTypedParameterizable(self):
2299             p = _TypedParameterizable("blah", b=int32(1))
2300             #see if copy works deeply
2301             other = p.copy()
2302             other.b = 2
2303             self.assertNotEqual(p.b,other.b)
2304 
2305         def testProcessInsertion(self):
2306             p = Process("test")
2307             p.a = EDAnalyzer("MyAnalyzer")
2308             self.assertTrue( 'a' in p.analyzers_() )
2309             self.assertTrue( 'a' in p.analyzers)
2310             p.add_(Service("SomeService"))
2311             self.assertTrue('SomeService' in p.services_())
2312             self.assertEqual(p.SomeService.type_(), "SomeService")
2313             p.Tracer = Service("Tracer")
2314             self.assertTrue('Tracer' in p.services_())
2315             self.assertRaises(TypeError, setattr, *(p,'b',"this should fail"))
2316             self.assertRaises(TypeError, setattr, *(p,'bad',Service("MessageLogger")))
2317             self.assertRaises(ValueError, setattr, *(p,'bad',Source("PoolSource")))
2318             p.out = OutputModule("Outer")
2319             self.assertEqual(p.out.type_(), 'Outer')
2320             self.assertTrue( 'out' in p.outputModules_() )
2321 
2322             p.geom = ESSource("GeomProd")
2323             self.assertTrue('geom' in p.es_sources_())
2324             p.add_(ESSource("ConfigDB"))
2325             self.assertTrue('ConfigDB' in p.es_sources_())
2326 
2327             p.aliasfoo1 = EDAlias(foo1 = VPSet(PSet(type = string("Foo1"))))
2328             self.assertTrue('aliasfoo1' in p.aliases_())
2329 
2330         def testProcessExtend(self):
2331             class FromArg(object):
2332                 def __init__(self,*arg,**args):
2333                     for name in args.keys():
2334                         self.__dict__[name]=args[name]
2335 
2336             a=EDAnalyzer("MyAnalyzer")
2337             t=EDAnalyzer("MyAnalyzer")
2338             t.setLabel("foo")
2339             s1 = Sequence(a)
2340             s2 = Sequence(s1)
2341             s3 = Sequence(s2)
2342             d = FromArg(
2343                     a=a,
2344                     b=Service("Full"),
2345                     c=Path(a),
2346                     d=s2,
2347                     e=s1,
2348                     f=s3,
2349                     g=Sequence(s1+s2+s3)
2350                 )
2351             p = Process("Test")
2352             p.extend(d)
2353             self.assertEqual(p.a.type_(),"MyAnalyzer")
2354             self.assertEqual(p.a.label_(),"a")
2355             self.assertRaises(AttributeError,getattr,p,'b')
2356             self.assertEqual(p.Full.type_(),"Full")
2357             self.assertEqual(str(p.c),'a')
2358             self.assertEqual(str(p.d),'a')
2359 
2360             z1 = FromArg(
2361                     a=a,
2362                     b=Service("Full"),
2363                     c=Path(a),
2364                     d=s2,
2365                     e=s1,
2366                     f=s3,
2367                     s4=s3,
2368                     g=Sequence(s1+s2+s3)
2369                  )
2370 
2371             p1 = Process("Test")
2372             #p1.extend(z1)
2373             self.assertRaises(ValueError, p1.extend, z1)
2374 
2375             z2 = FromArg(
2376                     a=a,
2377                     b=Service("Full"),
2378                     c=Path(a),
2379                     d=s2,
2380                     e=s1,
2381                     f=s3,
2382                     aaa=copy.deepcopy(a),
2383                     s4=copy.deepcopy(s3),
2384                     g=Sequence(s1+s2+s3),
2385                     t=t
2386                 )
2387             p2 = Process("Test")
2388             p2.extend(z2)
2389             #self.assertRaises(ValueError, p2.extend, z2)
2390             self.assertEqual(p2.s4.label_(),"s4")
2391             #p2.s4.setLabel("foo")
2392             self.assertRaises(ValueError, p2.s4.setLabel, "foo")
2393             p2.s4.setLabel("s4")
2394             p2.s4.setLabel(None)
2395             p2.s4.setLabel("foo")
2396             p2._Process__setObjectLabel(p2.s4, "foo")
2397             p2._Process__setObjectLabel(p2.s4, None)
2398             p2._Process__setObjectLabel(p2.s4, "bar")
2399 
2400 
2401             p = Process('test')
2402             p.a = EDProducer("MyProducer")
2403             p.t = Task(p.a)
2404             p.p = Path(p.t)
2405             self.assertRaises(ValueError, p.extend, FromArg(a = EDProducer("YourProducer")))
2406             self.assertRaises(ValueError, p.extend, FromArg(a = EDAlias()))
2407             self.assertRaises(ValueError, p.__setattr__, "a", EDAlias())
2408 
2409             p = Process('test')
2410             p.a = EDProducer("MyProducer")
2411             p.t = ConditionalTask(p.a)
2412             p.p = Path(p.t)
2413             self.assertRaises(ValueError, p.extend, FromArg(a = EDProducer("YourProducer")))
2414             self.assertRaises(ValueError, p.extend, FromArg(a = EDAlias()))
2415             self.assertRaises(ValueError, p.__setattr__, "a", EDAlias())
2416 
2417             p = Process('test')
2418             p.a = EDProducer("MyProducer")
2419             p.s = Sequence(p.a)
2420             p.p = Path(p.s)
2421             self.assertRaises(ValueError, p.extend, FromArg(a = EDProducer("YourProducer")))
2422             self.assertRaises(ValueError, p.extend, FromArg(a = EDAlias()))
2423             self.assertRaises(ValueError, p.__setattr__, "a", EDAlias())
2424 
2425         def testProcessDumpPython(self):
2426             self.assertEqual(Process("test").dumpPython(),
2427 """import FWCore.ParameterSet.Config as cms
2428 
2429 process = cms.Process("test")
2430 
2431 process.maxEvents = cms.untracked.PSet(
2432     input = cms.optional.untracked.int32,
2433     output = cms.optional.untracked.allowed(cms.int32,cms.PSet)
2434 )
2435 
2436 process.maxLuminosityBlocks = cms.untracked.PSet(
2437     input = cms.untracked.int32(-1)
2438 )
2439 
2440 process.options = cms.untracked.PSet(
2441     FailPath = cms.untracked.vstring(),
2442     IgnoreCompletely = cms.untracked.vstring(),
2443     Rethrow = cms.untracked.vstring(),
2444     SkipEvent = cms.untracked.vstring(),
2445     accelerators = cms.untracked.vstring('*'),
2446     allowUnscheduled = cms.obsolete.untracked.bool,
2447     canDeleteEarly = cms.untracked.vstring(),
2448     deleteNonConsumedUnscheduledModules = cms.untracked.bool(True),
2449     dumpOptions = cms.untracked.bool(False),
2450     emptyRunLumiMode = cms.obsolete.untracked.string,
2451     eventSetup = cms.untracked.PSet(
2452         forceNumberOfConcurrentIOVs = cms.untracked.PSet(
2453             allowAnyLabel_=cms.required.untracked.uint32
2454         ),
2455         numberOfConcurrentIOVs = cms.untracked.uint32(0)
2456     ),
2457     fileMode = cms.untracked.string('FULLMERGE'),
2458     forceEventSetupCacheClearOnNewRun = cms.untracked.bool(False),
2459     holdsReferencesToDeleteEarly = cms.untracked.VPSet(),
2460     makeTriggerResults = cms.obsolete.untracked.bool,
2461     modulesToIgnoreForDeleteEarly = cms.untracked.vstring(),
2462     numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(0),
2463     numberOfConcurrentRuns = cms.untracked.uint32(1),
2464     numberOfStreams = cms.untracked.uint32(0),
2465     numberOfThreads = cms.untracked.uint32(1),
2466     printDependencies = cms.untracked.bool(False),
2467     sizeOfStackForThreadsInKB = cms.optional.untracked.uint32,
2468     throwIfIllegalParameter = cms.untracked.bool(True),
2469     wantSummary = cms.untracked.bool(False)
2470 )
2471 
2472 process.MessageLogger = cms.Service("MessageLogger",
2473     cerr = cms.untracked.PSet(
2474         FwkReport = cms.untracked.PSet(
2475             limit = cms.untracked.int32(10000000),
2476             reportEvery = cms.untracked.int32(1)
2477         ),
2478         FwkSummary = cms.untracked.PSet(
2479             limit = cms.untracked.int32(10000000),
2480             reportEvery = cms.untracked.int32(1)
2481         ),
2482         INFO = cms.untracked.PSet(
2483             limit = cms.untracked.int32(0)
2484         ),
2485         Root_NoDictionary = cms.untracked.PSet(
2486             limit = cms.untracked.int32(0)
2487         ),
2488         default = cms.untracked.PSet(
2489             limit = cms.untracked.int32(10000000)
2490         ),
2491         enable = cms.untracked.bool(True),
2492         enableStatistics = cms.untracked.bool(False),
2493         lineLength = cms.optional.untracked.int32,
2494         noLineBreaks = cms.optional.untracked.bool,
2495         noTimeStamps = cms.untracked.bool(False),
2496         resetStatistics = cms.untracked.bool(False),
2497         statisticsThreshold = cms.untracked.string('WARNING'),
2498         threshold = cms.untracked.string('INFO'),
2499         allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2500             limit = cms.optional.untracked.int32,
2501             reportEvery = cms.untracked.int32(1),
2502             timespan = cms.optional.untracked.int32
2503         )
2504     ),
2505     cout = cms.untracked.PSet(
2506         enable = cms.untracked.bool(False),
2507         enableStatistics = cms.untracked.bool(False),
2508         lineLength = cms.optional.untracked.int32,
2509         noLineBreaks = cms.optional.untracked.bool,
2510         noTimeStamps = cms.optional.untracked.bool,
2511         resetStatistics = cms.untracked.bool(False),
2512         statisticsThreshold = cms.optional.untracked.string,
2513         threshold = cms.optional.untracked.string,
2514         allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2515             limit = cms.optional.untracked.int32,
2516             reportEvery = cms.untracked.int32(1),
2517             timespan = cms.optional.untracked.int32
2518         )
2519     ),
2520     debugModules = cms.untracked.vstring(),
2521     default = cms.untracked.PSet(
2522         limit = cms.optional.untracked.int32,
2523         lineLength = cms.untracked.int32(80),
2524         noLineBreaks = cms.untracked.bool(False),
2525         noTimeStamps = cms.untracked.bool(False),
2526         reportEvery = cms.untracked.int32(1),
2527         statisticsThreshold = cms.untracked.string('INFO'),
2528         threshold = cms.untracked.string('INFO'),
2529         timespan = cms.optional.untracked.int32,
2530         allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2531             limit = cms.optional.untracked.int32,
2532             reportEvery = cms.untracked.int32(1),
2533             timespan = cms.optional.untracked.int32
2534         )
2535     ),
2536     files = cms.untracked.PSet(
2537         allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2538             enableStatistics = cms.untracked.bool(False),
2539             extension = cms.optional.untracked.string,
2540             filename = cms.optional.untracked.string,
2541             lineLength = cms.optional.untracked.int32,
2542             noLineBreaks = cms.optional.untracked.bool,
2543             noTimeStamps = cms.optional.untracked.bool,
2544             output = cms.optional.untracked.string,
2545             resetStatistics = cms.untracked.bool(False),
2546             statisticsThreshold = cms.optional.untracked.string,
2547             threshold = cms.optional.untracked.string,
2548             allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2549                 limit = cms.optional.untracked.int32,
2550                 reportEvery = cms.untracked.int32(1),
2551                 timespan = cms.optional.untracked.int32
2552             )
2553         )
2554     ),
2555     suppressDebug = cms.untracked.vstring(),
2556     suppressFwkInfo = cms.untracked.vstring(),
2557     suppressInfo = cms.untracked.vstring(),
2558     suppressWarning = cms.untracked.vstring(),
2559     allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2560         limit = cms.optional.untracked.int32,
2561         reportEvery = cms.untracked.int32(1),
2562         timespan = cms.optional.untracked.int32
2563     )
2564 )
2565 
2566 
2567 """)
2568             p = Process("test")
2569             p.a = EDAnalyzer("MyAnalyzer")
2570             p.p = Path(p.a)
2571             p.s = Sequence(p.a)
2572             p.r = Sequence(p.s)
2573             p.p2 = Path(p.s)
2574             p.schedule = Schedule(p.p2,p.p)
2575             d=p.dumpPython()
2576             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2577 """process.a = cms.EDAnalyzer("MyAnalyzer")
2578 process.s = cms.Sequence(process.a)
2579 process.r = cms.Sequence(process.s)
2580 process.p = cms.Path(process.a)
2581 process.p2 = cms.Path(process.s)
2582 process.schedule = cms.Schedule(*[ process.p2, process.p ])""")
2583             #Reverse order of 'r' and 's'
2584             p = Process("test")
2585             p.a = EDAnalyzer("MyAnalyzer")
2586             p.p = Path(p.a)
2587             p.r = Sequence(p.a)
2588             p.s = Sequence(p.r)
2589             p.p2 = Path(p.r)
2590             p.schedule = Schedule(p.p2,p.p)
2591             p.b = EDAnalyzer("YourAnalyzer")
2592             d=p.dumpPython()
2593             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2594 """process.a = cms.EDAnalyzer("MyAnalyzer")
2595 process.b = cms.EDAnalyzer("YourAnalyzer")
2596 process.r = cms.Sequence(process.a)
2597 process.s = cms.Sequence(process.r)
2598 process.p = cms.Path(process.a)
2599 process.p2 = cms.Path(process.r)
2600 process.schedule = cms.Schedule(*[ process.p2, process.p ])""")
2601         #use an anonymous sequence
2602             p = Process("test")
2603             p.a = EDAnalyzer("MyAnalyzer")
2604             p.p = Path(p.a)
2605             s = Sequence(p.a)
2606             p.r = Sequence(s)
2607             p.p2 = Path(p.r)
2608             p.schedule = Schedule(p.p2,p.p)
2609             d=p.dumpPython()
2610             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2611 """process.a = cms.EDAnalyzer("MyAnalyzer")
2612 process.r = cms.Sequence((process.a))
2613 process.p = cms.Path(process.a)
2614 process.p2 = cms.Path(process.r)
2615 process.schedule = cms.Schedule(*[ process.p2, process.p ])""")
2616 
2617             # include some tasks
2618             p = Process("test")
2619             p.a = EDAnalyzer("MyAnalyzer")
2620             p.b = EDProducer("bProducer")
2621             p.c = EDProducer("cProducer")
2622             p.d = EDProducer("dProducer")
2623             p.e = EDProducer("eProducer")
2624             p.f = EDProducer("fProducer")
2625             p.g = EDProducer("gProducer")
2626             p.task5 = Task()
2627             p.task3 = Task()
2628             p.task2 = Task(p.c, p.task3)
2629             p.task4 = Task(p.f, p.task2)
2630             p.task1 = Task(p.task5)
2631             p.task3.add(p.task1)
2632             p.p = Path(p.a)
2633             s = Sequence(p.a)
2634             p.r = Sequence(s)
2635             p.p2 = Path(p.r, p.task1, p.task2)
2636             p.schedule = Schedule(p.p2,p.p,tasks=[p.task3,p.task4, p.task5])
2637             d=p.dumpPython()
2638             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2639 """process.b = cms.EDProducer("bProducer")
2640 process.c = cms.EDProducer("cProducer")
2641 process.d = cms.EDProducer("dProducer")
2642 process.e = cms.EDProducer("eProducer")
2643 process.f = cms.EDProducer("fProducer")
2644 process.g = cms.EDProducer("gProducer")
2645 process.a = cms.EDAnalyzer("MyAnalyzer")
2646 process.task5 = cms.Task()
2647 process.task1 = cms.Task(process.task5)
2648 process.task3 = cms.Task(process.task1)
2649 process.task2 = cms.Task(process.c, process.task3)
2650 process.task4 = cms.Task(process.f, process.task2)
2651 process.r = cms.Sequence((process.a))
2652 process.p = cms.Path(process.a)
2653 process.p2 = cms.Path(process.r, process.task1, process.task2)
2654 process.schedule = cms.Schedule(*[ process.p2, process.p ], tasks=[process.task3, process.task4, process.task5])""")
2655             # include some conditional tasks
2656             p = Process("test")
2657             p.a = EDAnalyzer("MyAnalyzer")
2658             p.b = EDProducer("bProducer")
2659             p.c = EDProducer("cProducer")
2660             p.d = EDProducer("dProducer")
2661             p.e = EDProducer("eProducer")
2662             p.f = EDProducer("fProducer")
2663             p.g = EDProducer("gProducer")
2664             p.task5 = Task()
2665             p.task3 = Task()
2666             p.task2 = ConditionalTask(p.c, p.task3)
2667             p.task1 = ConditionalTask(p.task5)
2668             p.p = Path(p.a)
2669             s = Sequence(p.a)
2670             p.r = Sequence(s)
2671             p.p2 = Path(p.r, p.task1, p.task2)
2672             p.schedule = Schedule(p.p2,p.p,tasks=[p.task5])
2673             d=p.dumpPython()
2674             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2675 """process.b = cms.EDProducer("bProducer")
2676 process.c = cms.EDProducer("cProducer")
2677 process.d = cms.EDProducer("dProducer")
2678 process.e = cms.EDProducer("eProducer")
2679 process.f = cms.EDProducer("fProducer")
2680 process.g = cms.EDProducer("gProducer")
2681 process.a = cms.EDAnalyzer("MyAnalyzer")
2682 process.task5 = cms.Task()
2683 process.task3 = cms.Task()
2684 process.task2 = cms.ConditionalTask(process.c, process.task3)
2685 process.task1 = cms.ConditionalTask(process.task5)
2686 process.r = cms.Sequence((process.a))
2687 process.p = cms.Path(process.a)
2688 process.p2 = cms.Path(process.r, process.task1, process.task2)
2689 process.schedule = cms.Schedule(*[ process.p2, process.p ], tasks=[process.task5])""")
2690             # only tasks
2691             p = Process("test")
2692             p.d = EDProducer("dProducer")
2693             p.e = EDProducer("eProducer")
2694             p.f = EDProducer("fProducer")
2695             p.g = EDProducer("gProducer")
2696             p.task1 = Task(p.d, p.e)
2697             task2 = Task(p.f, p.g)
2698             p.schedule = Schedule(tasks=[p.task1,task2])
2699             d=p.dumpPython()
2700             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2701 """process.d = cms.EDProducer("dProducer")
2702 process.e = cms.EDProducer("eProducer")
2703 process.f = cms.EDProducer("fProducer")
2704 process.g = cms.EDProducer("gProducer")
2705 process.task1 = cms.Task(process.d, process.e)
2706 process.schedule = cms.Schedule(tasks=[cms.Task(process.f, process.g), process.task1])""")
2707             # empty schedule
2708             p = Process("test")
2709             p.schedule = Schedule()
2710             d=p.dumpPython()
2711             self.assertEqual(_lineDiff(d,Process('test').dumpPython()),
2712 """process.schedule = cms.Schedule()""")
2713 
2714             s = Sequence()
2715             a = EDProducer("A")
2716             s2 = Sequence(a)
2717             s2 += s
2718             process = Process("DUMP")
2719             process.a = a
2720             process.s2 = s2
2721             d=process.dumpPython()
2722             self.assertEqual(_lineDiff(d,Process('DUMP').dumpPython()),
2723 """process.a = cms.EDProducer("A")
2724 process.s2 = cms.Sequence(process.a)""")
2725             s = Sequence()
2726             s1 = Sequence(s)
2727             a = EDProducer("A")
2728             s3 = Sequence(a+a)
2729             s2 = Sequence(a+s3)
2730             s2 += s1
2731             process = Process("DUMP")
2732             process.a = a
2733             process.s2 = s2
2734             d=process.dumpPython()
2735             self.assertEqual(_lineDiff(d,Process('DUMP').dumpPython()),
2736 """process.a = cms.EDProducer("A")
2737 process.s2 = cms.Sequence(process.a+(process.a+process.a))""")
2738 
2739         def testSecSource(self):
2740             p = Process('test')
2741             p.a = SecSource("MySecSource")
2742             self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),'process.a = cms.SecSource("MySecSource")')
2743 
2744         def testGlobalReplace(self):
2745             p = Process('test')
2746             p.a = EDAnalyzer("MyAnalyzer")
2747             old = p.a
2748             p.b = EDAnalyzer("YourAnalyzer")
2749             p.c = EDAnalyzer("OurAnalyzer")
2750             p.d = EDProducer("MyProducer")
2751             old2 = p.d
2752             p.t1 = Task(p.d)
2753             t2 = Task(p.d)
2754             t3 = Task(p.d)
2755             t4 = Task(p.d)
2756             t5 = Task(p.d)
2757             t6 = Task(p.d)
2758             p.ct1 = ConditionalTask(p.d)
2759             s = Sequence(p.a*p.b)
2760             p.s4 = Sequence(p.a*p.b, p.ct1)
2761             s.associate(t2)
2762             p.s4.associate(t2)
2763             p.p = Path(p.c+s+p.a)
2764             p.p2 = Path(p.c+p.s4+p.a, p.ct1)
2765             p.e3 = EndPath(p.c+s+p.a)
2766             new = EDAnalyzer("NewAnalyzer")
2767             new2 = EDProducer("NewProducer")
2768             visitor1 = NodeVisitor()
2769             p.p.visit(visitor1)
2770             self.assertTrue(visitor1.modules == set([old,old2,p.b,p.c]))
2771             p.schedule = Schedule(tasks=[t6])
2772             p.globalReplace("a",new)
2773             p.globalReplace("d",new2)
2774             visitor2 = NodeVisitor()
2775             p.p.visit(visitor2)
2776             self.assertTrue(visitor2.modules == set([new,new2,p.b,p.c]))
2777             self.assertEqual(p.p.dumpPython()[:-1], "cms.Path(process.c+process.a+process.b+process.a, cms.Task(process.d))")
2778             visitor_p2 = NodeVisitor()
2779             p.p2.visit(visitor_p2)
2780             self.assertTrue(visitor_p2.modules == set([new,new2,p.b,p.c]))
2781             self.assertEqual(p.p2.dumpPython()[:-1], "cms.Path(process.c+process.s4+process.a, process.ct1)")
2782             visitor3 = NodeVisitor()
2783             p.e3.visit(visitor3)
2784             self.assertTrue(visitor3.modules == set([new,new2,p.b,p.c]))
2785             visitor4 = NodeVisitor()
2786             p.s4.visit(visitor4)
2787             self.assertTrue(visitor4.modules == set([new,new2,p.b]))
2788             self.assertEqual(p.s4.dumpPython()[:-1],"cms.Sequence(process.a+process.b, cms.Task(process.d), process.ct1)")
2789             visitor5 = NodeVisitor()
2790             p.t1.visit(visitor5)
2791             self.assertTrue(visitor5.modules == set([new2]))
2792             visitor6 = NodeVisitor()
2793             listOfTasks = list(p.schedule._tasks)
2794             listOfTasks[0].visit(visitor6)
2795             self.assertTrue(visitor6.modules == set([new2]))
2796             visitor7 = NodeVisitor()
2797             p.ct1.visit(visitor7)
2798             self.assertTrue(visitor7.modules == set([new2]))
2799             visitor8 = NodeVisitor()
2800             listOfConditionalTasks = list(p.conditionaltasks.values())
2801             listOfConditionalTasks[0].visit(visitor8)
2802             self.assertTrue(visitor8.modules == set([new2]))
2803 
2804 
2805             p.d2 = EDProducer("YourProducer")
2806             p.schedule = Schedule(p.p, p.p2, p.e3, tasks=[p.t1])
2807             self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2808             p.p = Path(p.c+s)
2809             self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2810             p.e3 = EndPath(p.c)
2811             self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2812             p.t1 = Task(p.d2)
2813             self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2814 
2815         def testSequence(self):
2816             p = Process('test')
2817             p.a = EDAnalyzer("MyAnalyzer")
2818             p.b = EDAnalyzer("YourAnalyzer")
2819             p.c = EDAnalyzer("OurAnalyzer")
2820             p.s = Sequence(p.a*p.b)
2821             self.assertEqual(str(p.s),'a+b')
2822             self.assertEqual(p.s.label_(),'s')
2823             path = Path(p.c+p.s)
2824             self.assertEqual(str(path),'c+a+b')
2825             p._validateSequence(path, 'p1')
2826             notInProcess = EDAnalyzer('NotInProcess')
2827             p2 = Path(p.c+p.s*notInProcess)
2828             self.assertRaises(RuntimeError, p._validateSequence, p2, 'p2')
2829 
2830         def testSequence2(self):
2831             p = Process('test')
2832             p.a = EDAnalyzer("MyAnalyzer")
2833             p.b = EDAnalyzer("YourAnalyzer")
2834             p.c = EDAnalyzer("OurAnalyzer")
2835             testseq = Sequence(p.a*p.b)
2836             p.s = testseq
2837             #p.y = testseq
2838             self.assertRaises(ValueError, p.__setattr__, "y", testseq) 
2839 
2840         def testServiceInProcess(self):
2841             service = Service("d")
2842             self.assertFalse(service._inProcess)
2843             process = Process("test")
2844             process.d = service
2845             self.assertTrue(service._inProcess)
2846             service2 = Service("d")
2847             process.d = service2
2848             self.assertFalse(service._inProcess)
2849             self.assertTrue(service2._inProcess)
2850             del process.d
2851             self.assertFalse(service2._inProcess)
2852 
2853         def testTask(self):
2854 
2855             # create some objects to use in tests
2856             edanalyzer = EDAnalyzer("a")
2857             edproducer = EDProducer("b")
2858             edproducer2 = EDProducer("b2")
2859             edproducer3 = EDProducer("b3")
2860             edproducer4 = EDProducer("b4")
2861             edproducer8 = EDProducer("b8")
2862             edproducer9 = EDProducer("b9")
2863             edfilter = EDFilter("c")
2864             service = Service("d")
2865             service3 = Service("d", v = untracked.uint32(3))
2866             essource = ESSource("e")
2867             esproducer = ESProducer("f")
2868             testTask2 = Task()
2869 
2870             # test adding things to Tasks
2871             testTask1 = Task(edproducer, edfilter)
2872             self.assertRaises(RuntimeError, testTask1.add, edanalyzer)
2873             testTask1.add(essource, service)
2874             testTask1.add(essource, esproducer)
2875             testTask1.add(testTask2)
2876             coll = testTask1._collection
2877             self.assertTrue(edproducer in coll)
2878             self.assertTrue(edfilter in coll)
2879             self.assertTrue(service in coll)
2880             self.assertTrue(essource in coll)
2881             self.assertTrue(esproducer in coll)
2882             self.assertTrue(testTask2 in coll)
2883             self.assertTrue(len(coll) == 6)
2884             self.assertTrue(len(testTask2._collection) == 0)
2885 
2886             taskContents = []
2887             for i in testTask1:
2888                 taskContents.append(i)
2889             self.assertTrue(taskContents == [edproducer, edfilter, essource, service, esproducer, testTask2])
2890 
2891             # test attaching Task to Process
2892             process = Process("test")
2893 
2894             process.mproducer = edproducer
2895             process.mproducer2 = edproducer2
2896             process.mfilter = edfilter
2897             process.messource = essource
2898             process.mesproducer = esproducer
2899             process.d = service
2900 
2901             testTask3 = Task(edproducer, edproducer2)
2902             testTask1.add(testTask3)
2903             process.myTask1 = testTask1
2904 
2905             # test the validation that occurs when attaching a Task to a Process
2906             # first a case that passes, then one the fails on an EDProducer
2907             # then one that fails on a service
2908             l = set()
2909             visitor = NodeNameVisitor(l)
2910             testTask1.visit(visitor)
2911             self.assertTrue(l == set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
2912             l2 = testTask1.moduleNames
2913             self.assertTrue(l == set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
2914 
2915             testTask4 = Task(edproducer3)
2916             l.clear()
2917             self.assertRaises(RuntimeError, testTask4.visit, visitor)
2918             try:
2919                 process.myTask4 = testTask4
2920                 self.assertTrue(False)
2921             except RuntimeError:
2922                 pass
2923 
2924             testTask5 = Task(service3)
2925             l.clear()
2926             self.assertRaises(RuntimeError, testTask5.visit, visitor)
2927             try:
2928                 process.myTask5 = testTask5
2929                 self.assertTrue(False)
2930             except RuntimeError:
2931                 pass
2932 
2933             process.d = service3
2934             process.myTask5 = testTask5
2935 
2936             # test placement into the Process and the tasks property
2937             expectedDict = { 'myTask1' : testTask1, 'myTask5' : testTask5 }
2938             expectedFixedDict = DictTypes.FixedKeysDict(expectedDict);
2939             self.assertTrue(process.tasks == expectedFixedDict)
2940             self.assertTrue(process.tasks['myTask1'] == testTask1)
2941             self.assertTrue(process.myTask1 == testTask1)
2942 
2943             # test replacing an EDProducer in a Task when calling __settattr__
2944             # for the EDProducer on the Process.
2945             process.mproducer2 = edproducer4
2946             process.d = service
2947             l = list()
2948             visitor1 = ModuleNodeVisitor(l)
2949             testTask1.visit(visitor1)
2950             l.sort(key=lambda mod: mod.__str__())
2951             expectedList = sorted([edproducer,essource,esproducer,service,edfilter,edproducer,edproducer4],key=lambda mod: mod.__str__())
2952             self.assertTrue(expectedList == l)
2953             process.myTask6 = Task()
2954             process.myTask7 = Task()
2955             process.mproducer8 = edproducer8
2956             process.myTask8 = Task(process.mproducer8)
2957             process.myTask6.add(process.myTask7)
2958             process.myTask7.add(process.myTask8)
2959             process.myTask1.add(process.myTask6)
2960             process.myTask8.add(process.myTask5)
2961 
2962             testDict = process._itemsInDependencyOrder(process.tasks)
2963             expectedLabels = ["myTask5", "myTask8", "myTask7", "myTask6", "myTask1"]
2964             expectedTasks = [process.myTask5, process.myTask8, process.myTask7, process.myTask6, process.myTask1]
2965             index = 0
2966             for testLabel, testTask in testDict.items():
2967                 self.assertTrue(testLabel == expectedLabels[index])
2968                 self.assertTrue(testTask == expectedTasks[index])
2969                 index += 1
2970 
2971             pythonDump = testTask1.dumpPython(PrintOptions())
2972 
2973 
2974             expectedPythonDump = 'cms.Task(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer, process.mproducer2, process.myTask6)\n'
2975             self.assertTrue(pythonDump == expectedPythonDump)
2976 
2977             process.myTask5 = Task()
2978             process.myTask100 = Task()
2979             process.mproducer9 = edproducer9
2980             sequence1 = Sequence(process.mproducer8, process.myTask1, process.myTask5, testTask2, testTask3)
2981             sequence2 = Sequence(process.mproducer8 + process.mproducer9)
2982             process.sequence3 = Sequence((process.mproducer8 + process.mfilter))
2983             sequence4 = Sequence()
2984             process.path1 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+sequence4)
2985             process.path1.associate(process.myTask1, process.myTask5, testTask2, testTask3)
2986             process.path11 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+ sequence4,process.myTask1, process.myTask5, testTask2, testTask3, process.myTask100)
2987             process.path2 = Path(process.mproducer)
2988             process.path3 = Path(process.mproducer9+process.mproducer8,testTask2)
2989 
2990             self.assertTrue(process.path1.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)\n')
2991 
2992             self.assertTrue(process.path11.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask100, process.myTask5)\n')
2993 
2994             # test NodeNameVisitor and moduleNames
2995             l = set()
2996             nameVisitor = NodeNameVisitor(l)
2997             process.path1.visit(nameVisitor)
2998             self.assertTrue(l == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
2999             self.assertTrue(process.path1.moduleNames() == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3000 
3001             # test copy
3002             process.mproducer10 = EDProducer("b10")
3003             process.path21 = process.path11.copy()
3004             process.path21.replace(process.mproducer, process.mproducer10)
3005 
3006             self.assertTrue(process.path11.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask100, process.myTask5)\n')
3007 
3008             # Some peculiarities of the way things work show up here. dumpPython sorts tasks and
3009             # removes duplication at the level of strings. The Task and Sequence objects themselves
3010             # remove duplicate tasks in their contents if the instances are the same (exact same python
3011             # object id which is not the same as the string representation being the same).
3012             # Also note that the mutating visitor replaces sequences and tasks that have
3013             # modified contents with their modified contents, it does not modify the sequence
3014             # or task itself.
3015             self.assertTrue(process.path21.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer10), cms.Task(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), process.myTask100, process.myTask5)\n')
3016 
3017             process.path22 = process.path21.copyAndExclude([process.d, process.mesproducer, process.mfilter])
3018             self.assertTrue(process.path22.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer10), cms.Task(process.messource, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), process.myTask100, process.myTask5)\n')
3019 
3020             process.path23 = process.path22.copyAndExclude([process.messource, process.mproducer10])
3021             self.assertTrue(process.path23.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.Task(), cms.Task(process.None), cms.Task(process.mproducer2, process.mproducer8, process.myTask5), process.myTask100, process.myTask5)\n')
3022 
3023             process.a = EDAnalyzer("MyAnalyzer")
3024             process.b = OutputModule("MyOutputModule")
3025             process.c = EDFilter("MyFilter")
3026             process.d = EDProducer("MyProducer")
3027             process.e = ESProducer("MyESProducer")
3028             process.f = ESSource("MyESSource")
3029             process.g = ESProducer("g")
3030             process.path24 = Path(process.a+process.b+process.c+process.d)
3031             process.path25 = process.path24.copyAndExclude([process.a,process.b,process.c])
3032             self.assertTrue(process.path25.dumpPython() == 'cms.Path(process.d)\n')
3033             #print process.path3
3034             #print process.dumpPython()
3035 
3036             process.path200 = EndPath(Sequence(process.c,Task(process.e)))
3037             process.path200.replace(process.c,process.b)
3038             process.path200.replace(process.e,process.f)
3039             self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.b, cms.Task(process.f))\n")
3040             process.path200.replace(process.b,process.c)
3041             process.path200.replace(process.f,process.e)
3042             self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.c, cms.Task(process.e))\n")
3043             process.path200.replace(process.c,process.a)
3044             process.path200.replace(process.e,process.g)
3045             self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.a, cms.Task(process.g))\n")
3046             process.path200.replace(process.a,process.c)
3047             process.path200.replace(process.g,process.e)
3048             self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.c, cms.Task(process.e))\n")
3049 
3050         def testConditionalTask(self):
3051 
3052             # create some objects to use in tests
3053             edanalyzer = EDAnalyzer("a")
3054             edproducer = EDProducer("b")
3055             edproducer2 = EDProducer("b2")
3056             edproducer3 = EDProducer("b3")
3057             edproducer4 = EDProducer("b4")
3058             edproducer8 = EDProducer("b8")
3059             edproducer9 = EDProducer("b9")
3060             edfilter = EDFilter("c")
3061             service = Service("d")
3062             service3 = Service("d", v = untracked.uint32(3))
3063             essource = ESSource("e")
3064             esproducer = ESProducer("f")
3065             testTask2 = Task()
3066             testCTask2 = ConditionalTask()
3067 
3068             # test adding things to Tasks
3069             testTask1 = ConditionalTask(edproducer, edfilter)
3070             self.assertRaises(RuntimeError, testTask1.add, edanalyzer)
3071             testTask1.add(essource, service)
3072             testTask1.add(essource, esproducer)
3073             testTask1.add(testTask2)
3074             testTask1.add(testCTask2)
3075             coll = testTask1._collection
3076             self.assertTrue(edproducer in coll)
3077             self.assertTrue(edfilter in coll)
3078             self.assertTrue(service in coll)
3079             self.assertTrue(essource in coll)
3080             self.assertTrue(esproducer in coll)
3081             self.assertTrue(testTask2 in coll)
3082             self.assertTrue(testCTask2 in coll)
3083             self.assertTrue(len(coll) == 7)
3084             self.assertTrue(len(testTask2._collection) == 0)
3085 
3086             taskContents = []
3087             for i in testTask1:
3088                 taskContents.append(i)
3089             self.assertEqual(taskContents, [edproducer, edfilter, essource, service, esproducer, testTask2, testCTask2])
3090 
3091             # test attaching Task to Process
3092             process = Process("test")
3093 
3094             process.mproducer = edproducer
3095             process.mproducer2 = edproducer2
3096             process.mfilter = edfilter
3097             process.messource = essource
3098             process.mesproducer = esproducer
3099             process.d = service
3100 
3101             testTask3 = ConditionalTask(edproducer, edproducer2)
3102             testTask1.add(testTask3)
3103             process.myTask1 = testTask1
3104 
3105             # test the validation that occurs when attaching a ConditionalTask to a Process
3106             # first a case that passes, then one the fails on an EDProducer
3107             # then one that fails on a service
3108             l = set()
3109             visitor = NodeNameVisitor(l)
3110             testTask1.visit(visitor)
3111             self.assertEqual(l, set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
3112             l2 = testTask1.moduleNames()
3113             self.assertEqual(l2, set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
3114 
3115             testTask4 = ConditionalTask(edproducer3)
3116             l.clear()
3117             self.assertRaises(RuntimeError, testTask4.visit, visitor)
3118             try:
3119                 process.myTask4 = testTask4
3120                 self.assertTrue(False)
3121             except RuntimeError:
3122                 pass
3123 
3124             testTask5 = ConditionalTask(service3)
3125             l.clear()
3126             self.assertRaises(RuntimeError, testTask5.visit, visitor)
3127             try:
3128                 process.myTask5 = testTask5
3129                 self.assertTrue(False)
3130             except RuntimeError:
3131                 pass
3132 
3133             process.d = service3
3134             process.myTask5 = testTask5
3135 
3136             # test placement into the Process and the tasks property
3137             expectedDict = { 'myTask1' : testTask1, 'myTask5' : testTask5 }
3138             expectedFixedDict = DictTypes.FixedKeysDict(expectedDict);
3139             self.assertEqual(process.conditionaltasks, expectedFixedDict)
3140             self.assertEqual(process.conditionaltasks['myTask1'], testTask1)
3141             self.assertEqual(process.myTask1, testTask1)
3142 
3143             # test replacing an EDProducer in a ConditionalTask when calling __settattr__
3144             # for the EDProducer on the Process.
3145             process.mproducer2 = edproducer4
3146             process.d = service
3147             l = list()
3148             visitor1 = ModuleNodeVisitor(l)
3149             testTask1.visit(visitor1)
3150             l.sort(key=lambda mod: mod.__str__())
3151             expectedList = sorted([edproducer,essource,esproducer,service,edfilter,edproducer,edproducer4],key=lambda mod: mod.__str__())
3152             self.assertEqual(expectedList, l)
3153             process.myTask6 = ConditionalTask()
3154             process.myTask7 = ConditionalTask()
3155             process.mproducer8 = edproducer8
3156             process.myTask8 = ConditionalTask(process.mproducer8)
3157             process.myTask6.add(process.myTask7)
3158             process.myTask7.add(process.myTask8)
3159             process.myTask1.add(process.myTask6)
3160             process.myTask8.add(process.myTask5)
3161             self.assertEqual(process.myTask8.dumpPython(), "cms.ConditionalTask(process.mproducer8, process.myTask5)\n")
3162 
3163             testDict = process._itemsInDependencyOrder(process.conditionaltasks)
3164             expectedLabels = ["myTask5", "myTask8", "myTask7", "myTask6", "myTask1"]
3165             expectedTasks = [process.myTask5, process.myTask8, process.myTask7, process.myTask6, process.myTask1]
3166             index = 0
3167             for testLabel, testTask in testDict.items():
3168                 self.assertEqual(testLabel, expectedLabels[index])
3169                 self.assertEqual(testTask, expectedTasks[index])
3170                 index += 1
3171 
3172             pythonDump = testTask1.dumpPython(PrintOptions())
3173 
3174 
3175             expectedPythonDump = 'cms.ConditionalTask(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer, process.mproducer2, process.myTask6)\n'
3176             self.assertEqual(pythonDump, expectedPythonDump)
3177 
3178             process.myTask5 = ConditionalTask()
3179             self.assertEqual(process.myTask8.dumpPython(), "cms.ConditionalTask(process.mproducer8, process.myTask5)\n")
3180             process.myTask100 = ConditionalTask()
3181             process.mproducer9 = edproducer9
3182             sequence1 = Sequence(process.mproducer8, process.myTask1, process.myTask5, testTask2, testTask3)
3183             sequence2 = Sequence(process.mproducer8 + process.mproducer9)
3184             process.sequence3 = Sequence((process.mproducer8 + process.mfilter))
3185             sequence4 = Sequence()
3186             process.path1 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+sequence4)
3187             process.path1.associate(process.myTask1, process.myTask5, testTask2, testTask3)
3188             process.path11 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+ sequence4,process.myTask1, process.myTask5, testTask2, testTask3, process.myTask100)
3189             process.path2 = Path(process.mproducer)
3190             process.path3 = Path(process.mproducer9+process.mproducer8,testTask2)
3191 
3192             self.assertEqual(process.path1.dumpPython(PrintOptions()), 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)\n')
3193 
3194             self.assertEqual(process.path11.dumpPython(PrintOptions()), 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask100, process.myTask5)\n')
3195 
3196             # test NodeNameVisitor and moduleNames
3197             l = set()
3198             nameVisitor = NodeNameVisitor(l)
3199             process.path1.visit(nameVisitor)
3200             self.assertTrue(l == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3201             self.assertTrue(process.path1.moduleNames() == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3202 
3203             # test copy
3204             process.mproducer10 = EDProducer("b10")
3205             process.path21 = process.path11.copy()
3206             process.path21.replace(process.mproducer, process.mproducer10)
3207 
3208             self.assertEqual(process.path11.dumpPython(PrintOptions()), 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask100, process.myTask5)\n')
3209 
3210             # Some peculiarities of the way things work show up here. dumpPython sorts tasks and
3211             # removes duplication at the level of strings. The Task and Sequence objects themselves
3212             # remove duplicate tasks in their contents if the instances are the same (exact same python
3213             # object id which is not the same as the string representation being the same).
3214             # Also note that the mutating visitor replaces sequences and tasks that have
3215             # modified contents with their modified contents, it does not modify the sequence
3216             # or task itself.
3217             self.assertEqual(process.path21.dumpPython(PrintOptions()), 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer10), cms.ConditionalTask(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), cms.Task(), process.myTask100, process.myTask5)\n')
3218 
3219             process.path22 = process.path21.copyAndExclude([process.d, process.mesproducer, process.mfilter])
3220             self.assertEqual(process.path22.dumpPython(PrintOptions()), 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.ConditionalTask(process.None, process.mproducer10), cms.ConditionalTask(process.messource, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), cms.Task(), process.myTask100, process.myTask5)\n')
3221 
3222             process.path23 = process.path22.copyAndExclude([process.messource, process.mproducer10])
3223             self.assertEqual(process.path23.dumpPython(PrintOptions()), 'cms.Path(process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.ConditionalTask(process.None), cms.ConditionalTask(process.mproducer2, process.mproducer8, process.myTask5), cms.Task(), process.myTask100, process.myTask5)\n')
3224 
3225             process = Process("Test")
3226 
3227             process.b = EDProducer("b")
3228             process.b2 = EDProducer("b2")
3229             process.b3 = EDProducer("b3")
3230             process.p = Path(process.b, ConditionalTask(process.b3, process.b2))
3231             p = TestMakePSet()
3232             process.fillProcessDesc(p)
3233             self.assertEqual(p.values["@all_modules"], (True, ['b', 'b2', 'b3']))
3234             self.assertEqual(p.values["@paths"], (True, ['p']))
3235             self.assertEqual(p.values["p"], (True, ['b','#','b2','b3','@']))
3236             
3237 
3238         def testPath(self):
3239             p = Process("test")
3240             p.a = EDAnalyzer("MyAnalyzer")
3241             p.b = EDAnalyzer("YourAnalyzer")
3242             p.c = EDAnalyzer("OurAnalyzer")
3243             path = Path(p.a)
3244             path *= p.b
3245             path += p.c
3246             self.assertEqual(str(path),'a+b+c')
3247             path = Path(p.a*p.b+p.c)
3248             self.assertEqual(str(path),'a+b+c')
3249 #            path = Path(p.a)*p.b+p.c #This leads to problems with sequences
3250 #            self.assertEqual(str(path),'((a*b)+c)')
3251             path = Path(p.a+ p.b*p.c)
3252             self.assertEqual(str(path),'a+b+c')
3253             path = Path(p.a*(p.b+p.c))
3254             self.assertEqual(str(path),'a+b+c')
3255             path = Path(p.a*(p.b+~p.c))
3256             pathx = Path(p.a*(p.b+ignore(p.c)))
3257             self.assertEqual(str(path),'a+b+~c')
3258             p.es = ESProducer("AnESProducer")
3259             self.assertRaises(TypeError,Path,p.es)
3260 
3261             t = Path()
3262             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path()\n')
3263 
3264             t = Path(p.a)
3265             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a)\n')
3266 
3267             t = Path(Task())
3268             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(cms.Task())\n')
3269 
3270             t = Path(p.a, Task())
3271             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.Task())\n')
3272 
3273             p.prod = EDProducer("prodName")
3274             p.t1 = Task(p.prod)
3275             t = Path(p.a, p.t1, Task(), p.t1)
3276             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.Task(), process.t1)\n')
3277 
3278             t = Path(ConditionalTask())
3279             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(cms.ConditionalTask())\n')
3280 
3281             t = Path(p.a, ConditionalTask())
3282             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.ConditionalTask())\n')
3283 
3284             p.prod = EDProducer("prodName")
3285             p.t1 = ConditionalTask(p.prod)
3286             t = Path(p.a, p.t1, Task(), p.t1)
3287             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.Task(), process.t1)\n')
3288 
3289         def testFinalPath(self):
3290             p = Process("test")
3291             p.a = OutputModule("MyOutputModule")
3292             p.b = OutputModule("YourOutputModule")
3293             p.c = OutputModule("OurOutputModule")
3294             path = FinalPath(p.a)
3295             path *= p.b
3296             path += p.c
3297             self.assertEqual(str(path),'a+b+c')
3298             path = FinalPath(p.a*p.b+p.c)
3299             self.assertEqual(str(path),'a+b+c')
3300             path = FinalPath(p.a+ p.b*p.c)
3301             self.assertEqual(str(path),'a+b+c')
3302             path = FinalPath(p.a*(p.b+p.c))
3303             self.assertEqual(str(path),'a+b+c')
3304             p.es = ESProducer("AnESProducer")
3305             self.assertRaises(TypeError,FinalPath,p.es)
3306 
3307             t = FinalPath()
3308             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.FinalPath()\n')
3309 
3310             t = FinalPath(p.a)
3311             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.FinalPath(process.a)\n')
3312 
3313             self.assertRaises(TypeError, FinalPath, Task())
3314             self.assertRaises(TypeError, FinalPath, p.a, Task())
3315 
3316             p.prod = EDProducer("prodName")
3317             p.t1 = Task(p.prod)
3318             self.assertRaises(TypeError, FinalPath, p.a, p.t1, Task(), p.t1)
3319 
3320             p.prod = EDProducer("prodName")
3321             p.t1 = ConditionalTask(p.prod)
3322             self.assertRaises(TypeError, FinalPath, p.a, p.t1, ConditionalTask(), p.t1)
3323 
3324             p.t = FinalPath(p.a)
3325             p.a = OutputModule("ReplacedOutputModule")
3326             self.assertEqual(p.t.dumpPython(PrintOptions()), 'cms.FinalPath(process.a)\n')
3327             
3328         def testCloneSequence(self):
3329             p = Process("test")
3330             a = EDAnalyzer("MyAnalyzer")
3331             p.a = a
3332             a.setLabel("a")
3333             b = EDAnalyzer("YOurAnalyzer")
3334             p.b = b
3335             b.setLabel("b")
3336             path = Path(a * b)
3337             p.path = Path(p.a*p.b)
3338             lookuptable = {id(a): p.a, id(b): p.b}
3339             #self.assertEqual(str(path),str(path._postProcessFixup(lookuptable)))
3340             #lookuptable = p._cloneToObjectDict
3341             #self.assertEqual(str(path),str(path._postProcessFixup(lookuptable)))
3342             self.assertEqual(str(path),str(p.path))
3343 
3344         def testContains(self):
3345 
3346             a = EDProducer("a")
3347             b = EDProducer("b")
3348             c = EDProducer("c")
3349             d = EDProducer("d")
3350             e = EDProducer("e")
3351             f = EDProducer("f")
3352             g = EDProducer("g")
3353             h = EDProducer("h")
3354             i = EDProducer("i")
3355             j = EDProducer("j")
3356             k = EDProducer("k")
3357             l = EDProducer("l")
3358             m = EDProducer("m")
3359             n = EDProducer("n")
3360 
3361             seq1 = Sequence(e)
3362             task1 = Task(g)
3363             ctask1 = ConditionalTask(h)
3364             path = Path(a * c * seq1, task1, ctask1)
3365 
3366             self.assertTrue(path.contains(a))
3367             self.assertFalse(path.contains(b))
3368             self.assertTrue(path.contains(c))
3369             self.assertFalse(path.contains(d))
3370             self.assertTrue(path.contains(e))
3371             self.assertFalse(path.contains(f))
3372             self.assertTrue(path.contains(g))
3373             self.assertTrue(path.contains(h))
3374 
3375             endpath = EndPath(h * i)
3376             self.assertFalse(endpath.contains(b))
3377             self.assertTrue(endpath.contains(i))
3378 
3379             seq = Sequence(a * c)
3380             self.assertFalse(seq.contains(b))
3381             self.assertTrue(seq.contains(c))
3382 
3383             task2 = Task(l)
3384             task = Task(j, k, task2)
3385             self.assertFalse(task.contains(b))
3386             self.assertTrue(task.contains(j))
3387             self.assertTrue(task.contains(k))
3388             self.assertTrue(task.contains(l))
3389 
3390             task3 = Task(m)
3391             path2 = Path(n)
3392             sch = Schedule(path, path2, tasks=[task,task3])
3393             self.assertFalse(sch.contains(b))
3394             self.assertTrue(sch.contains(a))
3395             self.assertTrue(sch.contains(c))
3396             self.assertTrue(sch.contains(e))
3397             self.assertTrue(sch.contains(g))
3398             self.assertTrue(sch.contains(n))
3399             self.assertTrue(sch.contains(j))
3400             self.assertTrue(sch.contains(k))
3401             self.assertTrue(sch.contains(l))
3402             self.assertTrue(sch.contains(m))
3403 
3404             ctask2 = ConditionalTask(l, task1)
3405             ctask = ConditionalTask(j, k, ctask2)
3406             self.assertFalse(ctask.contains(b))
3407             self.assertTrue(ctask.contains(j))
3408             self.assertTrue(ctask.contains(k))
3409             self.assertTrue(ctask.contains(l))
3410             self.assertTrue(ctask.contains(g))
3411 
3412         def testSchedule(self):
3413             p = Process("test")
3414             p.a = EDAnalyzer("MyAnalyzer")
3415             p.b = EDAnalyzer("YourAnalyzer")
3416             p.c = EDAnalyzer("OurAnalyzer")
3417             p.d = EDAnalyzer("OurAnalyzer")
3418             p.path1 = Path(p.a)
3419             p.path2 = Path(p.b)
3420             p.path3 = Path(p.d)
3421 
3422             s = Schedule(p.path1,p.path2)
3423             self.assertEqual(s[0],p.path1)
3424             self.assertEqual(s[1],p.path2)
3425             p.schedule = s
3426             self.assertTrue('b' in p.schedule.moduleNames())
3427             self.assertTrue(hasattr(p, 'b'))
3428             self.assertTrue(hasattr(p, 'c'))
3429             self.assertTrue(hasattr(p, 'd'))
3430             self.assertTrue(hasattr(p, 'path1'))
3431             self.assertTrue(hasattr(p, 'path2'))
3432             self.assertTrue(hasattr(p, 'path3'))
3433             p.prune()
3434             self.assertTrue('b' in p.schedule.moduleNames())
3435             self.assertTrue(hasattr(p, 'b'))
3436             self.assertTrue(not hasattr(p, 'c'))
3437             self.assertTrue(not hasattr(p, 'd'))
3438             self.assertTrue(hasattr(p, 'path1'))
3439             self.assertTrue(hasattr(p, 'path2'))
3440             self.assertTrue(not hasattr(p, 'path3'))
3441 
3442             self.assertTrue(len(p.schedule._tasks) == 0)
3443 
3444             p = Process("test")
3445             p.a = EDAnalyzer("MyAnalyzer")
3446             p.b = EDAnalyzer("YourAnalyzer")
3447             p.c = EDAnalyzer("OurAnalyzer")
3448             p.d = EDAnalyzer("dAnalyzer")
3449             p.e = EDProducer("eProducer")
3450             p.f = EDProducer("fProducer")
3451             p.Tracer = Service("Tracer")
3452             p.path1 = Path(p.a)
3453             p.path2 = Path(p.b)
3454             p.path3 = Path(p.d)
3455             p.task1 = Task(p.e)
3456             p.task2 = Task(p.f, p.Tracer)
3457             s = Schedule(p.path1,p.path2,tasks=[p.task1,p.task2,p.task1])
3458             self.assertEqual(s[0],p.path1)
3459             self.assertEqual(s[1],p.path2)
3460             self.assertTrue(len(s._tasks) == 2)
3461             self.assertTrue(p.task1 in s._tasks)
3462             self.assertTrue(p.task2 in s._tasks)
3463             listOfTasks = list(s._tasks)
3464             self.assertTrue(len(listOfTasks) == 2)
3465             self.assertTrue(p.task1 == listOfTasks[0])
3466             self.assertTrue(p.task2 == listOfTasks[1])
3467             p.schedule = s
3468             self.assertTrue('b' in p.schedule.moduleNames())
3469 
3470             process2 = Process("test")
3471             process2.a = EDAnalyzer("MyAnalyzer")
3472             process2.e = EDProducer("eProducer")
3473             process2.path1 = Path(process2.a)
3474             process2.task1 = Task(process2.e)
3475             process2.schedule = Schedule(process2.path1,tasks=process2.task1)
3476             listOfTasks = list(process2.schedule._tasks)
3477             self.assertTrue(listOfTasks[0] == process2.task1)
3478 
3479             # test Schedule copy
3480             s2 = s.copy()
3481             self.assertEqual(s2[0],p.path1)
3482             self.assertEqual(s2[1],p.path2)
3483             self.assertTrue(len(s2._tasks) == 2)
3484             self.assertTrue(p.task1 in s2._tasks)
3485             self.assertTrue(p.task2 in s2._tasks)
3486             listOfTasks = list(s2._tasks)
3487             self.assertTrue(len(listOfTasks) == 2)
3488             self.assertTrue(p.task1 == listOfTasks[0])
3489             self.assertTrue(p.task2 == listOfTasks[1])
3490 
3491             names = s.moduleNames()
3492             self.assertTrue(names == set(['a', 'b', 'e', 'Tracer', 'f']))
3493             #adding a path not attached to the Process should cause an exception
3494             p = Process("test")
3495             p.a = EDAnalyzer("MyAnalyzer")
3496             path1 = Path(p.a)
3497             s = Schedule(path1)
3498             self.assertRaises(RuntimeError, lambda : p.setSchedule_(s) )
3499 
3500             #make sure anonymous sequences work
3501             p = Process("test")
3502             p.a = EDAnalyzer("MyAnalyzer")
3503             p.b = EDAnalyzer("MyOtherAnalyzer")
3504             p.c = EDProducer("MyProd")
3505             path1 = Path(p.c*Sequence(p.a+p.b))
3506             s = Schedule(path1)
3507             self.assertTrue('a' in s.moduleNames())
3508             self.assertTrue('b' in s.moduleNames())
3509             self.assertTrue('c' in s.moduleNames())
3510             p.path1 = path1
3511             p.schedule = s
3512             p.prune()
3513             self.assertTrue('a' in s.moduleNames())
3514             self.assertTrue('b' in s.moduleNames())
3515             self.assertTrue('c' in s.moduleNames())
3516 
3517         def testImplicitSchedule(self):
3518             p = Process("test")
3519             p.a = EDAnalyzer("MyAnalyzer")
3520             p.b = EDAnalyzer("YourAnalyzer")
3521             p.c = EDAnalyzer("OurAnalyzer")
3522             p.path1 = Path(p.a)
3523             p.path2 = Path(p.b)
3524             self.assertTrue(p.schedule is None)
3525             pths = p.paths
3526             keys = pths.keys()
3527             self.assertEqual(pths[keys[0]],p.path1)
3528             self.assertEqual(pths[keys[1]],p.path2)
3529             p.prune()
3530             self.assertTrue(hasattr(p, 'a'))
3531             self.assertTrue(hasattr(p, 'b'))
3532             self.assertTrue(not hasattr(p, 'c'))
3533             self.assertTrue(hasattr(p, 'path1'))
3534             self.assertTrue(hasattr(p, 'path2'))
3535 
3536 
3537             p = Process("test")
3538             p.a = EDAnalyzer("MyAnalyzer")
3539             p.b = EDAnalyzer("YourAnalyzer")
3540             p.c = EDAnalyzer("OurAnalyzer")
3541             p.path2 = Path(p.b)
3542             p.path1 = Path(p.a)
3543             self.assertTrue(p.schedule is None)
3544             pths = p.paths
3545             keys = pths.keys()
3546             self.assertEqual(pths[keys[1]],p.path1)
3547             self.assertEqual(pths[keys[0]],p.path2)
3548 
3549 
3550         def testUsing(self):
3551             p = Process('test')
3552             p.block = PSet(a = int32(1))
3553             p.modu = EDAnalyzer('Analyzer', p.block, b = int32(2))
3554             self.assertEqual(p.modu.a.value(),1)
3555             self.assertEqual(p.modu.b.value(),2)
3556 
3557         def testOverride(self):
3558             p = Process('test')
3559             a = EDProducer("A", a1=int32(0))
3560             self.assertTrue(not a.isModified())
3561             a.a1 = 1
3562             self.assertTrue(a.isModified())
3563             p.a = a
3564             self.assertEqual(p.a.a1.value(), 1)
3565             # try adding an unmodified module.
3566             # should accept it
3567             p.a = EDProducer("A", a1=int32(2))
3568             self.assertEqual(p.a.a1.value(), 2)
3569             # try adding a modified module.  Should throw
3570             # no longer, since the same (modified) say, geometry
3571             # could come from more than one cff
3572             b = EDProducer("A", a1=int32(3))
3573             b.a1 = 4
3574             #self.assertRaises(RuntimeError, setattr, *(p,'a',b))
3575             ps1 = PSet(a = int32(1))
3576             ps2 = PSet(a = int32(2))
3577             self.assertRaises(ValueError, EDProducer, 'C', ps1, ps2)
3578             self.assertRaises(ValueError, EDProducer, 'C', ps1, a=int32(3))
3579         
3580         def testOptions(self):
3581             p = Process('test')
3582             self.assertEqual(p.options.numberOfThreads.value(),1)
3583             p.options.numberOfThreads = 8
3584             self.assertEqual(p.options.numberOfThreads.value(),8)
3585             p.options = PSet()
3586             self.assertEqual(p.options.numberOfThreads.value(),1)
3587             p.options = dict(numberOfStreams =2,
3588                              numberOfThreads =2)
3589             self.assertEqual(p.options.numberOfThreads.value(),2)
3590             self.assertEqual(p.options.numberOfStreams.value(),2)
3591 
3592         def testMaxEvents(self):
3593             p = Process("Test")
3594             p.maxEvents.input = 10
3595             self.assertEqual(p.maxEvents.input.value(),10)
3596             p = Process("Test")
3597             p.maxEvents.output = 10
3598             self.assertEqual(p.maxEvents.output.value(),10)
3599             p = Process("Test")
3600             p.maxEvents.output = PSet(out=untracked.int32(10))
3601             self.assertEqual(p.maxEvents.output.out.value(), 10)
3602             p = Process("Test")
3603             p.maxEvents = untracked.PSet(input = untracked.int32(5))
3604             self.assertEqual(p.maxEvents.input.value(), 5)
3605 
3606         
3607         def testExamples(self):
3608             p = Process("Test")
3609             p.source = Source("PoolSource",fileNames = untracked(string("file:reco.root")))
3610             p.foos = EDProducer("FooProducer")
3611             p.bars = EDProducer("BarProducer", foos=InputTag("foos"))
3612             p.out = OutputModule("PoolOutputModule",fileName=untracked(string("file:foos.root")))
3613             p.bars.foos = 'Foosball'
3614             self.assertEqual(p.bars.foos, InputTag('Foosball'))
3615             p.p = Path(p.foos*p.bars)
3616             p.e = EndPath(p.out)
3617             p.add_(Service("MessageLogger"))
3618 
3619         def testPrefers(self):
3620             p = Process("Test")
3621             p.add_(ESSource("ForceSource"))
3622             p.juicer = ESProducer("JuicerProducer")
3623             p.prefer("ForceSource")
3624             p.prefer("juicer")
3625             self.assertEqual(_lineDiff(p.dumpPython(), Process('Test').dumpPython()),
3626 """process.juicer = cms.ESProducer("JuicerProducer")
3627 process.ForceSource = cms.ESSource("ForceSource")
3628 process.prefer("ForceSource")
3629 process.prefer("juicer")""")
3630             p.prefer("juicer",fooRcd=vstring("Foo"))
3631             self.assertEqual(_lineDiff(p.dumpPython(), Process('Test').dumpPython()),
3632 """process.juicer = cms.ESProducer("JuicerProducer")
3633 process.ForceSource = cms.ESSource("ForceSource")
3634 process.prefer("ForceSource")
3635 process.prefer("juicer",
3636     fooRcd = cms.vstring('Foo')
3637 )""")
3638 
3639         def testFreeze(self):
3640             process = Process("Freeze")
3641             m = EDProducer("M", p=PSet(i = int32(1)))
3642             m.p.i = 2
3643             process.m = m
3644             # should be frozen
3645             #self.assertRaises(ValueError, setattr, m.p, 'i', 3)
3646             #self.assertRaises(ValueError, setattr, m, 'p', PSet(i=int32(1)))
3647             #self.assertRaises(ValueError, setattr, m.p, 'j', 1)
3648             #self.assertRaises(ValueError, setattr, m, 'j', 1)
3649             # But OK to change through the process
3650             process.m.p.i = 4
3651             self.assertEqual(process.m.p.i.value(), 4)
3652             process.m.p = PSet(j=int32(1))
3653             # should work to clone it, though
3654             m2 = m.clone(p = PSet(i = int32(5)), j = int32(8))
3655             m2.p.i = 6
3656             m2.j = 8
3657         def testSubProcess(self):
3658             process = Process("Parent")
3659             subProcess = Process("Child")
3660             subProcess.a = EDProducer("A")
3661             subProcess.p = Path(subProcess.a)
3662             subProcess.add_(Service("Foo"))
3663             process.addSubProcess(SubProcess(subProcess))
3664             d = process.dumpPython()
3665             equalD ="""parentProcess = process
3666 process.a = cms.EDProducer("A")
3667 process.Foo = cms.Service("Foo")
3668 process.p = cms.Path(process.a)
3669 childProcess = process
3670 process = parentProcess
3671 process.addSubProcess(cms.SubProcess(process = childProcess, SelectEvents = cms.untracked.PSet(
3672 ), outputCommands = cms.untracked.vstring()))"""
3673             equalD = equalD.replace("parentProcess","parentProcess"+str(hash(process.subProcesses_()[0])))
3674             # SubProcesses are dumped before Services, so in order to
3675             # craft the dump of the Parent and Child manually the dump
3676             # of the Parent needs to be split at the MessageLogger
3677             # boundary (now when it is part of Process by default),
3678             # and insert the dump of the Child between the top part of
3679             # the Parent (before MessageLogger) and the bottom part of
3680             # the Parent (after and including MessageLogger)
3681             messageLoggerSplit = 'process.MessageLogger = cms.Service'
3682             parentDumpSplit = Process('Parent').dumpPython().split(messageLoggerSplit)
3683             childProcess = Process('Child')
3684             del childProcess.MessageLogger
3685             combinedDump = parentDumpSplit[0] + childProcess.dumpPython() + messageLoggerSplit + parentDumpSplit[1]
3686             self.assertEqual(_lineDiff(d, combinedDump), equalD)
3687             p = TestMakePSet()
3688             process.fillProcessDesc(p)
3689             self.assertEqual((True,['a']),p.values["subProcesses"][1][0].values["process"][1].values['@all_modules'])
3690             self.assertEqual((True,['p']),p.values["subProcesses"][1][0].values["process"][1].values['@paths'])
3691             self.assertEqual({'@service_type':(True,'Foo')}, p.values["subProcesses"][1][0].values["process"][1].values["services"][1][0].values)
3692         def testRefToPSet(self):
3693             proc = Process("test")
3694             proc.top = PSet(a = int32(1))
3695             proc.ref = PSet(refToPSet_ = string("top"))
3696             proc.ref2 = PSet( a = int32(1), b = PSet( refToPSet_ = string("top")))
3697             proc.ref3 = PSet(refToPSet_ = string("ref"))
3698             proc.ref4 = VPSet(PSet(refToPSet_ = string("top")),
3699                               PSet(refToPSet_ = string("ref2")))
3700             p = TestMakePSet()
3701             proc.fillProcessDesc(p)
3702             self.assertEqual((True,1),p.values["ref"][1].values["a"])
3703             self.assertEqual((True,1),p.values["ref3"][1].values["a"])
3704             self.assertEqual((True,1),p.values["ref2"][1].values["a"])
3705             self.assertEqual((True,1),p.values["ref2"][1].values["b"][1].values["a"])
3706             self.assertEqual((True,1),p.values["ref4"][1][0].values["a"])
3707             self.assertEqual((True,1),p.values["ref4"][1][1].values["a"])
3708         def testSwitchProducer(self):
3709             proc = Process("test")
3710             proc.sp = SwitchProducerTest(test2 = EDProducer("Foo",
3711                                                             a = int32(1),
3712                                                             b = PSet(c = int32(2))),
3713                                          test1 = EDProducer("Bar",
3714                                                             aa = int32(11),
3715                                                             bb = PSet(cc = int32(12))))
3716             self.assertEqual(proc.sp.label_(), "sp")
3717             self.assertEqual(proc.sp.test1.label_(), "sp@test1")
3718             self.assertEqual(proc.sp.test2.label_(), "sp@test2")
3719 
3720             proc.a = EDProducer("A")
3721             proc.s = Sequence(proc.a + proc.sp)
3722             proc.t = Task(proc.a, proc.sp)
3723             proc.p = Path()
3724             proc.p.associate(proc.t)
3725             p = TestMakePSet()
3726             proc.fillProcessDesc(p)
3727             self.assertEqual((True,"EDProducer"), p.values["sp"][1].values["@module_edm_type"])
3728             self.assertEqual((True, "SwitchProducer"), p.values["sp"][1].values["@module_type"])
3729             self.assertEqual((True, "sp"), p.values["sp"][1].values["@module_label"])
3730             all_cases = copy.deepcopy(p.values["sp"][1].values["@all_cases"])
3731             all_cases[1].sort() # names of all cases come via dict, i.e. their order is undefined
3732             self.assertEqual((True, ["sp@test1", "sp@test2"]), all_cases)
3733             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
3734             self.assertEqual(["a", "sp", "sp@test1", "sp@test2"], p.values["@all_modules"][1])
3735             self.assertEqual((True,"EDProducer"), p.values["sp@test1"][1].values["@module_edm_type"])
3736             self.assertEqual((True,"Bar"), p.values["sp@test1"][1].values["@module_type"])
3737             self.assertEqual((True,"EDProducer"), p.values["sp@test2"][1].values["@module_edm_type"])
3738             self.assertEqual((True,"Foo"), p.values["sp@test2"][1].values["@module_type"])
3739             dump = proc.dumpPython()
3740             self.assertEqual(dump.find('@'), -1)
3741             self.assertEqual(specialImportRegistry.getSpecialImports(), ["from test import SwitchProducerTest"])
3742             self.assertTrue(dump.find("\nfrom test import SwitchProducerTest\n") != -1)
3743 
3744             # EDAlias as non-chosen case
3745             proc = Process("test")
3746             proc.sp = SwitchProducerTest(test2 = EDProducer("Foo",
3747                                                             a = int32(1),
3748                                                             b = PSet(c = int32(2))),
3749                                          test1 = EDAlias(a = VPSet(PSet(type = string("Bar")))))
3750             proc.a = EDProducer("A")
3751             proc.s = Sequence(proc.a + proc.sp)
3752             proc.t = Task(proc.a, proc.sp)
3753             proc.p = Path()
3754             proc.p.associate(proc.t)
3755             p = TestMakePSet()
3756             proc.fillProcessDesc(p)
3757             self.assertEqual((True,"EDProducer"), p.values["sp"][1].values["@module_edm_type"])
3758             self.assertEqual((True, "SwitchProducer"), p.values["sp"][1].values["@module_type"])
3759             self.assertEqual((True, "sp"), p.values["sp"][1].values["@module_label"])
3760             all_cases = copy.deepcopy(p.values["sp"][1].values["@all_cases"])
3761             all_cases[1].sort()
3762             self.assertEqual((True, ["sp@test1", "sp@test2"]), all_cases)
3763             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
3764             self.assertEqual(["a", "sp", "sp@test2"], p.values["@all_modules"][1])
3765             self.assertEqual(["sp@test1"], p.values["@all_aliases"][1])
3766             self.assertEqual((True,"EDProducer"), p.values["sp@test2"][1].values["@module_edm_type"])
3767             self.assertEqual((True,"Foo"), p.values["sp@test2"][1].values["@module_type"])
3768             self.assertEqual((True,"EDAlias"), p.values["sp@test1"][1].values["@module_edm_type"])
3769             self.assertEqual((True,"Bar"), p.values["sp@test1"][1].values["a"][1][0].values["type"])
3770 
3771             # EDAlias as chosen case
3772             proc = Process("test")
3773             proc.sp = SwitchProducerTest(test1 = EDProducer("Foo",
3774                                                             a = int32(1),
3775                                                             b = PSet(c = int32(2))),
3776                                          test2 = EDAlias(a = VPSet(PSet(type = string("Bar")))))
3777             proc.a = EDProducer("A")
3778             proc.s = Sequence(proc.a + proc.sp)
3779             proc.t = Task(proc.a, proc.sp)
3780             proc.p = Path()
3781             proc.p.associate(proc.t)
3782             p = TestMakePSet()
3783             proc.fillProcessDesc(p)
3784             self.assertEqual((True,"EDProducer"), p.values["sp"][1].values["@module_edm_type"])
3785             self.assertEqual((True, "SwitchProducer"), p.values["sp"][1].values["@module_type"])
3786             self.assertEqual((True, "sp"), p.values["sp"][1].values["@module_label"])
3787             self.assertEqual((True, ["sp@test1", "sp@test2"]), p.values["sp"][1].values["@all_cases"])
3788             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
3789             self.assertEqual(["a", "sp", "sp@test1"], p.values["@all_modules"][1])
3790             self.assertEqual(["sp@test2"], p.values["@all_aliases"][1])
3791             self.assertEqual((True,"EDProducer"), p.values["sp@test1"][1].values["@module_edm_type"])
3792             self.assertEqual((True,"Foo"), p.values["sp@test1"][1].values["@module_type"])
3793             self.assertEqual((True,"EDAlias"), p.values["sp@test2"][1].values["@module_edm_type"])
3794             self.assertEqual((True,"Bar"), p.values["sp@test2"][1].values["a"][1][0].values["type"])
3795 
3796             # ConditionalTask
3797             proc = Process("test")
3798             proc.spct = SwitchProducerTest(test2 = EDProducer("Foo",
3799                                                               a = int32(1),
3800                                                               b = PSet(c = int32(2))),
3801                                            test1 = EDProducer("Bar",
3802                                                               aa = int32(11),
3803                                                               bb = PSet(cc = int32(12))),
3804                                            test3 = EDAlias(a = VPSet(PSet(type = string("Bar")))))
3805             proc.spp = proc.spct.clone()
3806             proc.a = EDProducer("A")
3807             proc.ct = ConditionalTask(proc.spct)
3808             proc.p = Path(proc.a, proc.ct)
3809             proc.pp = Path(proc.a + proc.spp)
3810             p = TestMakePSet()
3811             proc.fillProcessDesc(p)
3812             self.assertEqual(["a", "spct", "spct@test1", "spct@test2", "spp", "spp@test1", "spp@test2"], p.values["@all_modules"][1])
3813             self.assertEqual(["a", "#", "spct", "spct@test1", "spct@test2", "@"], p.values["p"][1])
3814             self.assertEqual(["a", "spp", "#", "spp@test1", "spp@test2", "@"], p.values["pp"][1])
3815 
3816         def testPrune(self):
3817             p = Process("test")
3818             p.a = EDAnalyzer("MyAnalyzer")
3819             p.b = EDAnalyzer("YourAnalyzer")
3820             p.c = EDAnalyzer("OurAnalyzer")
3821             p.d = EDAnalyzer("OurAnalyzer")
3822             p.e = EDProducer("MyProducer")
3823             p.f = EDProducer("YourProducer")
3824             p.g = EDProducer("TheirProducer")
3825             p.h = EDProducer("OnesProducer")
3826             p.s = Sequence(p.d)
3827             p.t1 = Task(p.e)
3828             p.t2 = Task(p.f)
3829             p.t3 = Task(p.g, p.t1)
3830             p.ct1 = ConditionalTask(p.h)
3831             p.ct2 = ConditionalTask(p.f)
3832             p.ct3 = ConditionalTask(p.ct1)
3833             p.path1 = Path(p.a, p.t3, p.ct3)
3834             p.path2 = Path(p.b)
3835             self.assertTrue(p.schedule is None)
3836             pths = p.paths
3837             keys = pths.keys()
3838             self.assertEqual(pths[keys[0]],p.path1)
3839             self.assertEqual(pths[keys[1]],p.path2)
3840             p.pset1 = PSet(parA = string("pset1"))
3841             p.pset2 = untracked.PSet(parA = string("pset2"))
3842             p.vpset1 = VPSet()
3843             p.vpset2 = untracked.VPSet()
3844             p.prune()
3845             self.assertTrue(hasattr(p, 'a'))
3846             self.assertTrue(hasattr(p, 'b'))
3847             self.assertTrue(not hasattr(p, 'c'))
3848             self.assertTrue(not hasattr(p, 'd'))
3849             self.assertTrue(hasattr(p, 'e'))
3850             self.assertTrue(not hasattr(p, 'f'))
3851             self.assertTrue(hasattr(p, 'g'))
3852             self.assertTrue(hasattr(p, 'h'))
3853             self.assertTrue(not hasattr(p, 's'))
3854             self.assertTrue(hasattr(p, 't1'))
3855             self.assertTrue(not hasattr(p, 't2'))
3856             self.assertTrue(hasattr(p, 't3'))
3857             self.assertTrue(hasattr(p, 'path1'))
3858             self.assertTrue(hasattr(p, 'path2'))
3859 #            self.assertTrue(not hasattr(p, 'pset1'))
3860 #            self.assertTrue(hasattr(p, 'pset2'))
3861 #            self.assertTrue(not hasattr(p, 'vpset1'))
3862 #            self.assertTrue(not hasattr(p, 'vpset2'))
3863 
3864             p = Process("test")
3865             p.a = EDAnalyzer("MyAnalyzer")
3866             p.b = EDAnalyzer("YourAnalyzer")
3867             p.c = EDAnalyzer("OurAnalyzer")
3868             p.d = EDAnalyzer("OurAnalyzer")
3869             p.e = EDAnalyzer("OurAnalyzer")
3870             p.f = EDProducer("MyProducer")
3871             p.g = EDProducer("YourProducer")
3872             p.h = EDProducer("TheirProducer")
3873             p.i = EDProducer("OurProducer")
3874             p.j = EDProducer("OurProducer")
3875             p.k = EDProducer("OurProducer")
3876             p.l = EDProducer("OurProducer")
3877             p.t1 = Task(p.f)
3878             p.t2 = Task(p.g)
3879             p.t3 = Task(p.h)
3880             p.t4 = Task(p.i)
3881             p.ct1 = Task(p.f)
3882             p.ct2 = Task(p.j)
3883             p.ct3 = Task(p.k)
3884             p.ct4 = Task(p.l)
3885             p.s = Sequence(p.d, p.t1, p.ct1)
3886             p.s2 = Sequence(p.b, p.t2, p.ct2)
3887             p.s3 = Sequence(p.e)
3888             p.path1 = Path(p.a, p.t3, p.ct3)
3889             p.path2 = Path(p.b)
3890             p.path3 = Path(p.b+p.s2)
3891             p.path4 = Path(p.b+p.s3)
3892             p.schedule = Schedule(p.path1,p.path2,p.path3)
3893             p.schedule.associate(p.t4)
3894             pths = p.paths
3895             keys = pths.keys()
3896             self.assertEqual(pths[keys[0]],p.path1)
3897             self.assertEqual(pths[keys[1]],p.path2)
3898             p.prune()
3899             self.assertTrue(hasattr(p, 'a'))
3900             self.assertTrue(hasattr(p, 'b'))
3901             self.assertTrue(not hasattr(p, 'c'))
3902             self.assertTrue(not hasattr(p, 'd'))
3903             self.assertTrue(not hasattr(p, 'e'))
3904             self.assertTrue(not hasattr(p, 'f'))
3905             self.assertTrue(hasattr(p, 'g'))
3906             self.assertTrue(hasattr(p, 'h'))
3907             self.assertTrue(hasattr(p, 'i'))
3908             self.assertTrue(hasattr(p, 'j'))
3909             self.assertTrue(hasattr(p, 'k'))
3910             self.assertTrue(not hasattr(p, 'l'))
3911             self.assertTrue(not hasattr(p, 't1'))
3912             self.assertTrue(hasattr(p, 't2'))
3913             self.assertTrue(hasattr(p, 't3'))
3914             self.assertTrue(hasattr(p, 't4'))
3915             self.assertTrue(not hasattr(p, 'ct1'))
3916             self.assertTrue(hasattr(p, 'ct2'))
3917             self.assertTrue(hasattr(p, 'ct3'))
3918             self.assertTrue(not hasattr(p, 'ct4'))
3919             self.assertTrue(not hasattr(p, 's'))
3920             self.assertTrue(hasattr(p, 's2'))
3921             self.assertTrue(not hasattr(p, 's3'))
3922             self.assertTrue(hasattr(p, 'path1'))
3923             self.assertTrue(hasattr(p, 'path2'))
3924             self.assertTrue(hasattr(p, 'path3'))
3925             self.assertTrue(not hasattr(p, 'path4'))
3926             #test SequencePlaceholder
3927             p = Process("test")
3928             p.a = EDAnalyzer("MyAnalyzer")
3929             p.b = EDAnalyzer("YourAnalyzer")
3930             p.s = Sequence(SequencePlaceholder("a")+p.b)
3931             p.pth = Path(p.s)
3932             p.prune()
3933             self.assertTrue(hasattr(p, 'a'))
3934             self.assertTrue(hasattr(p, 'b'))
3935             self.assertTrue(hasattr(p, 's'))
3936             self.assertTrue(hasattr(p, 'pth'))
3937             #test unresolved SequencePlaceholder
3938             p = Process("test")
3939             p.b = EDAnalyzer("YourAnalyzer")
3940             p.s = Sequence(SequencePlaceholder("a")+p.b)
3941             p.pth = Path(p.s)
3942             p.prune(keepUnresolvedSequencePlaceholders=True)
3943             self.assertTrue(hasattr(p, 'b'))
3944             self.assertTrue(hasattr(p, 's'))
3945             self.assertTrue(hasattr(p, 'pth'))
3946             self.assertEqual(p.s.dumpPython(),'cms.Sequence(cms.SequencePlaceholder("a")+process.b)\n')
3947             #test TaskPlaceholder
3948             p = Process("test")
3949             p.a = EDProducer("MyProducer")
3950             p.b = EDProducer("YourProducer")
3951             p.s = Task(TaskPlaceholder("a"),p.b)
3952             p.pth = Path(p.s)
3953             p.prune()
3954             self.assertTrue(hasattr(p, 'a'))
3955             self.assertTrue(hasattr(p, 'b'))
3956             self.assertTrue(hasattr(p, 's'))
3957             self.assertTrue(hasattr(p, 'pth'))
3958             #test ConditionalTaskPlaceholder
3959             p = Process("test")
3960             p.a = EDProducer("MyProducer")
3961             p.b = EDProducer("YourProducer")
3962             p.s = ConditionalTask(ConditionalTaskPlaceholder("a"),p.b)
3963             p.pth = Path(p.s)
3964             p.prune()
3965             self.assertTrue(hasattr(p, 'a'))
3966             self.assertTrue(hasattr(p, 'b'))
3967             self.assertTrue(hasattr(p, 's'))
3968             self.assertTrue(hasattr(p, 'pth'))
3969             #test unresolved SequencePlaceholder
3970             p = Process("test")
3971             p.b = EDProducer("YourAnalyzer")
3972             p.s = Task(TaskPlaceholder("a"),p.b)
3973             p.pth = Path(p.s)
3974             p.prune(keepUnresolvedSequencePlaceholders=True)
3975             self.assertTrue(hasattr(p, 'b'))
3976             self.assertTrue(hasattr(p, 's'))
3977             self.assertTrue(hasattr(p, 'pth'))
3978             self.assertEqual(p.s.dumpPython(),'cms.Task(cms.TaskPlaceholder("a"), process.b)\n')
3979         def testTaskPlaceholder(self):
3980             p = Process("test")
3981             p.a = EDProducer("ma")
3982             p.b = EDAnalyzer("mb")
3983             p.t1 = Task(TaskPlaceholder("c"))
3984             p.t2 = Task(p.a, TaskPlaceholder("d"), p.t1)
3985             p.t3 = Task(TaskPlaceholder("e"))
3986             p.path1 = Path(p.b, p.t2, p.t3)
3987             p.t5 = Task(p.a, TaskPlaceholder("g"), TaskPlaceholder("t4"))
3988             p.t4 = Task(TaskPlaceholder("f"))
3989             p.endpath1 = EndPath(p.b, p.t5)
3990             p.t6 = Task(TaskPlaceholder("h"))
3991             p.t7 = Task(p.a, TaskPlaceholder("i"), p.t6)
3992             p.t8 = Task(TaskPlaceholder("j"))
3993             p.schedule = Schedule(p.path1, p.endpath1,tasks=[p.t7,p.t8])
3994             p.c = EDProducer("mc")
3995             p.d = EDProducer("md")
3996             p.e = EDProducer("me")
3997             p.f = EDProducer("mf")
3998             p.g = EDProducer("mg")
3999             p.h = EDProducer("mh")
4000             p.i = EDProducer("mi")
4001             p.j = EDProducer("mj")
4002             self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4003 """process.a = cms.EDProducer("ma")
4004 process.c = cms.EDProducer("mc")
4005 process.d = cms.EDProducer("md")
4006 process.e = cms.EDProducer("me")
4007 process.f = cms.EDProducer("mf")
4008 process.g = cms.EDProducer("mg")
4009 process.h = cms.EDProducer("mh")
4010 process.i = cms.EDProducer("mi")
4011 process.j = cms.EDProducer("mj")
4012 process.b = cms.EDAnalyzer("mb")
4013 process.t1 = cms.Task(cms.TaskPlaceholder("c"))
4014 process.t2 = cms.Task(cms.TaskPlaceholder("d"), process.a, process.t1)
4015 process.t3 = cms.Task(cms.TaskPlaceholder("e"))
4016 process.t5 = cms.Task(cms.TaskPlaceholder("g"), cms.TaskPlaceholder("t4"), process.a)
4017 process.t4 = cms.Task(cms.TaskPlaceholder("f"))
4018 process.t6 = cms.Task(cms.TaskPlaceholder("h"))
4019 process.t7 = cms.Task(cms.TaskPlaceholder("i"), process.a, process.t6)
4020 process.t8 = cms.Task(cms.TaskPlaceholder("j"))
4021 process.path1 = cms.Path(process.b, process.t2, process.t3)
4022 process.endpath1 = cms.EndPath(process.b, process.t5)
4023 process.schedule = cms.Schedule(*[ process.path1, process.endpath1 ], tasks=[process.t7, process.t8])""")
4024             p.resolve()
4025             self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4026 """process.a = cms.EDProducer("ma")
4027 process.c = cms.EDProducer("mc")
4028 process.d = cms.EDProducer("md")
4029 process.e = cms.EDProducer("me")
4030 process.f = cms.EDProducer("mf")
4031 process.g = cms.EDProducer("mg")
4032 process.h = cms.EDProducer("mh")
4033 process.i = cms.EDProducer("mi")
4034 process.j = cms.EDProducer("mj")
4035 process.b = cms.EDAnalyzer("mb")
4036 process.t1 = cms.Task(process.c)
4037 process.t2 = cms.Task(process.a, process.d, process.t1)
4038 process.t3 = cms.Task(process.e)
4039 process.t4 = cms.Task(process.f)
4040 process.t6 = cms.Task(process.h)
4041 process.t7 = cms.Task(process.a, process.i, process.t6)
4042 process.t8 = cms.Task(process.j)
4043 process.t5 = cms.Task(process.a, process.g, process.t4)
4044 process.path1 = cms.Path(process.b, process.t2, process.t3)
4045 process.endpath1 = cms.EndPath(process.b, process.t5)
4046 process.schedule = cms.Schedule(*[ process.path1, process.endpath1 ], tasks=[process.t7, process.t8])""")
4047         def testConditionalTaskPlaceholder(self):
4048             p = Process("test")
4049             p.a = EDProducer("ma")
4050             p.b = EDAnalyzer("mb")
4051             p.t1 = ConditionalTask(ConditionalTaskPlaceholder("c"))
4052             p.t2 = ConditionalTask(p.a, ConditionalTaskPlaceholder("d"), p.t1)
4053             p.t3 = ConditionalTask(ConditionalTaskPlaceholder("e"))
4054             p.path1 = Path(p.b, p.t2, p.t3)
4055             p.t5 = ConditionalTask(p.a, ConditionalTaskPlaceholder("g"), ConditionalTaskPlaceholder("t4"))
4056             p.t4 = ConditionalTask(ConditionalTaskPlaceholder("f"))
4057             p.path2 = Path(p.b, p.t5)
4058             p.schedule = Schedule(p.path1, p.path2)
4059             p.c = EDProducer("mc")
4060             p.d = EDProducer("md")
4061             p.e = EDProducer("me")
4062             p.f = EDProducer("mf")
4063             p.g = EDProducer("mg")
4064             p.h = EDProducer("mh")
4065             p.i = EDProducer("mi")
4066             p.j = EDProducer("mj")
4067             self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4068 """process.a = cms.EDProducer("ma")
4069 process.c = cms.EDProducer("mc")
4070 process.d = cms.EDProducer("md")
4071 process.e = cms.EDProducer("me")
4072 process.f = cms.EDProducer("mf")
4073 process.g = cms.EDProducer("mg")
4074 process.h = cms.EDProducer("mh")
4075 process.i = cms.EDProducer("mi")
4076 process.j = cms.EDProducer("mj")
4077 process.b = cms.EDAnalyzer("mb")
4078 process.t1 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("c"))
4079 process.t2 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("d"), process.a, process.t1)
4080 process.t3 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("e"))
4081 process.t5 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("g"), cms.ConditionalTaskPlaceholder("t4"), process.a)
4082 process.t4 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("f"))
4083 process.path1 = cms.Path(process.b, process.t2, process.t3)
4084 process.path2 = cms.Path(process.b, process.t5)
4085 process.schedule = cms.Schedule(*[ process.path1, process.path2 ])""")
4086             p.resolve()
4087             self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
4088 """process.a = cms.EDProducer("ma")
4089 process.c = cms.EDProducer("mc")
4090 process.d = cms.EDProducer("md")
4091 process.e = cms.EDProducer("me")
4092 process.f = cms.EDProducer("mf")
4093 process.g = cms.EDProducer("mg")
4094 process.h = cms.EDProducer("mh")
4095 process.i = cms.EDProducer("mi")
4096 process.j = cms.EDProducer("mj")
4097 process.b = cms.EDAnalyzer("mb")
4098 process.t1 = cms.ConditionalTask(process.c)
4099 process.t2 = cms.ConditionalTask(process.a, process.d, process.t1)
4100 process.t3 = cms.ConditionalTask(process.e)
4101 process.t4 = cms.ConditionalTask(process.f)
4102 process.t5 = cms.ConditionalTask(process.a, process.g, process.t4)
4103 process.path1 = cms.Path(process.b, process.t2, process.t3)
4104 process.path2 = cms.Path(process.b, process.t5)
4105 process.schedule = cms.Schedule(*[ process.path1, process.path2 ])""")
4106 
4107         def testDelete(self):
4108             p = Process("test")
4109             p.a = EDAnalyzer("MyAnalyzer")
4110             p.b = EDAnalyzer("YourAnalyzer")
4111             p.c = EDAnalyzer("OurAnalyzer")
4112             p.d = EDAnalyzer("OurAnalyzer")
4113             p.e = EDAnalyzer("OurAnalyzer")
4114             p.f = EDAnalyzer("OurAnalyzer")
4115             p.g = EDProducer("OurProducer")
4116             p.h = EDProducer("YourProducer")
4117             p.i = SwitchProducerTest(
4118                 test1 = EDProducer("OneProducer"),
4119                 test2 = EDProducer("TwoProducer")
4120             )
4121             p.t1 = Task(p.g, p.h, p.i)
4122             t2 = Task(p.g, p.h, p.i)
4123             t3 = Task(p.g, p.h)
4124             p.t4 = Task(p.h)
4125             p.ct1 = ConditionalTask(p.g, p.h, p.i)
4126             ct2 = ConditionalTask(p.g, p.h)
4127             ct3 = ConditionalTask(p.g, p.h)
4128             p.ct4 = ConditionalTask(p.h)
4129             p.s = Sequence(p.d+p.e)
4130             p.path1 = Path(p.a+p.f+p.s,t2,ct2)
4131             p.path2 = Path(p.a)
4132             p.path3 = Path(ct3, p.ct4)
4133             p.endpath2 = EndPath(p.b)
4134             p.endpath1 = EndPath(p.b+p.f)
4135             p.schedule = Schedule(p.path2, p.path3, p.endpath2, tasks=[t3, p.t4])
4136             self.assertTrue(hasattr(p, 'f'))
4137             self.assertTrue(hasattr(p, 'g'))
4138             self.assertTrue(hasattr(p, 'i'))
4139             del p.e
4140             del p.f
4141             del p.g
4142             del p.i
4143             self.assertFalse(hasattr(p, 'f'))
4144             self.assertFalse(hasattr(p, 'g'))
4145             self.assertEqual(p.t1.dumpPython(), 'cms.Task(process.h)\n')
4146             self.assertEqual(p.ct1.dumpPython(), 'cms.ConditionalTask(process.h)\n')
4147             self.assertEqual(p.s.dumpPython(), 'cms.Sequence(process.d)\n')
4148             self.assertEqual(p.path1.dumpPython(), 'cms.Path(process.a+process.s, cms.ConditionalTask(process.h), cms.Task(process.h))\n')
4149             self.assertEqual(p.endpath1.dumpPython(), 'cms.EndPath(process.b)\n')
4150             self.assertEqual(p.path3.dumpPython(), 'cms.Path(cms.ConditionalTask(process.h), process.ct4)\n')
4151             del p.s
4152             self.assertEqual(p.path1.dumpPython(), 'cms.Path(process.a+(process.d), cms.ConditionalTask(process.h), cms.Task(process.h))\n')
4153             self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(*[ process.path2, process.path3, process.endpath2 ], tasks=[cms.Task(process.h), process.t4])\n')
4154             del p.path2
4155             self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(*[ process.path3, process.endpath2 ], tasks=[cms.Task(process.h), process.t4])\n')
4156             del p.path3
4157             self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(*[ process.endpath2 ], tasks=[cms.Task(process.h), process.t4])\n')
4158             del p.endpath2
4159             self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(tasks=[cms.Task(process.h), process.t4])\n')
4160             del p.t4
4161             self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(tasks=[cms.Task(process.h)])\n')
4162         def testModifier(self):
4163             m1 = Modifier()
4164             Process._firstProcess = True
4165             p = Process("test")
4166             self.assertRaises(RuntimeError, lambda: Process("test2", m1))
4167             m1 = Modifier()
4168             Process._firstProcess = True
4169             p = Process("test",m1)
4170             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1))
4171             def _mod_fred(obj):
4172                 obj.fred = 2
4173             m1.toModify(p.a,_mod_fred)
4174             self.assertEqual(p.a.fred.value(),2)
4175             p.b = EDAnalyzer("YourAnalyzer", wilma = int32(1))
4176             m1.toModify(p.b, wilma = 2)
4177             self.assertEqual(p.b.wilma.value(),2)
4178             self.assertTrue(p.isUsingModifier(m1))
4179             #check that Modifier not attached to a process doesn't run
4180             m1 = Modifier()
4181             Process._firstProcess = True
4182             p = Process("test")
4183             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1))
4184             m1.toModify(p.a,_mod_fred)
4185             p.b = EDAnalyzer("YourAnalyzer", wilma = int32(1))
4186             m1.toModify(p.b, wilma = 2)
4187             self.assertEqual(p.a.fred.value(),1)
4188             self.assertEqual(p.b.wilma.value(),1)
4189             self.assertEqual(p.isUsingModifier(m1),False)
4190             #make sure clones get the changes
4191             m1 = Modifier()
4192             Process._firstProcess = True
4193             p = Process("test",m1)
4194             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4195             m1.toModify(p.a, fred = int32(2))
4196             p.b = p.a.clone(wilma = int32(3))
4197             self.assertEqual(p.a.fred.value(),2)
4198             self.assertEqual(p.a.wilma.value(),1)
4199             self.assertEqual(p.b.fred.value(),2)
4200             self.assertEqual(p.b.wilma.value(),3)
4201             #test removal of parameter
4202             m1 = Modifier()
4203             Process._firstProcess = True
4204             p = Process("test",m1)
4205             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1), fintstones = PSet(fred = int32(1)))
4206             m1.toModify(p.a, fred = None, fintstones = dict(fred = None))
4207             self.assertEqual(hasattr(p.a, "fred"), False)
4208             self.assertEqual(hasattr(p.a.fintstones, "fred"), False)
4209             self.assertEqual(p.a.wilma.value(),1)
4210             #test adding a parameter
4211             m1 = Modifier()
4212             Process._firstProcess = True
4213             p = Process("test",m1)
4214             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1))
4215             m1.toModify(p.a, wilma = int32(2))
4216             self.assertEqual(p.a.fred.value(), 1)
4217             self.assertEqual(p.a.wilma.value(),2)
4218             #test setting of value in PSet
4219             m1 = Modifier()
4220             Process._firstProcess = True
4221             p = Process("test",m1)
4222             p.a = EDAnalyzer("MyAnalyzer", flintstones = PSet(fred = int32(1), wilma = int32(1)))
4223             m1.toModify(p.a, flintstones = dict(fred = int32(2)))
4224             self.assertEqual(p.a.flintstones.fred.value(),2)
4225             self.assertEqual(p.a.flintstones.wilma.value(),1)
4226             #test proper exception from nonexisting parameter name
4227             m1 = Modifier()
4228             Process._firstProcess = True
4229             p = Process("test",m1)
4230             p.a = EDAnalyzer("MyAnalyzer", flintstones = PSet(fred = PSet(wilma = int32(1))))
4231             self.assertRaises(KeyError, lambda: m1.toModify(p.a, flintstones = dict(imnothere = dict(wilma=2))))
4232             self.assertRaises(KeyError, lambda: m1.toModify(p.a, foo = 1))
4233             #test setting a value in a VPSet
4234             m1 = Modifier()
4235             Process._firstProcess = True
4236             p = Process("test",m1)
4237             p.a = EDAnalyzer("MyAnalyzer", flintstones = VPSet(PSet(fred = int32(1)), PSet(wilma = int32(1))))
4238             m1.toModify(p.a, flintstones = {1:dict(wilma = int32(2))})
4239             self.assertEqual(p.a.flintstones[0].fred.value(),1)
4240             self.assertEqual(p.a.flintstones[1].wilma.value(),2)
4241             #test setting a value in a list of values
4242             m1 = Modifier()
4243             Process._firstProcess = True
4244             p = Process("test",m1)
4245             p.a = EDAnalyzer("MyAnalyzer", fred = vuint32(1,2,3))
4246             m1.toModify(p.a, fred = {1:7})
4247             self.assertEqual(p.a.fred[0],1)
4248             self.assertEqual(p.a.fred[1],7)
4249             self.assertEqual(p.a.fred[2],3)
4250             #test IndexError setting a value in a list to an item key not in the list
4251             m1 = Modifier()
4252             Process._firstProcess = True
4253             p = Process("test",m1)
4254             p.a = EDAnalyzer("MyAnalyzer", fred = vuint32(1,2,3))
4255             raised = False
4256             try: m1.toModify(p.a, fred = {5:7})
4257             except IndexError as e: raised = True
4258             self.assertEqual(raised, True)
4259             #test TypeError setting a value in a list using a key that is not an int
4260             m1 = Modifier()
4261             Process._firstProcess = True
4262             p = Process("test",m1)
4263             p.a = EDAnalyzer("MyAnalyzer", flintstones = VPSet(PSet(fred = int32(1)), PSet(wilma = int32(1))))
4264             raised = False
4265             try: m1.toModify(p.a, flintstones = dict(bogus = int32(37)))
4266             except TypeError as e: raised = True
4267             self.assertEqual(raised, True)
4268             #test that load causes process wide methods to run
4269             def _rem_a(proc):
4270                 del proc.a
4271             class ProcModifierMod(object):
4272                 def __init__(self,modifier,func):
4273                     self.proc_mod_ = modifier.makeProcessModifier(func)
4274             class DummyMod(object):
4275                 def __init__(self):
4276                     self.a = EDAnalyzer("Dummy")
4277             testMod = DummyMod()
4278             p.extend(testMod)
4279             self.assertTrue(hasattr(p,"a"))
4280             m1 = Modifier()
4281             Process._firstProcess = True
4282             p = Process("test",m1)
4283             testProcMod = ProcModifierMod(m1,_rem_a)
4284             p.extend(testMod)
4285             p.extend(testProcMod)
4286             self.assertTrue(not hasattr(p,"a"))
4287             #test ModifierChain
4288             m1 = Modifier()
4289             mc = ModifierChain(m1)
4290             Process._firstProcess = True
4291             p = Process("test",mc)
4292             self.assertTrue(p.isUsingModifier(m1))
4293             self.assertTrue(p.isUsingModifier(mc))
4294             testMod = DummyMod()
4295             p.b = EDAnalyzer("Dummy2", fred = int32(1))
4296             m1.toModify(p.b, fred = int32(3))
4297             p.extend(testMod)
4298             testProcMod = ProcModifierMod(m1,_rem_a)
4299             p.extend(testProcMod)
4300             self.assertTrue(not hasattr(p,"a"))
4301             self.assertEqual(p.b.fred.value(),3)
4302             #check cloneAndExclude
4303             m1 = Modifier()
4304             m2 = Modifier()
4305             mc = ModifierChain(m1,m2)
4306             mclone = mc.copyAndExclude([m2])
4307             self.assertTrue(not mclone._isOrContains(m2))
4308             self.assertTrue(mclone._isOrContains(m1))
4309             m3 = Modifier()
4310             mc2 = ModifierChain(mc,m3)
4311             mclone = mc2.copyAndExclude([m2])
4312             self.assertTrue(not mclone._isOrContains(m2))
4313             self.assertTrue(mclone._isOrContains(m1))
4314             self.assertTrue(mclone._isOrContains(m3))
4315             #check combining
4316             m1 = Modifier()
4317             m2 = Modifier()
4318             Process._firstProcess = True
4319             p = Process("test",m1)
4320             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4321             (m1 & m2).toModify(p.a, fred = int32(2))
4322             self.assertRaises(TypeError, lambda: (m1 & m2).toModify(p.a, 1, wilma=2))
4323             self.assertEqual(p.a.fred, 1)
4324             m1 = Modifier()
4325             m2 = Modifier()
4326             Process._firstProcess = True
4327             p = Process("test",m1,m2)
4328             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4329             (m1 & m2).toModify(p.a, fred = int32(2))
4330             self.assertEqual(p.a.fred, 2)
4331             m1 = Modifier()
4332             m2 = Modifier()
4333             m3 = Modifier()
4334             Process._firstProcess = True
4335             p = Process("test",m1,m2,m3)
4336             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4337             (m1 & m2 & m3).toModify(p.a, fred = int32(2))
4338             self.assertEqual(p.a.fred, 2)
4339             (m1 & (m2 & m3)).toModify(p.a, fred = int32(3))
4340             self.assertEqual(p.a.fred, 3)
4341             ((m1 & m2) & m3).toModify(p.a, fred = int32(4))
4342             self.assertEqual(p.a.fred, 4)
4343             #check inverse
4344             m1 = Modifier()
4345             m2 = Modifier()
4346             Process._firstProcess = True
4347             p = Process("test", m1)
4348             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4349             (~m1).toModify(p.a, fred=2)
4350             self.assertEqual(p.a.fred, 1)
4351             (~m2).toModify(p.a, wilma=2)
4352             self.assertEqual(p.a.wilma, 2)
4353             self.assertRaises(TypeError, lambda: (~m1).toModify(p.a, 1, wilma=2))
4354             self.assertRaises(TypeError, lambda: (~m2).toModify(p.a, 1, wilma=2))
4355             # check or
4356             m1 = Modifier()
4357             m2 = Modifier()
4358             m3 = Modifier()
4359             Process._firstProcess = True
4360             p = Process("test", m1)
4361             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4362             (m1 | m2).toModify(p.a, fred=2)
4363             self.assertEqual(p.a.fred, 2)
4364             (m1 | m2 | m3).toModify(p.a, fred=3)
4365             self.assertEqual(p.a.fred, 3)
4366             (m3 | m2 | m1).toModify(p.a, fred=4)
4367             self.assertEqual(p.a.fred, 4)
4368             ((m1 | m2) | m3).toModify(p.a, fred=5)
4369             self.assertEqual(p.a.fred, 5)
4370             (m1 | (m2 | m3)).toModify(p.a, fred=6)
4371             self.assertEqual(p.a.fred, 6)
4372             (m2 | m3).toModify(p.a, fred=7)
4373             self.assertEqual(p.a.fred, 6)
4374             self.assertRaises(TypeError, lambda: (m1 | m2).toModify(p.a, 1, wilma=2))
4375             self.assertRaises(TypeError, lambda: (m2 | m3).toModify(p.a, 1, wilma=2))
4376             # check combinations
4377             m1 = Modifier()
4378             m2 = Modifier()
4379             m3 = Modifier()
4380             m4 = Modifier()
4381             Process._firstProcess = True
4382             p = Process("test", m1, m2)
4383             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4384             (m1 & ~m2).toModify(p.a, fred=2)
4385             self.assertEqual(p.a.fred, 1)
4386             (m1 & ~m3).toModify(p.a, fred=2)
4387             self.assertEqual(p.a.fred, 2)
4388             (m1 | ~m2).toModify(p.a, fred=3)
4389             self.assertEqual(p.a.fred, 3)
4390             (~m1 | ~m2).toModify(p.a, fred=4)
4391             self.assertEqual(p.a.fred, 3)
4392             (~m3 & ~m4).toModify(p.a, fred=4)
4393             self.assertEqual(p.a.fred, 4)
4394             ((m1 & m3) | ~m4).toModify(p.a, fred=5)
4395             self.assertEqual(p.a.fred, 5)
4396             #check toReplaceWith
4397             m1 = Modifier()
4398             Process._firstProcess = True
4399             p = Process("test",m1)
4400             p.a =EDAnalyzer("MyAnalyzer", fred = int32(1))
4401             m1.toReplaceWith(p.a, EDAnalyzer("YourAnalyzer", wilma = int32(3)))
4402             self.assertRaises(TypeError, lambda: m1.toReplaceWith(p.a, EDProducer("YourProducer")))
4403             #Task
4404             p.b =EDAnalyzer("BAn")
4405             p.c =EDProducer("c")
4406             p.d =EDProducer("d")
4407             p.tc = Task(p.c)
4408             p.td = Task(p.d)
4409             p.s = Sequence(p.a, p.tc)
4410             m1.toReplaceWith(p.s, Sequence(p.a+p.b, p.td))
4411             self.assertEqual(p.a.wilma.value(),3)
4412             self.assertEqual(p.a.type_(),"YourAnalyzer")
4413             self.assertEqual(hasattr(p,"fred"),False)
4414             self.assertTrue(p.s.dumpPython() == "cms.Sequence(process.a+process.b, process.td)\n")
4415             p.e =EDProducer("e")
4416             m1.toReplaceWith(p.td, Task(p.e))
4417             self.assertTrue(p.td._collection == OrderedSet([p.e]))
4418             #ConditionalTask
4419             p.b =EDAnalyzer("BAn")
4420             p.c =EDProducer("c")
4421             p.d =EDProducer("d")
4422             del p.tc
4423             del p.td
4424             p.tc = ConditionalTask(p.c)
4425             p.td = ConditionalTask(p.d)
4426             p.s = Sequence(p.a, p.tc)
4427             m1.toReplaceWith(p.s, Sequence(p.a+p.b, p.td))
4428             self.assertEqual(p.a.wilma.value(),3)
4429             self.assertEqual(p.a.type_(),"YourAnalyzer")
4430             self.assertEqual(hasattr(p,"fred"),False)
4431             self.assertTrue(p.s.dumpPython() == "cms.Sequence(process.a+process.b, process.td)\n")
4432             p.e =EDProducer("e")
4433             m1.toReplaceWith(p.td, ConditionalTask(p.e))
4434             self.assertTrue(p.td._collection == OrderedSet([p.e]))
4435             #check toReplaceWith doesn't activate not chosen
4436             m1 = Modifier()
4437             Process._firstProcess = True
4438             p = Process("test")
4439             p.a =EDAnalyzer("MyAnalyzer", fred = int32(1))
4440             m1.toReplaceWith(p.a, EDAnalyzer("YourAnalyzer", wilma = int32(3)))
4441             self.assertEqual(p.a.type_(),"MyAnalyzer")
4442             #check toReplaceWith and and/not/or combinations
4443             m1 = Modifier()
4444             m2 = Modifier()
4445             m3 = Modifier()
4446             m4 = Modifier()
4447             Process._firstProcess = True
4448             p = Process("test", m1, m2)
4449             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4450             self.assertRaises(TypeError, lambda: (m1 & m2).toReplaceWith(p.a, EDProducer("YourProducer")))
4451             self.assertRaises(TypeError, lambda: (m3 & m4).toReplaceWith(p.a, EDProducer("YourProducer")))
4452             self.assertRaises(TypeError, lambda: (~m3).toReplaceWith(p.a, EDProducer("YourProducer")))
4453             self.assertRaises(TypeError, lambda: (~m1).toReplaceWith(p.a, EDProducer("YourProducer")))
4454             self.assertRaises(TypeError, lambda: (m1 | m3).toReplaceWith(p.a, EDProducer("YourProducer")))
4455             self.assertRaises(TypeError, lambda: (m3 | m4).toReplaceWith(p.a, EDProducer("YourProducer")))
4456             (m1 & m2).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer1"))
4457             self.assertEqual(p.a.type_(), "YourAnalyzer1")
4458             (m1 & m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer2"))
4459             self.assertEqual(p.a.type_(), "YourAnalyzer1")
4460             (~m1).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer2"))
4461             self.assertEqual(p.a.type_(), "YourAnalyzer1")
4462             (~m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer2"))
4463             self.assertEqual(p.a.type_(), "YourAnalyzer2")
4464             (m1 | m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer3"))
4465             self.assertEqual(p.a.type_(), "YourAnalyzer3")
4466             (m3 | m4).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer4"))
4467             self.assertEqual(p.a.type_(), "YourAnalyzer3")
4468             #check chaining of toModify and toReplaceWith
4469             m1 = Modifier()
4470             m2 = Modifier()
4471             m3 = Modifier()
4472             Process._firstProcess = True
4473             p = Process("test", m1, m2)
4474             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4475             p.b = EDProducer("MyProducer", barney = int32(1), betty = int32(1))
4476             (m1 & m2).toModify(p.a, fred = 2).toModify(p.b, betty = 3)
4477             self.assertEqual(p.a.fred, 2)
4478             self.assertEqual(p.a.wilma, 1)
4479             self.assertEqual(p.b.barney, 1)
4480             self.assertEqual(p.b.betty, 3)
4481             (m1 | m3).toModify(p.a, wilma = 4).toModify(p.b, barney = 5)
4482             self.assertEqual(p.a.fred, 2)
4483             self.assertEqual(p.a.wilma, 4)
4484             self.assertEqual(p.b.barney, 5)
4485             self.assertEqual(p.b.betty, 3)
4486             (m2 & ~m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer")).toModify(p.b, barney = 6)
4487             self.assertEqual(p.a.type_(), "YourAnalyzer")
4488             self.assertEqual(p.b.barney, 6)
4489             self.assertEqual(p.b.betty, 3)
4490             (m1 & ~m3).toModify(p.a, param=int32(42)).toReplaceWith(p.b, EDProducer("YourProducer"))
4491             self.assertEqual(p.a.type_(), "YourAnalyzer")
4492             self.assertEqual(p.a.param, 42)
4493             self.assertEqual(p.b.type_(), "YourProducer")
4494 
4495             # EDAlias
4496             a = EDAlias(foo2 = VPSet(PSet(type = string("Foo2"))))
4497             m = Modifier()
4498             m._setChosen()
4499             # Modify parameters
4500             m.toModify(a, foo2 = {0: dict(type = "Foo3")})
4501             self.assertEqual(a.foo2[0].type, "Foo3")
4502             # Add an alias
4503             m.toModify(a, foo4 = VPSet(PSet(type = string("Foo4"))))
4504             self.assertEqual(a.foo2[0].type, "Foo3")
4505             self.assertEqual(a.foo4[0].type, "Foo4")
4506             # Remove an alias
4507             m.toModify(a, foo2 = None)
4508             self.assertFalse(hasattr(a, "foo2"))
4509             self.assertEqual(a.foo4[0].type, "Foo4")
4510             # Replace (doesn't work out of the box because EDAlias is not _Parameterizable
4511             m.toReplaceWith(a, EDAlias(bar = VPSet(PSet(type = string("Bar")))))
4512             self.assertFalse(hasattr(a, "foo2"))
4513             self.assertFalse(hasattr(a, "foo4"))
4514             self.assertTrue(hasattr(a, "bar"))
4515             self.assertEqual(a.bar[0].type, "Bar")
4516 
4517             # SwitchProducer
4518             sp = SwitchProducerTest(test1 = EDProducer("Foo",
4519                                                        a = int32(1),
4520                                                        b = PSet(c = int32(2))),
4521                                     test2 = EDProducer("Bar",
4522                                                        aa = int32(11),
4523                                                        bb = PSet(cc = int32(12))))
4524             m = Modifier()
4525             m._setChosen()
4526             # Modify parameters
4527             m.toModify(sp,
4528                        test1 = dict(a = 4, b = dict(c = None)),
4529                        test2 = dict(aa = 15, bb = dict(cc = 45, dd = string("foo"))))
4530             self.assertEqual(sp.test1.a.value(), 4)
4531             self.assertEqual(sp.test1.b.hasParameter("c"), False)
4532             self.assertEqual(sp.test2.aa.value(), 15)
4533             self.assertEqual(sp.test2.bb.cc.value(), 45)
4534             self.assertEqual(sp.test2.bb.dd.value(), "foo")
4535             # Replace a producer
4536             m.toReplaceWith(sp.test1, EDProducer("Fred", x = int32(42)))
4537             self.assertEqual(sp.test1.type_(), "Fred")
4538             self.assertEqual(sp.test1.x.value(), 42)
4539             self.assertRaises(TypeError, lambda: m.toReplaceWith(sp.test1, EDAnalyzer("Foo")))
4540             # Alternative way (only to be allow same syntax to be used as for adding)
4541             m.toModify(sp, test2 = EDProducer("Xyzzy", x = int32(24)))
4542             self.assertEqual(sp.test2.type_(), "Xyzzy")
4543             self.assertEqual(sp.test2.x.value(), 24)
4544             self.assertRaises(TypeError, lambda: m.toModify(sp, test2 = EDAnalyzer("Foo")))
4545             # Add a producer
4546             m.toModify(sp, test3 = EDProducer("Wilma", y = int32(24)))
4547             self.assertEqual(sp.test3.type_(), "Wilma")
4548             self.assertEqual(sp.test3.y.value(), 24)
4549             self.assertRaises(TypeError, lambda: m.toModify(sp, test4 = EDAnalyzer("Foo")))
4550             # Remove a producer
4551             m.toModify(sp, test2 = None)
4552             self.assertEqual(hasattr(sp, "test2"), False)
4553             # Add an alias
4554             m.toModify(sp, test2 = EDAlias(foo = VPSet(PSet(type = string("int")))))
4555             self.assertTrue(hasattr(sp.test2, "foo"))
4556             # Replace an alias
4557             m.toReplaceWith(sp.test2, EDAlias(bar = VPSet(PSet(type = string("int")))))
4558             self.assertTrue(hasattr(sp.test2, "bar"))
4559             # Alternative way
4560             m.toModify(sp, test2 = EDAlias(xyzzy = VPSet(PSet(type = string("int")))))
4561             self.assertTrue(hasattr(sp.test2, "xyzzy"))
4562             # Replace an alias with EDProducer
4563             self.assertRaises(TypeError, lambda: m.toReplaceWith(sp.test2, EDProducer("Foo")))
4564             m.toModify(sp, test2 = EDProducer("Foo"))
4565         def testProcessFragment(self):
4566             #check defaults are not overwritten
4567             f = ProcessFragment('Fragment')
4568             p = Process('PROCESS')
4569             p.maxEvents.input = 10
4570             p.options.numberOfThreads = 4
4571             p.maxLuminosityBlocks.input = 2
4572             p.extend(f)
4573             self.assertEqual(p.maxEvents.input.value(),10)
4574             self.assertEqual(p.options.numberOfThreads.value(), 4)
4575             self.assertEqual(p.maxLuminosityBlocks.input.value(),2)
4576             #general checks
4577             f = ProcessFragment("Fragment")
4578             f.fltr = EDFilter("Foo")
4579             p = Process('PROCESS')
4580             p.extend(f)
4581             self.assertTrue(hasattr(p,'fltr'))
4582         def testProcessForProcessAccelerator(self):
4583             proc = Process("TEST")
4584             p = ProcessForProcessAccelerator(proc)
4585             p.TestService = Service("TestService")
4586             self.assertTrue(hasattr(proc, "TestService"))
4587             self.assertEqual(proc.TestService.type_(), "TestService")
4588             self.assertRaises(TypeError, setattr, p, "a", EDProducer("Foo"))
4589             p.add_(Service("TestServiceTwo"))
4590             self.assertTrue(hasattr(proc, "TestServiceTwo"))
4591             self.assertEqual(proc.TestServiceTwo.type_(), "TestServiceTwo")
4592             p.TestService.foo = untracked.uint32(42)
4593             self.assertEqual(proc.TestService.foo.value(), 42)
4594             proc.mod = EDProducer("Producer")
4595             self.assertRaises(TypeError, getattr, p, "mod")
4596         def testProcessAccelerator(self):
4597             proc = Process("TEST")
4598             p = TestMakePSet()
4599             proc.fillProcessDesc(p)
4600             self.assertTrue(["cpu"], p.values["@available_accelerators"][1])
4601             self.assertFalse(p.values["@selected_accelerators"][0])
4602             self.assertTrue(["cpu"], p.values["@selected_accelerators"][1])
4603             self.assertFalse(p.values["@module_type_resolver"][0])
4604             self.assertEqual("", p.values["@module_type_resolver"][1])
4605 
4606             proc = Process("TEST")
4607             self.assertRaises(TypeError, setattr, proc, "processAcceleratorTest", ProcessAcceleratorTest())
4608             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4609             del proc.MessageLogger # remove boilerplate unnecessary for this test case
4610             self.maxDiff = None
4611             self.assertEqual(proc.dumpPython(),
4612 """import FWCore.ParameterSet.Config as cms
4613 from test import ProcessAcceleratorTest
4614 
4615 process = cms.Process("TEST")
4616 
4617 process.maxEvents = cms.untracked.PSet(
4618     input = cms.optional.untracked.int32,
4619     output = cms.optional.untracked.allowed(cms.int32,cms.PSet)
4620 )
4621 
4622 process.maxLuminosityBlocks = cms.untracked.PSet(
4623     input = cms.untracked.int32(-1)
4624 )
4625 
4626 process.options = cms.untracked.PSet(
4627     FailPath = cms.untracked.vstring(),
4628     IgnoreCompletely = cms.untracked.vstring(),
4629     Rethrow = cms.untracked.vstring(),
4630     SkipEvent = cms.untracked.vstring(),
4631     accelerators = cms.untracked.vstring('*'),
4632     allowUnscheduled = cms.obsolete.untracked.bool,
4633     canDeleteEarly = cms.untracked.vstring(),
4634     deleteNonConsumedUnscheduledModules = cms.untracked.bool(True),
4635     dumpOptions = cms.untracked.bool(False),
4636     emptyRunLumiMode = cms.obsolete.untracked.string,
4637     eventSetup = cms.untracked.PSet(
4638         forceNumberOfConcurrentIOVs = cms.untracked.PSet(
4639             allowAnyLabel_=cms.required.untracked.uint32
4640         ),
4641         numberOfConcurrentIOVs = cms.untracked.uint32(0)
4642     ),
4643     fileMode = cms.untracked.string('FULLMERGE'),
4644     forceEventSetupCacheClearOnNewRun = cms.untracked.bool(False),
4645     holdsReferencesToDeleteEarly = cms.untracked.VPSet(),
4646     makeTriggerResults = cms.obsolete.untracked.bool,
4647     modulesToIgnoreForDeleteEarly = cms.untracked.vstring(),
4648     numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(0),
4649     numberOfConcurrentRuns = cms.untracked.uint32(1),
4650     numberOfStreams = cms.untracked.uint32(0),
4651     numberOfThreads = cms.untracked.uint32(1),
4652     printDependencies = cms.untracked.bool(False),
4653     sizeOfStackForThreadsInKB = cms.optional.untracked.uint32,
4654     throwIfIllegalParameter = cms.untracked.bool(True),
4655     wantSummary = cms.untracked.bool(False)
4656 )
4657 
4658 process.ProcessAcceleratorTest = ProcessAcceleratorTest(
4659     enabled = ['test1', 'test2', 'anothertest3']
4660 )
4661 
4662 
4663 """)
4664             p = TestMakePSet()
4665             proc.fillProcessDesc(p)
4666             self.assertEqual(["*"], p.values["options"][1].values["accelerators"][1])
4667             self.assertFalse(p.values["options"][1].values["accelerators"][0])
4668             self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["@selected_accelerators"][1])
4669             self.assertEqual("AcceleratorTestService", p.values["services"][1][0].values["@service_type"][1])
4670             self.assertFalse(p.values["@available_accelerators"][0])
4671             self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4672             self.assertFalse(p.values["@module_type_resolver"][0])
4673             self.assertEqual("", p.values["@module_type_resolver"][1])
4674 
4675             proc = Process("TEST")
4676             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"])
4677             p = TestMakePSet()
4678             proc.fillProcessDesc(p)
4679             self.assertEqual(["cpu", "test1"], p.values["@selected_accelerators"][1])
4680             self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
4681 
4682             proc = Process("TEST")
4683             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4684             proc.options.accelerators = ["test2"]
4685             p = TestMakePSet()
4686             proc.fillProcessDesc(p)
4687             self.assertEqual(["test2"], p.values["@selected_accelerators"][1])
4688             self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4689 
4690             proc = Process("TEST")
4691             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4692             proc.options.accelerators = ["test*"]
4693             proc.fillProcessDesc(p)
4694             self.assertEqual(["test1", "test2"], p.values["@selected_accelerators"][1])
4695             self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4696 
4697             proc = Process("TEST")
4698             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"])
4699             proc.options.accelerators = ["test2"]
4700             p = TestMakePSet()
4701             proc.fillProcessDesc(p)
4702             self.assertEqual([], p.values["@selected_accelerators"][1])
4703             self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
4704 
4705             proc = Process("TEST")
4706             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4707             proc.options.accelerators = ["cpu*"]
4708             p = TestMakePSet()
4709             proc.fillProcessDesc(p)
4710             self.assertEqual(["cpu"], p.values["@selected_accelerators"][1])
4711             self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4712 
4713             proc = Process("TEST")
4714             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4715             proc.options.accelerators = ["test3"]
4716             p = TestMakePSet()
4717             self.assertRaises(ValueError, proc.fillProcessDesc, p)
4718 
4719             proc = Process("TEST")
4720             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4721             proc.options.accelerators = ["*", "test1"]
4722             p = TestMakePSet()
4723             self.assertRaises(ValueError, proc.fillProcessDesc, p)
4724 
4725             proc = Process("TEST")
4726             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4727             proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2()
4728             p = TestMakePSet()
4729             proc.fillProcessDesc(p)
4730             self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@selected_accelerators"][1])
4731             self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4732 
4733             proc = Process("TEST")
4734             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4735             proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2()
4736             proc.options.accelerators = ["*test3", "c*"]
4737             p = TestMakePSet()
4738             proc.fillProcessDesc(p)
4739             self.assertEqual(["anothertest3", "cpu"], p.values["@selected_accelerators"][1])
4740             self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4741 
4742             proc = Process("TEST")
4743             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4744             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4745                                                              a = int32(1),
4746                                                              b = PSet(c = int32(2))),
4747                                           test1 = EDProducer("Bar",
4748                                                              aa = int32(11),
4749                                                              bb = PSet(cc = int32(12))))
4750             proc.p = Path(proc.sp)
4751             p = TestMakePSet()
4752             proc.fillProcessDesc(p)
4753             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
4754 
4755             proc = Process("TEST")
4756             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"])
4757             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4758                                                              a = int32(1),
4759                                                              b = PSet(c = int32(2))),
4760                                           test1 = EDProducer("Bar",
4761                                                              aa = int32(11),
4762                                                              bb = PSet(cc = int32(12))))
4763             proc.p = Path(proc.sp)
4764             p = TestMakePSet()
4765             proc.fillProcessDesc(p)
4766             self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
4767 
4768             proc = Process("TEST")
4769             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4770             proc.options.accelerators = ["test1"]
4771             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4772                                                              a = int32(1),
4773                                                              b = PSet(c = int32(2))),
4774                                           test1 = EDProducer("Bar",
4775                                                              aa = int32(11),
4776                                                              bb = PSet(cc = int32(12))))
4777             proc.p = Path(proc.sp)
4778             p = TestMakePSet()
4779             proc.fillProcessDesc(p)
4780             self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
4781 
4782             proc = Process("TEST")
4783             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4784             proc.options.accelerators = ["test*"]
4785             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4786                                                              a = int32(1),
4787                                                              b = PSet(c = int32(2))),
4788                                           test1 = EDProducer("Bar",
4789                                                              aa = int32(11),
4790                                                              bb = PSet(cc = int32(12))))
4791             proc.p = Path(proc.sp)
4792             p = TestMakePSet()
4793             proc.fillProcessDesc(p)
4794             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
4795 
4796             proc = Process("TEST")
4797             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4798             proc.options.accelerators = ["anothertest3"]
4799             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4800                                                              a = int32(1),
4801                                                              b = PSet(c = int32(2))),
4802                                           test1 = EDProducer("Bar",
4803                                                              aa = int32(11),
4804                                                              bb = PSet(cc = int32(12))))
4805             proc.p = Path(proc.sp)
4806             p = TestMakePSet()
4807             self.assertRaises(RuntimeError, proc.fillProcessDesc, p)
4808 
4809             proc = Process("TEST")
4810             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4811                 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4812             proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2()
4813             proc.normalProducer = EDProducer("FooProducer")
4814             proc.testProducer = EDProducer("BarProducer@test")
4815             proc.test2Producer = EDProducer("BarProducer@test", test=untracked.PSet(backend=untracked.string("test2_backend")))
4816             proc.testAnalyzer = EDAnalyzer("Analyzer@test")
4817             proc.testFilter = EDAnalyzer("Filter@test")
4818             proc.testESProducer = ESProducer("ESProducer@test")
4819             proc.testESSource = ESSource("ESSource@test")
4820             proc.p = Path(proc.normalProducer+proc.testProducer+proc.test2Producer+proc.testAnalyzer+proc.testFilter)
4821             p = TestMakePSet()
4822             proc.fillProcessDesc(p)
4823             self.assertEqual("TestModuleTypeResolver", p.values["@module_type_resolver"][1])
4824             self.assertEqual("FooProducer", p.values["normalProducer"][1].values["@module_type"][1])
4825             self.assertEqual(len(list(filter(lambda x: not "@" in x, p.values["normalProducer"][1].values.keys()))), 0)
4826             self.assertEqual("BarProducer@test", p.values["testProducer"][1].values["@module_type"][1])
4827             self.assertEqual(False, p.values["testProducer"][1].values["test"][0])
4828             self.assertEqual(False, p.values["testProducer"][1].values["test"][1].values["backend"][0])
4829             self.assertEqual("test1_backend", p.values["testProducer"][1].values["test"][1].values["backend"][1])
4830             self.assertEqual("BarProducer@test", p.values["test2Producer"][1].values["@module_type"][1])
4831             self.assertEqual("test2_backend", p.values["test2Producer"][1].values["test"][1].values["backend"][1])
4832             self.assertEqual("Analyzer@test", p.values["testAnalyzer"][1].values["@module_type"][1])
4833             self.assertEqual("test1_backend", p.values["testAnalyzer"][1].values["test"][1].values["backend"][1])
4834             self.assertEqual("Filter@test", p.values["testFilter"][1].values["@module_type"][1])
4835             self.assertEqual("test1_backend", p.values["testFilter"][1].values["test"][1].values["backend"][1])
4836             self.assertEqual("ESProducer@test", p.values["ESProducer@test@testESProducer"][1].values["@module_type"][1])
4837             self.assertEqual("test1_backend", p.values["ESProducer@test@testESProducer"][1].values["test"][1].values["backend"][1])
4838             self.assertEqual("ESSource@test", p.values["ESSource@test@testESSource"][1].values["@module_type"][1])
4839             self.assertEqual("test1_backend", p.values["ESSource@test@testESSource"][1].values["test"][1].values["backend"][1])
4840 
4841             # No required accelerators available
4842             proc = Process("Test")
4843             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4844                 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4845             proc.options.accelerators = []
4846             self.assertRaises(EDMException, proc.fillProcessDesc, p)
4847 
4848             proc = Process("Test")
4849             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4850                 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4851             proc.options.accelerators = ["test1"]
4852             proc.test2Producer = EDProducer("BarProducer@test", test=untracked.PSet(backend=untracked.string("test2_backend")))
4853             proc.p = Path(proc.test2Producer)
4854             self.assertRaises(EDMException, proc.fillProcessDesc, p)
4855 
4856             # Two ProcessAccelerators return type resolver
4857             proc = Process("TEST")
4858             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(
4859                 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4860             proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2(
4861                 moduleTypeResolverMaker=lambda accelerators: TestModuleTypeResolver(accelerators))
4862             p = TestMakePSet()
4863             self.assertRaises(RuntimeError, proc.fillProcessDesc, p)
4864 
4865             import pickle
4866             proc = Process("TEST")
4867             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4868             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4869                                                              a = int32(1),
4870                                                              b = PSet(c = int32(2))),
4871                                           test1 = EDProducer("Bar",
4872                                                              aa = int32(11),
4873                                                              bb = PSet(cc = int32(12))))
4874             proc.p = Path(proc.sp)
4875             pkl = pickle.dumps(proc)
4876             unpkl = pickle.loads(pkl)
4877             p = TestMakePSet()
4878             unpkl.fillProcessDesc(p)
4879             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
4880             self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4881             unpkl = pickle.loads(pkl)
4882             unpkl.ProcessAcceleratorTest.setEnabled(["test1"])
4883             p = TestMakePSet()
4884             unpkl.fillProcessDesc(p)
4885             self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
4886             self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
4887 
4888     unittest.main()