Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2022-05-14 00:30:21

0001 #!/usr/bin/env python3
0002 
0003 ### command line options helper
0004 from __future__ import print_function
0005 from __future__ import absolute_import
0006 import os
0007 from  .Options import Options
0008 options = Options()
0009 
0010 
0011 ## imports
0012 import sys
0013 from .Mixins import PrintOptions,_ParameterTypeBase,_SimpleParameterTypeBase, _Parameterizable, _ConfigureComponent, _TypedParameterizable, _Labelable,  _Unlabelable,  _ValidatingListBase, _modifyParametersFromDict
0014 from .Mixins import *
0015 from .Types import *
0016 from .Modules import *
0017 from .Modules import _Module
0018 from .SequenceTypes import *
0019 from .SequenceTypes import _ModuleSequenceType, _Sequenceable  #extend needs it
0020 from .SequenceVisitors import PathValidator, EndPathValidator, FinalPathValidator, ScheduleTaskValidator, NodeVisitor, CompositeVisitor, ModuleNamesFromGlobalsVisitor
0021 from .MessageLogger import MessageLogger
0022 from . import DictTypes
0023 
0024 from .ExceptionHandling import *
0025 
0026 #when building RECO paths we have hit the default recursion limit
0027 if sys.getrecursionlimit()<5000:
0028     sys.setrecursionlimit(5000)
0029 
0030 def checkImportPermission(minLevel = 2, allowedPatterns = []):
0031     """
0032     Raise an exception if called by special config files. This checks
0033     the call or import stack for the importing file. An exception is raised if
0034     the importing module is not in allowedPatterns and if it is called too deeply:
0035     minLevel = 2: inclusion by top lvel cfg only
0036     minLevel = 1: No inclusion allowed
0037     allowedPatterns = ['Module1','Module2/SubModule1'] allows import
0038     by any module in Module1 or Submodule1
0039     """
0040 
0041     import inspect
0042     import os
0043 
0044     ignorePatterns = ['FWCore/ParameterSet/Config.py', 'FWCore/ParameterSet/python/Config.py','<string>','<frozen ']
0045     CMSSWPath = [os.environ['CMSSW_BASE'],os.environ['CMSSW_RELEASE_BASE']]
0046 
0047     # Filter the stack to things in CMSSWPath and not in ignorePatterns
0048     trueStack = []
0049     for item in inspect.stack():
0050         inPath = False
0051         ignore = False
0052 
0053         for pattern in CMSSWPath:
0054             if item[1].find(pattern) != -1:
0055                 inPath = True
0056                 break
0057         if item[1].find('/') == -1: # The base file, no path
0058             inPath = True
0059 
0060         for pattern in ignorePatterns:
0061             if item[1].find(pattern) != -1:
0062                 ignore = True
0063                 break
0064 
0065         if inPath and not ignore:
0066             trueStack.append(item[1])
0067 
0068     importedFile = trueStack[0]
0069     importedBy   = ''
0070     if len(trueStack) > 1:
0071         importedBy = trueStack[1]
0072 
0073     for pattern in allowedPatterns:
0074         if importedBy.find(pattern) > -1:
0075             return True
0076 
0077     if len(trueStack) <= minLevel: # Imported directly
0078         return True
0079 
0080     raise ImportError("Inclusion of %s is allowed only by cfg or specified cfi files."
0081                       % importedFile)
0082 
0083 def findProcess(module):
0084     """Look inside the module and find the Processes it contains"""
0085     class Temp(object):
0086         pass
0087     process = None
0088     if isinstance(module,dict):
0089         if 'process' in module:
0090             p = module['process']
0091             module = Temp()
0092             module.process = p
0093     if hasattr(module,'process'):
0094         if isinstance(module.process,Process):
0095             process = module.process
0096         else:
0097             raise RuntimeError("The attribute named 'process' does not inherit from the Process class")
0098     else:
0099         raise RuntimeError("no 'process' attribute found in the module, please add one")
0100     return process
0101 
0102 class Process(object):
0103     """Root class for a CMS configuration process"""
0104     _firstProcess = True
0105     def __init__(self,name,*Mods):
0106         """The argument 'name' will be the name applied to this Process
0107             Can optionally pass as additional arguments cms.Modifier instances
0108             that will be used to modify the Process as it is built
0109             """
0110         self.__dict__['_Process__name'] = name
0111         if not name.isalnum():
0112             raise RuntimeError("Error: The process name is an empty string or contains non-alphanumeric characters")
0113         self.__dict__['_Process__filters'] = {}
0114         self.__dict__['_Process__producers'] = {}
0115         self.__dict__['_Process__switchproducers'] = {}
0116         self.__dict__['_Process__source'] = None
0117         self.__dict__['_Process__looper'] = None
0118         self.__dict__['_Process__subProcesses'] = []
0119         self.__dict__['_Process__schedule'] = None
0120         self.__dict__['_Process__analyzers'] = {}
0121         self.__dict__['_Process__outputmodules'] = {}
0122         self.__dict__['_Process__paths'] = DictTypes.SortedKeysDict()    # have to keep the order
0123         self.__dict__['_Process__endpaths'] = DictTypes.SortedKeysDict() # of definition
0124         self.__dict__['_Process__finalpaths'] = DictTypes.SortedKeysDict() # of definition
0125         self.__dict__['_Process__sequences'] = {}
0126         self.__dict__['_Process__tasks'] = {}
0127         self.__dict__['_Process__conditionaltasks'] = {}
0128         self.__dict__['_Process__services'] = {}
0129         self.__dict__['_Process__essources'] = {}
0130         self.__dict__['_Process__esproducers'] = {}
0131         self.__dict__['_Process__esprefers'] = {}
0132         self.__dict__['_Process__aliases'] = {}
0133         self.__dict__['_Process__psets']={}
0134         self.__dict__['_Process__vpsets']={}
0135         self.__dict__['_cloneToObjectDict'] = {}
0136         # policy switch to avoid object overwriting during extend/load
0137         self.__dict__['_Process__InExtendCall'] = False
0138         self.__dict__['_Process__partialschedules'] = {}
0139         self.__isStrict = False
0140         self.__dict__['_Process__modifiers'] = Mods
0141         self.__dict__['_Process__accelerators'] = {}
0142         self.options = Process.defaultOptions_()
0143         self.maxEvents = Process.defaultMaxEvents_()
0144         self.maxLuminosityBlocks = Process.defaultMaxLuminosityBlocks_()
0145         # intentionally not cloned to ensure that everyone taking
0146         # MessageLogger still via
0147         # FWCore.Message(Logger|Service).MessageLogger_cfi
0148         # use the very same MessageLogger object.
0149         self.MessageLogger = MessageLogger
0150         if Process._firstProcess:
0151             Process._firstProcess = False
0152         else:
0153             if len(Mods) > 0:
0154                 for m in self.__modifiers:
0155                     if not m._isChosen():
0156                         raise RuntimeError("The Process {} tried to redefine which Modifiers to use after another Process was already started".format(name))
0157         for m in self.__modifiers:
0158             m._setChosen()
0159 
0160     def setStrict(self, value):
0161         self.__isStrict = value
0162         _Module.__isStrict__ = True
0163 
0164     # some user-friendly methods for command-line browsing
0165     def producerNames(self):
0166         """Returns a string containing all the EDProducer labels separated by a blank"""
0167         return ' '.join(self.producers_().keys())
0168     def switchProducerNames(self):
0169         """Returns a string containing all the SwitchProducer labels separated by a blank"""
0170         return ' '.join(self.switchProducers_().keys())
0171     def analyzerNames(self):
0172         """Returns a string containing all the EDAnalyzer labels separated by a blank"""
0173         return ' '.join(self.analyzers_().keys())
0174     def filterNames(self):
0175         """Returns a string containing all the EDFilter labels separated by a blank"""
0176         return ' '.join(self.filters_().keys())
0177     def pathNames(self):
0178         """Returns a string containing all the Path names separated by a blank"""
0179         return ' '.join(self.paths_().keys())
0180 
0181     def __setstate__(self, pkldict):
0182         """
0183         Unpickling hook.
0184 
0185         Since cloneToObjectDict stores a hash of objects by their
0186         id() it needs to be updated when unpickling to use the
0187         new object id values instantiated during the unpickle.
0188 
0189         """
0190         self.__dict__.update(pkldict)
0191         tmpDict = {}
0192         for value in self._cloneToObjectDict.values():
0193             tmpDict[id(value)] = value
0194         self.__dict__['_cloneToObjectDict'] = tmpDict
0195 
0196 
0197 
0198     def filters_(self):
0199         """returns a dict of the filters that have been added to the Process"""
0200         return DictTypes.FixedKeysDict(self.__filters)
0201     filters = property(filters_, doc="dictionary containing the filters for the process")
0202     def name_(self):
0203         return self.__name
0204     def setName_(self,name):
0205         if not name.isalnum():
0206             raise RuntimeError("Error: The process name is an empty string or contains non-alphanumeric characters")
0207         self.__dict__['_Process__name'] = name
0208     process = property(name_,setName_, doc="name of the process")
0209     def producers_(self):
0210         """returns a dict of the producers that have been added to the Process"""
0211         return DictTypes.FixedKeysDict(self.__producers)
0212     producers = property(producers_,doc="dictionary containing the producers for the process")
0213     def switchProducers_(self):
0214         """returns a dict of the SwitchProducers that have been added to the Process"""
0215         return DictTypes.FixedKeysDict(self.__switchproducers)
0216     switchProducers = property(switchProducers_,doc="dictionary containing the SwitchProducers for the process")
0217     def source_(self):
0218         """returns the source that has been added to the Process or None if none have been added"""
0219         return self.__source
0220     def setSource_(self,src):
0221         self._placeSource('source',src)
0222     source = property(source_,setSource_,doc='the main source or None if not set')
0223     def looper_(self):
0224         """returns the looper that has been added to the Process or None if none have been added"""
0225         return self.__looper
0226     def setLooper_(self,lpr):
0227         self._placeLooper('looper',lpr)
0228     looper = property(looper_,setLooper_,doc='the main looper or None if not set')
0229     @staticmethod
0230     def defaultOptions_():
0231         return untracked.PSet(numberOfThreads = untracked.uint32(1),
0232                               numberOfStreams = untracked.uint32(0),
0233                               numberOfConcurrentRuns = untracked.uint32(1),
0234                               numberOfConcurrentLuminosityBlocks = untracked.uint32(0),
0235                               eventSetup = untracked.PSet(
0236                                   numberOfConcurrentIOVs = untracked.uint32(0),
0237                                   forceNumberOfConcurrentIOVs = untracked.PSet(
0238                                       allowAnyLabel_ = required.untracked.uint32
0239                                   )
0240                               ),
0241                               accelerators = untracked.vstring('*'),
0242                               wantSummary = untracked.bool(False),
0243                               fileMode = untracked.string('FULLMERGE'),
0244                               forceEventSetupCacheClearOnNewRun = untracked.bool(False),
0245                               throwIfIllegalParameter = untracked.bool(True),
0246                               printDependencies = untracked.bool(False),
0247                               deleteNonConsumedUnscheduledModules = untracked.bool(True),
0248                               sizeOfStackForThreadsInKB = optional.untracked.uint32,
0249                               Rethrow = untracked.vstring(),
0250                               SkipEvent = untracked.vstring(),
0251                               FailPath = untracked.vstring(),
0252                               IgnoreCompletely = untracked.vstring(),
0253                               canDeleteEarly = untracked.vstring(),
0254                               dumpOptions = untracked.bool(False),
0255                               allowUnscheduled = obsolete.untracked.bool,
0256                               emptyRunLumiMode = obsolete.untracked.string,
0257                               makeTriggerResults = obsolete.untracked.bool
0258                               )
0259     def __updateOptions(self,opt):
0260         newOpts = self.defaultOptions_()
0261         if isinstance(opt,dict):
0262             for k,v in opt.items():
0263                 setattr(newOpts,k,v)
0264         else:
0265             for p in opt.parameters_():
0266                 setattr(newOpts, p, getattr(opt,p))
0267         return newOpts
0268     @staticmethod
0269     def defaultMaxEvents_():
0270         return untracked.PSet(input=optional.untracked.int32,
0271                               output=optional.untracked.allowed(int32,PSet))
0272     def __updateMaxEvents(self,ps):
0273         newMax = self.defaultMaxEvents_()
0274         if isinstance(ps,dict):
0275             for k,v in ps.items():
0276                 setattr(newMax,k,v)
0277         else:
0278             for p in ps.parameters_():
0279                 setattr(newMax, p, getattr(ps,p))
0280         return newMax
0281     @staticmethod
0282     def defaultMaxLuminosityBlocks_():
0283         return untracked.PSet(input=untracked.int32(-1))
0284     def subProcesses_(self):
0285         """returns a list of the subProcesses that have been added to the Process"""
0286         return self.__subProcesses
0287     subProcesses = property(subProcesses_,doc='the SubProcesses that have been added to the Process')
0288     def analyzers_(self):
0289         """returns a dict of the analyzers that have been added to the Process"""
0290         return DictTypes.FixedKeysDict(self.__analyzers)
0291     analyzers = property(analyzers_,doc="dictionary containing the analyzers for the process")
0292     def outputModules_(self):
0293         """returns a dict of the output modules that have been added to the Process"""
0294         return DictTypes.FixedKeysDict(self.__outputmodules)
0295     outputModules = property(outputModules_,doc="dictionary containing the output_modules for the process")
0296     def paths_(self):
0297         """returns a dict of the paths that have been added to the Process"""
0298         return DictTypes.SortedAndFixedKeysDict(self.__paths)
0299     paths = property(paths_,doc="dictionary containing the paths for the process")
0300     def endpaths_(self):
0301         """returns a dict of the endpaths that have been added to the Process"""
0302         return DictTypes.SortedAndFixedKeysDict(self.__endpaths)
0303     endpaths = property(endpaths_,doc="dictionary containing the endpaths for the process")
0304     def finalpaths_(self):
0305         """returns a dict of the finalpaths that have been added to the Process"""
0306         return DictTypes.SortedAndFixedKeysDict(self.__finalpaths)
0307     finalpaths = property(finalpaths_,doc="dictionary containing the finalpaths for the process")
0308     def sequences_(self):
0309         """returns a dict of the sequences that have been added to the Process"""
0310         return DictTypes.FixedKeysDict(self.__sequences)
0311     sequences = property(sequences_,doc="dictionary containing the sequences for the process")
0312     def tasks_(self):
0313         """returns a dict of the tasks that have been added to the Process"""
0314         return DictTypes.FixedKeysDict(self.__tasks)
0315     tasks = property(tasks_,doc="dictionary containing the tasks for the process")
0316     def conditionaltasks_(self):
0317         """returns a dict of the conditionaltasks that have been added to the Process"""
0318         return DictTypes.FixedKeysDict(self.__conditionaltasks)
0319     conditionaltasks = property(conditionaltasks_,doc="dictionary containing the conditionatasks for the process")
0320     def schedule_(self):
0321         """returns the schedule that has been added to the Process or None if none have been added"""
0322         return self.__schedule
0323     def setPartialSchedule_(self,sch,label):
0324         if label == "schedule":
0325             self.setSchedule_(sch)
0326         else:
0327             self._place(label, sch, self.__partialschedules)
0328     def setSchedule_(self,sch):
0329         # See if every path and endpath has been inserted into the process
0330         index = 0
0331         try:
0332             for p in sch:
0333                 p.label_()
0334                 index +=1
0335         except:
0336             raise RuntimeError("The path at index "+str(index)+" in the Schedule was not attached to the process.")
0337         self.__dict__['_Process__schedule'] = sch
0338     schedule = property(schedule_,setSchedule_,doc='the schedule or None if not set')
0339     def services_(self):
0340         """returns a dict of the services that have been added to the Process"""
0341         return DictTypes.FixedKeysDict(self.__services)
0342     services = property(services_,doc="dictionary containing the services for the process")
0343     def processAccelerators_(self):
0344         """returns a dict of the ProcessAccelerators that have been added to the Process"""
0345         return DictTypes.FixedKeysDict(self.__accelerators)
0346     processAccelerators = property(processAccelerators_,doc="dictionary containing the ProcessAccelerators for the process")
0347     def es_producers_(self):
0348         """returns a dict of the esproducers that have been added to the Process"""
0349         return DictTypes.FixedKeysDict(self.__esproducers)
0350     es_producers = property(es_producers_,doc="dictionary containing the es_producers for the process")
0351     def es_sources_(self):
0352         """returns a the es_sources that have been added to the Process"""
0353         return DictTypes.FixedKeysDict(self.__essources)
0354     es_sources = property(es_sources_,doc="dictionary containing the es_sources for the process")
0355     def es_prefers_(self):
0356         """returns a dict of the es_prefers that have been added to the Process"""
0357         return DictTypes.FixedKeysDict(self.__esprefers)
0358     es_prefers = property(es_prefers_,doc="dictionary containing the es_prefers for the process")
0359     def aliases_(self):
0360         """returns a dict of the aliases that have been added to the Process"""
0361         return DictTypes.FixedKeysDict(self.__aliases)
0362     aliases = property(aliases_,doc="dictionary containing the aliases for the process")
0363     def psets_(self):
0364         """returns a dict of the PSets that have been added to the Process"""
0365         return DictTypes.FixedKeysDict(self.__psets)
0366     psets = property(psets_,doc="dictionary containing the PSets for the process")
0367     def vpsets_(self):
0368         """returns a dict of the VPSets that have been added to the Process"""
0369         return DictTypes.FixedKeysDict(self.__vpsets)
0370     vpsets = property(vpsets_,doc="dictionary containing the PSets for the process")
0371 
0372     def isUsingModifier(self,mod):
0373         """returns True if the Modifier is in used by this Process"""
0374         if mod._isChosen():
0375             for m in self.__modifiers:
0376                 if m._isOrContains(mod):
0377                     return True
0378         return False
0379 
0380     def __setObjectLabel(self, object, newLabel) :
0381         if not object.hasLabel_() :
0382             object.setLabel(newLabel)
0383             return
0384         if newLabel == object.label_() :
0385             return
0386         if newLabel is None :
0387             object.setLabel(None)
0388             return
0389         if (hasattr(self, object.label_()) and id(getattr(self, object.label_())) == id(object)) :
0390             msg100 = "Attempting to change the label of an attribute of the Process\n"
0391             msg101 = "Old label = "+object.label_()+"  New label = "+newLabel+"\n"
0392             msg102 = "Type = "+str(type(object))+"\n"
0393             msg103 = "Some possible solutions:\n"
0394             msg104 = "  1. Clone modules instead of using simple assignment. Cloning is\n"
0395             msg105 = "  also preferred for other types when possible.\n"
0396             msg106 = "  2. Declare new names starting with an underscore if they are\n"
0397             msg107 = "  for temporaries you do not want propagated into the Process. The\n"
0398             msg108 = "  underscore tells \"from x import *\" and process.load not to import\n"
0399             msg109 = "  the name.\n"
0400             msg110 = "  3. Reorganize so the assigment is not necessary. Giving a second\n"
0401             msg111 = "  name to the same object usually causes confusion and problems.\n"
0402             msg112 = "  4. Compose Sequences: newName = cms.Sequence(oldName)\n"
0403             raise ValueError(msg100+msg101+msg102+msg103+msg104+msg105+msg106+msg107+msg108+msg109+msg110+msg111+msg112)
0404         object.setLabel(None)
0405         object.setLabel(newLabel)
0406 
0407     def __setattr__(self,name,value):
0408         # check if the name is well-formed (only _ and alphanumerics are allowed)
0409         if not name.replace('_','').isalnum():
0410             raise ValueError('The label '+name+' contains forbiden characters')
0411 
0412         if name == 'options':
0413             value = self.__updateOptions(value)
0414         if name == 'maxEvents':
0415             value = self.__updateMaxEvents(value)
0416 
0417         # private variable exempt from all this
0418         if name.startswith('_Process__'):
0419             self.__dict__[name]=value
0420             return
0421         if not isinstance(value,_ConfigureComponent):
0422             raise TypeError("can only assign labels to an object that inherits from '_ConfigureComponent'\n"
0423                             +"an instance of "+str(type(value))+" will not work - requested label is "+name)
0424         if not isinstance(value,_Labelable) and not isinstance(value,Source) and not isinstance(value,Looper) and not isinstance(value,Schedule):
0425             if name == value.type_():
0426                 if hasattr(self,name) and (getattr(self,name)!=value):
0427                     self._replaceInTasks(name, value)
0428                     self._replaceInConditionalTasks(name, value)
0429                 # Only Services get handled here
0430                 self.add_(value)
0431                 return
0432             else:
0433                 raise TypeError("an instance of "+str(type(value))+" can not be assigned the label '"+name+"'.\n"+
0434                                 "Please either use the label '"+value.type_()+" or use the 'add_' method instead.")
0435         #clone the item
0436         if self.__isStrict:
0437             newValue =value.copy()
0438             try:
0439                 newValue._filename = value._filename
0440             except:
0441                 pass
0442             value.setIsFrozen()
0443         else:
0444             newValue =value
0445         if not self._okToPlace(name, value, self.__dict__):
0446             newFile='top level config'
0447             if hasattr(value,'_filename'):
0448                 newFile = value._filename
0449             oldFile='top level config'
0450             oldValue = getattr(self,name)
0451             if hasattr(oldValue,'_filename'):
0452                 oldFile = oldValue._filename
0453             msg = "Trying to override definition of process."+name
0454             msg += "\n new object defined in: "+newFile
0455             msg += "\n existing object defined in: "+oldFile
0456             raise ValueError(msg)
0457         # remove the old object of the name (if there is one)
0458         if hasattr(self,name) and not (getattr(self,name)==newValue):
0459             # Complain if items in sequences or tasks from load() statements have
0460             # degenerate names, but if the user overwrites a name in the
0461             # main config, replace it everywhere
0462             if newValue._isTaskComponent():
0463                 if not self.__InExtendCall:
0464                     self._replaceInTasks(name, newValue)
0465                     self._replaceInConditionalTasks(name, newValue)
0466                     self._replaceInSchedule(name, newValue)
0467                 else:
0468                     if not isinstance(newValue, Task):
0469                         #should check to see if used in task before complaining
0470                         newFile='top level config'
0471                         if hasattr(value,'_filename'):
0472                             newFile = value._filename
0473                         oldFile='top level config'
0474                         oldValue = getattr(self,name)
0475                         if hasattr(oldValue,'_filename'):
0476                             oldFile = oldValue._filename
0477                         msg1 = "Trying to override definition of "+name+" while it is used by the task "
0478                         msg2 = "\n new object defined in: "+newFile
0479                         msg2 += "\n existing object defined in: "+oldFile
0480                         s = self.__findFirstUsingModule(self.tasks,oldValue)
0481                         if s is not None:
0482                             raise ValueError(msg1+s.label_()+msg2)
0483 
0484             if isinstance(newValue, _Sequenceable) or newValue._isTaskComponent() or isinstance(newValue, ConditionalTask):
0485                 if not self.__InExtendCall:
0486                     if isinstance(newValue, ConditionalTask):
0487                         self._replaceInConditionalTasks(name, newValue)
0488                     self._replaceInSequences(name, newValue)
0489                 else:
0490                     #should check to see if used in sequence before complaining
0491                     newFile='top level config'
0492                     if hasattr(value,'_filename'):
0493                         newFile = value._filename
0494                     oldFile='top level config'
0495                     oldValue = getattr(self,name)
0496                     if hasattr(oldValue,'_filename'):
0497                         oldFile = oldValue._filename
0498                     msg1 = "Trying to override definition of "+name+" while it is used by the "
0499                     msg2 = "\n new object defined in: "+newFile
0500                     msg2 += "\n existing object defined in: "+oldFile
0501                     s = self.__findFirstUsingModule(self.sequences,oldValue)
0502                     if s is not None:
0503                         raise ValueError(msg1+"sequence "+s.label_()+msg2)
0504                     s = self.__findFirstUsingModule(self.paths,oldValue)
0505                     if s is not None:
0506                         raise ValueError(msg1+"path "+s.label_()+msg2)
0507                     s = self.__findFirstUsingModule(self.endpaths,oldValue)
0508                     if s is not None:
0509                         raise ValueError(msg1+"endpath "+s.label_()+msg2)
0510                     s = self.__findFirstUsingModule(self.finalpaths,oldValue)
0511                     if s is not None:
0512                         raise ValueError(msg1+"finalpath "+s.label_()+msg2)
0513 
0514             # In case of EDAlias, raise Exception always to avoid surprises
0515             if isinstance(newValue, EDAlias):
0516                 oldValue = getattr(self, name)
0517                 #should check to see if used in task/sequence before complaining
0518                 newFile='top level config'
0519                 if hasattr(value,'_filename'):
0520                     newFile = value._filename
0521                 oldFile='top level config'
0522                 if hasattr(oldValue,'_filename'):
0523                     oldFile = oldValue._filename
0524                 msg1 = "Trying to override definition of "+name+" with an EDAlias while it is used by the "
0525                 msg2 = "\n new object defined in: "+newFile
0526                 msg2 += "\n existing object defined in: "+oldFile
0527                 s = self.__findFirstUsingModule(self.tasks,oldValue)
0528                 if s is not None:
0529                     raise ValueError(msg1+"task "+s.label_()+msg2)
0530                 s = self.__findFirstUsingModule(self.sequences,oldValue)
0531                 if s is not None:
0532                     raise ValueError(msg1+"sequence "+s.label_()+msg2)
0533                 s = self.__findFirstUsingModule(self.paths,oldValue)
0534                 if s is not None:
0535                     raise ValueError(msg1+"path "+s.label_()+msg2)
0536                 s = self.__findFirstUsingModule(self.endpaths,oldValue)
0537                 if s is not None:
0538                     raise ValueError(msg1+"endpath "+s.label_()+msg2)
0539                 s = self.__findFirstUsingModule(self.finalpaths,oldValue)
0540                 if s is not None:
0541                     raise ValueError(msg1+"finalpath "+s.label_()+msg2)
0542 
0543             if not self.__InExtendCall and (Schedule._itemIsValid(newValue) or isinstance(newValue, Task)):
0544                 self._replaceInScheduleDirectly(name, newValue)
0545 
0546             self._delattrFromSetattr(name)
0547         self.__dict__[name]=newValue
0548         if isinstance(newValue,_Labelable):
0549             self.__setObjectLabel(newValue, name)
0550             self._cloneToObjectDict[id(value)] = newValue
0551             self._cloneToObjectDict[id(newValue)] = newValue
0552         #now put in proper bucket
0553         newValue._place(name,self)
0554     def __findFirstUsingModule(self, seqsOrTasks, mod):
0555         """Given a container of sequences or tasks, find the first sequence or task
0556         containing mod and return it. If none is found, return None"""
0557         from FWCore.ParameterSet.SequenceTypes import ModuleNodeVisitor
0558         l = list()
0559         for seqOrTask in seqsOrTasks.values():
0560             l[:] = []
0561             v = ModuleNodeVisitor(l)
0562             seqOrTask.visit(v)
0563             if mod in l:
0564                 return seqOrTask
0565         return None
0566 
0567     def _delHelper(self,name):
0568         if not hasattr(self,name):
0569             raise KeyError('process does not know about '+name)
0570         elif name.startswith('_Process__'):
0571             raise ValueError('this attribute cannot be deleted')
0572 
0573         # we have to remove it from all dictionaries/registries
0574         dicts = [item for item in self.__dict__.values() if (isinstance(item, dict) or isinstance(item, DictTypes.SortedKeysDict))]
0575         for reg in dicts:
0576             if name in reg: del reg[name]
0577         # if it was a labelable object, the label needs to be removed
0578         obj = getattr(self,name)
0579         if isinstance(obj,_Labelable):
0580             obj.setLabel(None)
0581         if isinstance(obj,Service):
0582             obj._inProcess = False
0583 
0584     def __delattr__(self,name):
0585         self._delHelper(name)
0586         obj = getattr(self,name)
0587         if not obj is None:
0588             if not isinstance(obj, Sequence) and not isinstance(obj, Task) and not isinstance(obj,ConditionalTask):
0589                 # For modules, ES modules and services we can also remove
0590                 # the deleted object from Sequences, Paths, EndPaths, and
0591                 # Tasks. Note that for Sequences and Tasks that cannot be done
0592                 # reliably as the places where the Sequence or Task was used
0593                 # might have been expanded so we do not even try. We considered
0594                 # raising an exception if a Sequences or Task was explicitly
0595                 # deleted, but did not because when done carefully deletion
0596                 # is sometimes OK (for example in the prune function where it
0597                 # has been checked that the deleted Sequence is not used).
0598                 if obj._isTaskComponent():
0599                     self._replaceInTasks(name, None)
0600                     self._replaceInConditionalTasks(name, None)
0601                     self._replaceInSchedule(name, None)
0602                 if isinstance(obj, _Sequenceable) or obj._isTaskComponent():
0603                     self._replaceInSequences(name, None)
0604                 if Schedule._itemIsValid(obj) or isinstance(obj, Task):
0605                     self._replaceInScheduleDirectly(name, None)
0606         # now remove it from the process itself
0607         try:
0608             del self.__dict__[name]
0609         except:
0610             pass
0611 
0612     def _delattrFromSetattr(self,name):
0613         """Similar to __delattr__ but we need different behavior when called from __setattr__"""
0614         self._delHelper(name)
0615         # now remove it from the process itself
0616         try:
0617             del self.__dict__[name]
0618         except:
0619             pass
0620 
0621     def add_(self,value):
0622         """Allows addition of components that do not have to have a label, e.g. Services"""
0623         if not isinstance(value,_ConfigureComponent):
0624             raise TypeError
0625         if not isinstance(value,_Unlabelable):
0626             raise TypeError
0627         #clone the item
0628         if self.__isStrict:
0629             newValue =value.copy()
0630             value.setIsFrozen()
0631         else:
0632             newValue =value
0633         newValue._place('',self)
0634 
0635     def _okToPlace(self, name, mod, d):
0636         if not self.__InExtendCall:
0637             # if going
0638             return True
0639         elif not self.__isStrict:
0640             return True
0641         elif name in d:
0642             # if there's an old copy, and the new one
0643             # hasn't been modified, we're done.  Still
0644             # not quite safe if something has been defined twice.
0645             #  Need to add checks
0646             if mod._isModified:
0647                 if d[name]._isModified:
0648                     return False
0649                 else:
0650                     return True
0651             else:
0652                 return True
0653         else:
0654             return True
0655 
0656     def _place(self, name, mod, d):
0657         if self._okToPlace(name, mod, d):
0658             if self.__isStrict and isinstance(mod, _ModuleSequenceType):
0659                 d[name] = mod._postProcessFixup(self._cloneToObjectDict)
0660             else:
0661                 d[name] = mod
0662             if isinstance(mod,_Labelable):
0663                 self.__setObjectLabel(mod, name)
0664     def _placeOutputModule(self,name,mod):
0665         self._place(name, mod, self.__outputmodules)
0666     def _placeProducer(self,name,mod):
0667         self._place(name, mod, self.__producers)
0668     def _placeSwitchProducer(self,name,mod):
0669         self._place(name, mod, self.__switchproducers)
0670     def _placeFilter(self,name,mod):
0671         self._place(name, mod, self.__filters)
0672     def _placeAnalyzer(self,name,mod):
0673         self._place(name, mod, self.__analyzers)
0674     def _placePath(self,name,mod):
0675         self._validateSequence(mod, name)
0676         try:
0677             self._place(name, mod, self.__paths)
0678         except ModuleCloneError as msg:
0679             context = format_outerframe(4)
0680             raise Exception("%sThe module %s in path %s is unknown to the process %s." %(context, msg, name, self._Process__name))
0681     def _placeEndPath(self,name,mod):
0682         self._validateSequence(mod, name)
0683         try:
0684             self._place(name, mod, self.__endpaths)
0685         except ModuleCloneError as msg:
0686             context = format_outerframe(4)
0687             raise Exception("%sThe module %s in endpath %s is unknown to the process %s." %(context, msg, name, self._Process__name))
0688     def _placeFinalPath(self,name,mod):
0689         self._validateSequence(mod, name)
0690         try:
0691             self._place(name, mod, self.__finalpaths)
0692         except ModuleCloneError as msg:
0693             context = format_outerframe(4)
0694             raise Exception("%sThe module %s in finalpath %s is unknown to the process %s." %(context, msg, name, self._Process__name))
0695     def _placeSequence(self,name,mod):
0696         self._validateSequence(mod, name)
0697         self._place(name, mod, self.__sequences)
0698     def _placeESProducer(self,name,mod):
0699         self._place(name, mod, self.__esproducers)
0700     def _placeESPrefer(self,name,mod):
0701         self._place(name, mod, self.__esprefers)
0702     def _placeESSource(self,name,mod):
0703         self._place(name, mod, self.__essources)
0704     def _placeTask(self,name,task):
0705         self._validateTask(task, name)
0706         self._place(name, task, self.__tasks)
0707     def _placeConditionalTask(self,name,task):
0708         self._validateConditionalTask(task, name)
0709         self._place(name, task, self.__conditionaltasks)
0710     def _placeAlias(self,name,mod):
0711         self._place(name, mod, self.__aliases)
0712     def _placePSet(self,name,mod):
0713         self._place(name, mod, self.__psets)
0714     def _placeVPSet(self,name,mod):
0715         self._place(name, mod, self.__vpsets)
0716     def _placeSource(self,name,mod):
0717         """Allow the source to be referenced by 'source' or by type name"""
0718         if name != 'source':
0719             raise ValueError("The label '"+name+"' can not be used for a Source.  Only 'source' is allowed.")
0720         if self.__dict__['_Process__source'] is not None :
0721             del self.__dict__[self.__dict__['_Process__source'].type_()]
0722         self.__dict__['_Process__source'] = mod
0723         self.__dict__[mod.type_()] = mod
0724     def _placeLooper(self,name,mod):
0725         if name != 'looper':
0726             raise ValueError("The label '"+name+"' can not be used for a Looper.  Only 'looper' is allowed.")
0727         self.__dict__['_Process__looper'] = mod
0728         self.__dict__[mod.type_()] = mod
0729     def _placeSubProcess(self,name,mod):
0730         self.__dict__['_Process__subProcess'] = mod
0731         self.__dict__[mod.type_()] = mod
0732     def addSubProcess(self,mod):
0733         self.__subProcesses.append(mod)
0734     def _placeService(self,typeName,mod):
0735         self._place(typeName, mod, self.__services)
0736         if typeName in self.__dict__:
0737             self.__dict__[typeName]._inProcess = False
0738         self.__dict__[typeName]=mod
0739     def _placeAccelerator(self,typeName,mod):
0740         self._place(typeName, mod, self.__accelerators)
0741         self.__dict__[typeName]=mod
0742     def load(self, moduleName):
0743         moduleName = moduleName.replace("/",".")
0744         module = __import__(moduleName)
0745         self.extend(sys.modules[moduleName])
0746     def extend(self,other,items=()):
0747         """Look in other and find types that we can use"""
0748         # enable explicit check to avoid overwriting of existing objects
0749         self.__dict__['_Process__InExtendCall'] = True
0750 
0751         seqs = dict()
0752         tasksToAttach = dict()
0753         mods = []
0754         for name in dir(other):
0755             #'from XX import *' ignores these, and so should we.
0756             if name.startswith('_'):
0757                 continue
0758             item = getattr(other,name)
0759             if name == "source" or name == "looper":
0760                 # In these cases 'item' could be None if the specific object was not defined
0761                 if item is not None:
0762                     self.__setattr__(name,item)
0763             elif isinstance(item,_ModuleSequenceType):
0764                 seqs[name]=item
0765             elif isinstance(item,Task) or isinstance(item, ConditionalTask):
0766                 tasksToAttach[name] = item
0767             elif isinstance(item,_Labelable):
0768                 self.__setattr__(name,item)
0769                 if not item.hasLabel_() :
0770                     item.setLabel(name)
0771             elif isinstance(item,Schedule):
0772                 self.__setattr__(name,item)
0773             elif isinstance(item,_Unlabelable):
0774                 self.add_(item)
0775             elif isinstance(item,ProcessModifier):
0776                 mods.append(item)
0777             elif isinstance(item,ProcessFragment):
0778                 self.extend(item)
0779 
0780         #now create a sequence that uses the newly made items
0781         for name,seq in seqs.items():
0782             if id(seq) not in self._cloneToObjectDict:
0783                 self.__setattr__(name,seq)
0784             else:
0785                 newSeq = self._cloneToObjectDict[id(seq)]
0786                 self.__dict__[name]=newSeq
0787                 self.__setObjectLabel(newSeq, name)
0788                 #now put in proper bucket
0789                 newSeq._place(name,self)
0790 
0791         for name, task in tasksToAttach.items():
0792             self.__setattr__(name, task)
0793 
0794         #apply modifiers now that all names have been added
0795         for item in mods:
0796             item.apply(self)
0797 
0798         self.__dict__['_Process__InExtendCall'] = False
0799 
0800     def _dumpConfigNamedList(self,items,typeName,options):
0801         returnValue = ''
0802         for name,item in items:
0803             returnValue +=options.indentation()+typeName+' '+name+' = '+item.dumpConfig(options)
0804         return returnValue
0805 
0806     def _dumpConfigUnnamedList(self,items,typeName,options):
0807         returnValue = ''
0808         for name,item in items:
0809             returnValue +=options.indentation()+typeName+' = '+item.dumpConfig(options)
0810         return returnValue
0811 
0812     def _dumpConfigOptionallyNamedList(self,items,typeName,options):
0813         returnValue = ''
0814         for name,item in items:
0815             if name == item.type_():
0816                 name = ''
0817             returnValue +=options.indentation()+typeName+' '+name+' = '+item.dumpConfig(options)
0818         return returnValue
0819 
0820     def dumpConfig(self, options=PrintOptions()):
0821         """return a string containing the equivalent process defined using the old configuration language"""
0822         config = "process "+self.__name+" = {\n"
0823         options.indent()
0824         if self.source_():
0825             config += options.indentation()+"source = "+self.source_().dumpConfig(options)
0826         if self.looper_():
0827             config += options.indentation()+"looper = "+self.looper_().dumpConfig(options)
0828 
0829         config+=self._dumpConfigNamedList(self.subProcesses_(),
0830                                   'subProcess',
0831                                   options)
0832         config+=self._dumpConfigNamedList(self.producers_().items(),
0833                                   'module',
0834                                   options)
0835         config+=self._dumpConfigNamedList(self.switchProducers_().items(),
0836                                   'module',
0837                                   options)
0838         config+=self._dumpConfigNamedList(self.filters_().items(),
0839                                   'module',
0840                                   options)
0841         config+=self._dumpConfigNamedList(self.analyzers_().items(),
0842                                   'module',
0843                                   options)
0844         config+=self._dumpConfigNamedList(self.outputModules_().items(),
0845                                   'module',
0846                                   options)
0847         config+=self._dumpConfigNamedList(self.sequences_().items(),
0848                                   'sequence',
0849                                   options)
0850         config+=self._dumpConfigNamedList(self.paths_().items(),
0851                                   'path',
0852                                   options)
0853         config+=self._dumpConfigNamedList(self.endpaths_().items(),
0854                                   'endpath',
0855                                   options)
0856         config+=self._dumpConfigNamedList(self.finalpaths_().items(),
0857                                   'finalpath',
0858                                   options)
0859         config+=self._dumpConfigUnnamedList(self.services_().items(),
0860                                   'service',
0861                                   options)
0862         config+=self._dumpConfigNamedList(self.aliases_().items(),
0863                                   'alias',
0864                                   options)
0865         config+=self._dumpConfigOptionallyNamedList(
0866             self.es_producers_().items(),
0867             'es_module',
0868             options)
0869         config+=self._dumpConfigOptionallyNamedList(
0870             self.es_sources_().items(),
0871             'es_source',
0872             options)
0873         config += self._dumpConfigESPrefers(options)
0874         for name,item in self.psets.items():
0875             config +=options.indentation()+item.configTypeName()+' '+name+' = '+item.configValue(options)
0876         for name,item in self.vpsets.items():
0877             config +=options.indentation()+'VPSet '+name+' = '+item.configValue(options)
0878         if self.schedule:
0879             pathNames = [p.label_() for p in self.schedule]
0880             config +=options.indentation()+'schedule = {'+','.join(pathNames)+'}\n'
0881 
0882 #        config+=self._dumpConfigNamedList(self.vpsets.items(),
0883 #                                  'VPSet',
0884 #                                  options)
0885         config += "}\n"
0886         options.unindent()
0887         return config
0888 
0889     def _dumpConfigESPrefers(self, options):
0890         result = ''
0891         for item in self.es_prefers_().values():
0892             result +=options.indentation()+'es_prefer '+item.targetLabel_()+' = '+item.dumpConfig(options)
0893         return result
0894 
0895     def _dumpPythonSubProcesses(self, l, options):
0896         returnValue = ''
0897         for item in l:
0898             returnValue += item.dumpPython(options)+'\n\n'
0899         return returnValue
0900 
0901     def _dumpPythonList(self, d, options):
0902         returnValue = ''
0903         if isinstance(d, DictTypes.SortedKeysDict):
0904             for name,item in d.items():
0905                 returnValue +='process.'+name+' = '+item.dumpPython(options)+'\n\n'
0906         else:
0907             for name,item in sorted(d.items()):
0908                 returnValue +='process.'+name+' = '+item.dumpPython(options)+'\n\n'
0909         return returnValue
0910 
0911     def _splitPythonList(self, subfolder, d, options):
0912         parts = DictTypes.SortedKeysDict()
0913         for name, item in d.items() if isinstance(d, DictTypes.SortedKeysDict) else sorted(d.items()):
0914             code = ''
0915             dependencies = item.directDependencies()
0916             for module_subfolder, module in dependencies:
0917                 module = module + '_cfi'
0918                 if options.useSubdirectories and module_subfolder:
0919                     module = module_subfolder + '.' + module
0920                 if options.targetDirectory is not None:
0921                     if options.useSubdirectories and subfolder:
0922                       module = '..' + module
0923                     else:
0924                       module = '.' + module
0925                 code += 'from ' + module + ' import *\n'
0926             if dependencies:
0927                 code += '\n'
0928             code += name + ' = ' + item.dumpPython(options)
0929             parts[name] = subfolder, code
0930         return parts
0931 
0932     def _validateSequence(self, sequence, label):
0933         # See if every module has been inserted into the process
0934         try:
0935             l = set()
0936             visitor = NodeNameVisitor(l)
0937             sequence.visit(visitor)
0938         except Exception as e:
0939             raise RuntimeError("An entry in sequence {} has no label\n  Seen entries: {}\n  Error: {}".format(label, l, e))
0940 
0941     def _validateTask(self, task, label):
0942         # See if every module and service has been inserted into the process
0943         try:
0944             l = set()
0945             visitor = NodeNameVisitor(l)
0946             task.visit(visitor)
0947         except:
0948             raise RuntimeError("An entry in task " + label + ' has not been attached to the process')
0949     def _validateConditionalTask(self, task, label):
0950         # See if every module and service has been inserted into the process
0951         try:
0952             l = set()
0953             visitor = NodeNameVisitor(l)
0954             task.visit(visitor)
0955         except:
0956             raise RuntimeError("An entry in task " + label + ' has not been attached to the process')
0957 
0958     def _itemsInDependencyOrder(self, processDictionaryOfItems):
0959         # The items can be Sequences or Tasks and the input
0960         # argument should either be the dictionary of sequences
0961         # or the dictionary of tasks from the process.
0962 
0963         returnValue=DictTypes.SortedKeysDict()
0964 
0965         # For each item, see what other items it depends upon
0966         # For our purpose here, an item depends on the items it contains.
0967         dependencies = {}
0968         for label,item in processDictionaryOfItems.items():
0969             containedItems = []
0970             if isinstance(item, Task):
0971                 v = TaskVisitor(containedItems)
0972             elif isinstance(item, ConditionalTask):
0973                 v = ConditionalTaskVisitor(containedItems)
0974             else:
0975                 v = SequenceVisitor(containedItems)
0976             try:
0977                 item.visit(v)
0978             except RuntimeError:
0979                 if isinstance(item, Task):
0980                     raise RuntimeError("Failed in a Task visitor. Probably " \
0981                                        "a circular dependency discovered in Task with label " + label)
0982                 elif isinstance(item, ConditionalTask):
0983                     raise RuntimeError("Failed in a ConditionalTask visitor. Probably " \
0984                                        "a circular dependency discovered in ConditionalTask with label " + label)
0985                 else:
0986                     raise RuntimeError("Failed in a Sequence visitor. Probably a " \
0987                                        "circular dependency discovered in Sequence with label " + label)
0988             for containedItem in containedItems:
0989                 # Check for items that both have labels and are not in the process.
0990                 # This should not normally occur unless someone explicitly assigns a
0991                 # label without putting the item in the process (which should not ever
0992                 # be done). We check here because this problem could cause the code
0993                 # in the 'while' loop below to go into an infinite loop.
0994                 if containedItem.hasLabel_():
0995                     testItem = processDictionaryOfItems.get(containedItem.label_())
0996                     if testItem is None or containedItem != testItem:
0997                         if isinstance(item, Task):
0998                             raise RuntimeError("Task has a label, but using its label to get an attribute" \
0999                                                " from the process yields a different object or None\n"+
1000                                                "label = " + containedItem.label_())
1001                         if isinstance(item, ConditionalTask):
1002                             raise RuntimeError("ConditionalTask has a label, but using its label to get an attribute" \
1003                                                " from the process yields a different object or None\n"+
1004                                                "label = " + containedItem.label_())
1005                         else:
1006                             raise RuntimeError("Sequence has a label, but using its label to get an attribute" \
1007                                                " from the process yields a different object or None\n"+
1008                                                "label = " + containedItem.label_())
1009             dependencies[label]=[dep.label_() for dep in containedItems if dep.hasLabel_()]
1010 
1011         # keep looping until we get rid of all dependencies
1012         while dependencies:
1013             oldDeps = dict(dependencies)
1014             for label,deps in oldDeps.items():
1015                 if len(deps)==0:
1016                     returnValue[label]=processDictionaryOfItems[label]
1017                     #remove this as a dependency for all other tasks
1018                     del dependencies[label]
1019                     for lb2,deps2 in dependencies.items():
1020                         while deps2.count(label):
1021                             deps2.remove(label)
1022         return returnValue
1023 
1024     def _dumpPython(self, d, options):
1025         result = ''
1026         for name, value in sorted(d.items()):
1027             result += value.dumpPythonAs(name,options)+'\n'
1028         return result
1029 
1030     def _splitPython(self, subfolder, d, options):
1031         result = {}
1032         for name, value in sorted(d.items()):
1033             result[name] = subfolder, value.dumpPythonAs(name, options) + '\n'
1034         return result
1035 
1036     def dumpPython(self, options=PrintOptions()):
1037         """return a string containing the equivalent process defined using python"""
1038         specialImportRegistry._reset()
1039         header = "import FWCore.ParameterSet.Config as cms"
1040         result = "process = cms.Process(\""+self.__name+"\")\n\n"
1041         if self.source_():
1042             result += "process.source = "+self.source_().dumpPython(options)
1043         if self.looper_():
1044             result += "process.looper = "+self.looper_().dumpPython()
1045         result+=self._dumpPythonList(self.psets, options)
1046         result+=self._dumpPythonList(self.vpsets, options)
1047         result+=self._dumpPythonSubProcesses(self.subProcesses_(), options)
1048         result+=self._dumpPythonList(self.producers_(), options)
1049         result+=self._dumpPythonList(self.switchProducers_(), options)
1050         result+=self._dumpPythonList(self.filters_() , options)
1051         result+=self._dumpPythonList(self.analyzers_(), options)
1052         result+=self._dumpPythonList(self.outputModules_(), options)
1053         result+=self._dumpPythonList(self.services_(), options)
1054         result+=self._dumpPythonList(self.processAccelerators_(), options)
1055         result+=self._dumpPythonList(self.es_producers_(), options)
1056         result+=self._dumpPythonList(self.es_sources_(), options)
1057         result+=self._dumpPython(self.es_prefers_(), options)
1058         result+=self._dumpPythonList(self._itemsInDependencyOrder(self.tasks), options)
1059         result+=self._dumpPythonList(self._itemsInDependencyOrder(self.conditionaltasks), options)
1060         result+=self._dumpPythonList(self._itemsInDependencyOrder(self.sequences), options)
1061         result+=self._dumpPythonList(self.paths_(), options)
1062         result+=self._dumpPythonList(self.endpaths_(), options)
1063         result+=self._dumpPythonList(self.finalpaths_(), options)
1064         result+=self._dumpPythonList(self.aliases_(), options)
1065         if not self.schedule_() == None:
1066             result += 'process.schedule = ' + self.schedule.dumpPython(options)
1067         imports = specialImportRegistry.getSpecialImports()
1068         if len(imports) > 0:
1069             header += "\n" + "\n".join(imports)
1070         header += "\n\n"
1071         return header+result
1072 
1073     def splitPython(self, options = PrintOptions()):
1074         """return a map of file names to python configuration fragments"""
1075         specialImportRegistry._reset()
1076         # extract individual fragments
1077         options.isCfg = False
1078         header = "import FWCore.ParameterSet.Config as cms"
1079         result = ''
1080         parts = {}
1081         files = {}
1082 
1083         result = 'process = cms.Process("' + self.__name + '")\n\n'
1084 
1085         if self.source_():
1086             parts['source'] = (None, 'source = ' + self.source_().dumpPython(options))
1087 
1088         if self.looper_():
1089             parts['looper'] = (None, 'looper = ' + self.looper_().dumpPython())
1090 
1091         parts.update(self._splitPythonList('psets', self.psets, options))
1092         parts.update(self._splitPythonList('psets', self.vpsets, options))
1093         # FIXME
1094         #parts.update(self._splitPythonSubProcesses(self.subProcesses_(), options))
1095         if len(self.subProcesses_()):
1096           sys.stderr.write("error: subprocesses are not supported yet\n\n")
1097         parts.update(self._splitPythonList('modules', self.producers_(), options))
1098         parts.update(self._splitPythonList('modules', self.switchProducers_(), options))
1099         parts.update(self._splitPythonList('modules', self.filters_() , options))
1100         parts.update(self._splitPythonList('modules', self.analyzers_(), options))
1101         parts.update(self._splitPythonList('modules', self.outputModules_(), options))
1102         parts.update(self._splitPythonList('services', self.services_(), options))
1103         parts.update(self._splitPythonList('eventsetup', self.es_producers_(), options))
1104         parts.update(self._splitPythonList('eventsetup', self.es_sources_(), options))
1105         parts.update(self._splitPython('eventsetup', self.es_prefers_(), options))
1106         parts.update(self._splitPythonList('tasks', self._itemsInDependencyOrder(self.tasks), options))
1107         parts.update(self._splitPythonList('sequences', self._itemsInDependencyOrder(self.sequences), options))
1108         parts.update(self._splitPythonList('paths', self.paths_(), options))
1109         parts.update(self._splitPythonList('paths', self.endpaths_(), options))
1110         parts.update(self._splitPythonList('paths', self.finalpaths_(), options))
1111         parts.update(self._splitPythonList('modules', self.aliases_(), options))
1112 
1113         if options.targetDirectory is not None:
1114             files[options.targetDirectory + '/__init__.py'] = ''
1115 
1116         if options.useSubdirectories:
1117           for sub in 'psets', 'modules', 'services', 'eventsetup', 'tasks', 'sequences', 'paths':
1118             if options.targetDirectory is not None:
1119                 sub = options.targetDirectory + '/' + sub
1120             files[sub + '/__init__.py'] = ''
1121 
1122         # case insensitive sort by subfolder and module name
1123         parts = sorted(parts.items(), key = lambda nsc: (nsc[1][0].lower() if nsc[1][0] else '', nsc[0].lower()))
1124 
1125         for (name, (subfolder, code)) in parts:
1126             filename = name + '_cfi'
1127             if options.useSubdirectories and subfolder:
1128                 filename = subfolder + '/' + filename
1129             if options.targetDirectory is not None:
1130                 filename = options.targetDirectory + '/' + filename
1131             result += 'process.load("%s")\n' % filename
1132             files[filename + '.py'] = header + '\n\n' + code
1133 
1134         if self.schedule_() is not None:
1135             options.isCfg = True
1136             result += '\nprocess.schedule = ' + self.schedule.dumpPython(options)
1137 
1138         imports = specialImportRegistry.getSpecialImports()
1139         if len(imports) > 0:
1140             header += '\n' + '\n'.join(imports)
1141         files['-'] = header + '\n\n' + result
1142         return files
1143 
1144     def _replaceInSequences(self, label, new):
1145         old = getattr(self,label)
1146         #TODO - replace by iterator concatenation
1147         #to ovoid dependency problems between sequences, first modify
1148         # process known sequences to do a non-recursive change. Then do
1149         # a recursive change to get cases where a sub-sequence unknown to
1150         # the process has the item to be replaced
1151         for sequenceable in self.sequences.values():
1152             sequenceable._replaceIfHeldDirectly(old,new)
1153         for sequenceable in self.sequences.values():
1154             sequenceable.replace(old,new)
1155         for sequenceable in self.paths.values():
1156             sequenceable.replace(old,new)
1157         for sequenceable in self.endpaths.values():
1158             sequenceable.replace(old,new)
1159         for sequenceable in self.finalpaths.values():
1160             sequenceable.replace(old,new)
1161     def _replaceInTasks(self, label, new):
1162         old = getattr(self,label)
1163         for task in self.tasks.values():
1164             task.replace(old, new)
1165     def _replaceInConditionalTasks(self, label, new):
1166         old = getattr(self,label)
1167         for task in self.conditionaltasks.values():
1168             task.replace(old, new)
1169     def _replaceInSchedule(self, label, new):
1170         if self.schedule_() == None:
1171             return
1172         old = getattr(self,label)
1173         for task in self.schedule_()._tasks:
1174             task.replace(old, new)
1175     def _replaceInScheduleDirectly(self, label, new):
1176         if self.schedule_() == None:
1177             return
1178         old = getattr(self,label)
1179         self.schedule_()._replaceIfHeldDirectly(old, new)
1180     def globalReplace(self,label,new):
1181         """ Replace the item with label 'label' by object 'new' in the process and all sequences/paths/tasks"""
1182         if not hasattr(self,label):
1183             raise LookupError("process has no item of label "+label)
1184         setattr(self,label,new)
1185     def _insertInto(self, parameterSet, itemDict):
1186         for name,value in itemDict.items():
1187             value.insertInto(parameterSet, name)
1188     def _insertOneInto(self, parameterSet, label, item, tracked):
1189         vitems = []
1190         if not item == None:
1191             newlabel = item.nameInProcessDesc_(label)
1192             vitems = [newlabel]
1193             item.insertInto(parameterSet, newlabel)
1194         parameterSet.addVString(tracked, label, vitems)
1195     def _insertManyInto(self, parameterSet, label, itemDict, tracked):
1196         l = []
1197         for name,value in itemDict.items():
1198             value.appendToProcessDescList_(l, name)
1199             value.insertInto(parameterSet, name)
1200         # alphabetical order is easier to compare with old language
1201         l.sort()
1202         parameterSet.addVString(tracked, label, l)
1203     def _insertSwitchProducersInto(self, parameterSet, labelModules, labelAliases, itemDict, tracked):
1204         modules = parameterSet.getVString(tracked, labelModules)
1205         aliases = parameterSet.getVString(tracked, labelAliases)
1206         accelerators = parameterSet.getVString(False, "@selected_accelerators")
1207         for name,value in itemDict.items():
1208             value.appendToProcessDescLists_(modules, aliases, name)
1209             value.insertInto(parameterSet, name, accelerators)
1210         modules.sort()
1211         aliases.sort()
1212         parameterSet.addVString(tracked, labelModules, modules)
1213         parameterSet.addVString(tracked, labelAliases, aliases)
1214     def _insertSubProcessesInto(self, parameterSet, label, itemList, tracked):
1215         l = []
1216         subprocs = []
1217         for value in itemList:
1218             name = value.getProcessName()
1219             newLabel = value.nameInProcessDesc_(name)
1220             l.append(newLabel)
1221             pset = value.getSubProcessPSet(parameterSet)
1222             subprocs.append(pset)
1223         # alphabetical order is easier to compare with old language
1224         l.sort()
1225         parameterSet.addVString(tracked, label, l)
1226         parameterSet.addVPSet(False,"subProcesses",subprocs)
1227     def _insertPaths(self, processPSet, nodeVisitor):
1228         scheduledPaths = []
1229         triggerPaths = []
1230         endpaths = []
1231         finalpaths = []
1232         if self.schedule_() == None:
1233             # make one from triggerpaths & endpaths
1234             for name in self.paths_():
1235                 scheduledPaths.append(name)
1236                 triggerPaths.append(name)
1237             for name in self.endpaths_():
1238                 scheduledPaths.append(name)
1239                 endpaths.append(name)
1240             for name in self.finalpaths_():
1241                 finalpaths.append(name)
1242         else:
1243             for path in self.schedule_():
1244                 pathname = path.label_()
1245                 if pathname in self.endpaths_():
1246                     endpaths.append(pathname)
1247                     scheduledPaths.append(pathname)
1248                 elif pathname in self.finalpaths_():
1249                     finalpaths.append(pathname)
1250                 else:
1251                     scheduledPaths.append(pathname)
1252                     triggerPaths.append(pathname)
1253             for task in self.schedule_()._tasks:
1254                 task.resolve(self.__dict__)
1255                 scheduleTaskValidator = ScheduleTaskValidator()
1256                 task.visit(scheduleTaskValidator)
1257                 task.visit(nodeVisitor)
1258         # consolidate all final_paths into one EndPath
1259         endPathWithFinalPathModulesName ="@finalPath"
1260         finalPathEndPath = EndPath()
1261         if finalpaths:
1262           endpaths.append(endPathWithFinalPathModulesName)
1263           scheduledPaths.append(endPathWithFinalPathModulesName)
1264           finalpathValidator = FinalPathValidator()
1265           modulesOnFinalPath = []
1266           for finalpathname in finalpaths:
1267               iFinalPath = self.finalpaths_()[finalpathname]
1268               iFinalPath.resolve(self.__dict__)
1269               finalpathValidator.setLabel(finalpathname)
1270               iFinalPath.visit(finalpathValidator)
1271               if finalpathValidator.filtersOnFinalpaths or finalpathValidator.producersOnFinalpaths:
1272                   names = [p.label_ for p in finalpathValidator.filtersOnFinalpaths]
1273                   names.extend( [p.label_ for p in finalpathValidator.producersOnFinalpaths])
1274                   raise RuntimeError("FinalPath %s has non OutputModules %s" % (finalpathname, ",".join(names)))
1275               modulesOnFinalPath.extend(iFinalPath.moduleNames())
1276           for m in modulesOnFinalPath:
1277             mod = getattr(self, m)
1278             setattr(mod, "@onFinalPath", untracked.bool(True))
1279             finalPathEndPath += mod
1280             
1281         processPSet.addVString(True, "@end_paths", endpaths)
1282         processPSet.addVString(True, "@paths", scheduledPaths)
1283         # trigger_paths are a little different
1284         p = processPSet.newPSet()
1285         p.addVString(True, "@trigger_paths", triggerPaths)
1286         processPSet.addPSet(True, "@trigger_paths", p)
1287         # add all these paths
1288         pathValidator = PathValidator()
1289         endpathValidator = EndPathValidator()
1290         decoratedList = []
1291         lister = DecoratedNodeNameVisitor(decoratedList)
1292         condTaskModules = []
1293         condTaskVistor = ModuleNodeOnConditionalTaskVisitor(condTaskModules)
1294         pathCompositeVisitor = CompositeVisitor(pathValidator, nodeVisitor, lister, condTaskVistor)
1295         endpathCompositeVisitor = CompositeVisitor(endpathValidator, nodeVisitor, lister)
1296         for triggername in triggerPaths:
1297             iPath = self.paths_()[triggername]
1298             iPath.resolve(self.__dict__)
1299             pathValidator.setLabel(triggername)
1300             lister.initialize()
1301             condTaskModules[:] = []
1302             iPath.visit(pathCompositeVisitor)
1303             if condTaskModules:
1304               decoratedList.append("#")
1305               l = list({x.label_() for x in condTaskModules})
1306               l.sort()
1307               decoratedList.extend(l)
1308               decoratedList.append("@")
1309             iPath.insertInto(processPSet, triggername, decoratedList[:])
1310         for endpathname in endpaths:
1311             if endpathname is not endPathWithFinalPathModulesName:
1312               iEndPath = self.endpaths_()[endpathname]
1313             else:
1314               iEndPath = finalPathEndPath
1315             iEndPath.resolve(self.__dict__)
1316             endpathValidator.setLabel(endpathname)
1317             lister.initialize()
1318             iEndPath.visit(endpathCompositeVisitor)
1319             iEndPath.insertInto(processPSet, endpathname, decoratedList[:])
1320         processPSet.addVString(False, "@filters_on_endpaths", endpathValidator.filtersOnEndpaths)
1321           
1322 
1323     def resolve(self,keepUnresolvedSequencePlaceholders=False):
1324         for x in self.paths.values():
1325             x.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1326         for x in self.endpaths.values():
1327             x.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1328         for x in self.finalpaths.values():
1329             x.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1330         if not self.schedule_() == None:
1331             for task in self.schedule_()._tasks:
1332                 task.resolve(self.__dict__,keepUnresolvedSequencePlaceholders)
1333 
1334     def prune(self,verbose=False,keepUnresolvedSequencePlaceholders=False):
1335         """ Remove clutter from the process that we think is unnecessary:
1336         tracked PSets, VPSets and unused modules and sequences. If a Schedule has been set, then Paths and EndPaths
1337         not in the schedule will also be removed, along with an modules and sequences used only by
1338         those removed Paths and EndPaths. The keepUnresolvedSequencePlaceholders keeps also unresolved TaskPlaceholders."""
1339 # need to update this to only prune psets not on refToPSets
1340 # but for now, remove the delattr
1341 #        for name in self.psets_():
1342 #            if getattr(self,name).isTracked():
1343 #                delattr(self, name)
1344         for name in self.vpsets_():
1345             delattr(self, name)
1346         #first we need to resolve any SequencePlaceholders being used
1347         self.resolve(keepUnresolvedSequencePlaceholders)
1348         usedModules = set()
1349         unneededPaths = set()
1350         tasks = list()
1351         tv = TaskVisitor(tasks)
1352         if self.schedule_():
1353             usedModules=set(self.schedule_().moduleNames())
1354             #get rid of unused paths
1355             schedNames = set(( x.label_() for x in self.schedule_()))
1356             names = set(self.paths)
1357             names.update(set(self.endpaths))
1358             names.update(set(self.finalpaths))
1359             unneededPaths = names - schedNames
1360             for n in unneededPaths:
1361                 delattr(self,n)
1362             for t in self.schedule_().tasks():
1363                 tv.enter(t)
1364                 t.visit(tv)
1365                 tv.leave(t)
1366         else:
1367             pths = list(self.paths.values())
1368             pths.extend(self.endpaths.values())
1369             pths.extend(self.finalpaths.values())
1370             temp = Schedule(*pths)
1371             usedModules=set(temp.moduleNames())
1372         unneededModules = self._pruneModules(self.producers_(), usedModules)
1373         unneededModules.update(self._pruneModules(self.switchProducers_(), usedModules))
1374         unneededModules.update(self._pruneModules(self.filters_(), usedModules))
1375         unneededModules.update(self._pruneModules(self.analyzers_(), usedModules))
1376         #remove sequences and tasks that do not appear in remaining paths and endpaths
1377         seqs = list()
1378         sv = SequenceVisitor(seqs)
1379         for p in self.paths.values():
1380             p.visit(sv)
1381             p.visit(tv)
1382         for p in self.endpaths.values():
1383             p.visit(sv)
1384             p.visit(tv)
1385         for p in self.finalpaths.values():
1386             p.visit(sv)
1387             p.visit(tv)
1388         def removeUnneeded(seqOrTasks, allSequencesOrTasks):
1389             _keepSet = set(( s for s in seqOrTasks if s.hasLabel_()))
1390             _availableSet = set(allSequencesOrTasks.values())
1391             _unneededSet = _availableSet-_keepSet
1392             _unneededLabels = []
1393             for s in _unneededSet:
1394                 _unneededLabels.append(s.label_())
1395                 delattr(self,s.label_())
1396             return _unneededLabels
1397         unneededSeqLabels = removeUnneeded(seqs, self.sequences)
1398         unneededTaskLabels = removeUnneeded(tasks, self.tasks)
1399         if verbose:
1400             print("prune removed the following:")
1401             print("  modules:"+",".join(unneededModules))
1402             print("  tasks:"+",".join(unneededTaskLabels))
1403             print("  sequences:"+",".join(unneededSeqLabels))
1404             print("  paths/endpaths/finalpaths:"+",".join(unneededPaths))
1405     def _pruneModules(self, d, scheduledNames):
1406         moduleNames = set(d.keys())
1407         junk = moduleNames - scheduledNames
1408         for name in junk:
1409             delattr(self, name)
1410         return junk
1411 
1412     def fillProcessDesc(self, processPSet):
1413         """Used by the framework to convert python to C++ objects"""
1414         class ServiceInjectorAdaptor(object):
1415             def __init__(self,ppset,thelist):
1416                 self.__thelist = thelist
1417                 self.__processPSet = ppset
1418             def addService(self,pset):
1419                 self.__thelist.append(pset)
1420             def newPSet(self):
1421                 return self.__processPSet.newPSet()
1422         #This adaptor is used to 'add' the method 'getTopPSet_'
1423         # to the ProcessDesc and PythonParameterSet C++ classes.
1424         # This method is needed for the PSet refToPSet_ functionality.
1425         class TopLevelPSetAcessorAdaptor(object):
1426             def __init__(self,ppset,process):
1427                 self.__ppset = ppset
1428                 self.__process = process
1429             def __getattr__(self,attr):
1430                 return getattr(self.__ppset,attr)
1431             def getTopPSet_(self,label):
1432                 return getattr(self.__process,label)
1433             def newPSet(self):
1434                 return TopLevelPSetAcessorAdaptor(self.__ppset.newPSet(),self.__process)
1435             def addPSet(self,tracked,name,ppset):
1436                 return self.__ppset.addPSet(tracked,name,self.__extractPSet(ppset))
1437             def addVPSet(self,tracked,name,vpset):
1438                 return self.__ppset.addVPSet(tracked,name,[self.__extractPSet(x) for x in vpset])
1439             def __extractPSet(self,pset):
1440                 if isinstance(pset,TopLevelPSetAcessorAdaptor):
1441                     return pset.__ppset
1442                 return pset
1443 
1444         self.validate()
1445         processPSet.addString(True, "@process_name", self.name_())
1446         self.handleProcessAccelerators(processPSet)
1447         all_modules = self.producers_().copy()
1448         all_modules.update(self.filters_())
1449         all_modules.update(self.analyzers_())
1450         all_modules.update(self.outputModules_())
1451         adaptor = TopLevelPSetAcessorAdaptor(processPSet,self)
1452         self._insertInto(adaptor, self.psets_())
1453         self._insertInto(adaptor, self.vpsets_())
1454         self._insertOneInto(adaptor,  "@all_sources", self.source_(), True)
1455         self._insertOneInto(adaptor,  "@all_loopers", self.looper_(), True)
1456         self._insertSubProcessesInto(adaptor, "@all_subprocesses", self.subProcesses_(), False)
1457         self._insertManyInto(adaptor, "@all_esprefers", self.es_prefers_(), True)
1458         self._insertManyInto(adaptor, "@all_aliases", self.aliases_(), True)
1459         # This will visit all the paths and endpaths that are scheduled to run,
1460         # as well as the Tasks associated to them and the schedule. It remembers
1461         # the modules, ESSources, ESProducers, and services it visits.
1462         nodeVisitor = NodeVisitor()
1463         self._insertPaths(adaptor, nodeVisitor)
1464         all_modules_onTasksOrScheduled = { key:value for key, value in all_modules.items() if value in nodeVisitor.modules }
1465         self._insertManyInto(adaptor, "@all_modules", all_modules_onTasksOrScheduled, True)
1466         all_switches = self.switchProducers_().copy()
1467         all_switches_onTasksOrScheduled = {key:value for key, value in all_switches.items() if value in nodeVisitor.modules }
1468         self._insertSwitchProducersInto(adaptor, "@all_modules", "@all_aliases", all_switches_onTasksOrScheduled, True)
1469         # Same as nodeVisitor except this one visits all the Tasks attached
1470         # to the process.
1471         processNodeVisitor = NodeVisitor()
1472         for pTask in self.tasks.values():
1473             pTask.visit(processNodeVisitor)
1474         esProducersToEnable = {}
1475         for esProducerName, esProducer in self.es_producers_().items():
1476             if esProducer in nodeVisitor.esProducers or not (esProducer in processNodeVisitor.esProducers):
1477                 esProducersToEnable[esProducerName] = esProducer
1478         self._insertManyInto(adaptor, "@all_esmodules", esProducersToEnable, True)
1479         esSourcesToEnable = {}
1480         for esSourceName, esSource in self.es_sources_().items():
1481             if esSource in nodeVisitor.esSources or not (esSource in processNodeVisitor.esSources):
1482                 esSourcesToEnable[esSourceName] = esSource
1483         self._insertManyInto(adaptor, "@all_essources", esSourcesToEnable, True)
1484         #handle services differently
1485         services = []
1486         for serviceName, serviceObject in self.services_().items():
1487             if serviceObject in nodeVisitor.services or not (serviceObject in processNodeVisitor.services):
1488                 serviceObject.insertInto(ServiceInjectorAdaptor(adaptor,services))
1489         adaptor.addVPSet(False,"services",services)
1490         return processPSet
1491 
1492     def validate(self):
1493         # check if there's some input
1494         # Breaks too many unit tests for now
1495         #if self.source_() == None and self.looper_() == None:
1496         #    raise RuntimeError("No input source was found for this process")
1497         pass
1498 
1499     def handleProcessAccelerators(self, parameterSet):
1500         # 'cpu' accelerator is always implicitly there
1501         allAccelerators = set(["cpu"])
1502         availableAccelerators = set(["cpu"])
1503         for acc in self.__dict__['_Process__accelerators'].values():
1504             allAccelerators.update(acc.labels())
1505             availableAccelerators.update(acc.enabledLabels())
1506         availableAccelerators = sorted(list(availableAccelerators))
1507         parameterSet.addVString(False, "@available_accelerators", availableAccelerators)
1508 
1509         # Resolve wildcards
1510         selectedAccelerators = []
1511         if "*" in self.options.accelerators:
1512             if len(self.options.accelerators) >= 2:
1513                 raise ValueError("process.options.accelerators may contain '*' only as the only element, now it has {} elements".format(len(self.options.accelerators)))
1514             selectedAccelerators = availableAccelerators
1515         else:
1516             import fnmatch
1517             resolved = set()
1518             invalid = []
1519             for pattern in self.options.accelerators:
1520                 acc = [a for a in availableAccelerators if fnmatch.fnmatchcase(a, pattern)]
1521                 if len(acc) == 0:
1522                     if not any(fnmatch.fnmatchcase(a, pattern) for a in allAccelerators):
1523                         invalid.append(pattern)
1524                 else:
1525                    resolved.update(acc)
1526             # Sanity check
1527             if len(invalid) != 0:
1528                 raise ValueError("Invalid pattern{} of {} in process.options.accelerators, valid values are {} or a pattern matching to some of them.".format(
1529                     "s" if len(invalid) > 2 else "",
1530                     ",".join(invalid),
1531                     ",".join(sorted(list(allAccelerators)))))
1532             selectedAccelerators = sorted(list(resolved))
1533         parameterSet.addVString(False, "@selected_accelerators", selectedAccelerators)
1534 
1535         # Customize
1536         wrapped = ProcessForProcessAccelerator(self)
1537         for acc in self.__dict__['_Process__accelerators'].values():
1538             acc.apply(wrapped, selectedAccelerators)
1539 
1540     def prefer(self, esmodule,*args,**kargs):
1541         """Prefer this ES source or producer.  The argument can
1542            either be an object label, e.g.,
1543              process.prefer(process.juicerProducer) (not supported yet)
1544            or a name of an ESSource or ESProducer
1545              process.prefer("juicer")
1546            or a type of unnamed ESSource or ESProducer
1547              process.prefer("JuicerProducer")
1548            In addition, you can pass as a labelled arguments the name of the Record you wish to
1549            prefer where the type passed is a cms.vstring and that vstring can contain the
1550            name of the C++ types in the Record that are being preferred, e.g.,
1551               #prefer all data in record 'OrangeRecord' from 'juicer'
1552               process.prefer("juicer", OrangeRecord=cms.vstring())
1553            or
1554               #prefer only "Orange" data in "OrangeRecord" from "juicer"
1555               process.prefer("juicer", OrangeRecord=cms.vstring("Orange"))
1556            or
1557               #prefer only "Orange" data with label "ExtraPulp" in "OrangeRecord" from "juicer"
1558               ESPrefer("ESJuicerProd", OrangeRecord=cms.vstring("Orange/ExtraPulp"))
1559         """
1560         # see if this refers to a named ESProducer
1561         if isinstance(esmodule, ESSource) or isinstance(esmodule, ESProducer):
1562             raise RuntimeError("Syntax of process.prefer(process.esmodule) not supported yet")
1563         elif self._findPreferred(esmodule, self.es_producers_(),*args,**kargs) or \
1564                 self._findPreferred(esmodule, self.es_sources_(),*args,**kargs):
1565             pass
1566         else:
1567             raise RuntimeError("Cannot resolve prefer for "+repr(esmodule))
1568 
1569     def _findPreferred(self, esname, d,*args,**kargs):
1570         # is esname a name in the dictionary?
1571         if esname in d:
1572             typ = d[esname].type_()
1573             if typ == esname:
1574                 self.__setattr__( esname+"_prefer", ESPrefer(typ,*args,**kargs) )
1575             else:
1576                 self.__setattr__( esname+"_prefer", ESPrefer(typ, esname,*args,**kargs) )
1577             return True
1578         else:
1579             # maybe it's an unnamed ESModule?
1580             found = False
1581             for name, value in d.items():
1582                 if value.type_() == esname:
1583                     if found:
1584                         raise RuntimeError("More than one ES module for "+esname)
1585                     found = True
1586                     self.__setattr__(esname+"_prefer",  ESPrefer(d[esname].type_()) )
1587             return found
1588 
1589 
1590 class ProcessFragment(object):
1591     def __init__(self, process):
1592         if isinstance(process, Process):
1593             self.__process = process
1594         elif isinstance(process, str):
1595             self.__process = Process(process)
1596             #make sure we do not override the defaults
1597             del self.__process.options
1598             del self.__process.maxEvents
1599             del self.__process.maxLuminosityBlocks
1600         else:
1601             raise TypeError('a ProcessFragment can only be constructed from an existig Process or from process name')
1602     def __dir__(self):
1603         return [ x for x in dir(self.__process) if isinstance(getattr(self.__process, x), _ConfigureComponent) ]
1604     def __getattribute__(self, name):
1605         if name == '_ProcessFragment__process':
1606             return object.__getattribute__(self, '_ProcessFragment__process')
1607         else:
1608             return getattr(self.__process, name)
1609     def __setattr__(self, name, value):
1610         if name == '_ProcessFragment__process':
1611             object.__setattr__(self, name, value)
1612         else:
1613             setattr(self.__process, name, value)
1614     def __delattr__(self, name):
1615         if name == '_ProcessFragment__process':
1616             pass
1617         else:
1618             return delattr(self.__process, name)
1619 
1620 
1621 class FilteredStream(dict):
1622     """a dictionary with fixed keys"""
1623     def _blocked_attribute(obj):
1624         raise AttributeError("An FilteredStream defintion cannot be modified after creation.")
1625     _blocked_attribute = property(_blocked_attribute)
1626     __setattr__ = __delitem__ = __setitem__ = clear = _blocked_attribute
1627     pop = popitem = setdefault = update = _blocked_attribute
1628     def __new__(cls, *args, **kw):
1629         new = dict.__new__(cls)
1630         dict.__init__(new, *args, **kw)
1631         keys = sorted(kw.keys())
1632         if keys != ['content', 'dataTier', 'name', 'paths', 'responsible', 'selectEvents']:
1633             raise ValueError("The needed parameters are: content, dataTier, name, paths, responsible, selectEvents")
1634         if not isinstance(kw['name'],str):
1635             raise ValueError("name must be of type string")
1636         if not isinstance(kw['content'], vstring) and not isinstance(kw['content'],str):
1637             raise ValueError("content must be of type vstring or string")
1638         if not isinstance(kw['dataTier'], string):
1639             raise ValueError("dataTier must be of type string")
1640         if not isinstance(kw['selectEvents'], PSet):
1641             raise ValueError("selectEvents must be of type PSet")
1642         if not isinstance(kw['paths'],(tuple, Path)):
1643             raise ValueError("'paths' must be a tuple of paths")
1644         return new
1645     def __init__(self, *args, **kw):
1646         pass
1647     def __repr__(self):
1648         return "FilteredStream object: %s" %self["name"]
1649     def __getattr__(self,attr):
1650         return self[attr]
1651 
1652 class SubProcess(_Unlabelable):
1653     """Allows embedding another process within a parent process. This allows one to 
1654     chain processes together directly in one cmsRun job rather than having to run
1655     separate jobs that are connected via a temporary file.
1656     """
1657     def __init__(self,process, SelectEvents = untracked.PSet(), outputCommands = untracked.vstring()):
1658         """
1659         """
1660         if not isinstance(process, Process):
1661             raise ValueError("the 'process' argument must be of type cms.Process")
1662         if not isinstance(SelectEvents,PSet):
1663             raise ValueError("the 'SelectEvents' argument must be of type cms.untracked.PSet")
1664         if not isinstance(outputCommands,vstring):
1665             raise ValueError("the 'outputCommands' argument must be of type cms.untracked.vstring")
1666         self.__process = process
1667         self.__SelectEvents = SelectEvents
1668         self.__outputCommands = outputCommands
1669         # Need to remove MessageLogger from the subprocess now that MessageLogger is always present
1670         if self.__process.MessageLogger is not MessageLogger:
1671             print("""Warning: You have reconfigured service
1672 'edm::MessageLogger' in a subprocess.
1673 This service has already been configured.
1674 This particular service may not be reconfigured in a subprocess.
1675 The reconfiguration will be ignored.""")
1676         del self.__process.MessageLogger
1677     def dumpPython(self, options=PrintOptions()):
1678         out = "parentProcess"+str(hash(self))+" = process\n"
1679         out += self.__process.dumpPython()
1680         out += "childProcess = process\n"
1681         out += "process = parentProcess"+str(hash(self))+"\n"
1682         out += "process.addSubProcess(cms.SubProcess(process = childProcess, SelectEvents = "+self.__SelectEvents.dumpPython(options) +", outputCommands = "+self.__outputCommands.dumpPython(options) +"))"
1683         return out
1684     def getProcessName(self):
1685         return self.__process.name_()
1686     def process(self):
1687         return self.__process
1688     def SelectEvents(self):
1689         return self.__SelectEvents
1690     def outputCommands(self):
1691         return self.__outputCommands
1692     def type_(self):
1693         return 'subProcess'
1694     def nameInProcessDesc_(self,label):
1695         return label
1696     def _place(self,label,process):
1697         process._placeSubProcess('subProcess',self)
1698     def getSubProcessPSet(self,parameterSet):
1699         topPSet = parameterSet.newPSet()
1700         self.__process.fillProcessDesc(topPSet)
1701         subProcessPSet = parameterSet.newPSet()
1702         self.__SelectEvents.insertInto(subProcessPSet,"SelectEvents")
1703         self.__outputCommands.insertInto(subProcessPSet,"outputCommands")
1704         subProcessPSet.addPSet(False,"process",topPSet)
1705         return subProcessPSet
1706 
1707 class _ParameterModifier(object):
1708     """Helper class for Modifier that takes key/value pairs and uses them to reset parameters of the object"""
1709     def __init__(self,args):
1710         self.__args = args
1711     def __call__(self,obj):
1712         params = {}
1713         for k in self.__args.keys():
1714             if hasattr(obj,k):
1715                 params[k] = getattr(obj,k)
1716         _modifyParametersFromDict(params, self.__args, self._raiseUnknownKey)
1717         for k in self.__args.keys():
1718             if k in params:
1719                 setattr(obj,k,params[k])
1720             else:
1721                 #the parameter must have been removed
1722                 delattr(obj,k)
1723     @staticmethod
1724     def _raiseUnknownKey(key):
1725         raise KeyError("Unknown parameter name "+key+" specified while calling Modifier")
1726 
1727 class _BoolModifierBase(object):
1728     """A helper base class for _AndModifier, _InvertModifier, and _OrModifier to contain the common code"""
1729     def __init__(self, lhs, rhs=None):
1730         self._lhs = lhs
1731         if rhs is not None:
1732             self._rhs = rhs
1733     def toModify(self,obj, func=None,**kw):
1734         Modifier._toModifyCheck(obj,func,**kw)
1735         if not self._isChosen():
1736             return
1737         Modifier._toModify(obj,func,**kw)
1738     def toReplaceWith(self,toObj,fromObj):
1739         Modifier._toReplaceWithCheck(toObj,fromObj)
1740         if not self._isChosen():
1741             return
1742         Modifier._toReplaceWith(toObj,fromObj)
1743     def makeProcessModifier(self,func):
1744         """This is used to create a ProcessModifer that can perform actions on the process as a whole.
1745             This takes as argument a callable object (e.g. function) that takes as its sole argument an instance of Process.
1746             In order to work, the value returned from this function must be assigned to a uniquely named variable."""
1747         return ProcessModifier(self,func)
1748     def __and__(self, other):
1749         return _AndModifier(self,other)
1750     def __invert__(self):
1751         return _InvertModifier(self)
1752     def __or__(self, other):
1753         return _OrModifier(self,other)
1754 
1755 class _AndModifier(_BoolModifierBase):
1756     """A modifier which only applies if multiple Modifiers are chosen"""
1757     def __init__(self, lhs, rhs):
1758         super(_AndModifier,self).__init__(lhs, rhs)
1759     def _isChosen(self):
1760         return self._lhs._isChosen() and self._rhs._isChosen()
1761 
1762 class _InvertModifier(_BoolModifierBase):
1763     """A modifier which only applies if a Modifier is not chosen"""
1764     def __init__(self, lhs):
1765         super(_InvertModifier,self).__init__(lhs)
1766     def _isChosen(self):
1767         return not self._lhs._isChosen()
1768 
1769 class _OrModifier(_BoolModifierBase):
1770     """A modifier which only applies if at least one of multiple Modifiers is chosen"""
1771     def __init__(self, lhs, rhs):
1772         super(_OrModifier,self).__init__(lhs, rhs)
1773     def _isChosen(self):
1774         return self._lhs._isChosen() or self._rhs._isChosen()
1775 
1776 
1777 class Modifier(object):
1778     """This class is used to define standard modifications to a Process.
1779     An instance of this class is declared to denote a specific modification,e.g. era2017 could
1780     reconfigure items in a process to match our expectation of running in 2017. Once declared,
1781     these Modifier instances are imported into a configuration and items that need to be modified
1782     are then associated with the Modifier and with the action to do the modification.
1783     The registered modifications will only occur if the Modifier was passed to 
1784     the cms.Process' constructor.
1785     """
1786     def __init__(self):
1787         self.__processModifiers = []
1788         self.__chosen = False
1789     def makeProcessModifier(self,func):
1790         """This is used to create a ProcessModifer that can perform actions on the process as a whole.
1791            This takes as argument a callable object (e.g. function) that takes as its sole argument an instance of Process.
1792            In order to work, the value returned from this function must be assigned to a uniquely named variable.
1793         """
1794         return ProcessModifier(self,func)
1795     @staticmethod
1796     def _toModifyCheck(obj,func,**kw):
1797         if func is not None and len(kw) != 0:
1798             raise TypeError("toModify takes either two arguments or one argument and key/value pairs")
1799     def toModify(self,obj, func=None,**kw):
1800         """This is used to register an action to be performed on the specific object. Two different forms are allowed
1801         Form 1: A callable object (e.g. function) can be passed as the second. This callable object is expected to take one argument
1802         that will be the object passed in as the first argument.
1803         Form 2: A list of parameter name, value pairs can be passed
1804            mod.toModify(foo, fred=cms.int32(7), barney = cms.double(3.14))
1805         This form can also be used to remove a parameter by passing the value of None
1806             #remove the parameter foo.fred       
1807             mod.toModify(foo, fred = None)
1808         Additionally, parameters embedded within PSets can also be modified using a dictionary
1809             #change foo.fred.pebbles to 3 and foo.fred.friend to "barney"
1810             mod.toModify(foo, fred = dict(pebbles = 3, friend = "barney)) )
1811         """
1812         Modifier._toModifyCheck(obj,func,**kw)
1813         if not self._isChosen():
1814             return
1815         Modifier._toModify(obj,func,**kw)
1816     @staticmethod
1817     def _toModify(obj,func,**kw):
1818         if func is not None:
1819             func(obj)
1820         else:
1821             temp =_ParameterModifier(kw)
1822             temp(obj)
1823     @staticmethod
1824     def _toReplaceWithCheck(toObj,fromObj):
1825         if not isinstance(fromObj, type(toObj)):
1826             raise TypeError("toReplaceWith requires both arguments to be the same class type")
1827     def toReplaceWith(self,toObj,fromObj):
1828         """If the Modifier is chosen the internals of toObj will be associated with the internals of fromObj
1829         """
1830         Modifier._toReplaceWithCheck(toObj,fromObj)
1831         if not self._isChosen():
1832             return
1833         Modifier._toReplaceWith(toObj,fromObj)
1834     @staticmethod
1835     def _toReplaceWith(toObj,fromObj):
1836         if isinstance(fromObj,_ModuleSequenceType):
1837             toObj._seq = fromObj._seq
1838             toObj._tasks = fromObj._tasks
1839         elif isinstance(fromObj,Task):
1840             toObj._collection = fromObj._collection
1841         elif isinstance(fromObj,ConditionalTask):
1842             toObj._collection = fromObj._collection
1843         elif isinstance(fromObj,_Parameterizable):
1844             #clear old items just incase fromObj is not a complete superset of toObj
1845             for p in toObj.parameterNames_():
1846                 delattr(toObj,p)
1847             for p in fromObj.parameterNames_():
1848                 setattr(toObj,p,getattr(fromObj,p))
1849             if isinstance(fromObj,_TypedParameterizable):
1850                 toObj._TypedParameterizable__type = fromObj._TypedParameterizable__type
1851 
1852         else:
1853             raise TypeError("toReplaceWith does not work with type "+str(type(toObj)))
1854 
1855     def _setChosen(self):
1856         """Should only be called by cms.Process instances"""
1857         self.__chosen = True
1858     def _isChosen(self):
1859         return self.__chosen
1860     def __and__(self, other):
1861         return _AndModifier(self,other)
1862     def __invert__(self):
1863         return _InvertModifier(self)
1864     def __or__(self, other):
1865         return _OrModifier(self,other)
1866     def _isOrContains(self, other):
1867         return self == other
1868 
1869 
1870 class ModifierChain(object):
1871     """A Modifier made up of a list of Modifiers
1872     """
1873     def __init__(self, *chainedModifiers):
1874         self.__chosen = False
1875         self.__chain = chainedModifiers
1876     def _applyNewProcessModifiers(self,process):
1877         """Should only be called by cms.Process instances
1878         applies list of accumulated changes to the process"""
1879         for m in self.__chain:
1880             m._applyNewProcessModifiers(process)
1881     def _setChosen(self):
1882         """Should only be called by cms.Process instances"""
1883         self.__chosen = True
1884         for m in self.__chain:
1885             m._setChosen()
1886     def _isChosen(self):
1887         return self.__chosen
1888     def copyAndExclude(self, toExclude):
1889         """Creates a new ModifierChain which is a copy of
1890           this ModifierChain but excludes any Modifier or
1891           ModifierChain in the list toExclude.
1892           The exclusion is done recursively down the chain.
1893           """
1894         newMods = []
1895         for m in self.__chain:
1896             if m not in toExclude:
1897                 s = m
1898                 if isinstance(m,ModifierChain):
1899                     s = m.__copyIfExclude(toExclude)
1900                 newMods.append(s)
1901         return ModifierChain(*newMods)
1902     def __copyIfExclude(self,toExclude):
1903         shouldCopy = False
1904         for m in toExclude:
1905             if self._isOrContains(m):
1906                 shouldCopy = True
1907                 break
1908         if shouldCopy:
1909             return self.copyAndExclude(toExclude)
1910         return self
1911     def _isOrContains(self, other):
1912         if self is other:
1913             return True
1914         for m in self.__chain:
1915             if m._isOrContains(other):
1916                 return True
1917         return False
1918 
1919 class ProcessModifier(object):
1920     """A class used by a Modifier to affect an entire Process instance.
1921     When a Process 'loads' a module containing a ProcessModifier, that
1922     ProcessModifier will be applied to the Process if and only if the 
1923     Modifier passed to the constructor has been chosen.
1924     """
1925     def __init__(self, modifier, func):
1926         self.__modifier = modifier
1927         self.__func = func
1928         self.__seenProcesses = set()
1929     def apply(self,process):
1930         if self.__modifier._isChosen():
1931             if process not in self.__seenProcesses:
1932                 self.__func(process)
1933                 self.__seenProcesses.add(process)
1934 
1935 class ProcessAccelerator(_ConfigureComponent,_Unlabelable):
1936     """A class used to specify possible compute accelerators in a Process
1937     instance. It is intended to be derived for any
1938     accelerator/portability technology, and provides hooks such that a
1939     specific customization can be applied to the Process on a worker
1940     node at the point where the python configuration is serialized for C++.
1941 
1942     The customization must not change the configuration hash. To
1943     enforce this reuirement, the customization gets a
1944     ProcessForProcessAccelerator wrapper that gives access to only
1945     those parts of the configuration that can be changed. Nevertheless
1946     it would be good to have specific unit test for each deriving
1947     class to ensure that all combinations of the enabled accelerators
1948     give the same configuration hash.
1949     """
1950     def __init__(self):
1951         pass
1952     def _place(self, name, proc):
1953         proc._placeAccelerator(self.type_(), self)
1954     def type_(self):
1955         return type(self).__name__
1956     def dumpPython(self, options=PrintOptions()):
1957         specialImportRegistry.registerUse(self)
1958         result = self.__class__.__name__+"(" # not including cms. since the deriving classes are not in cms "namespace"
1959         options.indent()
1960         res = self.dumpPythonImpl(options)
1961         options.unindent()
1962         if len(res) > 0:
1963             result += "\n"+res+"\n"
1964         result += ")\n"
1965         return result
1966 
1967     # The following methods are hooks to be overridden (if needed) in the deriving class
1968     def dumpPythonImpl(self, options):
1969         """Override if need to add any 'body' content to dumpPython(). Returns a string."""
1970         return ""
1971     def labels(self):
1972         """Override to return a list of strings for the accelerator labels."""
1973         return []
1974     def enabledLabels(self):
1975         """Override to return a list of strings for the accelerator labels
1976         that are enabled in the system the job is being run on."""
1977         return []
1978     def apply(self, process, accelerators):
1979         """Override if need to customize the Process at worker node. The
1980         selected available accelerator labels are given in the
1981         'accelerators' argument (the patterns, e.g. '*' have been
1982         expanded to concrete labels).
1983 
1984         This function may touch only untracked parameters.
1985         """
1986         pass
1987 
1988 class ProcessForProcessAccelerator(object):
1989     """This class is inteded to wrap the Process object to constrain the
1990     available functionality for ProcessAccelerator.apply()"""
1991     def  __init__(self, process):
1992         self.__process = process
1993     def __getattr__(self, label):
1994         value = getattr(self.__process, label)
1995         if not isinstance(value, Service):
1996             raise TypeError("ProcessAccelerator.apply() can get only Services. Tried to get {} with label {}".format(str(type(value)), label))
1997         return value
1998     def __setattr__(self, label, value):
1999         if label == "_ProcessForProcessAccelerator__process":
2000             super().__setattr__(label, value)
2001         else:
2002             if not isinstance(value, Service):
2003                 raise TypeError("ProcessAccelerator.apply() can only set Services. Tried to set {} with label {}".format(str(type(value)), label))
2004             setattr(self.__process, label, value)
2005     def add_(self, value):
2006         if not isinstance(value, Service):
2007             raise TypeError("ProcessAccelerator.apply() can only add Services. Tried to set {} with label {}".format(str(type(value)), label))
2008         self.__process.add_(value)
2009 
2010 # Need to be a module-level function for the configuration with a
2011 # SwitchProducer to be pickleable.
2012 def _switchproducer_test2_case1(accelerators):
2013     return ("test1" in accelerators, -10)
2014 def _switchproducer_test2_case2(accelerators):
2015     return ("test2" in accelerators, -9)
2016 
2017 if __name__=="__main__":
2018     import unittest
2019     import copy
2020 
2021     def _lineDiff(newString, oldString):
2022         newString = ( x for x in newString.split('\n') if len(x) > 0)
2023         oldString = [ x for x in oldString.split('\n') if len(x) > 0]
2024         diff = []
2025         oldStringLine = 0
2026         for l in newString:
2027             if oldStringLine >= len(oldString):
2028                 diff.append(l)
2029                 continue
2030             if l == oldString[oldStringLine]:
2031                 oldStringLine +=1
2032                 continue
2033             diff.append(l)
2034         return "\n".join( diff )
2035 
2036     class TestMakePSet(object):
2037         """Has same interface as the C++ object that creates PSets
2038         """
2039         def __init__(self):
2040             self.values = dict()
2041         def __insertValue(self,tracked,label,value):
2042             self.values[label]=(tracked,value)
2043         def __getValue(self,tracked,label):
2044             pair = self.values[label]
2045             if pair[0] != tracked:
2046                raise Exception("Asked for %s parameter '%s', but it is %s" % ("tracked" if tracked else "untracked",
2047                                                                               label,
2048                                                                               "tracked" if pair[0] else "untracked"))
2049             return pair[1]
2050         def addInt32(self,tracked,label,value):
2051             self.__insertValue(tracked,label,value)
2052         def addVInt32(self,tracked,label,value):
2053             self.__insertValue(tracked,label,value)
2054         def addUInt32(self,tracked,label,value):
2055             self.__insertValue(tracked,label,value)
2056         def addVUInt32(self,tracked,label,value):
2057             self.__insertValue(tracked,label,value)
2058         def addInt64(self,tracked,label,value):
2059             self.__insertValue(tracked,label,value)
2060         def addVInt64(self,tracked,label,value):
2061             self.__insertValue(tracked,label,value)
2062         def addUInt64(self,tracked,label,value):
2063             self.__insertValue(tracked,label,value)
2064         def addVUInt64(self,tracked,label,value):
2065             self.__insertValue(tracked,label,value)
2066         def addDouble(self,tracked,label,value):
2067             self.__insertValue(tracked,label,value)
2068         def addVDouble(self,tracked,label,value):
2069             self.__insertValue(tracked,label,value)
2070         def addBool(self,tracked,label,value):
2071             self.__insertValue(tracked,label,value)
2072         def addString(self,tracked,label,value):
2073             self.__insertValue(tracked,label,value)
2074         def addVString(self,tracked,label,value):
2075             self.__insertValue(tracked,label,value)
2076         def getVString(self,tracked,label):
2077             return self.__getValue(tracked, label)
2078         def addInputTag(self,tracked,label,value):
2079             self.__insertValue(tracked,label,value)
2080         def addVInputTag(self,tracked,label,value):
2081             self.__insertValue(tracked,label,value)
2082         def addESInputTag(self,tracked,label,value):
2083             self.__insertValue(tracked,label,value)
2084         def addVESInputTag(self,tracked,label,value):
2085             self.__insertValue(tracked,label,value)
2086         def addEventID(self,tracked,label,value):
2087             self.__insertValue(tracked,label,value)
2088         def addVEventID(self,tracked,label,value):
2089             self.__insertValue(tracked,label,value)
2090         def addLuminosityBlockID(self,tracked,label,value):
2091             self.__insertValue(tracked,label,value)
2092         def addLuminosityBlockID(self,tracked,label,value):
2093             self.__insertValue(tracked,label,value)
2094         def addEventRange(self,tracked,label,value):
2095             self.__insertValue(tracked,label,value)
2096         def addVEventRange(self,tracked,label,value):
2097             self.__insertValue(tracked,label,value)
2098         def addPSet(self,tracked,label,value):
2099             self.__insertValue(tracked,label,value)
2100         def addVPSet(self,tracked,label,value):
2101             self.__insertValue(tracked,label,value)
2102         def addFileInPath(self,tracked,label,value):
2103             self.__insertValue(tracked,label,value)
2104         def newPSet(self):
2105             return TestMakePSet()
2106 
2107     class SwitchProducerTest(SwitchProducer):
2108         def __init__(self, **kargs):
2109             super(SwitchProducerTest,self).__init__(
2110                 dict(
2111                     test1 = lambda accelerators: (True, -10),
2112                     test2 = lambda accelerators: (True, -9),
2113                     test3 = lambda accelerators: (True, -8),
2114                     test4 = lambda accelerators: (True, -7)
2115                 ), **kargs)
2116     specialImportRegistry.registerSpecialImportForType(SwitchProducerTest, "from test import SwitchProducerTest")
2117 
2118     class SwitchProducerTest2(SwitchProducer):
2119         def __init__(self, **kargs):
2120             super(SwitchProducerTest2,self).__init__(
2121                 dict(
2122                     test1 = _switchproducer_test2_case1,
2123                     test2 = _switchproducer_test2_case2,
2124                 ), **kargs)
2125     specialImportRegistry.registerSpecialImportForType(SwitchProducerTest2, "from test import SwitchProducerTest2")
2126 
2127     class ProcessAcceleratorTest(ProcessAccelerator):
2128         def __init__(self, enabled=["test1", "test2", "anothertest3"]):
2129             super(ProcessAcceleratorTest,self).__init__()
2130             self._labels = ["test1", "test2", "anothertest3"]
2131             self.setEnabled(enabled)
2132         def setEnabled(self, enabled):
2133             invalid = set(enabled).difference(set(self._labels))
2134             if len(invalid) > 0:
2135                 raise Exception("Tried to enabled nonexistent test accelerators {}".format(",".join(invalid)))
2136             self._enabled = enabled[:]
2137         def dumpPythonImpl(self,options):
2138             result = "{}enabled = [{}]".format(options.indentation(),
2139                                                ", ".join(["'{}'".format(e) for e in self._enabled]))
2140             return result
2141         def labels(self):
2142             return self._labels
2143         def enabledLabels(self):
2144             return self._enabled
2145         def apply(self, process, accelerators):
2146             process.AcceleratorTestService = Service("AcceleratorTestService")
2147     specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorTest, "from test import ProcessAcceleratorTest")
2148 
2149     class ProcessAcceleratorTest2(ProcessAccelerator):
2150         def __init__(self, enabled=["anothertest3", "anothertest4"]):
2151             super(ProcessAcceleratorTest2,self).__init__()
2152             self._labels = ["anothertest3", "anothertest4"]
2153             self.setEnabled(enabled)
2154         def setEnabled(self, enabled):
2155             invalid = set(enabled).difference(set(self._labels))
2156             if len(invalid) > 0:
2157                 raise Exception("Tried to enabled nonexistent test accelerators {}".format(",".join(invalid)))
2158             self._enabled = enabled[:]
2159         def dumpPythonImpl(self,options):
2160             result = "{}enabled = [{}]".format(options.indentation(),
2161                                                ", ".join(["'{}'".format(e) for e in self._enabled]))
2162             return result
2163         def labels(self):
2164             return self._labels
2165         def enabledLabels(self):
2166             return self._enabled
2167         def apply(self, process, accelerators):
2168             pass
2169     specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorTest2, "from test import ProcessAcceleratorTest2")
2170 
2171     class TestModuleCommand(unittest.TestCase):
2172         def setUp(self):
2173             """Nothing to do """
2174             None
2175         def testParameterizable(self):
2176             p = _Parameterizable()
2177             self.assertEqual(len(p.parameterNames_()),0)
2178             p.a = int32(1)
2179             self.assertTrue('a' in p.parameterNames_())
2180             self.assertEqual(p.a.value(), 1)
2181             p.a = 10
2182             self.assertEqual(p.a.value(), 10)
2183             p.a = untracked(int32(1))
2184             self.assertEqual(p.a.value(), 1)
2185             self.assertFalse(p.a.isTracked())
2186             p.a = untracked.int32(1)
2187             self.assertEqual(p.a.value(), 1)
2188             self.assertFalse(p.a.isTracked())
2189             p = _Parameterizable(foo=int32(10), bar = untracked(double(1.0)))
2190             self.assertEqual(p.foo.value(), 10)
2191             self.assertEqual(p.bar.value(),1.0)
2192             self.assertFalse(p.bar.isTracked())
2193             self.assertRaises(TypeError,setattr,(p,'c',1))
2194             p = _Parameterizable(a=PSet(foo=int32(10), bar = untracked(double(1.0))))
2195             self.assertEqual(p.a.foo.value(),10)
2196             self.assertEqual(p.a.bar.value(),1.0)
2197             p.b = untracked(PSet(fii = int32(1)))
2198             self.assertEqual(p.b.fii.value(),1)
2199             self.assertFalse(p.b.isTracked())
2200             #test the fact that values can be shared
2201             v = int32(10)
2202             p=_Parameterizable(a=v)
2203             v.setValue(11)
2204             self.assertEqual(p.a.value(),11)
2205             p.a = 12
2206             self.assertEqual(p.a.value(),12)
2207             self.assertEqual(v.value(),12)
2208         def testTypedParameterizable(self):
2209             p = _TypedParameterizable("blah", b=int32(1))
2210             #see if copy works deeply
2211             other = p.copy()
2212             other.b = 2
2213             self.assertNotEqual(p.b,other.b)
2214 
2215         def testProcessInsertion(self):
2216             p = Process("test")
2217             p.a = EDAnalyzer("MyAnalyzer")
2218             self.assertTrue( 'a' in p.analyzers_() )
2219             self.assertTrue( 'a' in p.analyzers)
2220             p.add_(Service("SomeService"))
2221             self.assertTrue('SomeService' in p.services_())
2222             self.assertEqual(p.SomeService.type_(), "SomeService")
2223             p.Tracer = Service("Tracer")
2224             self.assertTrue('Tracer' in p.services_())
2225             self.assertRaises(TypeError, setattr, *(p,'b',"this should fail"))
2226             self.assertRaises(TypeError, setattr, *(p,'bad',Service("MessageLogger")))
2227             self.assertRaises(ValueError, setattr, *(p,'bad',Source("PoolSource")))
2228             p.out = OutputModule("Outer")
2229             self.assertEqual(p.out.type_(), 'Outer')
2230             self.assertTrue( 'out' in p.outputModules_() )
2231 
2232             p.geom = ESSource("GeomProd")
2233             self.assertTrue('geom' in p.es_sources_())
2234             p.add_(ESSource("ConfigDB"))
2235             self.assertTrue('ConfigDB' in p.es_sources_())
2236 
2237             p.aliasfoo1 = EDAlias(foo1 = VPSet(PSet(type = string("Foo1"))))
2238             self.assertTrue('aliasfoo1' in p.aliases_())
2239 
2240         def testProcessExtend(self):
2241             class FromArg(object):
2242                 def __init__(self,*arg,**args):
2243                     for name in args.keys():
2244                         self.__dict__[name]=args[name]
2245 
2246             a=EDAnalyzer("MyAnalyzer")
2247             t=EDAnalyzer("MyAnalyzer")
2248             t.setLabel("foo")
2249             s1 = Sequence(a)
2250             s2 = Sequence(s1)
2251             s3 = Sequence(s2)
2252             d = FromArg(
2253                     a=a,
2254                     b=Service("Full"),
2255                     c=Path(a),
2256                     d=s2,
2257                     e=s1,
2258                     f=s3,
2259                     g=Sequence(s1+s2+s3)
2260                 )
2261             p = Process("Test")
2262             p.extend(d)
2263             self.assertEqual(p.a.type_(),"MyAnalyzer")
2264             self.assertEqual(p.a.label_(),"a")
2265             self.assertRaises(AttributeError,getattr,p,'b')
2266             self.assertEqual(p.Full.type_(),"Full")
2267             self.assertEqual(str(p.c),'a')
2268             self.assertEqual(str(p.d),'a')
2269 
2270             z1 = FromArg(
2271                     a=a,
2272                     b=Service("Full"),
2273                     c=Path(a),
2274                     d=s2,
2275                     e=s1,
2276                     f=s3,
2277                     s4=s3,
2278                     g=Sequence(s1+s2+s3)
2279                  )
2280 
2281             p1 = Process("Test")
2282             #p1.extend(z1)
2283             self.assertRaises(ValueError, p1.extend, z1)
2284 
2285             z2 = FromArg(
2286                     a=a,
2287                     b=Service("Full"),
2288                     c=Path(a),
2289                     d=s2,
2290                     e=s1,
2291                     f=s3,
2292                     aaa=copy.deepcopy(a),
2293                     s4=copy.deepcopy(s3),
2294                     g=Sequence(s1+s2+s3),
2295                     t=t
2296                 )
2297             p2 = Process("Test")
2298             p2.extend(z2)
2299             #self.assertRaises(ValueError, p2.extend, z2)
2300             self.assertEqual(p2.s4.label_(),"s4")
2301             #p2.s4.setLabel("foo")
2302             self.assertRaises(ValueError, p2.s4.setLabel, "foo")
2303             p2.s4.setLabel("s4")
2304             p2.s4.setLabel(None)
2305             p2.s4.setLabel("foo")
2306             p2._Process__setObjectLabel(p2.s4, "foo")
2307             p2._Process__setObjectLabel(p2.s4, None)
2308             p2._Process__setObjectLabel(p2.s4, "bar")
2309 
2310 
2311             p = Process('test')
2312             p.a = EDProducer("MyProducer")
2313             p.t = Task(p.a)
2314             p.p = Path(p.t)
2315             self.assertRaises(ValueError, p.extend, FromArg(a = EDProducer("YourProducer")))
2316             self.assertRaises(ValueError, p.extend, FromArg(a = EDAlias()))
2317             self.assertRaises(ValueError, p.__setattr__, "a", EDAlias())
2318 
2319             p = Process('test')
2320             p.a = EDProducer("MyProducer")
2321             p.t = ConditionalTask(p.a)
2322             p.p = Path(p.t)
2323             self.assertRaises(ValueError, p.extend, FromArg(a = EDProducer("YourProducer")))
2324             self.assertRaises(ValueError, p.extend, FromArg(a = EDAlias()))
2325             self.assertRaises(ValueError, p.__setattr__, "a", EDAlias())
2326 
2327             p = Process('test')
2328             p.a = EDProducer("MyProducer")
2329             p.s = Sequence(p.a)
2330             p.p = Path(p.s)
2331             self.assertRaises(ValueError, p.extend, FromArg(a = EDProducer("YourProducer")))
2332             self.assertRaises(ValueError, p.extend, FromArg(a = EDAlias()))
2333             self.assertRaises(ValueError, p.__setattr__, "a", EDAlias())
2334 
2335         def testProcessDumpPython(self):
2336             self.assertEqual(Process("test").dumpPython(),
2337 """import FWCore.ParameterSet.Config as cms
2338 
2339 process = cms.Process("test")
2340 
2341 process.maxEvents = cms.untracked.PSet(
2342     input = cms.optional.untracked.int32,
2343     output = cms.optional.untracked.allowed(cms.int32,cms.PSet)
2344 )
2345 
2346 process.maxLuminosityBlocks = cms.untracked.PSet(
2347     input = cms.untracked.int32(-1)
2348 )
2349 
2350 process.options = cms.untracked.PSet(
2351     FailPath = cms.untracked.vstring(),
2352     IgnoreCompletely = cms.untracked.vstring(),
2353     Rethrow = cms.untracked.vstring(),
2354     SkipEvent = cms.untracked.vstring(),
2355     accelerators = cms.untracked.vstring('*'),
2356     allowUnscheduled = cms.obsolete.untracked.bool,
2357     canDeleteEarly = cms.untracked.vstring(),
2358     deleteNonConsumedUnscheduledModules = cms.untracked.bool(True),
2359     dumpOptions = cms.untracked.bool(False),
2360     emptyRunLumiMode = cms.obsolete.untracked.string,
2361     eventSetup = cms.untracked.PSet(
2362         forceNumberOfConcurrentIOVs = cms.untracked.PSet(
2363             allowAnyLabel_=cms.required.untracked.uint32
2364         ),
2365         numberOfConcurrentIOVs = cms.untracked.uint32(0)
2366     ),
2367     fileMode = cms.untracked.string('FULLMERGE'),
2368     forceEventSetupCacheClearOnNewRun = cms.untracked.bool(False),
2369     makeTriggerResults = cms.obsolete.untracked.bool,
2370     numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(0),
2371     numberOfConcurrentRuns = cms.untracked.uint32(1),
2372     numberOfStreams = cms.untracked.uint32(0),
2373     numberOfThreads = cms.untracked.uint32(1),
2374     printDependencies = cms.untracked.bool(False),
2375     sizeOfStackForThreadsInKB = cms.optional.untracked.uint32,
2376     throwIfIllegalParameter = cms.untracked.bool(True),
2377     wantSummary = cms.untracked.bool(False)
2378 )
2379 
2380 process.MessageLogger = cms.Service("MessageLogger",
2381     cerr = cms.untracked.PSet(
2382         FwkReport = cms.untracked.PSet(
2383             limit = cms.untracked.int32(10000000),
2384             reportEvery = cms.untracked.int32(1)
2385         ),
2386         FwkSummary = cms.untracked.PSet(
2387             limit = cms.untracked.int32(10000000),
2388             reportEvery = cms.untracked.int32(1)
2389         ),
2390         INFO = cms.untracked.PSet(
2391             limit = cms.untracked.int32(0)
2392         ),
2393         Root_NoDictionary = cms.untracked.PSet(
2394             limit = cms.untracked.int32(0)
2395         ),
2396         default = cms.untracked.PSet(
2397             limit = cms.untracked.int32(10000000)
2398         ),
2399         enable = cms.untracked.bool(True),
2400         enableStatistics = cms.untracked.bool(True),
2401         lineLength = cms.optional.untracked.int32,
2402         noLineBreaks = cms.optional.untracked.bool,
2403         noTimeStamps = cms.untracked.bool(False),
2404         resetStatistics = cms.untracked.bool(False),
2405         statisticsThreshold = cms.untracked.string('WARNING'),
2406         threshold = cms.untracked.string('INFO'),
2407         allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2408             limit = cms.optional.untracked.int32,
2409             reportEvery = cms.untracked.int32(1),
2410             timespan = cms.optional.untracked.int32
2411         )
2412     ),
2413     cout = cms.untracked.PSet(
2414         enable = cms.untracked.bool(False),
2415         enableStatistics = cms.untracked.bool(False),
2416         lineLength = cms.optional.untracked.int32,
2417         noLineBreaks = cms.optional.untracked.bool,
2418         noTimeStamps = cms.optional.untracked.bool,
2419         resetStatistics = cms.untracked.bool(False),
2420         statisticsThreshold = cms.optional.untracked.string,
2421         threshold = cms.optional.untracked.string,
2422         allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2423             limit = cms.optional.untracked.int32,
2424             reportEvery = cms.untracked.int32(1),
2425             timespan = cms.optional.untracked.int32
2426         )
2427     ),
2428     debugModules = cms.untracked.vstring(),
2429     default = cms.untracked.PSet(
2430         limit = cms.optional.untracked.int32,
2431         lineLength = cms.untracked.int32(80),
2432         noLineBreaks = cms.untracked.bool(False),
2433         noTimeStamps = cms.untracked.bool(False),
2434         reportEvery = cms.untracked.int32(1),
2435         statisticsThreshold = cms.untracked.string('INFO'),
2436         threshold = cms.untracked.string('INFO'),
2437         timespan = cms.optional.untracked.int32,
2438         allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2439             limit = cms.optional.untracked.int32,
2440             reportEvery = cms.untracked.int32(1),
2441             timespan = cms.optional.untracked.int32
2442         )
2443     ),
2444     files = cms.untracked.PSet(
2445         allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2446             enableStatistics = cms.untracked.bool(False),
2447             extension = cms.optional.untracked.string,
2448             filename = cms.optional.untracked.string,
2449             lineLength = cms.optional.untracked.int32,
2450             noLineBreaks = cms.optional.untracked.bool,
2451             noTimeStamps = cms.optional.untracked.bool,
2452             output = cms.optional.untracked.string,
2453             resetStatistics = cms.untracked.bool(False),
2454             statisticsThreshold = cms.optional.untracked.string,
2455             threshold = cms.optional.untracked.string,
2456             allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2457                 limit = cms.optional.untracked.int32,
2458                 reportEvery = cms.untracked.int32(1),
2459                 timespan = cms.optional.untracked.int32
2460             )
2461         )
2462     ),
2463     suppressDebug = cms.untracked.vstring(),
2464     suppressFwkInfo = cms.untracked.vstring(),
2465     suppressInfo = cms.untracked.vstring(),
2466     suppressWarning = cms.untracked.vstring(),
2467     allowAnyLabel_=cms.optional.untracked.PSetTemplate(
2468         limit = cms.optional.untracked.int32,
2469         reportEvery = cms.untracked.int32(1),
2470         timespan = cms.optional.untracked.int32
2471     )
2472 )
2473 
2474 
2475 """)
2476             p = Process("test")
2477             p.a = EDAnalyzer("MyAnalyzer")
2478             p.p = Path(p.a)
2479             p.s = Sequence(p.a)
2480             p.r = Sequence(p.s)
2481             p.p2 = Path(p.s)
2482             p.schedule = Schedule(p.p2,p.p)
2483             d=p.dumpPython()
2484             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2485 """process.a = cms.EDAnalyzer("MyAnalyzer")
2486 process.s = cms.Sequence(process.a)
2487 process.r = cms.Sequence(process.s)
2488 process.p = cms.Path(process.a)
2489 process.p2 = cms.Path(process.s)
2490 process.schedule = cms.Schedule(*[ process.p2, process.p ])""")
2491             #Reverse order of 'r' and 's'
2492             p = Process("test")
2493             p.a = EDAnalyzer("MyAnalyzer")
2494             p.p = Path(p.a)
2495             p.r = Sequence(p.a)
2496             p.s = Sequence(p.r)
2497             p.p2 = Path(p.r)
2498             p.schedule = Schedule(p.p2,p.p)
2499             p.b = EDAnalyzer("YourAnalyzer")
2500             d=p.dumpPython()
2501             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2502 """process.a = cms.EDAnalyzer("MyAnalyzer")
2503 process.b = cms.EDAnalyzer("YourAnalyzer")
2504 process.r = cms.Sequence(process.a)
2505 process.s = cms.Sequence(process.r)
2506 process.p = cms.Path(process.a)
2507 process.p2 = cms.Path(process.r)
2508 process.schedule = cms.Schedule(*[ process.p2, process.p ])""")
2509         #use an anonymous sequence
2510             p = Process("test")
2511             p.a = EDAnalyzer("MyAnalyzer")
2512             p.p = Path(p.a)
2513             s = Sequence(p.a)
2514             p.r = Sequence(s)
2515             p.p2 = Path(p.r)
2516             p.schedule = Schedule(p.p2,p.p)
2517             d=p.dumpPython()
2518             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2519 """process.a = cms.EDAnalyzer("MyAnalyzer")
2520 process.r = cms.Sequence((process.a))
2521 process.p = cms.Path(process.a)
2522 process.p2 = cms.Path(process.r)
2523 process.schedule = cms.Schedule(*[ process.p2, process.p ])""")
2524 
2525             # include some tasks
2526             p = Process("test")
2527             p.a = EDAnalyzer("MyAnalyzer")
2528             p.b = EDProducer("bProducer")
2529             p.c = EDProducer("cProducer")
2530             p.d = EDProducer("dProducer")
2531             p.e = EDProducer("eProducer")
2532             p.f = EDProducer("fProducer")
2533             p.g = EDProducer("gProducer")
2534             p.task5 = Task()
2535             p.task3 = Task()
2536             p.task2 = Task(p.c, p.task3)
2537             p.task4 = Task(p.f, p.task2)
2538             p.task1 = Task(p.task5)
2539             p.task3.add(p.task1)
2540             p.p = Path(p.a)
2541             s = Sequence(p.a)
2542             p.r = Sequence(s)
2543             p.p2 = Path(p.r, p.task1, p.task2)
2544             p.schedule = Schedule(p.p2,p.p,tasks=[p.task3,p.task4, p.task5])
2545             d=p.dumpPython()
2546             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2547 """process.b = cms.EDProducer("bProducer")
2548 process.c = cms.EDProducer("cProducer")
2549 process.d = cms.EDProducer("dProducer")
2550 process.e = cms.EDProducer("eProducer")
2551 process.f = cms.EDProducer("fProducer")
2552 process.g = cms.EDProducer("gProducer")
2553 process.a = cms.EDAnalyzer("MyAnalyzer")
2554 process.task5 = cms.Task()
2555 process.task1 = cms.Task(process.task5)
2556 process.task3 = cms.Task(process.task1)
2557 process.task2 = cms.Task(process.c, process.task3)
2558 process.task4 = cms.Task(process.f, process.task2)
2559 process.r = cms.Sequence((process.a))
2560 process.p = cms.Path(process.a)
2561 process.p2 = cms.Path(process.r, process.task1, process.task2)
2562 process.schedule = cms.Schedule(*[ process.p2, process.p ], tasks=[process.task3, process.task4, process.task5])""")
2563             # include some conditional tasks
2564             p = Process("test")
2565             p.a = EDAnalyzer("MyAnalyzer")
2566             p.b = EDProducer("bProducer")
2567             p.c = EDProducer("cProducer")
2568             p.d = EDProducer("dProducer")
2569             p.e = EDProducer("eProducer")
2570             p.f = EDProducer("fProducer")
2571             p.g = EDProducer("gProducer")
2572             p.task5 = Task()
2573             p.task3 = Task()
2574             p.task2 = ConditionalTask(p.c, p.task3)
2575             p.task1 = ConditionalTask(p.task5)
2576             p.p = Path(p.a)
2577             s = Sequence(p.a)
2578             p.r = Sequence(s)
2579             p.p2 = Path(p.r, p.task1, p.task2)
2580             p.schedule = Schedule(p.p2,p.p,tasks=[p.task5])
2581             d=p.dumpPython()
2582             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2583 """process.b = cms.EDProducer("bProducer")
2584 process.c = cms.EDProducer("cProducer")
2585 process.d = cms.EDProducer("dProducer")
2586 process.e = cms.EDProducer("eProducer")
2587 process.f = cms.EDProducer("fProducer")
2588 process.g = cms.EDProducer("gProducer")
2589 process.a = cms.EDAnalyzer("MyAnalyzer")
2590 process.task5 = cms.Task()
2591 process.task3 = cms.Task()
2592 process.task2 = cms.ConditionalTask(process.c, process.task3)
2593 process.task1 = cms.ConditionalTask(process.task5)
2594 process.r = cms.Sequence((process.a))
2595 process.p = cms.Path(process.a)
2596 process.p2 = cms.Path(process.r, process.task1, process.task2)
2597 process.schedule = cms.Schedule(*[ process.p2, process.p ], tasks=[process.task5])""")
2598             # only tasks
2599             p = Process("test")
2600             p.d = EDProducer("dProducer")
2601             p.e = EDProducer("eProducer")
2602             p.f = EDProducer("fProducer")
2603             p.g = EDProducer("gProducer")
2604             p.task1 = Task(p.d, p.e)
2605             task2 = Task(p.f, p.g)
2606             p.schedule = Schedule(tasks=[p.task1,task2])
2607             d=p.dumpPython()
2608             self.assertEqual(_lineDiff(d,Process("test").dumpPython()),
2609 """process.d = cms.EDProducer("dProducer")
2610 process.e = cms.EDProducer("eProducer")
2611 process.f = cms.EDProducer("fProducer")
2612 process.g = cms.EDProducer("gProducer")
2613 process.task1 = cms.Task(process.d, process.e)
2614 process.schedule = cms.Schedule(tasks=[cms.Task(process.f, process.g), process.task1])""")
2615             # empty schedule
2616             p = Process("test")
2617             p.schedule = Schedule()
2618             d=p.dumpPython()
2619             self.assertEqual(_lineDiff(d,Process('test').dumpPython()),
2620 """process.schedule = cms.Schedule()""")
2621 
2622             s = Sequence()
2623             a = EDProducer("A")
2624             s2 = Sequence(a)
2625             s2 += s
2626             process = Process("DUMP")
2627             process.a = a
2628             process.s2 = s2
2629             d=process.dumpPython()
2630             self.assertEqual(_lineDiff(d,Process('DUMP').dumpPython()),
2631 """process.a = cms.EDProducer("A")
2632 process.s2 = cms.Sequence(process.a)""")
2633             s = Sequence()
2634             s1 = Sequence(s)
2635             a = EDProducer("A")
2636             s3 = Sequence(a+a)
2637             s2 = Sequence(a+s3)
2638             s2 += s1
2639             process = Process("DUMP")
2640             process.a = a
2641             process.s2 = s2
2642             d=process.dumpPython()
2643             self.assertEqual(_lineDiff(d,Process('DUMP').dumpPython()),
2644 """process.a = cms.EDProducer("A")
2645 process.s2 = cms.Sequence(process.a+(process.a+process.a))""")
2646 
2647         def testSecSource(self):
2648             p = Process('test')
2649             p.a = SecSource("MySecSource")
2650             self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),'process.a = cms.SecSource("MySecSource")')
2651 
2652         def testGlobalReplace(self):
2653             p = Process('test')
2654             p.a = EDAnalyzer("MyAnalyzer")
2655             old = p.a
2656             p.b = EDAnalyzer("YourAnalyzer")
2657             p.c = EDAnalyzer("OurAnalyzer")
2658             p.d = EDProducer("MyProducer")
2659             old2 = p.d
2660             p.t1 = Task(p.d)
2661             t2 = Task(p.d)
2662             t3 = Task(p.d)
2663             t4 = Task(p.d)
2664             t5 = Task(p.d)
2665             t6 = Task(p.d)
2666             p.ct1 = ConditionalTask(p.d)
2667             s = Sequence(p.a*p.b)
2668             p.s4 = Sequence(p.a*p.b, p.ct1)
2669             s.associate(t2)
2670             p.s4.associate(t2)
2671             p.p = Path(p.c+s+p.a)
2672             p.p2 = Path(p.c+p.s4+p.a, p.ct1)
2673             p.e3 = EndPath(p.c+s+p.a)
2674             new = EDAnalyzer("NewAnalyzer")
2675             new2 = EDProducer("NewProducer")
2676             visitor1 = NodeVisitor()
2677             p.p.visit(visitor1)
2678             self.assertTrue(visitor1.modules == set([old,old2,p.b,p.c]))
2679             p.schedule = Schedule(tasks=[t6])
2680             p.globalReplace("a",new)
2681             p.globalReplace("d",new2)
2682             visitor2 = NodeVisitor()
2683             p.p.visit(visitor2)
2684             self.assertTrue(visitor2.modules == set([new,new2,p.b,p.c]))
2685             self.assertEqual(p.p.dumpPython()[:-1], "cms.Path(process.c+process.a+process.b+process.a, cms.Task(process.d))")
2686             visitor_p2 = NodeVisitor()
2687             p.p2.visit(visitor_p2)
2688             self.assertTrue(visitor_p2.modules == set([new,new2,p.b,p.c]))
2689             self.assertEqual(p.p2.dumpPython()[:-1], "cms.Path(process.c+process.s4+process.a, process.ct1)")
2690             visitor3 = NodeVisitor()
2691             p.e3.visit(visitor3)
2692             self.assertTrue(visitor3.modules == set([new,new2,p.b,p.c]))
2693             visitor4 = NodeVisitor()
2694             p.s4.visit(visitor4)
2695             self.assertTrue(visitor4.modules == set([new,new2,p.b]))
2696             self.assertEqual(p.s4.dumpPython()[:-1],"cms.Sequence(process.a+process.b, cms.Task(process.d), process.ct1)")
2697             visitor5 = NodeVisitor()
2698             p.t1.visit(visitor5)
2699             self.assertTrue(visitor5.modules == set([new2]))
2700             visitor6 = NodeVisitor()
2701             listOfTasks = list(p.schedule._tasks)
2702             listOfTasks[0].visit(visitor6)
2703             self.assertTrue(visitor6.modules == set([new2]))
2704             visitor7 = NodeVisitor()
2705             p.ct1.visit(visitor7)
2706             self.assertTrue(visitor7.modules == set([new2]))
2707             visitor8 = NodeVisitor()
2708             listOfConditionalTasks = list(p.conditionaltasks.values())
2709             listOfConditionalTasks[0].visit(visitor8)
2710             self.assertTrue(visitor8.modules == set([new2]))
2711 
2712 
2713             p.d2 = EDProducer("YourProducer")
2714             p.schedule = Schedule(p.p, p.p2, p.e3, tasks=[p.t1])
2715             self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2716             p.p = Path(p.c+s)
2717             self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2718             p.e3 = EndPath(p.c)
2719             self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2720             p.t1 = Task(p.d2)
2721             self.assertEqual(p.schedule.dumpPython()[:-1], "cms.Schedule(*[ process.p, process.p2, process.e3 ], tasks=[process.t1])")
2722 
2723         def testSequence(self):
2724             p = Process('test')
2725             p.a = EDAnalyzer("MyAnalyzer")
2726             p.b = EDAnalyzer("YourAnalyzer")
2727             p.c = EDAnalyzer("OurAnalyzer")
2728             p.s = Sequence(p.a*p.b)
2729             self.assertEqual(str(p.s),'a+b')
2730             self.assertEqual(p.s.label_(),'s')
2731             path = Path(p.c+p.s)
2732             self.assertEqual(str(path),'c+a+b')
2733             p._validateSequence(path, 'p1')
2734             notInProcess = EDAnalyzer('NotInProcess')
2735             p2 = Path(p.c+p.s*notInProcess)
2736             self.assertRaises(RuntimeError, p._validateSequence, p2, 'p2')
2737 
2738         def testSequence2(self):
2739             p = Process('test')
2740             p.a = EDAnalyzer("MyAnalyzer")
2741             p.b = EDAnalyzer("YourAnalyzer")
2742             p.c = EDAnalyzer("OurAnalyzer")
2743             testseq = Sequence(p.a*p.b)
2744             p.s = testseq
2745             #p.y = testseq
2746             self.assertRaises(ValueError, p.__setattr__, "y", testseq) 
2747 
2748         def testServiceInProcess(self):
2749             service = Service("d")
2750             self.assertFalse(service._inProcess)
2751             process = Process("test")
2752             process.d = service
2753             self.assertTrue(service._inProcess)
2754             service2 = Service("d")
2755             process.d = service2
2756             self.assertFalse(service._inProcess)
2757             self.assertTrue(service2._inProcess)
2758             del process.d
2759             self.assertFalse(service2._inProcess)
2760 
2761         def testTask(self):
2762 
2763             # create some objects to use in tests
2764             edanalyzer = EDAnalyzer("a")
2765             edproducer = EDProducer("b")
2766             edproducer2 = EDProducer("b2")
2767             edproducer3 = EDProducer("b3")
2768             edproducer4 = EDProducer("b4")
2769             edproducer8 = EDProducer("b8")
2770             edproducer9 = EDProducer("b9")
2771             edfilter = EDFilter("c")
2772             service = Service("d")
2773             service3 = Service("d", v = untracked.uint32(3))
2774             essource = ESSource("e")
2775             esproducer = ESProducer("f")
2776             testTask2 = Task()
2777 
2778             # test adding things to Tasks
2779             testTask1 = Task(edproducer, edfilter)
2780             self.assertRaises(RuntimeError, testTask1.add, edanalyzer)
2781             testTask1.add(essource, service)
2782             testTask1.add(essource, esproducer)
2783             testTask1.add(testTask2)
2784             coll = testTask1._collection
2785             self.assertTrue(edproducer in coll)
2786             self.assertTrue(edfilter in coll)
2787             self.assertTrue(service in coll)
2788             self.assertTrue(essource in coll)
2789             self.assertTrue(esproducer in coll)
2790             self.assertTrue(testTask2 in coll)
2791             self.assertTrue(len(coll) == 6)
2792             self.assertTrue(len(testTask2._collection) == 0)
2793 
2794             taskContents = []
2795             for i in testTask1:
2796                 taskContents.append(i)
2797             self.assertTrue(taskContents == [edproducer, edfilter, essource, service, esproducer, testTask2])
2798 
2799             # test attaching Task to Process
2800             process = Process("test")
2801 
2802             process.mproducer = edproducer
2803             process.mproducer2 = edproducer2
2804             process.mfilter = edfilter
2805             process.messource = essource
2806             process.mesproducer = esproducer
2807             process.d = service
2808 
2809             testTask3 = Task(edproducer, edproducer2)
2810             testTask1.add(testTask3)
2811             process.myTask1 = testTask1
2812 
2813             # test the validation that occurs when attaching a Task to a Process
2814             # first a case that passes, then one the fails on an EDProducer
2815             # then one that fails on a service
2816             l = set()
2817             visitor = NodeNameVisitor(l)
2818             testTask1.visit(visitor)
2819             self.assertTrue(l == set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
2820             l2 = testTask1.moduleNames
2821             self.assertTrue(l == set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
2822 
2823             testTask4 = Task(edproducer3)
2824             l.clear()
2825             self.assertRaises(RuntimeError, testTask4.visit, visitor)
2826             try:
2827                 process.myTask4 = testTask4
2828                 self.assertTrue(False)
2829             except RuntimeError:
2830                 pass
2831 
2832             testTask5 = Task(service3)
2833             l.clear()
2834             self.assertRaises(RuntimeError, testTask5.visit, visitor)
2835             try:
2836                 process.myTask5 = testTask5
2837                 self.assertTrue(False)
2838             except RuntimeError:
2839                 pass
2840 
2841             process.d = service3
2842             process.myTask5 = testTask5
2843 
2844             # test placement into the Process and the tasks property
2845             expectedDict = { 'myTask1' : testTask1, 'myTask5' : testTask5 }
2846             expectedFixedDict = DictTypes.FixedKeysDict(expectedDict);
2847             self.assertTrue(process.tasks == expectedFixedDict)
2848             self.assertTrue(process.tasks['myTask1'] == testTask1)
2849             self.assertTrue(process.myTask1 == testTask1)
2850 
2851             # test replacing an EDProducer in a Task when calling __settattr__
2852             # for the EDProducer on the Process.
2853             process.mproducer2 = edproducer4
2854             process.d = service
2855             l = list()
2856             visitor1 = ModuleNodeVisitor(l)
2857             testTask1.visit(visitor1)
2858             l.sort(key=lambda mod: mod.__str__())
2859             expectedList = sorted([edproducer,essource,esproducer,service,edfilter,edproducer,edproducer4],key=lambda mod: mod.__str__())
2860             self.assertTrue(expectedList == l)
2861             process.myTask6 = Task()
2862             process.myTask7 = Task()
2863             process.mproducer8 = edproducer8
2864             process.myTask8 = Task(process.mproducer8)
2865             process.myTask6.add(process.myTask7)
2866             process.myTask7.add(process.myTask8)
2867             process.myTask1.add(process.myTask6)
2868             process.myTask8.add(process.myTask5)
2869 
2870             testDict = process._itemsInDependencyOrder(process.tasks)
2871             expectedLabels = ["myTask5", "myTask8", "myTask7", "myTask6", "myTask1"]
2872             expectedTasks = [process.myTask5, process.myTask8, process.myTask7, process.myTask6, process.myTask1]
2873             index = 0
2874             for testLabel, testTask in testDict.items():
2875                 self.assertTrue(testLabel == expectedLabels[index])
2876                 self.assertTrue(testTask == expectedTasks[index])
2877                 index += 1
2878 
2879             pythonDump = testTask1.dumpPython(PrintOptions())
2880 
2881 
2882             expectedPythonDump = 'cms.Task(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer, process.mproducer2, process.myTask6)\n'
2883             self.assertTrue(pythonDump == expectedPythonDump)
2884 
2885             process.myTask5 = Task()
2886             process.myTask100 = Task()
2887             process.mproducer9 = edproducer9
2888             sequence1 = Sequence(process.mproducer8, process.myTask1, process.myTask5, testTask2, testTask3)
2889             sequence2 = Sequence(process.mproducer8 + process.mproducer9)
2890             process.sequence3 = Sequence((process.mproducer8 + process.mfilter))
2891             sequence4 = Sequence()
2892             process.path1 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+sequence4)
2893             process.path1.associate(process.myTask1, process.myTask5, testTask2, testTask3)
2894             process.path11 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+ sequence4,process.myTask1, process.myTask5, testTask2, testTask3, process.myTask100)
2895             process.path2 = Path(process.mproducer)
2896             process.path3 = Path(process.mproducer9+process.mproducer8,testTask2)
2897 
2898             self.assertTrue(process.path1.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)\n')
2899 
2900             self.assertTrue(process.path11.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask100, process.myTask5)\n')
2901 
2902             # test NodeNameVisitor and moduleNames
2903             l = set()
2904             nameVisitor = NodeNameVisitor(l)
2905             process.path1.visit(nameVisitor)
2906             self.assertTrue(l == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
2907             self.assertTrue(process.path1.moduleNames() == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
2908 
2909             # test copy
2910             process.mproducer10 = EDProducer("b10")
2911             process.path21 = process.path11.copy()
2912             process.path21.replace(process.mproducer, process.mproducer10)
2913 
2914             self.assertTrue(process.path11.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer), process.myTask1, process.myTask100, process.myTask5)\n')
2915 
2916             # Some peculiarities of the way things work show up here. dumpPython sorts tasks and
2917             # removes duplication at the level of strings. The Task and Sequence objects themselves
2918             # remove duplicate tasks in their contents if the instances are the same (exact same python
2919             # object id which is not the same as the string representation being the same).
2920             # Also note that the mutating visitor replaces sequences and tasks that have
2921             # modified contents with their modified contents, it does not modify the sequence
2922             # or task itself.
2923             self.assertTrue(process.path21.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.sequence3, cms.Task(), cms.Task(process.None, process.mproducer10), cms.Task(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), process.myTask100, process.myTask5)\n')
2924 
2925             process.path22 = process.path21.copyAndExclude([process.d, process.mesproducer, process.mfilter])
2926             self.assertTrue(process.path22.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.Task(), cms.Task(process.None, process.mproducer10), cms.Task(process.messource, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), process.myTask100, process.myTask5)\n')
2927 
2928             process.path23 = process.path22.copyAndExclude([process.messource, process.mproducer10])
2929             self.assertTrue(process.path23.dumpPython(PrintOptions()) == 'cms.Path(process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.Task(), cms.Task(process.None), cms.Task(process.mproducer2, process.mproducer8, process.myTask5), process.myTask100, process.myTask5)\n')
2930 
2931             process.a = EDAnalyzer("MyAnalyzer")
2932             process.b = OutputModule("MyOutputModule")
2933             process.c = EDFilter("MyFilter")
2934             process.d = EDProducer("MyProducer")
2935             process.e = ESProducer("MyESProducer")
2936             process.f = ESSource("MyESSource")
2937             process.g = ESProducer("g")
2938             process.path24 = Path(process.a+process.b+process.c+process.d)
2939             process.path25 = process.path24.copyAndExclude([process.a,process.b,process.c])
2940             self.assertTrue(process.path25.dumpPython() == 'cms.Path(process.d)\n')
2941             #print process.path3
2942             #print process.dumpPython()
2943 
2944             process.path200 = EndPath(Sequence(process.c,Task(process.e)))
2945             process.path200.replace(process.c,process.b)
2946             process.path200.replace(process.e,process.f)
2947             self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.b, cms.Task(process.f))\n")
2948             process.path200.replace(process.b,process.c)
2949             process.path200.replace(process.f,process.e)
2950             self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.c, cms.Task(process.e))\n")
2951             process.path200.replace(process.c,process.a)
2952             process.path200.replace(process.e,process.g)
2953             self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.a, cms.Task(process.g))\n")
2954             process.path200.replace(process.a,process.c)
2955             process.path200.replace(process.g,process.e)
2956             self.assertEqual(process.path200.dumpPython(), "cms.EndPath(process.c, cms.Task(process.e))\n")
2957 
2958         def testConditionalTask(self):
2959 
2960             # create some objects to use in tests
2961             edanalyzer = EDAnalyzer("a")
2962             edproducer = EDProducer("b")
2963             edproducer2 = EDProducer("b2")
2964             edproducer3 = EDProducer("b3")
2965             edproducer4 = EDProducer("b4")
2966             edproducer8 = EDProducer("b8")
2967             edproducer9 = EDProducer("b9")
2968             edfilter = EDFilter("c")
2969             service = Service("d")
2970             service3 = Service("d", v = untracked.uint32(3))
2971             essource = ESSource("e")
2972             esproducer = ESProducer("f")
2973             testTask2 = Task()
2974             testCTask2 = ConditionalTask()
2975 
2976             # test adding things to Tasks
2977             testTask1 = ConditionalTask(edproducer, edfilter)
2978             self.assertRaises(RuntimeError, testTask1.add, edanalyzer)
2979             testTask1.add(essource, service)
2980             testTask1.add(essource, esproducer)
2981             testTask1.add(testTask2)
2982             testTask1.add(testCTask2)
2983             coll = testTask1._collection
2984             self.assertTrue(edproducer in coll)
2985             self.assertTrue(edfilter in coll)
2986             self.assertTrue(service in coll)
2987             self.assertTrue(essource in coll)
2988             self.assertTrue(esproducer in coll)
2989             self.assertTrue(testTask2 in coll)
2990             self.assertTrue(testCTask2 in coll)
2991             self.assertTrue(len(coll) == 7)
2992             self.assertTrue(len(testTask2._collection) == 0)
2993 
2994             taskContents = []
2995             for i in testTask1:
2996                 taskContents.append(i)
2997             self.assertEqual(taskContents, [edproducer, edfilter, essource, service, esproducer, testTask2, testCTask2])
2998 
2999             # test attaching Task to Process
3000             process = Process("test")
3001 
3002             process.mproducer = edproducer
3003             process.mproducer2 = edproducer2
3004             process.mfilter = edfilter
3005             process.messource = essource
3006             process.mesproducer = esproducer
3007             process.d = service
3008 
3009             testTask3 = ConditionalTask(edproducer, edproducer2)
3010             testTask1.add(testTask3)
3011             process.myTask1 = testTask1
3012 
3013             # test the validation that occurs when attaching a ConditionalTask to a Process
3014             # first a case that passes, then one the fails on an EDProducer
3015             # then one that fails on a service
3016             l = set()
3017             visitor = NodeNameVisitor(l)
3018             testTask1.visit(visitor)
3019             self.assertEqual(l, set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
3020             l2 = testTask1.moduleNames()
3021             self.assertEqual(l2, set(['mesproducer', 'mproducer', 'mproducer2', 'mfilter', 'd', 'messource']))
3022 
3023             testTask4 = ConditionalTask(edproducer3)
3024             l.clear()
3025             self.assertRaises(RuntimeError, testTask4.visit, visitor)
3026             try:
3027                 process.myTask4 = testTask4
3028                 self.assertTrue(False)
3029             except RuntimeError:
3030                 pass
3031 
3032             testTask5 = ConditionalTask(service3)
3033             l.clear()
3034             self.assertRaises(RuntimeError, testTask5.visit, visitor)
3035             try:
3036                 process.myTask5 = testTask5
3037                 self.assertTrue(False)
3038             except RuntimeError:
3039                 pass
3040 
3041             process.d = service3
3042             process.myTask5 = testTask5
3043 
3044             # test placement into the Process and the tasks property
3045             expectedDict = { 'myTask1' : testTask1, 'myTask5' : testTask5 }
3046             expectedFixedDict = DictTypes.FixedKeysDict(expectedDict);
3047             self.assertEqual(process.conditionaltasks, expectedFixedDict)
3048             self.assertEqual(process.conditionaltasks['myTask1'], testTask1)
3049             self.assertEqual(process.myTask1, testTask1)
3050 
3051             # test replacing an EDProducer in a ConditionalTask when calling __settattr__
3052             # for the EDProducer on the Process.
3053             process.mproducer2 = edproducer4
3054             process.d = service
3055             l = list()
3056             visitor1 = ModuleNodeVisitor(l)
3057             testTask1.visit(visitor1)
3058             l.sort(key=lambda mod: mod.__str__())
3059             expectedList = sorted([edproducer,essource,esproducer,service,edfilter,edproducer,edproducer4],key=lambda mod: mod.__str__())
3060             self.assertEqual(expectedList, l)
3061             process.myTask6 = ConditionalTask()
3062             process.myTask7 = ConditionalTask()
3063             process.mproducer8 = edproducer8
3064             process.myTask8 = ConditionalTask(process.mproducer8)
3065             process.myTask6.add(process.myTask7)
3066             process.myTask7.add(process.myTask8)
3067             process.myTask1.add(process.myTask6)
3068             process.myTask8.add(process.myTask5)
3069             self.assertEqual(process.myTask8.dumpPython(), "cms.ConditionalTask(process.mproducer8, process.myTask5)\n")
3070 
3071             testDict = process._itemsInDependencyOrder(process.conditionaltasks)
3072             expectedLabels = ["myTask5", "myTask8", "myTask7", "myTask6", "myTask1"]
3073             expectedTasks = [process.myTask5, process.myTask8, process.myTask7, process.myTask6, process.myTask1]
3074             index = 0
3075             for testLabel, testTask in testDict.items():
3076                 self.assertEqual(testLabel, expectedLabels[index])
3077                 self.assertEqual(testTask, expectedTasks[index])
3078                 index += 1
3079 
3080             pythonDump = testTask1.dumpPython(PrintOptions())
3081 
3082 
3083             expectedPythonDump = 'cms.ConditionalTask(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer, process.mproducer2, process.myTask6)\n'
3084             self.assertEqual(pythonDump, expectedPythonDump)
3085 
3086             process.myTask5 = ConditionalTask()
3087             self.assertEqual(process.myTask8.dumpPython(), "cms.ConditionalTask(process.mproducer8, process.myTask5)\n")
3088             process.myTask100 = ConditionalTask()
3089             process.mproducer9 = edproducer9
3090             sequence1 = Sequence(process.mproducer8, process.myTask1, process.myTask5, testTask2, testTask3)
3091             sequence2 = Sequence(process.mproducer8 + process.mproducer9)
3092             process.sequence3 = Sequence((process.mproducer8 + process.mfilter))
3093             sequence4 = Sequence()
3094             process.path1 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+sequence4)
3095             process.path1.associate(process.myTask1, process.myTask5, testTask2, testTask3)
3096             process.path11 = Path(process.mproducer+process.mproducer8+sequence1+sequence2+process.sequence3+ sequence4,process.myTask1, process.myTask5, testTask2, testTask3, process.myTask100)
3097             process.path2 = Path(process.mproducer)
3098             process.path3 = Path(process.mproducer9+process.mproducer8,testTask2)
3099 
3100             self.assertEqual(process.path1.dumpPython(PrintOptions()), 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)\n')
3101 
3102             self.assertEqual(process.path11.dumpPython(PrintOptions()), 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask100, process.myTask5)\n')
3103 
3104             # test NodeNameVisitor and moduleNames
3105             l = set()
3106             nameVisitor = NodeNameVisitor(l)
3107             process.path1.visit(nameVisitor)
3108             self.assertTrue(l == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3109             self.assertTrue(process.path1.moduleNames() == set(['mproducer', 'd', 'mesproducer', None, 'mproducer9', 'mproducer8', 'messource', 'mproducer2', 'mfilter']))
3110 
3111             # test copy
3112             process.mproducer10 = EDProducer("b10")
3113             process.path21 = process.path11.copy()
3114             process.path21.replace(process.mproducer, process.mproducer10)
3115 
3116             self.assertEqual(process.path11.dumpPython(PrintOptions()), 'cms.Path(process.mproducer+process.mproducer8+cms.Sequence(process.mproducer8, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask5)+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer), cms.Task(), process.myTask1, process.myTask100, process.myTask5)\n')
3117 
3118             # Some peculiarities of the way things work show up here. dumpPython sorts tasks and
3119             # removes duplication at the level of strings. The Task and Sequence objects themselves
3120             # remove duplicate tasks in their contents if the instances are the same (exact same python
3121             # object id which is not the same as the string representation being the same).
3122             # Also note that the mutating visitor replaces sequences and tasks that have
3123             # modified contents with their modified contents, it does not modify the sequence
3124             # or task itself.
3125             self.assertEqual(process.path21.dumpPython(PrintOptions()), 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.sequence3, cms.ConditionalTask(process.None, process.mproducer10), cms.ConditionalTask(process.d, process.mesproducer, process.messource, process.mfilter, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), cms.Task(), process.myTask100, process.myTask5)\n')
3126 
3127             process.path22 = process.path21.copyAndExclude([process.d, process.mesproducer, process.mfilter])
3128             self.assertEqual(process.path22.dumpPython(PrintOptions()), 'cms.Path(process.mproducer10+process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.ConditionalTask(process.None, process.mproducer10), cms.ConditionalTask(process.messource, process.mproducer10, process.mproducer2, process.mproducer8, process.myTask5), cms.Task(), process.myTask100, process.myTask5)\n')
3129 
3130             process.path23 = process.path22.copyAndExclude([process.messource, process.mproducer10])
3131             self.assertEqual(process.path23.dumpPython(PrintOptions()), 'cms.Path(process.mproducer8+process.mproducer8+(process.mproducer8+process.mproducer9)+process.mproducer8, cms.ConditionalTask(process.None), cms.ConditionalTask(process.mproducer2, process.mproducer8, process.myTask5), cms.Task(), process.myTask100, process.myTask5)\n')
3132 
3133             process = Process("Test")
3134 
3135             process.b = EDProducer("b")
3136             process.b2 = EDProducer("b2")
3137             process.b3 = EDProducer("b3")
3138             process.p = Path(process.b, ConditionalTask(process.b3, process.b2))
3139             p = TestMakePSet()
3140             process.fillProcessDesc(p)
3141             self.assertEqual(p.values["@all_modules"], (True, ['b', 'b2', 'b3']))
3142             self.assertEqual(p.values["@paths"], (True, ['p']))
3143             self.assertEqual(p.values["p"], (True, ['b','#','b2','b3','@']))
3144             
3145 
3146         def testPath(self):
3147             p = Process("test")
3148             p.a = EDAnalyzer("MyAnalyzer")
3149             p.b = EDAnalyzer("YourAnalyzer")
3150             p.c = EDAnalyzer("OurAnalyzer")
3151             path = Path(p.a)
3152             path *= p.b
3153             path += p.c
3154             self.assertEqual(str(path),'a+b+c')
3155             path = Path(p.a*p.b+p.c)
3156             self.assertEqual(str(path),'a+b+c')
3157 #            path = Path(p.a)*p.b+p.c #This leads to problems with sequences
3158 #            self.assertEqual(str(path),'((a*b)+c)')
3159             path = Path(p.a+ p.b*p.c)
3160             self.assertEqual(str(path),'a+b+c')
3161             path = Path(p.a*(p.b+p.c))
3162             self.assertEqual(str(path),'a+b+c')
3163             path = Path(p.a*(p.b+~p.c))
3164             pathx = Path(p.a*(p.b+ignore(p.c)))
3165             self.assertEqual(str(path),'a+b+~c')
3166             p.es = ESProducer("AnESProducer")
3167             self.assertRaises(TypeError,Path,p.es)
3168 
3169             t = Path()
3170             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path()\n')
3171 
3172             t = Path(p.a)
3173             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a)\n')
3174 
3175             t = Path(Task())
3176             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(cms.Task())\n')
3177 
3178             t = Path(p.a, Task())
3179             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.Task())\n')
3180 
3181             p.prod = EDProducer("prodName")
3182             p.t1 = Task(p.prod)
3183             t = Path(p.a, p.t1, Task(), p.t1)
3184             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.Task(), process.t1)\n')
3185 
3186             t = Path(ConditionalTask())
3187             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(cms.ConditionalTask())\n')
3188 
3189             t = Path(p.a, ConditionalTask())
3190             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.ConditionalTask())\n')
3191 
3192             p.prod = EDProducer("prodName")
3193             p.t1 = ConditionalTask(p.prod)
3194             t = Path(p.a, p.t1, Task(), p.t1)
3195             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.Path(process.a, cms.Task(), process.t1)\n')
3196 
3197         def testFinalPath(self):
3198             p = Process("test")
3199             p.a = OutputModule("MyOutputModule")
3200             p.b = OutputModule("YourOutputModule")
3201             p.c = OutputModule("OurOutputModule")
3202             path = FinalPath(p.a)
3203             path *= p.b
3204             path += p.c
3205             self.assertEqual(str(path),'a+b+c')
3206             path = FinalPath(p.a*p.b+p.c)
3207             self.assertEqual(str(path),'a+b+c')
3208             path = FinalPath(p.a+ p.b*p.c)
3209             self.assertEqual(str(path),'a+b+c')
3210             path = FinalPath(p.a*(p.b+p.c))
3211             self.assertEqual(str(path),'a+b+c')
3212             p.es = ESProducer("AnESProducer")
3213             self.assertRaises(TypeError,FinalPath,p.es)
3214 
3215             t = FinalPath()
3216             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.FinalPath()\n')
3217 
3218             t = FinalPath(p.a)
3219             self.assertEqual(t.dumpPython(PrintOptions()), 'cms.FinalPath(process.a)\n')
3220 
3221             self.assertRaises(TypeError, FinalPath, Task())
3222             self.assertRaises(TypeError, FinalPath, p.a, Task())
3223 
3224             p.prod = EDProducer("prodName")
3225             p.t1 = Task(p.prod)
3226             self.assertRaises(TypeError, FinalPath, p.a, p.t1, Task(), p.t1)
3227 
3228             p.prod = EDProducer("prodName")
3229             p.t1 = ConditionalTask(p.prod)
3230             self.assertRaises(TypeError, FinalPath, p.a, p.t1, ConditionalTask(), p.t1)
3231 
3232             p.t = FinalPath(p.a)
3233             p.a = OutputModule("ReplacedOutputModule")
3234             self.assertEqual(p.t.dumpPython(PrintOptions()), 'cms.FinalPath(process.a)\n')
3235             
3236         def testCloneSequence(self):
3237             p = Process("test")
3238             a = EDAnalyzer("MyAnalyzer")
3239             p.a = a
3240             a.setLabel("a")
3241             b = EDAnalyzer("YOurAnalyzer")
3242             p.b = b
3243             b.setLabel("b")
3244             path = Path(a * b)
3245             p.path = Path(p.a*p.b)
3246             lookuptable = {id(a): p.a, id(b): p.b}
3247             #self.assertEqual(str(path),str(path._postProcessFixup(lookuptable)))
3248             #lookuptable = p._cloneToObjectDict
3249             #self.assertEqual(str(path),str(path._postProcessFixup(lookuptable)))
3250             self.assertEqual(str(path),str(p.path))
3251 
3252         def testContains(self):
3253 
3254             a = EDProducer("a")
3255             b = EDProducer("b")
3256             c = EDProducer("c")
3257             d = EDProducer("d")
3258             e = EDProducer("e")
3259             f = EDProducer("f")
3260             g = EDProducer("g")
3261             h = EDProducer("h")
3262             i = EDProducer("i")
3263             j = EDProducer("j")
3264             k = EDProducer("k")
3265             l = EDProducer("l")
3266             m = EDProducer("m")
3267             n = EDProducer("n")
3268 
3269             seq1 = Sequence(e)
3270             task1 = Task(g)
3271             ctask1 = ConditionalTask(h)
3272             path = Path(a * c * seq1, task1, ctask1)
3273 
3274             self.assertTrue(path.contains(a))
3275             self.assertFalse(path.contains(b))
3276             self.assertTrue(path.contains(c))
3277             self.assertFalse(path.contains(d))
3278             self.assertTrue(path.contains(e))
3279             self.assertFalse(path.contains(f))
3280             self.assertTrue(path.contains(g))
3281             self.assertTrue(path.contains(h))
3282 
3283             endpath = EndPath(h * i)
3284             self.assertFalse(endpath.contains(b))
3285             self.assertTrue(endpath.contains(i))
3286 
3287             seq = Sequence(a * c)
3288             self.assertFalse(seq.contains(b))
3289             self.assertTrue(seq.contains(c))
3290 
3291             task2 = Task(l)
3292             task = Task(j, k, task2)
3293             self.assertFalse(task.contains(b))
3294             self.assertTrue(task.contains(j))
3295             self.assertTrue(task.contains(k))
3296             self.assertTrue(task.contains(l))
3297 
3298             task3 = Task(m)
3299             path2 = Path(n)
3300             sch = Schedule(path, path2, tasks=[task,task3])
3301             self.assertFalse(sch.contains(b))
3302             self.assertTrue(sch.contains(a))
3303             self.assertTrue(sch.contains(c))
3304             self.assertTrue(sch.contains(e))
3305             self.assertTrue(sch.contains(g))
3306             self.assertTrue(sch.contains(n))
3307             self.assertTrue(sch.contains(j))
3308             self.assertTrue(sch.contains(k))
3309             self.assertTrue(sch.contains(l))
3310             self.assertTrue(sch.contains(m))
3311 
3312             ctask2 = ConditionalTask(l, task1)
3313             ctask = ConditionalTask(j, k, ctask2)
3314             self.assertFalse(ctask.contains(b))
3315             self.assertTrue(ctask.contains(j))
3316             self.assertTrue(ctask.contains(k))
3317             self.assertTrue(ctask.contains(l))
3318             self.assertTrue(ctask.contains(g))
3319 
3320         def testSchedule(self):
3321             p = Process("test")
3322             p.a = EDAnalyzer("MyAnalyzer")
3323             p.b = EDAnalyzer("YourAnalyzer")
3324             p.c = EDAnalyzer("OurAnalyzer")
3325             p.d = EDAnalyzer("OurAnalyzer")
3326             p.path1 = Path(p.a)
3327             p.path2 = Path(p.b)
3328             p.path3 = Path(p.d)
3329 
3330             s = Schedule(p.path1,p.path2)
3331             self.assertEqual(s[0],p.path1)
3332             self.assertEqual(s[1],p.path2)
3333             p.schedule = s
3334             self.assertTrue('b' in p.schedule.moduleNames())
3335             self.assertTrue(hasattr(p, 'b'))
3336             self.assertTrue(hasattr(p, 'c'))
3337             self.assertTrue(hasattr(p, 'd'))
3338             self.assertTrue(hasattr(p, 'path1'))
3339             self.assertTrue(hasattr(p, 'path2'))
3340             self.assertTrue(hasattr(p, 'path3'))
3341             p.prune()
3342             self.assertTrue('b' in p.schedule.moduleNames())
3343             self.assertTrue(hasattr(p, 'b'))
3344             self.assertTrue(not hasattr(p, 'c'))
3345             self.assertTrue(not hasattr(p, 'd'))
3346             self.assertTrue(hasattr(p, 'path1'))
3347             self.assertTrue(hasattr(p, 'path2'))
3348             self.assertTrue(not hasattr(p, 'path3'))
3349 
3350             self.assertTrue(len(p.schedule._tasks) == 0)
3351 
3352             p = Process("test")
3353             p.a = EDAnalyzer("MyAnalyzer")
3354             p.b = EDAnalyzer("YourAnalyzer")
3355             p.c = EDAnalyzer("OurAnalyzer")
3356             p.d = EDAnalyzer("dAnalyzer")
3357             p.e = EDProducer("eProducer")
3358             p.f = EDProducer("fProducer")
3359             p.Tracer = Service("Tracer")
3360             p.path1 = Path(p.a)
3361             p.path2 = Path(p.b)
3362             p.path3 = Path(p.d)
3363             p.task1 = Task(p.e)
3364             p.task2 = Task(p.f, p.Tracer)
3365             s = Schedule(p.path1,p.path2,tasks=[p.task1,p.task2,p.task1])
3366             self.assertEqual(s[0],p.path1)
3367             self.assertEqual(s[1],p.path2)
3368             self.assertTrue(len(s._tasks) == 2)
3369             self.assertTrue(p.task1 in s._tasks)
3370             self.assertTrue(p.task2 in s._tasks)
3371             listOfTasks = list(s._tasks)
3372             self.assertTrue(len(listOfTasks) == 2)
3373             self.assertTrue(p.task1 == listOfTasks[0])
3374             self.assertTrue(p.task2 == listOfTasks[1])
3375             p.schedule = s
3376             self.assertTrue('b' in p.schedule.moduleNames())
3377 
3378             process2 = Process("test")
3379             process2.a = EDAnalyzer("MyAnalyzer")
3380             process2.e = EDProducer("eProducer")
3381             process2.path1 = Path(process2.a)
3382             process2.task1 = Task(process2.e)
3383             process2.schedule = Schedule(process2.path1,tasks=process2.task1)
3384             listOfTasks = list(process2.schedule._tasks)
3385             self.assertTrue(listOfTasks[0] == process2.task1)
3386 
3387             # test Schedule copy
3388             s2 = s.copy()
3389             self.assertEqual(s2[0],p.path1)
3390             self.assertEqual(s2[1],p.path2)
3391             self.assertTrue(len(s2._tasks) == 2)
3392             self.assertTrue(p.task1 in s2._tasks)
3393             self.assertTrue(p.task2 in s2._tasks)
3394             listOfTasks = list(s2._tasks)
3395             self.assertTrue(len(listOfTasks) == 2)
3396             self.assertTrue(p.task1 == listOfTasks[0])
3397             self.assertTrue(p.task2 == listOfTasks[1])
3398 
3399             names = s.moduleNames()
3400             self.assertTrue(names == set(['a', 'b', 'e', 'Tracer', 'f']))
3401             #adding a path not attached to the Process should cause an exception
3402             p = Process("test")
3403             p.a = EDAnalyzer("MyAnalyzer")
3404             path1 = Path(p.a)
3405             s = Schedule(path1)
3406             self.assertRaises(RuntimeError, lambda : p.setSchedule_(s) )
3407 
3408             #make sure anonymous sequences work
3409             p = Process("test")
3410             p.a = EDAnalyzer("MyAnalyzer")
3411             p.b = EDAnalyzer("MyOtherAnalyzer")
3412             p.c = EDProducer("MyProd")
3413             path1 = Path(p.c*Sequence(p.a+p.b))
3414             s = Schedule(path1)
3415             self.assertTrue('a' in s.moduleNames())
3416             self.assertTrue('b' in s.moduleNames())
3417             self.assertTrue('c' in s.moduleNames())
3418             p.path1 = path1
3419             p.schedule = s
3420             p.prune()
3421             self.assertTrue('a' in s.moduleNames())
3422             self.assertTrue('b' in s.moduleNames())
3423             self.assertTrue('c' in s.moduleNames())
3424 
3425         def testImplicitSchedule(self):
3426             p = Process("test")
3427             p.a = EDAnalyzer("MyAnalyzer")
3428             p.b = EDAnalyzer("YourAnalyzer")
3429             p.c = EDAnalyzer("OurAnalyzer")
3430             p.path1 = Path(p.a)
3431             p.path2 = Path(p.b)
3432             self.assertTrue(p.schedule is None)
3433             pths = p.paths
3434             keys = pths.keys()
3435             self.assertEqual(pths[keys[0]],p.path1)
3436             self.assertEqual(pths[keys[1]],p.path2)
3437             p.prune()
3438             self.assertTrue(hasattr(p, 'a'))
3439             self.assertTrue(hasattr(p, 'b'))
3440             self.assertTrue(not hasattr(p, 'c'))
3441             self.assertTrue(hasattr(p, 'path1'))
3442             self.assertTrue(hasattr(p, 'path2'))
3443 
3444 
3445             p = Process("test")
3446             p.a = EDAnalyzer("MyAnalyzer")
3447             p.b = EDAnalyzer("YourAnalyzer")
3448             p.c = EDAnalyzer("OurAnalyzer")
3449             p.path2 = Path(p.b)
3450             p.path1 = Path(p.a)
3451             self.assertTrue(p.schedule is None)
3452             pths = p.paths
3453             keys = pths.keys()
3454             self.assertEqual(pths[keys[1]],p.path1)
3455             self.assertEqual(pths[keys[0]],p.path2)
3456 
3457 
3458         def testUsing(self):
3459             p = Process('test')
3460             p.block = PSet(a = int32(1))
3461             p.modu = EDAnalyzer('Analyzer', p.block, b = int32(2))
3462             self.assertEqual(p.modu.a.value(),1)
3463             self.assertEqual(p.modu.b.value(),2)
3464 
3465         def testOverride(self):
3466             p = Process('test')
3467             a = EDProducer("A", a1=int32(0))
3468             self.assertTrue(not a.isModified())
3469             a.a1 = 1
3470             self.assertTrue(a.isModified())
3471             p.a = a
3472             self.assertEqual(p.a.a1.value(), 1)
3473             # try adding an unmodified module.
3474             # should accept it
3475             p.a = EDProducer("A", a1=int32(2))
3476             self.assertEqual(p.a.a1.value(), 2)
3477             # try adding a modified module.  Should throw
3478             # no longer, since the same (modified) say, geometry
3479             # could come from more than one cff
3480             b = EDProducer("A", a1=int32(3))
3481             b.a1 = 4
3482             #self.assertRaises(RuntimeError, setattr, *(p,'a',b))
3483             ps1 = PSet(a = int32(1))
3484             ps2 = PSet(a = int32(2))
3485             self.assertRaises(ValueError, EDProducer, 'C', ps1, ps2)
3486             self.assertRaises(ValueError, EDProducer, 'C', ps1, a=int32(3))
3487         
3488         def testOptions(self):
3489             p = Process('test')
3490             self.assertEqual(p.options.numberOfThreads.value(),1)
3491             p.options.numberOfThreads = 8
3492             self.assertEqual(p.options.numberOfThreads.value(),8)
3493             p.options = PSet()
3494             self.assertEqual(p.options.numberOfThreads.value(),1)
3495             p.options = dict(numberOfStreams =2,
3496                              numberOfThreads =2)
3497             self.assertEqual(p.options.numberOfThreads.value(),2)
3498             self.assertEqual(p.options.numberOfStreams.value(),2)
3499 
3500         def testMaxEvents(self):
3501             p = Process("Test")
3502             p.maxEvents.input = 10
3503             self.assertEqual(p.maxEvents.input.value(),10)
3504             p = Process("Test")
3505             p.maxEvents.output = 10
3506             self.assertEqual(p.maxEvents.output.value(),10)
3507             p = Process("Test")
3508             p.maxEvents.output = PSet(out=untracked.int32(10))
3509             self.assertEqual(p.maxEvents.output.out.value(), 10)
3510             p = Process("Test")
3511             p.maxEvents = untracked.PSet(input = untracked.int32(5))
3512             self.assertEqual(p.maxEvents.input.value(), 5)
3513 
3514         
3515         def testExamples(self):
3516             p = Process("Test")
3517             p.source = Source("PoolSource",fileNames = untracked(string("file:reco.root")))
3518             p.foos = EDProducer("FooProducer")
3519             p.bars = EDProducer("BarProducer", foos=InputTag("foos"))
3520             p.out = OutputModule("PoolOutputModule",fileName=untracked(string("file:foos.root")))
3521             p.bars.foos = 'Foosball'
3522             self.assertEqual(p.bars.foos, InputTag('Foosball'))
3523             p.p = Path(p.foos*p.bars)
3524             p.e = EndPath(p.out)
3525             p.add_(Service("MessageLogger"))
3526 
3527         def testPrefers(self):
3528             p = Process("Test")
3529             p.add_(ESSource("ForceSource"))
3530             p.juicer = ESProducer("JuicerProducer")
3531             p.prefer("ForceSource")
3532             p.prefer("juicer")
3533             self.assertEqual(_lineDiff(p.dumpPython(), Process('Test').dumpPython()),
3534 """process.juicer = cms.ESProducer("JuicerProducer")
3535 process.ForceSource = cms.ESSource("ForceSource")
3536 process.prefer("ForceSource")
3537 process.prefer("juicer")""")
3538             p.prefer("juicer",fooRcd=vstring("Foo"))
3539             self.assertEqual(_lineDiff(p.dumpPython(), Process('Test').dumpPython()),
3540 """process.juicer = cms.ESProducer("JuicerProducer")
3541 process.ForceSource = cms.ESSource("ForceSource")
3542 process.prefer("ForceSource")
3543 process.prefer("juicer",
3544     fooRcd = cms.vstring('Foo')
3545 )""")
3546 
3547         def testFreeze(self):
3548             process = Process("Freeze")
3549             m = EDProducer("M", p=PSet(i = int32(1)))
3550             m.p.i = 2
3551             process.m = m
3552             # should be frozen
3553             #self.assertRaises(ValueError, setattr, m.p, 'i', 3)
3554             #self.assertRaises(ValueError, setattr, m, 'p', PSet(i=int32(1)))
3555             #self.assertRaises(ValueError, setattr, m.p, 'j', 1)
3556             #self.assertRaises(ValueError, setattr, m, 'j', 1)
3557             # But OK to change through the process
3558             process.m.p.i = 4
3559             self.assertEqual(process.m.p.i.value(), 4)
3560             process.m.p = PSet(j=int32(1))
3561             # should work to clone it, though
3562             m2 = m.clone(p = PSet(i = int32(5)), j = int32(8))
3563             m2.p.i = 6
3564             m2.j = 8
3565         def testSubProcess(self):
3566             process = Process("Parent")
3567             subProcess = Process("Child")
3568             subProcess.a = EDProducer("A")
3569             subProcess.p = Path(subProcess.a)
3570             subProcess.add_(Service("Foo"))
3571             process.addSubProcess(SubProcess(subProcess))
3572             d = process.dumpPython()
3573             equalD ="""parentProcess = process
3574 process.a = cms.EDProducer("A")
3575 process.Foo = cms.Service("Foo")
3576 process.p = cms.Path(process.a)
3577 childProcess = process
3578 process = parentProcess
3579 process.addSubProcess(cms.SubProcess(process = childProcess, SelectEvents = cms.untracked.PSet(
3580 ), outputCommands = cms.untracked.vstring()))"""
3581             equalD = equalD.replace("parentProcess","parentProcess"+str(hash(process.subProcesses_()[0])))
3582             # SubProcesses are dumped before Services, so in order to
3583             # craft the dump of the Parent and Child manually the dump
3584             # of the Parent needs to be split at the MessageLogger
3585             # boundary (now when it is part of Process by default),
3586             # and insert the dump of the Child between the top part of
3587             # the Parent (before MessageLogger) and the bottom part of
3588             # the Parent (after and including MessageLogger)
3589             messageLoggerSplit = 'process.MessageLogger = cms.Service'
3590             parentDumpSplit = Process('Parent').dumpPython().split(messageLoggerSplit)
3591             childProcess = Process('Child')
3592             del childProcess.MessageLogger
3593             combinedDump = parentDumpSplit[0] + childProcess.dumpPython() + messageLoggerSplit + parentDumpSplit[1]
3594             self.assertEqual(_lineDiff(d, combinedDump), equalD)
3595             p = TestMakePSet()
3596             process.fillProcessDesc(p)
3597             self.assertEqual((True,['a']),p.values["subProcesses"][1][0].values["process"][1].values['@all_modules'])
3598             self.assertEqual((True,['p']),p.values["subProcesses"][1][0].values["process"][1].values['@paths'])
3599             self.assertEqual({'@service_type':(True,'Foo')}, p.values["subProcesses"][1][0].values["process"][1].values["services"][1][0].values)
3600         def testRefToPSet(self):
3601             proc = Process("test")
3602             proc.top = PSet(a = int32(1))
3603             proc.ref = PSet(refToPSet_ = string("top"))
3604             proc.ref2 = PSet( a = int32(1), b = PSet( refToPSet_ = string("top")))
3605             proc.ref3 = PSet(refToPSet_ = string("ref"))
3606             proc.ref4 = VPSet(PSet(refToPSet_ = string("top")),
3607                               PSet(refToPSet_ = string("ref2")))
3608             p = TestMakePSet()
3609             proc.fillProcessDesc(p)
3610             self.assertEqual((True,1),p.values["ref"][1].values["a"])
3611             self.assertEqual((True,1),p.values["ref3"][1].values["a"])
3612             self.assertEqual((True,1),p.values["ref2"][1].values["a"])
3613             self.assertEqual((True,1),p.values["ref2"][1].values["b"][1].values["a"])
3614             self.assertEqual((True,1),p.values["ref4"][1][0].values["a"])
3615             self.assertEqual((True,1),p.values["ref4"][1][1].values["a"])
3616         def testSwitchProducer(self):
3617             proc = Process("test")
3618             proc.sp = SwitchProducerTest(test2 = EDProducer("Foo",
3619                                                             a = int32(1),
3620                                                             b = PSet(c = int32(2))),
3621                                          test1 = EDProducer("Bar",
3622                                                             aa = int32(11),
3623                                                             bb = PSet(cc = int32(12))))
3624             self.assertEqual(proc.sp.label_(), "sp")
3625             self.assertEqual(proc.sp.test1.label_(), "sp@test1")
3626             self.assertEqual(proc.sp.test2.label_(), "sp@test2")
3627 
3628             proc.a = EDProducer("A")
3629             proc.s = Sequence(proc.a + proc.sp)
3630             proc.t = Task(proc.a, proc.sp)
3631             proc.p = Path()
3632             proc.p.associate(proc.t)
3633             p = TestMakePSet()
3634             proc.fillProcessDesc(p)
3635             self.assertEqual((True,"EDProducer"), p.values["sp"][1].values["@module_edm_type"])
3636             self.assertEqual((True, "SwitchProducer"), p.values["sp"][1].values["@module_type"])
3637             self.assertEqual((True, "sp"), p.values["sp"][1].values["@module_label"])
3638             all_cases = copy.deepcopy(p.values["sp"][1].values["@all_cases"])
3639             all_cases[1].sort() # names of all cases come via dict, i.e. their order is undefined
3640             self.assertEqual((True, ["sp@test1", "sp@test2"]), all_cases)
3641             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
3642             self.assertEqual(["a", "sp", "sp@test1", "sp@test2"], p.values["@all_modules"][1])
3643             self.assertEqual((True,"EDProducer"), p.values["sp@test1"][1].values["@module_edm_type"])
3644             self.assertEqual((True,"Bar"), p.values["sp@test1"][1].values["@module_type"])
3645             self.assertEqual((True,"EDProducer"), p.values["sp@test2"][1].values["@module_edm_type"])
3646             self.assertEqual((True,"Foo"), p.values["sp@test2"][1].values["@module_type"])
3647             dump = proc.dumpPython()
3648             self.assertEqual(dump.find('@'), -1)
3649             self.assertEqual(specialImportRegistry.getSpecialImports(), ["from test import SwitchProducerTest"])
3650             self.assertTrue(dump.find("\nfrom test import SwitchProducerTest\n") != -1)
3651 
3652             # EDAlias as non-chosen case
3653             proc = Process("test")
3654             proc.sp = SwitchProducerTest(test2 = EDProducer("Foo",
3655                                                             a = int32(1),
3656                                                             b = PSet(c = int32(2))),
3657                                          test1 = EDAlias(a = VPSet(PSet(type = string("Bar")))))
3658             proc.a = EDProducer("A")
3659             proc.s = Sequence(proc.a + proc.sp)
3660             proc.t = Task(proc.a, proc.sp)
3661             proc.p = Path()
3662             proc.p.associate(proc.t)
3663             p = TestMakePSet()
3664             proc.fillProcessDesc(p)
3665             self.assertEqual((True,"EDProducer"), p.values["sp"][1].values["@module_edm_type"])
3666             self.assertEqual((True, "SwitchProducer"), p.values["sp"][1].values["@module_type"])
3667             self.assertEqual((True, "sp"), p.values["sp"][1].values["@module_label"])
3668             all_cases = copy.deepcopy(p.values["sp"][1].values["@all_cases"])
3669             all_cases[1].sort()
3670             self.assertEqual((True, ["sp@test1", "sp@test2"]), all_cases)
3671             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
3672             self.assertEqual(["a", "sp", "sp@test2"], p.values["@all_modules"][1])
3673             self.assertEqual(["sp@test1"], p.values["@all_aliases"][1])
3674             self.assertEqual((True,"EDProducer"), p.values["sp@test2"][1].values["@module_edm_type"])
3675             self.assertEqual((True,"Foo"), p.values["sp@test2"][1].values["@module_type"])
3676             self.assertEqual((True,"EDAlias"), p.values["sp@test1"][1].values["@module_edm_type"])
3677             self.assertEqual((True,"Bar"), p.values["sp@test1"][1].values["a"][1][0].values["type"])
3678 
3679             # EDAlias as chosen case
3680             proc = Process("test")
3681             proc.sp = SwitchProducerTest(test1 = EDProducer("Foo",
3682                                                             a = int32(1),
3683                                                             b = PSet(c = int32(2))),
3684                                          test2 = EDAlias(a = VPSet(PSet(type = string("Bar")))))
3685             proc.a = EDProducer("A")
3686             proc.s = Sequence(proc.a + proc.sp)
3687             proc.t = Task(proc.a, proc.sp)
3688             proc.p = Path()
3689             proc.p.associate(proc.t)
3690             p = TestMakePSet()
3691             proc.fillProcessDesc(p)
3692             self.assertEqual((True,"EDProducer"), p.values["sp"][1].values["@module_edm_type"])
3693             self.assertEqual((True, "SwitchProducer"), p.values["sp"][1].values["@module_type"])
3694             self.assertEqual((True, "sp"), p.values["sp"][1].values["@module_label"])
3695             self.assertEqual((True, ["sp@test1", "sp@test2"]), p.values["sp"][1].values["@all_cases"])
3696             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
3697             self.assertEqual(["a", "sp", "sp@test1"], p.values["@all_modules"][1])
3698             self.assertEqual(["sp@test2"], p.values["@all_aliases"][1])
3699             self.assertEqual((True,"EDProducer"), p.values["sp@test1"][1].values["@module_edm_type"])
3700             self.assertEqual((True,"Foo"), p.values["sp@test1"][1].values["@module_type"])
3701             self.assertEqual((True,"EDAlias"), p.values["sp@test2"][1].values["@module_edm_type"])
3702             self.assertEqual((True,"Bar"), p.values["sp@test2"][1].values["a"][1][0].values["type"])
3703 
3704             # ConditionalTask
3705             proc = Process("test")
3706             proc.spct = SwitchProducerTest(test2 = EDProducer("Foo",
3707                                                               a = int32(1),
3708                                                               b = PSet(c = int32(2))),
3709                                            test1 = EDProducer("Bar",
3710                                                               aa = int32(11),
3711                                                               bb = PSet(cc = int32(12))),
3712                                            test3 = EDAlias(a = VPSet(PSet(type = string("Bar")))))
3713             proc.spp = proc.spct.clone()
3714             proc.a = EDProducer("A")
3715             proc.ct = ConditionalTask(proc.spct)
3716             proc.p = Path(proc.a, proc.ct)
3717             proc.pp = Path(proc.a + proc.spp)
3718             p = TestMakePSet()
3719             proc.fillProcessDesc(p)
3720             self.assertEqual(["a", "spct", "spct@test1", "spct@test2", "spp", "spp@test1", "spp@test2"], p.values["@all_modules"][1])
3721             self.assertEqual(["a", "#", "spct", "spct@test1", "spct@test2", "@"], p.values["p"][1])
3722             self.assertEqual(["a", "spp", "#", "spp@test1", "spp@test2", "@"], p.values["pp"][1])
3723 
3724         def testPrune(self):
3725             p = Process("test")
3726             p.a = EDAnalyzer("MyAnalyzer")
3727             p.b = EDAnalyzer("YourAnalyzer")
3728             p.c = EDAnalyzer("OurAnalyzer")
3729             p.d = EDAnalyzer("OurAnalyzer")
3730             p.e = EDProducer("MyProducer")
3731             p.f = EDProducer("YourProducer")
3732             p.g = EDProducer("TheirProducer")
3733             p.h = EDProducer("OnesProducer")
3734             p.s = Sequence(p.d)
3735             p.t1 = Task(p.e)
3736             p.t2 = Task(p.f)
3737             p.t3 = Task(p.g, p.t1)
3738             p.ct1 = ConditionalTask(p.h)
3739             p.ct2 = ConditionalTask(p.f)
3740             p.ct3 = ConditionalTask(p.ct1)
3741             p.path1 = Path(p.a, p.t3, p.ct3)
3742             p.path2 = Path(p.b)
3743             self.assertTrue(p.schedule is None)
3744             pths = p.paths
3745             keys = pths.keys()
3746             self.assertEqual(pths[keys[0]],p.path1)
3747             self.assertEqual(pths[keys[1]],p.path2)
3748             p.pset1 = PSet(parA = string("pset1"))
3749             p.pset2 = untracked.PSet(parA = string("pset2"))
3750             p.vpset1 = VPSet()
3751             p.vpset2 = untracked.VPSet()
3752             p.prune()
3753             self.assertTrue(hasattr(p, 'a'))
3754             self.assertTrue(hasattr(p, 'b'))
3755             self.assertTrue(not hasattr(p, 'c'))
3756             self.assertTrue(not hasattr(p, 'd'))
3757             self.assertTrue(hasattr(p, 'e'))
3758             self.assertTrue(not hasattr(p, 'f'))
3759             self.assertTrue(hasattr(p, 'g'))
3760             self.assertTrue(hasattr(p, 'h'))
3761             self.assertTrue(not hasattr(p, 's'))
3762             self.assertTrue(hasattr(p, 't1'))
3763             self.assertTrue(not hasattr(p, 't2'))
3764             self.assertTrue(hasattr(p, 't3'))
3765             self.assertTrue(hasattr(p, 'path1'))
3766             self.assertTrue(hasattr(p, 'path2'))
3767 #            self.assertTrue(not hasattr(p, 'pset1'))
3768 #            self.assertTrue(hasattr(p, 'pset2'))
3769 #            self.assertTrue(not hasattr(p, 'vpset1'))
3770 #            self.assertTrue(not hasattr(p, 'vpset2'))
3771 
3772             p = Process("test")
3773             p.a = EDAnalyzer("MyAnalyzer")
3774             p.b = EDAnalyzer("YourAnalyzer")
3775             p.c = EDAnalyzer("OurAnalyzer")
3776             p.d = EDAnalyzer("OurAnalyzer")
3777             p.e = EDAnalyzer("OurAnalyzer")
3778             p.f = EDProducer("MyProducer")
3779             p.g = EDProducer("YourProducer")
3780             p.h = EDProducer("TheirProducer")
3781             p.i = EDProducer("OurProducer")
3782             p.j = EDProducer("OurProducer")
3783             p.k = EDProducer("OurProducer")
3784             p.l = EDProducer("OurProducer")
3785             p.t1 = Task(p.f)
3786             p.t2 = Task(p.g)
3787             p.t3 = Task(p.h)
3788             p.t4 = Task(p.i)
3789             p.ct1 = Task(p.f)
3790             p.ct2 = Task(p.j)
3791             p.ct3 = Task(p.k)
3792             p.ct4 = Task(p.l)
3793             p.s = Sequence(p.d, p.t1, p.ct1)
3794             p.s2 = Sequence(p.b, p.t2, p.ct2)
3795             p.s3 = Sequence(p.e)
3796             p.path1 = Path(p.a, p.t3, p.ct3)
3797             p.path2 = Path(p.b)
3798             p.path3 = Path(p.b+p.s2)
3799             p.path4 = Path(p.b+p.s3)
3800             p.schedule = Schedule(p.path1,p.path2,p.path3)
3801             p.schedule.associate(p.t4)
3802             pths = p.paths
3803             keys = pths.keys()
3804             self.assertEqual(pths[keys[0]],p.path1)
3805             self.assertEqual(pths[keys[1]],p.path2)
3806             p.prune()
3807             self.assertTrue(hasattr(p, 'a'))
3808             self.assertTrue(hasattr(p, 'b'))
3809             self.assertTrue(not hasattr(p, 'c'))
3810             self.assertTrue(not hasattr(p, 'd'))
3811             self.assertTrue(not hasattr(p, 'e'))
3812             self.assertTrue(not hasattr(p, 'f'))
3813             self.assertTrue(hasattr(p, 'g'))
3814             self.assertTrue(hasattr(p, 'h'))
3815             self.assertTrue(hasattr(p, 'i'))
3816             self.assertTrue(hasattr(p, 'j'))
3817             self.assertTrue(hasattr(p, 'k'))
3818             self.assertTrue(not hasattr(p, 'l'))
3819             self.assertTrue(not hasattr(p, 't1'))
3820             self.assertTrue(hasattr(p, 't2'))
3821             self.assertTrue(hasattr(p, 't3'))
3822             self.assertTrue(hasattr(p, 't4'))
3823             self.assertTrue(not hasattr(p, 'ct1'))
3824             self.assertTrue(hasattr(p, 'ct2'))
3825             self.assertTrue(hasattr(p, 'ct3'))
3826             self.assertTrue(not hasattr(p, 'ct4'))
3827             self.assertTrue(not hasattr(p, 's'))
3828             self.assertTrue(hasattr(p, 's2'))
3829             self.assertTrue(not hasattr(p, 's3'))
3830             self.assertTrue(hasattr(p, 'path1'))
3831             self.assertTrue(hasattr(p, 'path2'))
3832             self.assertTrue(hasattr(p, 'path3'))
3833             self.assertTrue(not hasattr(p, 'path4'))
3834             #test SequencePlaceholder
3835             p = Process("test")
3836             p.a = EDAnalyzer("MyAnalyzer")
3837             p.b = EDAnalyzer("YourAnalyzer")
3838             p.s = Sequence(SequencePlaceholder("a")+p.b)
3839             p.pth = Path(p.s)
3840             p.prune()
3841             self.assertTrue(hasattr(p, 'a'))
3842             self.assertTrue(hasattr(p, 'b'))
3843             self.assertTrue(hasattr(p, 's'))
3844             self.assertTrue(hasattr(p, 'pth'))
3845             #test unresolved SequencePlaceholder
3846             p = Process("test")
3847             p.b = EDAnalyzer("YourAnalyzer")
3848             p.s = Sequence(SequencePlaceholder("a")+p.b)
3849             p.pth = Path(p.s)
3850             p.prune(keepUnresolvedSequencePlaceholders=True)
3851             self.assertTrue(hasattr(p, 'b'))
3852             self.assertTrue(hasattr(p, 's'))
3853             self.assertTrue(hasattr(p, 'pth'))
3854             self.assertEqual(p.s.dumpPython(),'cms.Sequence(cms.SequencePlaceholder("a")+process.b)\n')
3855             #test TaskPlaceholder
3856             p = Process("test")
3857             p.a = EDProducer("MyProducer")
3858             p.b = EDProducer("YourProducer")
3859             p.s = Task(TaskPlaceholder("a"),p.b)
3860             p.pth = Path(p.s)
3861             p.prune()
3862             self.assertTrue(hasattr(p, 'a'))
3863             self.assertTrue(hasattr(p, 'b'))
3864             self.assertTrue(hasattr(p, 's'))
3865             self.assertTrue(hasattr(p, 'pth'))
3866             #test ConditionalTaskPlaceholder
3867             p = Process("test")
3868             p.a = EDProducer("MyProducer")
3869             p.b = EDProducer("YourProducer")
3870             p.s = ConditionalTask(ConditionalTaskPlaceholder("a"),p.b)
3871             p.pth = Path(p.s)
3872             p.prune()
3873             self.assertTrue(hasattr(p, 'a'))
3874             self.assertTrue(hasattr(p, 'b'))
3875             self.assertTrue(hasattr(p, 's'))
3876             self.assertTrue(hasattr(p, 'pth'))
3877             #test unresolved SequencePlaceholder
3878             p = Process("test")
3879             p.b = EDProducer("YourAnalyzer")
3880             p.s = Task(TaskPlaceholder("a"),p.b)
3881             p.pth = Path(p.s)
3882             p.prune(keepUnresolvedSequencePlaceholders=True)
3883             self.assertTrue(hasattr(p, 'b'))
3884             self.assertTrue(hasattr(p, 's'))
3885             self.assertTrue(hasattr(p, 'pth'))
3886             self.assertEqual(p.s.dumpPython(),'cms.Task(cms.TaskPlaceholder("a"), process.b)\n')
3887         def testTaskPlaceholder(self):
3888             p = Process("test")
3889             p.a = EDProducer("ma")
3890             p.b = EDAnalyzer("mb")
3891             p.t1 = Task(TaskPlaceholder("c"))
3892             p.t2 = Task(p.a, TaskPlaceholder("d"), p.t1)
3893             p.t3 = Task(TaskPlaceholder("e"))
3894             p.path1 = Path(p.b, p.t2, p.t3)
3895             p.t5 = Task(p.a, TaskPlaceholder("g"), TaskPlaceholder("t4"))
3896             p.t4 = Task(TaskPlaceholder("f"))
3897             p.endpath1 = EndPath(p.b, p.t5)
3898             p.t6 = Task(TaskPlaceholder("h"))
3899             p.t7 = Task(p.a, TaskPlaceholder("i"), p.t6)
3900             p.t8 = Task(TaskPlaceholder("j"))
3901             p.schedule = Schedule(p.path1, p.endpath1,tasks=[p.t7,p.t8])
3902             p.c = EDProducer("mc")
3903             p.d = EDProducer("md")
3904             p.e = EDProducer("me")
3905             p.f = EDProducer("mf")
3906             p.g = EDProducer("mg")
3907             p.h = EDProducer("mh")
3908             p.i = EDProducer("mi")
3909             p.j = EDProducer("mj")
3910             self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
3911 """process.a = cms.EDProducer("ma")
3912 process.c = cms.EDProducer("mc")
3913 process.d = cms.EDProducer("md")
3914 process.e = cms.EDProducer("me")
3915 process.f = cms.EDProducer("mf")
3916 process.g = cms.EDProducer("mg")
3917 process.h = cms.EDProducer("mh")
3918 process.i = cms.EDProducer("mi")
3919 process.j = cms.EDProducer("mj")
3920 process.b = cms.EDAnalyzer("mb")
3921 process.t1 = cms.Task(cms.TaskPlaceholder("c"))
3922 process.t2 = cms.Task(cms.TaskPlaceholder("d"), process.a, process.t1)
3923 process.t3 = cms.Task(cms.TaskPlaceholder("e"))
3924 process.t5 = cms.Task(cms.TaskPlaceholder("g"), cms.TaskPlaceholder("t4"), process.a)
3925 process.t4 = cms.Task(cms.TaskPlaceholder("f"))
3926 process.t6 = cms.Task(cms.TaskPlaceholder("h"))
3927 process.t7 = cms.Task(cms.TaskPlaceholder("i"), process.a, process.t6)
3928 process.t8 = cms.Task(cms.TaskPlaceholder("j"))
3929 process.path1 = cms.Path(process.b, process.t2, process.t3)
3930 process.endpath1 = cms.EndPath(process.b, process.t5)
3931 process.schedule = cms.Schedule(*[ process.path1, process.endpath1 ], tasks=[process.t7, process.t8])""")
3932             p.resolve()
3933             self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
3934 """process.a = cms.EDProducer("ma")
3935 process.c = cms.EDProducer("mc")
3936 process.d = cms.EDProducer("md")
3937 process.e = cms.EDProducer("me")
3938 process.f = cms.EDProducer("mf")
3939 process.g = cms.EDProducer("mg")
3940 process.h = cms.EDProducer("mh")
3941 process.i = cms.EDProducer("mi")
3942 process.j = cms.EDProducer("mj")
3943 process.b = cms.EDAnalyzer("mb")
3944 process.t1 = cms.Task(process.c)
3945 process.t2 = cms.Task(process.a, process.d, process.t1)
3946 process.t3 = cms.Task(process.e)
3947 process.t4 = cms.Task(process.f)
3948 process.t6 = cms.Task(process.h)
3949 process.t7 = cms.Task(process.a, process.i, process.t6)
3950 process.t8 = cms.Task(process.j)
3951 process.t5 = cms.Task(process.a, process.g, process.t4)
3952 process.path1 = cms.Path(process.b, process.t2, process.t3)
3953 process.endpath1 = cms.EndPath(process.b, process.t5)
3954 process.schedule = cms.Schedule(*[ process.path1, process.endpath1 ], tasks=[process.t7, process.t8])""")
3955         def testConditionalTaskPlaceholder(self):
3956             p = Process("test")
3957             p.a = EDProducer("ma")
3958             p.b = EDAnalyzer("mb")
3959             p.t1 = ConditionalTask(ConditionalTaskPlaceholder("c"))
3960             p.t2 = ConditionalTask(p.a, ConditionalTaskPlaceholder("d"), p.t1)
3961             p.t3 = ConditionalTask(ConditionalTaskPlaceholder("e"))
3962             p.path1 = Path(p.b, p.t2, p.t3)
3963             p.t5 = ConditionalTask(p.a, ConditionalTaskPlaceholder("g"), ConditionalTaskPlaceholder("t4"))
3964             p.t4 = ConditionalTask(ConditionalTaskPlaceholder("f"))
3965             p.path2 = Path(p.b, p.t5)
3966             p.schedule = Schedule(p.path1, p.path2)
3967             p.c = EDProducer("mc")
3968             p.d = EDProducer("md")
3969             p.e = EDProducer("me")
3970             p.f = EDProducer("mf")
3971             p.g = EDProducer("mg")
3972             p.h = EDProducer("mh")
3973             p.i = EDProducer("mi")
3974             p.j = EDProducer("mj")
3975             self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
3976 """process.a = cms.EDProducer("ma")
3977 process.c = cms.EDProducer("mc")
3978 process.d = cms.EDProducer("md")
3979 process.e = cms.EDProducer("me")
3980 process.f = cms.EDProducer("mf")
3981 process.g = cms.EDProducer("mg")
3982 process.h = cms.EDProducer("mh")
3983 process.i = cms.EDProducer("mi")
3984 process.j = cms.EDProducer("mj")
3985 process.b = cms.EDAnalyzer("mb")
3986 process.t1 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("c"))
3987 process.t2 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("d"), process.a, process.t1)
3988 process.t3 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("e"))
3989 process.t5 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("g"), cms.ConditionalTaskPlaceholder("t4"), process.a)
3990 process.t4 = cms.ConditionalTask(cms.ConditionalTaskPlaceholder("f"))
3991 process.path1 = cms.Path(process.b, process.t2, process.t3)
3992 process.path2 = cms.Path(process.b, process.t5)
3993 process.schedule = cms.Schedule(*[ process.path1, process.path2 ])""")
3994             p.resolve()
3995             self.assertEqual(_lineDiff(p.dumpPython(),Process('test').dumpPython()),
3996 """process.a = cms.EDProducer("ma")
3997 process.c = cms.EDProducer("mc")
3998 process.d = cms.EDProducer("md")
3999 process.e = cms.EDProducer("me")
4000 process.f = cms.EDProducer("mf")
4001 process.g = cms.EDProducer("mg")
4002 process.h = cms.EDProducer("mh")
4003 process.i = cms.EDProducer("mi")
4004 process.j = cms.EDProducer("mj")
4005 process.b = cms.EDAnalyzer("mb")
4006 process.t1 = cms.ConditionalTask(process.c)
4007 process.t2 = cms.ConditionalTask(process.a, process.d, process.t1)
4008 process.t3 = cms.ConditionalTask(process.e)
4009 process.t4 = cms.ConditionalTask(process.f)
4010 process.t5 = cms.ConditionalTask(process.a, process.g, process.t4)
4011 process.path1 = cms.Path(process.b, process.t2, process.t3)
4012 process.path2 = cms.Path(process.b, process.t5)
4013 process.schedule = cms.Schedule(*[ process.path1, process.path2 ])""")
4014 
4015         def testDelete(self):
4016             p = Process("test")
4017             p.a = EDAnalyzer("MyAnalyzer")
4018             p.b = EDAnalyzer("YourAnalyzer")
4019             p.c = EDAnalyzer("OurAnalyzer")
4020             p.d = EDAnalyzer("OurAnalyzer")
4021             p.e = EDAnalyzer("OurAnalyzer")
4022             p.f = EDAnalyzer("OurAnalyzer")
4023             p.g = EDProducer("OurProducer")
4024             p.h = EDProducer("YourProducer")
4025             p.i = SwitchProducerTest(
4026                 test1 = EDProducer("OneProducer"),
4027                 test2 = EDProducer("TwoProducer")
4028             )
4029             p.t1 = Task(p.g, p.h, p.i)
4030             t2 = Task(p.g, p.h, p.i)
4031             t3 = Task(p.g, p.h)
4032             p.t4 = Task(p.h)
4033             p.ct1 = ConditionalTask(p.g, p.h, p.i)
4034             ct2 = ConditionalTask(p.g, p.h)
4035             ct3 = ConditionalTask(p.g, p.h)
4036             p.ct4 = ConditionalTask(p.h)
4037             p.s = Sequence(p.d+p.e)
4038             p.path1 = Path(p.a+p.f+p.s,t2,ct2)
4039             p.path2 = Path(p.a)
4040             p.path3 = Path(ct3, p.ct4)
4041             p.endpath2 = EndPath(p.b)
4042             p.endpath1 = EndPath(p.b+p.f)
4043             p.schedule = Schedule(p.path2, p.path3, p.endpath2, tasks=[t3, p.t4])
4044             self.assertTrue(hasattr(p, 'f'))
4045             self.assertTrue(hasattr(p, 'g'))
4046             self.assertTrue(hasattr(p, 'i'))
4047             del p.e
4048             del p.f
4049             del p.g
4050             del p.i
4051             self.assertFalse(hasattr(p, 'f'))
4052             self.assertFalse(hasattr(p, 'g'))
4053             self.assertEqual(p.t1.dumpPython(), 'cms.Task(process.h)\n')
4054             self.assertEqual(p.ct1.dumpPython(), 'cms.ConditionalTask(process.h)\n')
4055             self.assertEqual(p.s.dumpPython(), 'cms.Sequence(process.d)\n')
4056             self.assertEqual(p.path1.dumpPython(), 'cms.Path(process.a+process.s, cms.ConditionalTask(process.h), cms.Task(process.h))\n')
4057             self.assertEqual(p.endpath1.dumpPython(), 'cms.EndPath(process.b)\n')
4058             self.assertEqual(p.path3.dumpPython(), 'cms.Path(cms.ConditionalTask(process.h), process.ct4)\n')
4059             del p.s
4060             self.assertEqual(p.path1.dumpPython(), 'cms.Path(process.a+(process.d), cms.ConditionalTask(process.h), cms.Task(process.h))\n')
4061             self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(*[ process.path2, process.path3, process.endpath2 ], tasks=[cms.Task(process.h), process.t4])\n')
4062             del p.path2
4063             self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(*[ process.path3, process.endpath2 ], tasks=[cms.Task(process.h), process.t4])\n')
4064             del p.path3
4065             self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(*[ process.endpath2 ], tasks=[cms.Task(process.h), process.t4])\n')
4066             del p.endpath2
4067             self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(tasks=[cms.Task(process.h), process.t4])\n')
4068             del p.t4
4069             self.assertEqual(p.schedule_().dumpPython(), 'cms.Schedule(tasks=[cms.Task(process.h)])\n')
4070         def testModifier(self):
4071             m1 = Modifier()
4072             Process._firstProcess = True
4073             p = Process("test")
4074             self.assertRaises(RuntimeError, lambda: Process("test2", m1))
4075             m1 = Modifier()
4076             Process._firstProcess = True
4077             p = Process("test",m1)
4078             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1))
4079             def _mod_fred(obj):
4080                 obj.fred = 2
4081             m1.toModify(p.a,_mod_fred)
4082             self.assertEqual(p.a.fred.value(),2)
4083             p.b = EDAnalyzer("YourAnalyzer", wilma = int32(1))
4084             m1.toModify(p.b, wilma = 2)
4085             self.assertEqual(p.b.wilma.value(),2)
4086             self.assertTrue(p.isUsingModifier(m1))
4087             #check that Modifier not attached to a process doesn't run
4088             m1 = Modifier()
4089             Process._firstProcess = True
4090             p = Process("test")
4091             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1))
4092             m1.toModify(p.a,_mod_fred)
4093             p.b = EDAnalyzer("YourAnalyzer", wilma = int32(1))
4094             m1.toModify(p.b, wilma = 2)
4095             self.assertEqual(p.a.fred.value(),1)
4096             self.assertEqual(p.b.wilma.value(),1)
4097             self.assertEqual(p.isUsingModifier(m1),False)
4098             #make sure clones get the changes
4099             m1 = Modifier()
4100             Process._firstProcess = True
4101             p = Process("test",m1)
4102             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4103             m1.toModify(p.a, fred = int32(2))
4104             p.b = p.a.clone(wilma = int32(3))
4105             self.assertEqual(p.a.fred.value(),2)
4106             self.assertEqual(p.a.wilma.value(),1)
4107             self.assertEqual(p.b.fred.value(),2)
4108             self.assertEqual(p.b.wilma.value(),3)
4109             #test removal of parameter
4110             m1 = Modifier()
4111             Process._firstProcess = True
4112             p = Process("test",m1)
4113             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1), fintstones = PSet(fred = int32(1)))
4114             m1.toModify(p.a, fred = None, fintstones = dict(fred = None))
4115             self.assertEqual(hasattr(p.a, "fred"), False)
4116             self.assertEqual(hasattr(p.a.fintstones, "fred"), False)
4117             self.assertEqual(p.a.wilma.value(),1)
4118             #test adding a parameter
4119             m1 = Modifier()
4120             Process._firstProcess = True
4121             p = Process("test",m1)
4122             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1))
4123             m1.toModify(p.a, wilma = int32(2))
4124             self.assertEqual(p.a.fred.value(), 1)
4125             self.assertEqual(p.a.wilma.value(),2)
4126             #test setting of value in PSet
4127             m1 = Modifier()
4128             Process._firstProcess = True
4129             p = Process("test",m1)
4130             p.a = EDAnalyzer("MyAnalyzer", flintstones = PSet(fred = int32(1), wilma = int32(1)))
4131             m1.toModify(p.a, flintstones = dict(fred = int32(2)))
4132             self.assertEqual(p.a.flintstones.fred.value(),2)
4133             self.assertEqual(p.a.flintstones.wilma.value(),1)
4134             #test proper exception from nonexisting parameter name
4135             m1 = Modifier()
4136             Process._firstProcess = True
4137             p = Process("test",m1)
4138             p.a = EDAnalyzer("MyAnalyzer", flintstones = PSet(fred = PSet(wilma = int32(1))))
4139             self.assertRaises(KeyError, lambda: m1.toModify(p.a, flintstones = dict(imnothere = dict(wilma=2))))
4140             self.assertRaises(KeyError, lambda: m1.toModify(p.a, foo = 1))
4141             #test setting a value in a VPSet
4142             m1 = Modifier()
4143             Process._firstProcess = True
4144             p = Process("test",m1)
4145             p.a = EDAnalyzer("MyAnalyzer", flintstones = VPSet(PSet(fred = int32(1)), PSet(wilma = int32(1))))
4146             m1.toModify(p.a, flintstones = {1:dict(wilma = int32(2))})
4147             self.assertEqual(p.a.flintstones[0].fred.value(),1)
4148             self.assertEqual(p.a.flintstones[1].wilma.value(),2)
4149             #test setting a value in a list of values
4150             m1 = Modifier()
4151             Process._firstProcess = True
4152             p = Process("test",m1)
4153             p.a = EDAnalyzer("MyAnalyzer", fred = vuint32(1,2,3))
4154             m1.toModify(p.a, fred = {1:7})
4155             self.assertEqual(p.a.fred[0],1)
4156             self.assertEqual(p.a.fred[1],7)
4157             self.assertEqual(p.a.fred[2],3)
4158             #test IndexError setting a value in a list to an item key not in the list
4159             m1 = Modifier()
4160             Process._firstProcess = True
4161             p = Process("test",m1)
4162             p.a = EDAnalyzer("MyAnalyzer", fred = vuint32(1,2,3))
4163             raised = False
4164             try: m1.toModify(p.a, fred = {5:7})
4165             except IndexError as e: raised = True
4166             self.assertEqual(raised, True)
4167             #test TypeError setting a value in a list using a key that is not an int
4168             m1 = Modifier()
4169             Process._firstProcess = True
4170             p = Process("test",m1)
4171             p.a = EDAnalyzer("MyAnalyzer", flintstones = VPSet(PSet(fred = int32(1)), PSet(wilma = int32(1))))
4172             raised = False
4173             try: m1.toModify(p.a, flintstones = dict(bogus = int32(37)))
4174             except TypeError as e: raised = True
4175             self.assertEqual(raised, True)
4176             #test that load causes process wide methods to run
4177             def _rem_a(proc):
4178                 del proc.a
4179             class ProcModifierMod(object):
4180                 def __init__(self,modifier,func):
4181                     self.proc_mod_ = modifier.makeProcessModifier(func)
4182             class DummyMod(object):
4183                 def __init__(self):
4184                     self.a = EDAnalyzer("Dummy")
4185             testMod = DummyMod()
4186             p.extend(testMod)
4187             self.assertTrue(hasattr(p,"a"))
4188             m1 = Modifier()
4189             Process._firstProcess = True
4190             p = Process("test",m1)
4191             testProcMod = ProcModifierMod(m1,_rem_a)
4192             p.extend(testMod)
4193             p.extend(testProcMod)
4194             self.assertTrue(not hasattr(p,"a"))
4195             #test ModifierChain
4196             m1 = Modifier()
4197             mc = ModifierChain(m1)
4198             Process._firstProcess = True
4199             p = Process("test",mc)
4200             self.assertTrue(p.isUsingModifier(m1))
4201             self.assertTrue(p.isUsingModifier(mc))
4202             testMod = DummyMod()
4203             p.b = EDAnalyzer("Dummy2", fred = int32(1))
4204             m1.toModify(p.b, fred = int32(3))
4205             p.extend(testMod)
4206             testProcMod = ProcModifierMod(m1,_rem_a)
4207             p.extend(testProcMod)
4208             self.assertTrue(not hasattr(p,"a"))
4209             self.assertEqual(p.b.fred.value(),3)
4210             #check cloneAndExclude
4211             m1 = Modifier()
4212             m2 = Modifier()
4213             mc = ModifierChain(m1,m2)
4214             mclone = mc.copyAndExclude([m2])
4215             self.assertTrue(not mclone._isOrContains(m2))
4216             self.assertTrue(mclone._isOrContains(m1))
4217             m3 = Modifier()
4218             mc2 = ModifierChain(mc,m3)
4219             mclone = mc2.copyAndExclude([m2])
4220             self.assertTrue(not mclone._isOrContains(m2))
4221             self.assertTrue(mclone._isOrContains(m1))
4222             self.assertTrue(mclone._isOrContains(m3))
4223             #check combining
4224             m1 = Modifier()
4225             m2 = Modifier()
4226             Process._firstProcess = True
4227             p = Process("test",m1)
4228             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4229             (m1 & m2).toModify(p.a, fred = int32(2))
4230             self.assertRaises(TypeError, lambda: (m1 & m2).toModify(p.a, 1, wilma=2))
4231             self.assertEqual(p.a.fred, 1)
4232             m1 = Modifier()
4233             m2 = Modifier()
4234             Process._firstProcess = True
4235             p = Process("test",m1,m2)
4236             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4237             (m1 & m2).toModify(p.a, fred = int32(2))
4238             self.assertEqual(p.a.fred, 2)
4239             m1 = Modifier()
4240             m2 = Modifier()
4241             m3 = Modifier()
4242             Process._firstProcess = True
4243             p = Process("test",m1,m2,m3)
4244             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4245             (m1 & m2 & m3).toModify(p.a, fred = int32(2))
4246             self.assertEqual(p.a.fred, 2)
4247             (m1 & (m2 & m3)).toModify(p.a, fred = int32(3))
4248             self.assertEqual(p.a.fred, 3)
4249             ((m1 & m2) & m3).toModify(p.a, fred = int32(4))
4250             self.assertEqual(p.a.fred, 4)
4251             #check inverse
4252             m1 = Modifier()
4253             m2 = Modifier()
4254             Process._firstProcess = True
4255             p = Process("test", m1)
4256             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4257             (~m1).toModify(p.a, fred=2)
4258             self.assertEqual(p.a.fred, 1)
4259             (~m2).toModify(p.a, wilma=2)
4260             self.assertEqual(p.a.wilma, 2)
4261             self.assertRaises(TypeError, lambda: (~m1).toModify(p.a, 1, wilma=2))
4262             self.assertRaises(TypeError, lambda: (~m2).toModify(p.a, 1, wilma=2))
4263             # check or
4264             m1 = Modifier()
4265             m2 = Modifier()
4266             m3 = Modifier()
4267             Process._firstProcess = True
4268             p = Process("test", m1)
4269             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4270             (m1 | m2).toModify(p.a, fred=2)
4271             self.assertEqual(p.a.fred, 2)
4272             (m1 | m2 | m3).toModify(p.a, fred=3)
4273             self.assertEqual(p.a.fred, 3)
4274             (m3 | m2 | m1).toModify(p.a, fred=4)
4275             self.assertEqual(p.a.fred, 4)
4276             ((m1 | m2) | m3).toModify(p.a, fred=5)
4277             self.assertEqual(p.a.fred, 5)
4278             (m1 | (m2 | m3)).toModify(p.a, fred=6)
4279             self.assertEqual(p.a.fred, 6)
4280             (m2 | m3).toModify(p.a, fred=7)
4281             self.assertEqual(p.a.fred, 6)
4282             self.assertRaises(TypeError, lambda: (m1 | m2).toModify(p.a, 1, wilma=2))
4283             self.assertRaises(TypeError, lambda: (m2 | m3).toModify(p.a, 1, wilma=2))
4284             # check combinations
4285             m1 = Modifier()
4286             m2 = Modifier()
4287             m3 = Modifier()
4288             m4 = Modifier()
4289             Process._firstProcess = True
4290             p = Process("test", m1, m2)
4291             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4292             (m1 & ~m2).toModify(p.a, fred=2)
4293             self.assertEqual(p.a.fred, 1)
4294             (m1 & ~m3).toModify(p.a, fred=2)
4295             self.assertEqual(p.a.fred, 2)
4296             (m1 | ~m2).toModify(p.a, fred=3)
4297             self.assertEqual(p.a.fred, 3)
4298             (~m1 | ~m2).toModify(p.a, fred=4)
4299             self.assertEqual(p.a.fred, 3)
4300             (~m3 & ~m4).toModify(p.a, fred=4)
4301             self.assertEqual(p.a.fred, 4)
4302             ((m1 & m3) | ~m4).toModify(p.a, fred=5)
4303             self.assertEqual(p.a.fred, 5)
4304             #check toReplaceWith
4305             m1 = Modifier()
4306             Process._firstProcess = True
4307             p = Process("test",m1)
4308             p.a =EDAnalyzer("MyAnalyzer", fred = int32(1))
4309             m1.toReplaceWith(p.a, EDAnalyzer("YourAnalyzer", wilma = int32(3)))
4310             self.assertRaises(TypeError, lambda: m1.toReplaceWith(p.a, EDProducer("YourProducer")))
4311             #Task
4312             p.b =EDAnalyzer("BAn")
4313             p.c =EDProducer("c")
4314             p.d =EDProducer("d")
4315             p.tc = Task(p.c)
4316             p.td = Task(p.d)
4317             p.s = Sequence(p.a, p.tc)
4318             m1.toReplaceWith(p.s, Sequence(p.a+p.b, p.td))
4319             self.assertEqual(p.a.wilma.value(),3)
4320             self.assertEqual(p.a.type_(),"YourAnalyzer")
4321             self.assertEqual(hasattr(p,"fred"),False)
4322             self.assertTrue(p.s.dumpPython() == "cms.Sequence(process.a+process.b, process.td)\n")
4323             p.e =EDProducer("e")
4324             m1.toReplaceWith(p.td, Task(p.e))
4325             self.assertTrue(p.td._collection == OrderedSet([p.e]))
4326             #ConditionalTask
4327             p.b =EDAnalyzer("BAn")
4328             p.c =EDProducer("c")
4329             p.d =EDProducer("d")
4330             del p.tc
4331             del p.td
4332             p.tc = ConditionalTask(p.c)
4333             p.td = ConditionalTask(p.d)
4334             p.s = Sequence(p.a, p.tc)
4335             m1.toReplaceWith(p.s, Sequence(p.a+p.b, p.td))
4336             self.assertEqual(p.a.wilma.value(),3)
4337             self.assertEqual(p.a.type_(),"YourAnalyzer")
4338             self.assertEqual(hasattr(p,"fred"),False)
4339             self.assertTrue(p.s.dumpPython() == "cms.Sequence(process.a+process.b, process.td)\n")
4340             p.e =EDProducer("e")
4341             m1.toReplaceWith(p.td, ConditionalTask(p.e))
4342             self.assertTrue(p.td._collection == OrderedSet([p.e]))
4343             #check toReplaceWith doesn't activate not chosen
4344             m1 = Modifier()
4345             Process._firstProcess = True
4346             p = Process("test")
4347             p.a =EDAnalyzer("MyAnalyzer", fred = int32(1))
4348             m1.toReplaceWith(p.a, EDAnalyzer("YourAnalyzer", wilma = int32(3)))
4349             self.assertEqual(p.a.type_(),"MyAnalyzer")
4350             #check toReplaceWith and and/not/or combinations
4351             m1 = Modifier()
4352             m2 = Modifier()
4353             m3 = Modifier()
4354             m4 = Modifier()
4355             Process._firstProcess = True
4356             p = Process("test", m1, m2)
4357             p.a = EDAnalyzer("MyAnalyzer", fred = int32(1), wilma = int32(1))
4358             self.assertRaises(TypeError, lambda: (m1 & m2).toReplaceWith(p.a, EDProducer("YourProducer")))
4359             self.assertRaises(TypeError, lambda: (m3 & m4).toReplaceWith(p.a, EDProducer("YourProducer")))
4360             self.assertRaises(TypeError, lambda: (~m3).toReplaceWith(p.a, EDProducer("YourProducer")))
4361             self.assertRaises(TypeError, lambda: (~m1).toReplaceWith(p.a, EDProducer("YourProducer")))
4362             self.assertRaises(TypeError, lambda: (m1 | m3).toReplaceWith(p.a, EDProducer("YourProducer")))
4363             self.assertRaises(TypeError, lambda: (m3 | m4).toReplaceWith(p.a, EDProducer("YourProducer")))
4364             (m1 & m2).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer1"))
4365             self.assertEqual(p.a.type_(), "YourAnalyzer1")
4366             (m1 & m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer2"))
4367             self.assertEqual(p.a.type_(), "YourAnalyzer1")
4368             (~m1).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer2"))
4369             self.assertEqual(p.a.type_(), "YourAnalyzer1")
4370             (~m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer2"))
4371             self.assertEqual(p.a.type_(), "YourAnalyzer2")
4372             (m1 | m3).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer3"))
4373             self.assertEqual(p.a.type_(), "YourAnalyzer3")
4374             (m3 | m4).toReplaceWith(p.a, EDAnalyzer("YourAnalyzer4"))
4375             self.assertEqual(p.a.type_(), "YourAnalyzer3")
4376 
4377             # EDAlias
4378             a = EDAlias(foo2 = VPSet(PSet(type = string("Foo2"))))
4379             m = Modifier()
4380             m._setChosen()
4381             # Modify parameters
4382             m.toModify(a, foo2 = {0: dict(type = "Foo3")})
4383             self.assertEqual(a.foo2[0].type, "Foo3")
4384             # Add an alias
4385             m.toModify(a, foo4 = VPSet(PSet(type = string("Foo4"))))
4386             self.assertEqual(a.foo2[0].type, "Foo3")
4387             self.assertEqual(a.foo4[0].type, "Foo4")
4388             # Remove an alias
4389             m.toModify(a, foo2 = None)
4390             self.assertFalse(hasattr(a, "foo2"))
4391             self.assertEqual(a.foo4[0].type, "Foo4")
4392             # Replace (doesn't work out of the box because EDAlias is not _Parameterizable
4393             m.toReplaceWith(a, EDAlias(bar = VPSet(PSet(type = string("Bar")))))
4394             self.assertFalse(hasattr(a, "foo2"))
4395             self.assertFalse(hasattr(a, "foo4"))
4396             self.assertTrue(hasattr(a, "bar"))
4397             self.assertEqual(a.bar[0].type, "Bar")
4398 
4399             # SwitchProducer
4400             sp = SwitchProducerTest(test1 = EDProducer("Foo",
4401                                                        a = int32(1),
4402                                                        b = PSet(c = int32(2))),
4403                                     test2 = EDProducer("Bar",
4404                                                        aa = int32(11),
4405                                                        bb = PSet(cc = int32(12))))
4406             m = Modifier()
4407             m._setChosen()
4408             # Modify parameters
4409             m.toModify(sp,
4410                        test1 = dict(a = 4, b = dict(c = None)),
4411                        test2 = dict(aa = 15, bb = dict(cc = 45, dd = string("foo"))))
4412             self.assertEqual(sp.test1.a.value(), 4)
4413             self.assertEqual(sp.test1.b.hasParameter("c"), False)
4414             self.assertEqual(sp.test2.aa.value(), 15)
4415             self.assertEqual(sp.test2.bb.cc.value(), 45)
4416             self.assertEqual(sp.test2.bb.dd.value(), "foo")
4417             # Replace a producer
4418             m.toReplaceWith(sp.test1, EDProducer("Fred", x = int32(42)))
4419             self.assertEqual(sp.test1.type_(), "Fred")
4420             self.assertEqual(sp.test1.x.value(), 42)
4421             self.assertRaises(TypeError, lambda: m.toReplaceWith(sp.test1, EDAnalyzer("Foo")))
4422             # Alternative way (only to be allow same syntax to be used as for adding)
4423             m.toModify(sp, test2 = EDProducer("Xyzzy", x = int32(24)))
4424             self.assertEqual(sp.test2.type_(), "Xyzzy")
4425             self.assertEqual(sp.test2.x.value(), 24)
4426             self.assertRaises(TypeError, lambda: m.toModify(sp, test2 = EDAnalyzer("Foo")))
4427             # Add a producer
4428             m.toModify(sp, test3 = EDProducer("Wilma", y = int32(24)))
4429             self.assertEqual(sp.test3.type_(), "Wilma")
4430             self.assertEqual(sp.test3.y.value(), 24)
4431             self.assertRaises(TypeError, lambda: m.toModify(sp, test4 = EDAnalyzer("Foo")))
4432             # Remove a producer
4433             m.toModify(sp, test2 = None)
4434             self.assertEqual(hasattr(sp, "test2"), False)
4435             # Add an alias
4436             m.toModify(sp, test2 = EDAlias(foo = VPSet(PSet(type = string("int")))))
4437             self.assertTrue(hasattr(sp.test2, "foo"))
4438             # Replace an alias
4439             m.toReplaceWith(sp.test2, EDAlias(bar = VPSet(PSet(type = string("int")))))
4440             self.assertTrue(hasattr(sp.test2, "bar"))
4441             # Alternative way
4442             m.toModify(sp, test2 = EDAlias(xyzzy = VPSet(PSet(type = string("int")))))
4443             self.assertTrue(hasattr(sp.test2, "xyzzy"))
4444             # Replace an alias with EDProducer
4445             self.assertRaises(TypeError, lambda: m.toReplaceWith(sp.test2, EDProducer("Foo")))
4446             m.toModify(sp, test2 = EDProducer("Foo"))
4447         def testProcessFragment(self):
4448             #check defaults are not overwritten
4449             f = ProcessFragment('Fragment')
4450             p = Process('PROCESS')
4451             p.maxEvents.input = 10
4452             p.options.numberOfThreads = 4
4453             p.maxLuminosityBlocks.input = 2
4454             p.extend(f)
4455             self.assertEqual(p.maxEvents.input.value(),10)
4456             self.assertEqual(p.options.numberOfThreads.value(), 4)
4457             self.assertEqual(p.maxLuminosityBlocks.input.value(),2)
4458             #general checks
4459             f = ProcessFragment("Fragment")
4460             f.fltr = EDFilter("Foo")
4461             p = Process('PROCESS')
4462             p.extend(f)
4463             self.assertTrue(hasattr(p,'fltr'))
4464         def testProcessForProcessAccelerator(self):
4465             proc = Process("TEST")
4466             p = ProcessForProcessAccelerator(proc)
4467             p.TestService = Service("TestService")
4468             self.assertTrue(hasattr(proc, "TestService"))
4469             self.assertEqual(proc.TestService.type_(), "TestService")
4470             self.assertRaises(TypeError, setattr, p, "a", EDProducer("Foo"))
4471             p.add_(Service("TestServiceTwo"))
4472             self.assertTrue(hasattr(proc, "TestServiceTwo"))
4473             self.assertEqual(proc.TestServiceTwo.type_(), "TestServiceTwo")
4474             p.TestService.foo = untracked.uint32(42)
4475             self.assertEqual(proc.TestService.foo.value(), 42)
4476             proc.mod = EDProducer("Producer")
4477             self.assertRaises(TypeError, getattr, p, "mod")
4478         def testProcessAccelerator(self):
4479             proc = Process("TEST")
4480             p = TestMakePSet()
4481             proc.fillProcessDesc(p)
4482             self.assertTrue(["cpu"], p.values["@available_accelerators"][1])
4483             self.assertFalse(p.values["@selected_accelerators"][0])
4484             self.assertTrue(["cpu"], p.values["@selected_accelerators"][1])
4485 
4486             proc = Process("TEST")
4487             self.assertRaises(TypeError, setattr, proc, "processAcceleratorTest", ProcessAcceleratorTest())
4488             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4489             del proc.MessageLogger # remove boilerplate unnecessary for this test case
4490             self.assertEqual(proc.dumpPython(),
4491 """import FWCore.ParameterSet.Config as cms
4492 from test import ProcessAcceleratorTest
4493 
4494 process = cms.Process("TEST")
4495 
4496 process.maxEvents = cms.untracked.PSet(
4497     input = cms.optional.untracked.int32,
4498     output = cms.optional.untracked.allowed(cms.int32,cms.PSet)
4499 )
4500 
4501 process.maxLuminosityBlocks = cms.untracked.PSet(
4502     input = cms.untracked.int32(-1)
4503 )
4504 
4505 process.options = cms.untracked.PSet(
4506     FailPath = cms.untracked.vstring(),
4507     IgnoreCompletely = cms.untracked.vstring(),
4508     Rethrow = cms.untracked.vstring(),
4509     SkipEvent = cms.untracked.vstring(),
4510     accelerators = cms.untracked.vstring('*'),
4511     allowUnscheduled = cms.obsolete.untracked.bool,
4512     canDeleteEarly = cms.untracked.vstring(),
4513     deleteNonConsumedUnscheduledModules = cms.untracked.bool(True),
4514     dumpOptions = cms.untracked.bool(False),
4515     emptyRunLumiMode = cms.obsolete.untracked.string,
4516     eventSetup = cms.untracked.PSet(
4517         forceNumberOfConcurrentIOVs = cms.untracked.PSet(
4518             allowAnyLabel_=cms.required.untracked.uint32
4519         ),
4520         numberOfConcurrentIOVs = cms.untracked.uint32(0)
4521     ),
4522     fileMode = cms.untracked.string('FULLMERGE'),
4523     forceEventSetupCacheClearOnNewRun = cms.untracked.bool(False),
4524     makeTriggerResults = cms.obsolete.untracked.bool,
4525     numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(0),
4526     numberOfConcurrentRuns = cms.untracked.uint32(1),
4527     numberOfStreams = cms.untracked.uint32(0),
4528     numberOfThreads = cms.untracked.uint32(1),
4529     printDependencies = cms.untracked.bool(False),
4530     sizeOfStackForThreadsInKB = cms.optional.untracked.uint32,
4531     throwIfIllegalParameter = cms.untracked.bool(True),
4532     wantSummary = cms.untracked.bool(False)
4533 )
4534 
4535 process.ProcessAcceleratorTest = ProcessAcceleratorTest(
4536     enabled = ['test1', 'test2', 'anothertest3']
4537 )
4538 
4539 
4540 """)
4541             p = TestMakePSet()
4542             proc.fillProcessDesc(p)
4543             self.assertEqual(["*"], p.values["options"][1].values["accelerators"][1])
4544             self.assertFalse(p.values["options"][1].values["accelerators"][0])
4545             self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["@selected_accelerators"][1])
4546             self.assertEqual("AcceleratorTestService", p.values["services"][1][0].values["@service_type"][1])
4547             self.assertFalse(p.values["@available_accelerators"][0])
4548             self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4549 
4550             proc = Process("TEST")
4551             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"])
4552             p = TestMakePSet()
4553             proc.fillProcessDesc(p)
4554             self.assertEqual(["cpu", "test1"], p.values["@selected_accelerators"][1])
4555             self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
4556 
4557             proc = Process("TEST")
4558             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4559             proc.options.accelerators = ["test2"]
4560             p = TestMakePSet()
4561             proc.fillProcessDesc(p)
4562             self.assertEqual(["test2"], p.values["@selected_accelerators"][1])
4563             self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4564 
4565             proc = Process("TEST")
4566             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4567             proc.options.accelerators = ["test*"]
4568             proc.fillProcessDesc(p)
4569             self.assertEqual(["test1", "test2"], p.values["@selected_accelerators"][1])
4570             self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4571 
4572             proc = Process("TEST")
4573             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"])
4574             proc.options.accelerators = ["test2"]
4575             p = TestMakePSet()
4576             proc.fillProcessDesc(p)
4577             self.assertEqual([], p.values["@selected_accelerators"][1])
4578             self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
4579 
4580             proc = Process("TEST")
4581             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4582             proc.options.accelerators = ["cpu*"]
4583             p = TestMakePSet()
4584             proc.fillProcessDesc(p)
4585             self.assertEqual(["cpu"], p.values["@selected_accelerators"][1])
4586             self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4587 
4588             proc = Process("TEST")
4589             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4590             proc.options.accelerators = ["test3"]
4591             p = TestMakePSet()
4592             self.assertRaises(ValueError, proc.fillProcessDesc, p)
4593 
4594             proc = Process("TEST")
4595             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4596             proc.options.accelerators = ["*", "test1"]
4597             p = TestMakePSet()
4598             self.assertRaises(ValueError, proc.fillProcessDesc, p)
4599 
4600             proc = Process("TEST")
4601             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4602             proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2()
4603             p = TestMakePSet()
4604             proc.fillProcessDesc(p)
4605             self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@selected_accelerators"][1])
4606             self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4607 
4608             proc = Process("TEST")
4609             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4610             proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2()
4611             proc.options.accelerators = ["*test3", "c*"]
4612             p = TestMakePSet()
4613             proc.fillProcessDesc(p)
4614             self.assertEqual(["anothertest3", "cpu"], p.values["@selected_accelerators"][1])
4615             self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4616 
4617             proc = Process("TEST")
4618             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4619             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4620                                                              a = int32(1),
4621                                                              b = PSet(c = int32(2))),
4622                                           test1 = EDProducer("Bar",
4623                                                              aa = int32(11),
4624                                                              bb = PSet(cc = int32(12))))
4625             proc.p = Path(proc.sp)
4626             p = TestMakePSet()
4627             proc.fillProcessDesc(p)
4628             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
4629 
4630             proc = Process("TEST")
4631             proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"])
4632             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4633                                                              a = int32(1),
4634                                                              b = PSet(c = int32(2))),
4635                                           test1 = EDProducer("Bar",
4636                                                              aa = int32(11),
4637                                                              bb = PSet(cc = int32(12))))
4638             proc.p = Path(proc.sp)
4639             p = TestMakePSet()
4640             proc.fillProcessDesc(p)
4641             self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
4642 
4643             proc = Process("TEST")
4644             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4645             proc.options.accelerators = ["test1"]
4646             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4647                                                              a = int32(1),
4648                                                              b = PSet(c = int32(2))),
4649                                           test1 = EDProducer("Bar",
4650                                                              aa = int32(11),
4651                                                              bb = PSet(cc = int32(12))))
4652             proc.p = Path(proc.sp)
4653             p = TestMakePSet()
4654             proc.fillProcessDesc(p)
4655             self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
4656 
4657             proc = Process("TEST")
4658             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4659             proc.options.accelerators = ["test*"]
4660             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4661                                                              a = int32(1),
4662                                                              b = PSet(c = int32(2))),
4663                                           test1 = EDProducer("Bar",
4664                                                              aa = int32(11),
4665                                                              bb = PSet(cc = int32(12))))
4666             proc.p = Path(proc.sp)
4667             p = TestMakePSet()
4668             proc.fillProcessDesc(p)
4669             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
4670 
4671             proc = Process("TEST")
4672             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4673             proc.options.accelerators = ["anothertest3"]
4674             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4675                                                              a = int32(1),
4676                                                              b = PSet(c = int32(2))),
4677                                           test1 = EDProducer("Bar",
4678                                                              aa = int32(11),
4679                                                              bb = PSet(cc = int32(12))))
4680             proc.p = Path(proc.sp)
4681             p = TestMakePSet()
4682             self.assertRaises(RuntimeError, proc.fillProcessDesc, p)
4683 
4684             import pickle
4685             proc = Process("TEST")
4686             proc.ProcessAcceleratorTest = ProcessAcceleratorTest()
4687             proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo",
4688                                                              a = int32(1),
4689                                                              b = PSet(c = int32(2))),
4690                                           test1 = EDProducer("Bar",
4691                                                              aa = int32(11),
4692                                                              bb = PSet(cc = int32(12))))
4693             proc.p = Path(proc.sp)
4694             pkl = pickle.dumps(proc)
4695             unpkl = pickle.loads(pkl)
4696             p = TestMakePSet()
4697             unpkl.fillProcessDesc(p)
4698             self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"])
4699             self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1])
4700             unpkl = pickle.loads(pkl)
4701             unpkl.ProcessAcceleratorTest.setEnabled(["test1"])
4702             p = TestMakePSet()
4703             unpkl.fillProcessDesc(p)
4704             self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
4705             self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
4706 
4707     unittest.main()