ParameterSet/python/Mixins.py

0001 from builtins import range, object
0002 import inspect
0003 from typing import Union
0004
0005 class _ConfigureComponent(object):
0006     """Denotes a class that can be used by the Processes class"""
0007     def _isTaskComponent(self) -> bool:
0008         return False
0009
0010 class PrintOptions(object):
0011     def __init__(self, indent:int = 0, deltaIndent:int = 4, process:bool = True, targetDirectory: Union[str, None] = None, useSubdirectories:bool = False):
0012         self.indent_= indent
0013         self.deltaIndent_ = deltaIndent
0014         self.isCfg = process
0015         self.targetDirectory = targetDirectory
0016         self.useSubdirectories = useSubdirectories
0017     def indentation(self) -> str:
0018         return ' '*self.indent_
0019     def indent(self):
0020         self.indent_ += self.deltaIndent_
0021     def unindent(self):
0022         self.indent_ -= self.deltaIndent_
0023
0024 class _SpecialImportRegistry(object):
0025     """This class collects special import statements of configuration types"""
0026     def __init__(self):
0027         self._registry = {}
0028
0029     def _reset(self):
0030         for lst in self._registry.values():
0031             lst[1] = False
0032
0033     def registerSpecialImportForType(self, cls, impStatement):
0034         className = cls.__name__
0035         if className in self._registry:
0036             raise RuntimeError("Error: the configuration type '%s' already has an import statement registered '%s'" % (className, self._registry[className][0]))
0037         self._registry[className] = [impStatement, False]
0038
0039     def registerUse(self, obj):
0040         className = obj.__class__.__name__
0041         try:
0042             self._registry[className][1] = True
0043         except KeyError:
0044             pass
0045
0046     def getSpecialImports(self):
0047         coll = set()
0048         for (imp, used) in self._registry.values():
0049             if used:
0050                 coll.add(imp)
0051         return sorted(coll)
0052
0053 specialImportRegistry = _SpecialImportRegistry()
0054
0055 class _ParameterTypeBase(object):
0056     """base class for classes which are used as the 'parameters' for a ParameterSet"""
0057     def __init__(self):
0058         self.__dict__["_isFrozen"] = False
0059         self.__isTracked = True
0060         self._isModified = False
0061     def isModified(self) -> bool:
0062         return self._isModified
0063     def resetModified(self):
0064         self._isModified=False
0065     def configTypeName(self) -> str:
0066         if self.isTracked():
0067             return type(self).__name__
0068         return 'untracked '+type(self).__name__
0069     def pythonTypeName(self) -> str:
0070         if self.isTracked():
0071             return 'cms.'+type(self).__name__
0072         return 'cms.untracked.'+type(self).__name__
0073     def dumpPython(self, options:PrintOptions=PrintOptions()) -> str:
0074         specialImportRegistry.registerUse(self)
0075         return self.pythonTypeName()+"("+self.pythonValue(options)+")"
0076     def __repr__(self) -> str:
0077         return self.dumpPython()
0078     def isTracked(self) -> bool:
0079         return self.__isTracked
0080     def setIsTracked(self,trackness:bool):
0081         self.__isTracked = trackness
0082     def isFrozen(self) -> bool:
0083         return self._isFrozen
0084     def setIsFrozen(self):
0085         self._isFrozen = True
0086     def isCompatibleCMSType(self,aType) -> bool:
0087         return isinstance(self,aType)
0088     def _checkAndReturnValueWithType(self, valueWithType):
0089         if isinstance(valueWithType, type(self)):
0090             return valueWithType
0091         raise TypeError("Attempted to assign type {from_} to type {to}".format(from_ = str(type(valueWithType)), to = str(type(self))) )
0092
0093
0094 class _SimpleParameterTypeBase(_ParameterTypeBase):
0095     """base class for parameter classes which only hold a single value"""
0096     def __init__(self,value):
0097         super(_SimpleParameterTypeBase,self).__init__()
0098         self._value = value
0099         if not self._isValid(value):
0100             raise ValueError(str(value)+" is not a valid "+str(type(self)))
0101     def value(self):
0102         return self._value
0103     def setValue(self,value):
0104         if not self._isValid(value):
0105             raise ValueError(str(value)+" is not a valid "+str(type(self)))
0106         if value!=self._value:
0107             self._isModified=True
0108             self._value=value
0109     def configValue(self, options:PrintOptions=PrintOptions()) -> str:
0110         return str(self._value)
0111     def pythonValue(self, options:PrintOptions=PrintOptions()) -> str:
0112         return self.configValue(options)
0113     def __eq__(self,other) -> bool:
0114         if isinstance(other,_SimpleParameterTypeBase):
0115             return self._value == other._value
0116         return self._value == other
0117     def __ne__(self,other) -> bool:
0118         if isinstance(other,_SimpleParameterTypeBase):
0119             return self._value != other._value
0120         return self._value != other
0121     def __lt__(self,other) -> bool:
0122         if isinstance(other,_SimpleParameterTypeBase):
0123             return self._value < other._value
0124         return self._value < other
0125     def __le__(self,other) -> bool:
0126         if isinstance(other,_SimpleParameterTypeBase):
0127             return self._value <= other._value
0128         return self._value <= other
0129     def __gt__(self,other) -> bool:
0130         if isinstance(other,_SimpleParameterTypeBase):
0131             return self._value > other._value
0132         return self._value > other
0133     def __ge__(self,other) -> bool:
0134         if isinstance(other,_SimpleParameterTypeBase):
0135             return self._value >= other._value
0136         return self._value >= other
0137
0138
0139 class UsingBlock(_SimpleParameterTypeBase):
0140     """For injection purposes, pretend this is a new parameter type
0141        then have a post process step which strips these out
0142     """
0143     def __init__(self,value, s:str='', loc:int=0, file:str=''):
0144         super(UsingBlock,self).__init__(value)
0145         self.s = s
0146         self.loc = loc
0147         self.file = file
0148         self.isResolved = False
0149     @staticmethod
0150     def _isValid(value) -> bool:
0151         return isinstance(value,str)
0152     def _valueFromString(value) -> str:
0153         """only used for cfg-parsing"""
0154         return str(value)
0155     def insertInto(self, parameterSet, myname:str):
0156         value = self.value()
0157         #  doesn't seem to handle \0 correctly
0158         #if value == '\0':
0159         #    value = ''
0160         parameterSet.addString(self.isTracked(), myname, value)
0161     def dumpPython(self, options:PrintOptions=PrintOptions()) -> str:
0162         if options.isCfg:
0163             return "process."+self.value()
0164         else:
0165             return self.value()
0166
0167
0168 class _Parameterizable(object):
0169     """Base class for classes which allow addition of _ParameterTypeBase data"""
0170     def __init__(self,*arg,**kargs):
0171         self.__dict__['_Parameterizable__parameterNames'] = []
0172         self.__dict__["_isFrozen"] = False
0173         self.__dict__['_Parameterizable__validator'] = None
0174         """The named arguments are the 'parameters' which are added as 'python attributes' to the object"""
0175         if len(arg) != 0:
0176             self.__setParametersFromArg(*arg)
0177         self.__setParameters(kargs)
0178         self._isModified = False
0179
0180     def parameterNames_(self):
0181         """Returns the name of the parameters"""
0182         return self.__parameterNames[:]
0183     def isModified(self) -> bool:
0184         if self._isModified:
0185             return True
0186         for name in self.parameterNames_():
0187             param = self.__dict__[name]
0188             if isinstance(param, _Parameterizable) and param.isModified():
0189                 self._isModified = True
0190                 return True
0191         return False
0192
0193     def hasParameter(self, params) -> bool:
0194         """
0195         _hasParameter_
0196
0197         check that pset provided has the attribute chain
0198         specified.
0199
0200         Eg, if params is [ 'attr1', 'attr2', 'attr3' ]
0201         check for pset.attr1.attr2.attr3
0202
0203         returns True if parameter exists, False if not
0204         """
0205         return (self.getParameter(params) != None)
0206
0207     def getParameter(self, params):
0208         """
0209         _getParameter_
0210
0211         Retrieve the specified parameter from the PSet Provided
0212         given the attribute chain
0213
0214         returns None if not found
0215         """
0216         lastParam = self
0217         # Don't accidentally iterate over letters in a string
0218         if type(params).__name__ == 'str':
0219             return getattr(self, params, None)
0220         for param in params:
0221             lastParam = getattr(lastParam, param, None)
0222             if lastParam == None:
0223                 return None
0224         return lastParam
0225
0226     def parameters_(self):
0227         """Returns a dictionary of copies of the user-set parameters"""
0228         import copy
0229         result = dict()
0230         for name in self.parameterNames_():
0231                result[name]=copy.deepcopy(self.__dict__[name])
0232         return result
0233
0234     def __addParameter(self, name:str, value):
0235         if name == 'allowAnyLabel_':
0236             self.__validator = value
0237             self._isModified = True
0238             return
0239         if not isinstance(value,_ParameterTypeBase):
0240             if self.__validator is not None:
0241                 value = self.__validator.convert_(value)
0242             else:
0243                 self.__raiseBadSetAttr(name)
0244         if name in self.__dict__:
0245             message = "Duplicate insert of member " + name
0246             message += "\nThe original parameters are:\n"
0247             message += self.dumpPython() + '\n'
0248             raise ValueError(message)
0249         self.__dict__[name]=value
0250         self.__parameterNames.append(name)
0251         self._isModified = True
0252     def __setParametersFromArg(self, *arg):
0253         for block in arg:
0254             # Allow __PSet for testing
0255             if type(block).__name__ not in ["PSet", "__PSet", "dict"]:
0256                 raise ValueError("Only PSets can be passed as unnamed argument blocks.  This is a "+type(block).__name__)
0257             if isinstance(block,dict):
0258                 self.__setParameters(block)
0259             else:
0260                 self.__setParameters(block.parameters_())
0261
0262     def __setParameters(self,parameters):
0263         v = None
0264         for name,value in parameters.items():
0265             if name == 'allowAnyLabel_':
0266                 v = value
0267                 continue
0268             self.__addParameter(name, value)
0269         if v is not None:
0270             self.__validator=v
0271     def hasNoParameters(self) -> bool:
0272         return len(self.__parameterNames) == 0
0273     def __setattr__(self,name:str,value):
0274         #since labels are not supposed to have underscores at the beginning
0275         # I will assume that if we have such then we are setting an internal variable
0276         if self.isFrozen() and not (name in ["_Labelable__label","_isFrozen"] or name.startswith('_')):
0277             message = "Object already added to a process. It is read only now\n"
0278             message +=  "    %s = %s" %(name, value)
0279             message += "\nThe original parameters are:\n"
0280             message += self.dumpPython() + '\n'
0281             raise ValueError(message)
0282         # underscored names bypass checking for _ParameterTypeBase
0283         if name[0]=='_':
0284             super(_Parameterizable,self).__setattr__(name,value)
0285         elif not name in self.__dict__:
0286             self.__addParameter(name, value)
0287             self._isModified = True
0288         else:
0289             # handle the case where users just replace with a value, a = 12, rather than a = cms.int32(12)
0290             if isinstance(value,_ParameterTypeBase):
0291                 self.__dict__[name] = self.__dict__[name]._checkAndReturnValueWithType(value)
0292             else:
0293                 self.__dict__[name].setValue(value)
0294             self._isModified = True
0295     def update_(self, d):
0296         """"Takes a PSet or dict and adds the entries as parameters. Already existing parameters will be overwritten.
0297         """
0298         if type(d).__name__ not in ["PSet", "__PSet", "dict"]:
0299             raise ValueError("Only PSets or dicts can be passed to update_.  This is a "+type(d).__name__)
0300
0301         items = d.items() if isinstance(d, dict) else d.parameters_().items()
0302         for k,v in items:
0303                 setattr(self, k, v)
0304
0305
0306
0307     def isFrozen(self) -> bool:
0308         return self._isFrozen
0309     def setIsFrozen(self):
0310         self._isFrozen = True
0311         for name in self.parameterNames_():
0312             self.__dict__[name].setIsFrozen()
0313     def __delattr__(self,name:str):
0314         if self.isFrozen():
0315             raise ValueError("Object already added to a process. It is read only now")
0316         super(_Parameterizable,self).__delattr__(name)
0317         self.__parameterNames.remove(name)
0318     @staticmethod
0319     def __raiseBadSetAttr(name:str):
0320         raise TypeError(name+" does not already exist, so it can only be set to a CMS python configuration type")
0321     def dumpPython(self, options:PrintOptions=PrintOptions()) -> str:
0322         specialImportRegistry.registerUse(self)
0323         sortedNames = sorted(self.parameterNames_())
0324         if len(sortedNames) > 200:
0325         #Too many parameters for a python function call
0326         # The solution is to create a temporary dictionary which
0327         # is constructed by concatenating long lists (with maximum
0328         # 200 entries each) together.
0329         # This looks like
0330         #  **dict( [(...,...), ...] + [...] + ... )
0331             others = []
0332             usings = []
0333             for name in sortedNames:
0334                 param = self.__dict__[name]
0335                 # we don't want minuses in names
0336                 name2 = name.replace('-','_')
0337                 options.indent()
0338                 #_UsingNodes don't get assigned variables
0339                 if name.startswith("using_"):
0340                     usings.append(options.indentation()+param.dumpPython(options))
0341                 else:
0342                     others.append((name2, param.dumpPython(options)))
0343                 options.unindent()
0344
0345             resultList = ',\n'.join(usings)
0346             longOthers = options.indentation()+"**dict(\n"
0347             options.indent()
0348             longOthers += options.indentation()+"[\n"
0349             entriesInList = 0
0350             options.indent()
0351             for n,v in others:
0352                 entriesInList +=1
0353                 if entriesInList > 200:
0354                     #need to start a new list
0355                     options.unindent()
0356                     longOthers += options.indentation()+"] +\n"+options.indentation()+"[\n"
0357                     entriesInList = 0
0358                     options.indent()
0359                 longOthers += options.indentation()+'("'+n+'" , '+v+' ),\n'
0360
0361             longOthers += options.indentation()+"]\n"
0362             options.unindent()
0363             longOthers +=options.indentation()+")\n"
0364             options.unindent()
0365             ret = []
0366             if resultList:
0367                 ret.append(resultList)
0368             if longOthers:
0369                 ret.append(longOthers)
0370             return ",\n".join(ret)
0371         #Standard case, small number of parameters
0372         others = []
0373         usings = []
0374         for name in sortedNames:
0375             param = self.__dict__[name]
0376             # we don't want minuses in names
0377             name2 = name.replace('-','_')
0378             options.indent()
0379             #_UsingNodes don't get assigned variables
0380             if name.startswith("using_"):
0381                 usings.append(options.indentation()+param.dumpPython(options))
0382             else:
0383                 others.append(options.indentation()+name2+' = '+param.dumpPython(options))
0384             options.unindent()
0385         # usings need to go first
0386         resultList = usings
0387         resultList.extend(others)
0388         if self.__validator is not None:
0389             options.indent()
0390             resultList.append(options.indentation()+"allowAnyLabel_="+self.__validator.dumpPython(options))
0391             options.unindent()
0392         return ',\n'.join(resultList)+'\n'
0393     def __repr__(self) -> str:
0394         return self.dumpPython()
0395     def insertContentsInto(self, parameterSet):
0396         for name in self.parameterNames_():
0397             param = getattr(self,name)
0398             param.insertInto(parameterSet, name)
0399
0400
0401 class _TypedParameterizable(_Parameterizable):
0402     """Base class for classes which are Parameterizable and have a 'type' assigned"""
0403     def __init__(self,type_,*arg,**kargs):
0404         self.__dict__['_TypedParameterizable__type'] = type_
0405         #the 'type' is also placed in the 'arg' list and we need to remove it
0406         #if 'type_' not in kargs:
0407         #    arg = arg[1:]
0408         #else:
0409         #    del args['type_']
0410         super(_TypedParameterizable,self).__init__(*arg,**kargs)
0411         saveOrigin(self, 1)
0412     def _place(self,name:str,proc):
0413         self._placeImpl(name,proc)
0414     def type_(self):
0415         """returns the type of the object, e.g. 'FooProducer'"""
0416         return self.__type
0417     def copy(self):
0418         returnValue =_TypedParameterizable.__new__(type(self))
0419         params = self.parameters_()
0420         returnValue.__init__(self.__type,**params)
0421         returnValue._isModified = self._isModified
0422         return returnValue
0423     def clone(self, *args, **params):
0424         """Copies the object and allows one to modify the parameters of the clone.
0425         New parameters may be added by specify the exact type
0426         Modifying existing parameters can be done by just specifying the new
0427           value without having to specify the type.
0428         A parameter may be removed from the clone using the value None.
0429            #remove the parameter foo.fred
0430            mod.toModify(foo, fred = None)
0431         A parameter embedded within a PSet may be changed via a dictionary
0432            #change foo.fred.pebbles to 3 and foo.fred.friend to "barney"
0433            mod.toModify(foo, fred = dict(pebbles = 3, friend = "barney)) )
0434         """
0435         returnValue =_TypedParameterizable.__new__(type(self))
0436         myparams = self.parameters_()
0437
0438         # Prefer parameters given in PSet blocks over those in clone-from module
0439         for block in args:
0440             # Allow __PSet for testing
0441             if type(block).__name__ not in ["PSet", "__PSet"]:
0442                 raise ValueError("Only PSets can be passed as unnamed argument blocks.  This is a "+type(block).__name__)
0443             for name in block.parameterNames_():
0444                 try:
0445                     del myparams[name]
0446                 except KeyError:
0447                     pass
0448
0449         _modifyParametersFromDict(myparams, params, self._Parameterizable__raiseBadSetAttr)
0450         if self._Parameterizable__validator is not None:
0451             myparams["allowAnyLabel_"] = self._Parameterizable__validator
0452
0453         returnValue.__init__(self.__type,*args,
0454                              **myparams)
0455         returnValue._isModified = False
0456         returnValue._isFrozen = False
0457         saveOrigin(returnValue, 1)
0458         return returnValue
0459
0460     @staticmethod
0461     def __findDefaultsFor(label:str,type):
0462         #This routine is no longer used, but I might revive it in the future
0463         import sys
0464         import glob
0465         choices = list()
0466         for d in sys.path:
0467             choices.extend(glob.glob(d+'/*/*/'+label+'.py'))
0468         if not choices:
0469             return None
0470         #now see if any of them have what we want
0471         #the use of __import__ is taken from an example
0472         # from the www.python.org documentation on __import__
0473         for c in choices:
0474             #print " found file "+c
0475             name='.'.join(c[:-3].split('/')[-3:])
0476             #name = c[:-3].replace('/','.')
0477             mod = __import__(name)
0478             components = name.split('.')
0479             for comp in components[1:]:
0480                 mod = getattr(mod,comp)
0481             if hasattr(mod,label):
0482                 default = getattr(mod,label)
0483                 if isinstance(default,_TypedParameterizable):
0484                     if(default.type_() == type):
0485                         params = dict()
0486                         for name in default.parameterNames_():
0487                             params[name] = getattr(default,name)
0488                         return params
0489         return None
0490
0491     def directDependencies(self):
0492         return []
0493
0494     def dumpConfig(self, options:PrintOptions=PrintOptions()) -> str:
0495         config = self.__type +' { \n'
0496         for name in self.parameterNames_():
0497             param = self.__dict__[name]
0498             options.indent()
0499             config+=options.indentation()+param.configTypeName()+' '+name+' = '+param.configValue(options)+'\n'
0500             options.unindent()
0501         config += options.indentation()+'}\n'
0502         return config
0503
0504     def dumpPython(self, options:PrintOptions=PrintOptions()) -> str:
0505         specialImportRegistry.registerUse(self)
0506         result = "cms."+str(type(self).__name__)+'("'+self.type_()+'"'
0507         nparam = len(self.parameterNames_())
0508         if nparam == 0:
0509             result += ")\n"
0510         else:
0511             result += ",\n"+_Parameterizable.dumpPython(self,options)+options.indentation() + ")\n"
0512         return result
0513
0514     def dumpPythonAttributes(self, myname:str, options:PrintOptions) -> str:
0515         """ dumps the object with all attributes declared after the constructor"""
0516         result = ""
0517         for name in sorted(self.parameterNames_()):
0518             param = self.__dict__[name]
0519             result += options.indentation() + myname + "." + name + " = " + param.dumpPython(options) + "\n"
0520         return result
0521
0522     def nameInProcessDesc_(self, myname:str):
0523         return myname;
0524     def moduleLabel_(self, myname:str):
0525         return myname
0526     def appendToProcessDescList_(self, lst, myname:str):
0527         lst.append(self.nameInProcessDesc_(myname))
0528     def insertInto(self, parameterSet, myname:str):
0529         newpset = parameterSet.newPSet()
0530         newpset.addString(True, "@module_label", self.moduleLabel_(myname))
0531         newpset.addString(True, "@module_type", self.type_())
0532         newpset.addString(True, "@module_edm_type", type(self).__name__)
0533         self.insertContentsInto(newpset)
0534         parameterSet.addPSet(True, self.nameInProcessDesc_(myname), newpset)
0535
0536
0537
0538 class _Labelable(object):
0539     """A 'mixin' used to denote that the class can be paired with a label (e.g. an EDProducer)"""
0540     def label_(self) -> str:
0541         if not hasattr(self, "_Labelable__label"):
0542            raise RuntimeError("module has no label.  Perhaps it wasn't inserted into the process?")
0543         return self.__label
0544     def hasLabel_(self) -> bool:
0545         return hasattr(self, "_Labelable__label") and self.__label is not None
0546     def setLabel(self,label:str):
0547         if self.hasLabel_() :
0548             if self.label_() != label and label is not None :
0549                 msg100 = "Attempting to change the label of a Labelable object, possibly an attribute of the Process\n"
0550                 msg101 = "Old label = "+self.label_()+"  New label = "+label+"\n"
0551                 msg102 = "Type = "+str(type(self))+"\n"
0552                 msg103 = "Some possible solutions:\n"
0553                 msg104 = "  1. Clone modules instead of using simple assignment. Cloning is\n"
0554                 msg105 = "  also preferred for other types when possible.\n"
0555                 msg106 = "  2. Declare new names starting with an underscore if they are\n"
0556                 msg107 = "  for temporaries you do not want propagated into the Process. The\n"
0557                 msg108 = "  underscore tells \"from x import *\" and process.load not to import\n"
0558                 msg109 = "  the name.\n"
0559                 msg110 = "  3. Reorganize so the assigment is not necessary. Giving a second\n"
0560                 msg111 = "  name to the same object usually causes confusion and problems.\n"
0561                 msg112 = "  4. Compose Sequences: newName = cms.Sequence(oldName)\n"
0562                 raise ValueError(msg100+msg101+msg102+msg103+msg104+msg105+msg106+msg107+msg108+msg109+msg110+msg111+msg112)
0563         self.__label = label
0564     def label(self) -> str:
0565         #print "WARNING: _Labelable::label() needs to be changed to label_()"
0566         return self.__label
0567     def __str__(self):
0568         #this is probably a bad idea
0569         # I added this so that when we ask a path to print
0570         # we will see the label that has been assigned
0571         return str(self.__label)
0572     def dumpSequenceConfig(self):
0573         return str(self.__label)
0574     def dumpSequencePython(self, options:PrintOptions=PrintOptions()):
0575         if options.isCfg:
0576             return 'process.'+str(self.__label)
0577         else:
0578             return str(self.__label)
0579     def _findDependencies(self,knownDeps,presentDeps):
0580         #print 'in labelled'
0581         myDeps=knownDeps.get(self.label_(),None)
0582         if myDeps!=None:
0583             if presentDeps != myDeps:
0584                 raise RuntimeError("the module "+self.label_()+" has two dependencies \n"
0585                                    +str(presentDeps)+"\n"
0586                                    +str(myDeps)+"\n"
0587                                    +"Please modify sequences to rectify this inconsistency")
0588         else:
0589             myDeps=set(presentDeps)
0590             knownDeps[self.label_()]=myDeps
0591         presentDeps.add(self.label_())
0592
0593
0594 class _Unlabelable(object):
0595     """A 'mixin' used to denote that the class can be used without a label (e.g. a Service)"""
0596     pass
0597
0598 class _ValidatingListBase(list):
0599     """Base class for a list which enforces that its entries pass a 'validity' test"""
0600     def __init__(self,*arg,**args):
0601         super(_ValidatingListBase,self).__init__(arg)
0602         if 0 != len(args):
0603             raise SyntaxError("named arguments ("+','.join([x for x in args])+") passsed to "+str(type(self)))
0604         if not type(self)._isValid(iter(self)):
0605             raise TypeError("wrong types ("+','.join([str(type(value)) for value in iter(self)])+
0606                             ") added to "+str(type(self)))
0607     def __setitem__(self,key,value):
0608         if isinstance(key,slice):
0609             if not self._isValid(value):
0610                 raise TypeError("wrong type being inserted into this container "+self._labelIfAny())
0611         else:
0612             if not self._itemIsValid(value):
0613                 raise TypeError("can not insert the type "+str(type(value))+" in container "+self._labelIfAny())
0614         super(_ValidatingListBase,self).__setitem__(key,value)
0615     @classmethod
0616     def _isValid(cls,seq) -> bool:
0617         # see if strings get reinterpreted as lists
0618         if isinstance(seq, str):
0619             return False
0620         for item in seq:
0621             if not cls._itemIsValid(item):
0622                 return False
0623         return True
0624     def _itemFromArgument(self, x):
0625         return x
0626     def _convertArguments(self, seq):
0627         if isinstance(seq, str):
0628             yield seq
0629         for x in seq:
0630             yield self._itemFromArgument(x)
0631     def append(self,x):
0632         if not self._itemIsValid(x):
0633             raise TypeError("wrong type being appended to container "+self._labelIfAny())
0634         super(_ValidatingListBase,self).append(self._itemFromArgument(x))
0635     def extend(self,x):
0636         if not self._isValid(x):
0637             raise TypeError("wrong type being extended to container "+self._labelIfAny())
0638         super(_ValidatingListBase,self).extend(self._convertArguments(x))
0639     def __add__(self,rhs):
0640         if not self._isValid(rhs):
0641             raise TypeError("wrong type being added to container "+self._labelIfAny())
0642         import copy
0643         value = copy.copy(self)
0644         value.extend(rhs)
0645         return value
0646     def insert(self,i,x):
0647         if not self._itemIsValid(x):
0648             raise TypeError("wrong type being inserted to container "+self._labelIfAny())
0649         super(_ValidatingListBase,self).insert(i,self._itemFromArgument(x))
0650     def _labelIfAny(self) -> str:
0651         result = type(self).__name__
0652         if hasattr(self, '__label'):
0653             result += ' ' + self.__label
0654         return result
0655
0656 class _ValidatingParameterListBase(_ValidatingListBase,_ParameterTypeBase):
0657     def __init__(self,*arg,**args):
0658         _ParameterTypeBase.__init__(self)
0659         if len (arg) == 1 and not isinstance(arg[0],str):
0660             try:
0661                 arg = iter(arg[0])
0662             except TypeError:
0663                 pass
0664         super(_ValidatingParameterListBase,self).__init__(*arg,**args)
0665     def value(self):
0666         return list(self)
0667     def setValue(self,v):
0668         self[:] = []
0669         self.extend(v)
0670         self._isModified=True
0671     def configValue(self, options:PrintOptions=PrintOptions()) -> str:
0672         config = '{\n'
0673         first = True
0674         for value in iter(self):
0675             options.indent()
0676             config += options.indentation()
0677             if not first:
0678                 config+=', '
0679             config+=  self.configValueForItem(value, options)+'\n'
0680             first = False
0681             options.unindent()
0682         config += options.indentation()+'}\n'
0683         return config
0684     def configValueForItem(self,item, options:PrintOptions) -> str:
0685         return str(item)
0686     def pythonValueForItem(self,item, options:PrintOptions) -> str:
0687         return self.configValueForItem(item, options)
0688     def __repr__(self):
0689         return self.dumpPython()
0690     def dumpPython(self, options:PrintOptions=PrintOptions()) -> str:
0691         specialImportRegistry.registerUse(self)
0692         result = self.pythonTypeName()+"("
0693         n = len(self)
0694         if hasattr(self, "_nPerLine"):
0695             nPerLine = self._nPerLine
0696         else:
0697             nPerLine = 5
0698         if n>nPerLine: options.indent()
0699         if n>=256:
0700             #wrap in a tuple since they don't have a size constraint
0701             result+=" ("
0702         wroteAtLeastOne = False
0703         for i, v in enumerate(self):
0704             wroteAtLeastOne = True
0705             if i == 0:
0706                 if n>nPerLine: result += '\n'+options.indentation()
0707             else:
0708                 if i % nPerLine == 0:
0709                     result += ',\n'+options.indentation()
0710                 else:
0711                     result += ', '
0712             result += self.pythonValueForItem(v,options)
0713         if n>=256:
0714             result +=' ) '
0715         moreArgs = self._additionalInitArguments(options)
0716         if moreArgs:
0717             if wroteAtLeastOne:
0718                 result += ', \n' + options.indentation()
0719             result += moreArgs
0720         if n>nPerLine:
0721             options.unindent()
0722             result += '\n'+options.indentation()
0723         result += ')'
0724         return result
0725     def _additionalInitArguments(self, options):
0726         return ''
0727     def directDependencies(self):
0728         return []
0729     @staticmethod
0730     def _itemsFromStrings(strings,converter):
0731         return (converter(x).value() for x in strings)
0732
0733 def saveOrigin(obj, level):
0734     import sys
0735     fInfo = inspect.getframeinfo(sys._getframe(level+1))
0736     obj._filename = fInfo.filename
0737     obj._lineNumber =fInfo.lineno
0738
0739 def _modifyParametersFromDict(params, newParams, errorRaiser, keyDepth=""):
0740     if len(newParams):
0741         #need to treat items both in params and myparams specially
0742         for key,value in newParams.items():
0743             if key in params:
0744                 if value is None:
0745                     del params[key]
0746                 elif isinstance(value, dict):
0747                     if isinstance(params[key],_Parameterizable):
0748                         pset = params[key]
0749                         p =pset.parameters_()
0750                         oldkeys = set(p.keys())
0751                         _modifyParametersFromDict(p,
0752                                                   value,errorRaiser,
0753                                                   ("%s.%s" if isinstance(key, str) else "%s[%s]")%(keyDepth,key))
0754                         for k,v in p.items():
0755                             setattr(pset,k,v)
0756                             oldkeys.discard(k)
0757                         for k in oldkeys:
0758                             delattr(pset,k)
0759                     elif isinstance(params[key],_ValidatingParameterListBase):
0760                         if any(not isinstance(k, int) for k in value.keys()):
0761                             raise TypeError("Attempted to change a list using a dict whose keys are not integers")
0762                         plist = params[key]
0763                         if any((k < 0 or k >= len(plist)) for k in value.keys()):
0764                             raise IndexError("Attempted to set an index which is not in the list")
0765                         p = dict(enumerate(plist))
0766                         _modifyParametersFromDict(p,
0767                                                   value,errorRaiser,
0768                                                   ("%s.%s" if isinstance(key, str) else "%s[%s]")%(keyDepth,key))
0769                         for k,v in p.items():
0770                             plist[k] = v
0771                     else:
0772                         raise ValueError("Attempted to change non PSet value "+keyDepth+" using a dictionary")
0773                 elif isinstance(value,_ParameterTypeBase) or (isinstance(key, int)) or isinstance(value, _Parameterizable):
0774                     params[key] = value
0775                 else:
0776                     params[key].setValue(value)
0777             else:
0778                 if isinstance(value,_ParameterTypeBase) or isinstance(value, _Parameterizable):
0779                     params[key]=value
0780                 else:
0781                     errorRaiser(key)
0782
0783
0784 if __name__ == "__main__":
0785
0786     import unittest
0787     class TestList(_ValidatingParameterListBase):
0788         @classmethod
0789         def _itemIsValid(cls,item):
0790             return True
0791     class testMixins(unittest.TestCase):
0792         def testListConstruction(self):
0793             t = TestList(1)
0794             self.assertEqual(t,[1])
0795             t = TestList((1,))
0796             self.assertEqual(t,[1])
0797             t = TestList("one")
0798             self.assertEqual(t,["one"])
0799             t = TestList( [1,])
0800             self.assertEqual(t,[1])
0801             t = TestList( (x for x in [1]) )
0802             self.assertEqual(t,[1])
0803
0804             t = TestList(1,2)
0805             self.assertEqual(t,[1,2])
0806             t = TestList((1,2))
0807             self.assertEqual(t,[1,2])
0808             t = TestList("one","two")
0809             self.assertEqual(t,["one","two"])
0810             t = TestList(("one","two"))
0811             self.assertEqual(t,["one","two"])
0812             t = TestList( [1,2])
0813             self.assertEqual(t,[1,2])
0814             t = TestList( (x for x in [1,2]) )
0815             self.assertEqual(t,[1,2])
0816             t = TestList( iter((1,2)) )
0817             self.assertEqual(t,[1,2])
0818
0819
0820         def testLargeList(self):
0821             #lists larger than 255 entries can not be initialized
0822             #using the constructor
0823             args = [i for i in range(0,300)]
0824
0825             t = TestList(*args)
0826             pdump= t.dumpPython()
0827             class cms(object):
0828                 def __init__(self):
0829                     self.TestList = TestList
0830             pythonized = eval( pdump, globals(),{'cms':cms()} )
0831             self.assertEqual(t,pythonized)
0832         def testUsingBlock(self):
0833             a = UsingBlock("a")
0834             self.assertTrue(isinstance(a, _ParameterTypeBase))
0835         def testConstruction(self):
0836             class __Test(_TypedParameterizable):
0837                 pass
0838             class __TestType(_SimpleParameterTypeBase):
0839                 def _isValid(self,value):
0840                     return True
0841             class __PSet(_ParameterTypeBase,_Parameterizable):
0842                 def __init__(self,*arg,**args):
0843                     #need to call the inits separately
0844                     _ParameterTypeBase.__init__(self)
0845                     _Parameterizable.__init__(self,*arg,**args)
0846
0847             a = __Test("MyType", __PSet(a=__TestType(1)))
0848             self.assertEqual(a.a.value(), 1)
0849             b = __Test("MyType", __PSet(a=__TestType(1)), __PSet(b=__TestType(2)))
0850             self.assertEqual(b.a.value(), 1)
0851             self.assertEqual(b.b.value(), 2)
0852             self.assertRaises(ValueError, lambda: __Test("MyType", __PSet(a=__TestType(1)), __PSet(a=__TestType(2))))
0853             c = __Test("MyType", dict(a=__TestType(1)), dict(b=__TestType(2)))
0854             self.assertEqual(c.a.value(), 1)
0855             self.assertEqual(c.b.value(), 2)
0856             self.assertRaises(ValueError, lambda: __Test("MyType", dict(a=__TestType(1)), dict(a=__TestType(2))))
0857         def testUpdate_(self):
0858             class __Test(_TypedParameterizable):
0859                 pass
0860             class __TestType(_SimpleParameterTypeBase):
0861                 def _isValid(self,value):
0862                     return True
0863             class __PSet(_ParameterTypeBase,_Parameterizable):
0864                 def __init__(self,*arg,**args):
0865                     #need to call the inits separately
0866                     _ParameterTypeBase.__init__(self)
0867                     _Parameterizable.__init__(self,*arg,**args)
0868             a = __Test("MyType", a = __TestType(1))
0869             a.update_(dict(b=__TestType(2)))
0870             self.assertEqual(a.a.value(), 1)
0871             self.assertEqual(a.b.value(), 2)
0872             a.update_(dict(a=3))
0873             self.assertEqual(a.a.value(), 3)
0874             a.update_(__PSet(a=__TestType(5)))
0875             self.assertEqual(a.a.value(), 5)
0876             self.assertRaises(TypeError, lambda: a.update_(dict(c=6)))
0877
0878         def testCopy(self):
0879             class __Test(_TypedParameterizable):
0880                 pass
0881             class __TestType(_SimpleParameterTypeBase):
0882                 def _isValid(self,value):
0883                     return True
0884             a = __Test("MyType",t=__TestType(1), u=__TestType(2))
0885             b = a.copy()
0886             self.assertEqual(b.t.value(),1)
0887             self.assertEqual(b.u.value(),2)
0888
0889             c = __Test("MyType")
0890             self.assertEqual(len(c.parameterNames_()), 0)
0891             d = c.copy()
0892             self.assertEqual(len(d.parameterNames_()), 0)
0893         def testClone(self):
0894             class __Test(_TypedParameterizable):
0895                 pass
0896             class __TestType(_SimpleParameterTypeBase):
0897                 def _isValid(self,value):
0898                     return True
0899             class __PSet(_ParameterTypeBase,_Parameterizable):
0900                 def __init__(self,*arg,**args):
0901                     #need to call the inits separately
0902                     _ParameterTypeBase.__init__(self)
0903                     _Parameterizable.__init__(self,*arg,**args)
0904                 def dumpPython(self,options=PrintOptions()):
0905                     return "__PSet(\n"+_Parameterizable.dumpPython(self, options)+options.indentation()+")"
0906
0907             a = __Test("MyType",
0908                        t=__TestType(1),
0909                        u=__TestType(2),
0910                        w = __TestType(3),
0911                        x = __PSet(a = __TestType(4),
0912                                   b = __TestType(6),
0913                                   c = __PSet(gamma = __TestType(5))))
0914             b = a.clone(t=3,
0915                         v=__TestType(4),
0916                         w= None,
0917                         x = dict(a = 7,
0918                                  c = dict(gamma = 8),
0919                                  d = __TestType(9)))
0920             c = a.clone(x = dict(a=None, c=None))
0921             self.assertEqual(a.t.value(),1)
0922             self.assertEqual(a.u.value(),2)
0923             self.assertEqual(b.t.value(),3)
0924             self.assertEqual(b.u.value(),2)
0925             self.assertEqual(b.v.value(),4)
0926             self.assertEqual(b.x.a.value(),7)
0927             self.assertEqual(b.x.b.value(),6)
0928             self.assertEqual(b.x.c.gamma.value(),8)
0929             self.assertEqual(b.x.d.value(),9)
0930             self.assertEqual(hasattr(b,"w"), False)
0931             self.assertEqual(hasattr(c.x,"a"), False)
0932             self.assertEqual(hasattr(c.x,"c"), False)
0933             self.assertRaises(TypeError,a.clone,**{"v":1})
0934             d = a.clone(__PSet(k=__TestType(42)))
0935             self.assertEqual(d.t.value(), 1)
0936             self.assertEqual(d.k.value(), 42)
0937             d2 = a.clone(__PSet(t=__TestType(42)))
0938             self.assertEqual(d2.t.value(), 42)
0939             d3 = a.clone(__PSet(t=__TestType(42)),
0940                          __PSet(u=__TestType(56)))
0941             self.assertEqual(d3.t.value(), 42)
0942             self.assertEqual(d3.u.value(), 56)
0943             self.assertRaises(ValueError,a.clone,
0944                               __PSet(t=__TestType(42)),
0945                               __PSet(t=__TestType(56)))
0946             d4 = a.clone(__PSet(t=__TestType(43)), u = 57)
0947             self.assertEqual(d4.t.value(), 43)
0948             self.assertEqual(d4.u.value(), 57)
0949             self.assertRaises(TypeError,a.clone,t=__TestType(43),**{"doesNotExist":57})
0950
0951             e = __Test("MyType")
0952             self.assertEqual(len(e.parameterNames_()), 0)
0953             f = e.clone(__PSet(a = __TestType(1)), b = __TestType(2))
0954             self.assertEqual(f.a.value(), 1)
0955             self.assertEqual(f.b.value(), 2)
0956             g = e.clone()
0957             self.assertEqual(len(g.parameterNames_()), 0)
0958
0959         def testModified(self):
0960             class __TestType(_SimpleParameterTypeBase):
0961                 def _isValid(self,value):
0962                     return True
0963             a = __TestType(1)
0964             self.assertEqual(a.isModified(),False)
0965             a.setValue(1)
0966             self.assertEqual(a.isModified(),False)
0967             a.setValue(2)
0968             self.assertEqual(a.isModified(),True)
0969             a.resetModified()
0970             self.assertEqual(a.isModified(),False)
0971         def testLargeParameterizable(self):
0972             class tLPTest(_TypedParameterizable):
0973                 pass
0974             class tLPTestType(_SimpleParameterTypeBase):
0975                 def _isValid(self,value):
0976                     return True
0977             class __DummyModule(object):
0978                 def __init__(self):
0979                     self.tLPTest = tLPTest
0980                     self.tLPTestType = tLPTestType
0981             p = tLPTest("MyType",** dict( [ ("a"+str(x), tLPTestType(x)) for x in range(0,300) ] ) )
0982             #check they are the same
0983             self.assertEqual(p.dumpPython(), eval(p.dumpPython(),{"cms": __DummyModule()}).dumpPython())
0984         def testSpecialImportRegistry(self):
0985             reg = _SpecialImportRegistry()
0986             reg.registerSpecialImportForType(int, "import foo")
0987             self.assertRaises(RuntimeError, lambda: reg.registerSpecialImportForType(int, "import bar"))
0988             reg.registerSpecialImportForType(str, "import bar")
0989             self.assertEqual(reg.getSpecialImports(), [])
0990             reg.registerUse([1])
0991             self.assertEqual(reg.getSpecialImports(), [])
0992             reg.registerUse(1)
0993             self.assertEqual(reg.getSpecialImports(), ["import foo"])
0994             reg.registerUse(1)
0995             self.assertEqual(reg.getSpecialImports(), ["import foo"])
0996             reg.registerUse("a")
0997             self.assertEqual(reg.getSpecialImports(), ["import bar", "import foo"])
0998         def testInvalidTypeChange(self):
0999             class __Test(_TypedParameterizable):
1000                 pass
1001             class __TestTypeA(_SimpleParameterTypeBase):
1002                 def _isValid(self,value):
1003                     return True
1004             class __TestTypeB(_SimpleParameterTypeBase):
1005                 def _isValid(self,value):
1006                     return True
1007                 pass
1008             a = __Test("MyType",
1009                        t=__TestTypeA(1))
1010             self.assertRaises(TypeError, lambda : setattr(a,'t',__TestTypeB(2)))
1011
1012
1013     unittest.main()