Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-11-25 02:29:06

0001 
0002 import configparser as ConfigParser
0003 import os
0004 import re
0005 import copy
0006 import collections
0007 from .TkAlExceptions import AllInOneError
0008 from future.utils import PY3
0009 
0010 if PY3:
0011     unicode = str
0012 
0013 class AdaptedDict(collections.OrderedDict):
0014     """
0015     Dictionary which handles updates of values for already existing keys
0016     in a modified way.
0017     adapteddict[key] returns a list of all values associated with key
0018     This dictionary is used in the class `BetterConfigParser` instead of the
0019     default `dict_type` of the `ConfigParser` class.
0020     """
0021 
0022     def __init__(self, *args, **kwargs):
0023         self.validationslist = []
0024         collections.OrderedDict.__init__(self, *args, **kwargs)
0025 
0026     def __setitem__(self, key, value, dict_setitem=collections.OrderedDict.__setitem__):
0027         """
0028         od.__setitem__(i, y) <==> od[i]=y
0029         Updating an existing key appends the new value to the old value
0030         instead of replacing it.
0031 
0032         Arguments:
0033         - `key`: key part of the key-value pair
0034         - `value`: value part of the key-value pair
0035         - `dict_item`: method which is used for finally setting the item
0036         """
0037 
0038         if key != "__name__" and "__name__" in self and self["__name__"]=="validation":
0039             if isinstance(value, (str, unicode)):
0040                 for index, item in enumerate(self.validationslist[:]):
0041                     if item == (key, value.split("\n")):
0042                         self.validationslist[index] = (key, value)
0043                         return
0044             self.validationslist.append((key, value))
0045         else:
0046             dict_setitem(self, key, value)
0047 
0048     def __getitem__(self, key):
0049         if key != "__name__" and "__name__" in self and self["__name__"]=="validation":
0050             return [validation[1] for validation in self.validationslist if validation[0] == key]
0051         else:
0052             return collections.OrderedDict.__getitem__(self, key)
0053 
0054     def items(self):
0055         if "__name__" in self and self["__name__"]=="validation":
0056             return self.validationslist
0057         else:
0058             return collections.OrderedDict.items(self)
0059 
0060 class BetterConfigParser(ConfigParser.ConfigParser):
0061     def __init__(self):
0062         ConfigParser.ConfigParser.__init__(self,dict_type=AdaptedDict)
0063         self._optcre = self.OPTCRE_VALIDATION
0064 
0065     def optionxform(self, optionstr):
0066         return optionstr
0067     
0068     def exists( self, section, option):
0069         try:
0070             items = self.items(section) 
0071         except ConfigParser.NoSectionError:
0072             return False
0073         for item in items:
0074             if item[0] == option:
0075                 return True
0076         return False
0077         
0078     def __updateDict( self, dictionary, section ):
0079         result = dictionary
0080         try:
0081             for option in self.options( section ):
0082                 result[option] = self.get( section, option )
0083             if "local"+section.title() in self.sections():
0084                 for option in self.options( "local"+section.title() ):
0085                     result[option] = self.get( "local"+section.title(),
0086                                                    option )
0087         except ConfigParser.NoSectionError as section:
0088             msg = ("%s in configuration files. This section is mandatory."
0089                    %(str(section).replace(":", "", 1)))
0090             raise AllInOneError(msg)
0091         return result
0092 
0093     def getResultingSection( self, section, defaultDict = {}, demandPars = [] ):
0094         result = copy.deepcopy(defaultDict)
0095         for option in demandPars:
0096             try:
0097                 result[option] = self.get( section, option )
0098             except ConfigParser.NoOptionError as globalSectionError:
0099                 globalSection = str( globalSectionError ).split( "'" )[-2]
0100                 splittedSectionName = section.split( ":" )
0101                 if len( splittedSectionName ) > 1:
0102                     localSection = ("local"+section.split( ":" )[0].title()+":"
0103                                     +section.split(":")[1])
0104                 else:
0105                     localSection = ("local"+section.split( ":" )[0].title())
0106                 if self.has_section( localSection ):
0107                     try:
0108                         result[option] = self.get( localSection, option )
0109                     except ConfigParser.NoOptionError as option:
0110                         msg = ("%s. This option is mandatory."
0111                                %(str(option).replace(":", "", 1).replace(
0112                                     "section",
0113                                     "section '"+globalSection+"' or", 1)))
0114                         raise AllInOneError(msg)
0115                 else:
0116                     msg = ("%s. This option is mandatory."
0117                            %(str(globalSectionError).replace(":", "", 1)))
0118                     raise AllInOneError(msg)
0119         try:
0120             result = self.__updateDict( result, section )
0121         except AllInOneError:   #section doesn't exist
0122             if demandPars:      #then there's at least one mandatory parameter, which means the section needs to be there
0123                 raise           #otherwise all the parameters are optional, so it's ok
0124         return result
0125 
0126     def getAlignments( self ):
0127         alignments = []
0128         for section in self.sections():
0129             if "alignment:" in section:
0130                 alignments.append( Alignment( section.split( "alignment:" )[1],
0131                                               self ) )
0132         names_after_cleaning = [alignment.name for alignment in alignments]
0133         duplicates = [name
0134                       for name, count
0135                       in collections.Counter(names_after_cleaning).items()
0136                       if count > 1]
0137         if len(duplicates) > 0:
0138             msg = "Duplicate alignment names after removing invalid characters: "
0139             msg += ", ".join(duplicates) +"\n"
0140             msg += "Please rename the alignments to avoid name clashes."
0141             raise AllInOneError(msg)
0142         return alignments
0143 
0144     def getCompares( self ):
0145         compares = {}
0146         for section in self.sections():
0147             if "compare:" in section:
0148                 self.checkInput(section,
0149                                 knownSimpleOptions = ["levels", "dbOutput","moduleList","modulesToPlot","useDefaultRange","plotOnlyGlobal","plotPng","makeProfilePlots",
0150                                                       "dx_min","dx_max","dy_min","dy_max","dz_min","dz_max","dr_min","dr_max","rdphi_min","rdphi_max",
0151                                                       "dalpha_min","dalpha_max","dbeta_min","dbeta_max","dgamma_min","dgamma_max",
0152                                                       "jobmode", "3DSubdetector1", "3Dubdetector2", "3DTranslationalScaleFactor", "jobid", "multiIOV"])
0153                 levels = self.get( section, "levels" )
0154                 dbOutput = self.get( section, "dbOutput" )
0155                 compares[section.split(":")[1]] = ( levels, dbOutput )
0156         return compares
0157 
0158     def getGeneral( self ):
0159         defaults = {
0160             "jobmode":"interactive",
0161             "datadir":os.getcwd(),
0162             "logdir":os.getcwd(),
0163             }
0164         mandatories = ["eosdir",]
0165         self.checkInput("general", knownSimpleOptions = list(defaults.keys()) + mandatories )
0166         general = self.getResultingSection( "general", defaultDict = defaults, demandPars = mandatories )
0167         internal_section = "internals"
0168         if not self.has_section(internal_section):
0169             self.add_section(internal_section)
0170         if not self.has_option(internal_section, "workdir"):
0171             self.set(internal_section, "workdir", "/tmp/$USER")
0172         if not self.has_option(internal_section, "scriptsdir"):
0173             self.set(internal_section, "scriptsdir", "")
0174             #replaceByMap will fail if this is not replaced (which it is in validateAlignments.py)
0175 
0176         general["workdir"] = self.get(internal_section, "workdir")
0177         general["scriptsdir"] = self.get(internal_section, "scriptsdir")
0178         for folder in "workdir", "datadir", "logdir", "eosdir":
0179             general[folder] = os.path.expandvars(general[folder])
0180 
0181         return general
0182     
0183     def checkInput(self, section, knownSimpleOptions=[], knownKeywords=[],
0184                    ignoreOptions=[]):
0185         """
0186         Method which checks, if the given options in `section` are in the
0187         list of `knownSimpleOptions` or match an item of `knownKeywords`.
0188         This is basically a check for typos and wrong parameters.
0189         
0190         Arguments:
0191         - `section`: Section of a configuration file
0192         - `knownSimpleOptions`: List of allowed simple options in `section`.
0193         - `knownKeywords`: List of allowed keywords in `section`.
0194         """
0195 
0196         try:
0197             for option in self.options( section ):
0198                 if option in knownSimpleOptions:
0199                     continue
0200                 elif option.split()[0] in knownKeywords:
0201                     continue
0202                 elif option in ignoreOptions:
0203                     print ("Ignoring option '%s' in section '[%s]'."
0204                            %(option, section))
0205                 else:
0206                     msg = ("Invalid or unknown parameter '%s' in section '%s'!"
0207                            %(option, section))
0208                     raise AllInOneError(msg)
0209         except ConfigParser.NoSectionError:
0210             pass
0211 
0212     def set(self, section, option, value=None):
0213         try:
0214             ConfigParser.ConfigParser.set(self, section, option, value)
0215         except ConfigParser.NoSectionError:
0216             self.add_section(section)
0217             ConfigParser.ConfigParser.set(self, section, option, value)
0218 
0219     def items(self, section, raw=False, vars=None):
0220         if section == "validation":
0221             if raw or vars:
0222                 raise NotImplementedError("'raw' and 'vars' do not work for betterConfigParser.items()!")
0223             items = self._sections["validation"].items()
0224             return items
0225         else:
0226             return ConfigParser.ConfigParser.items(self, section, raw, vars)
0227 
0228     def write(self, fp):
0229         """Write an .ini-format representation of the configuration state."""
0230         for section in self._sections:
0231             fp.write("[%s]\n" % section)
0232             for (key, value) in self._sections[section].items():
0233                 if key == "__name__" or not isinstance(value, (str, unicode)):
0234                     continue
0235                 if value is not None:
0236                     key = " = ".join((key, str(value).replace('\n', '\n\t')))
0237                 fp.write("%s\n" % (key))
0238             fp.write("\n")
0239 
0240 
0241     #Preexisting validations in the validation section have syntax:
0242     #  preexistingoffline myoffline
0243     #with no = or :.  This regex takes care of that.
0244     OPTCRE_VALIDATION = re.compile(
0245         r'(?P<option>'
0246         r'(?P<preexisting>preexisting)?'
0247         r'[^:=\s][^:=]*)'                     # very permissive!
0248         r'\s*(?(preexisting)|'                # IF preexisting does not exist:
0249         r'(?P<vi>[:=])\s*'                    #   any number of space/tab,
0250                                               #   followed by separator
0251                                               #   (either : or =), followed
0252                                               #   by any # space/tab
0253         r'(?P<value>.*))$'                    #   everything up to eol
0254         )