Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2021-07-23 02:25:14

0001 from __future__ import print_function
0002 from __future__ import absolute_import
0003 from builtins import range
0004 from abc import ABCMeta, abstractmethod, abstractproperty
0005 import os
0006 import re
0007 import json
0008 from . import globalDictionaries
0009 from . import configTemplates
0010 from .dataset import Dataset
0011 from .helperFunctions import replaceByMap, addIndex, getCommandOutput2, boolfromstring, pythonboolstring
0012 from .TkAlExceptions import AllInOneError
0013 
0014 class ValidationMetaClass(ABCMeta):
0015     sets = ["mandatories", "optionals", "needpackages"]
0016     dicts = ["defaults"]
0017     def __new__(cls, clsname, bases, dct):
0018         for setname in cls.sets:
0019             if setname not in dct: dct[setname] = set()
0020             dct[setname] = set.union(dct[setname], *(getattr(base, setname) for base in bases if hasattr(base, setname)))
0021 
0022         for dictname in cls.dicts:
0023             if dictname not in dct: dct[dictname] = {}
0024             for base in bases:
0025                 if not hasattr(base, dictname): continue
0026                 newdict = getattr(base, dictname)
0027                 for key in set(newdict) & set(dct[dictname]):
0028                     if newdict[key] != dct[dictname][key]:
0029                         raise ValueError("Inconsistent values of defaults[{}]: {}, {}".format(key, newdict[key], dct[dictname][key]))
0030                 dct[dictname].update(newdict)
0031 
0032         for setname in cls.sets:      #e.g. removemandatories, used in preexistingvalidation
0033                                       #use with caution
0034             if "remove"+setname not in dct: dct["remove"+setname] = set()
0035             dct["remove"+setname] = set.union(dct["remove"+setname], *(getattr(base, "remove"+setname) for base in bases if hasattr(base, "remove"+setname)))
0036 
0037             dct[setname] -= dct["remove"+setname]
0038 
0039         return super(ValidationMetaClass, cls).__new__(cls, clsname, bases, dct)
0040 
0041 class GenericValidation(object, metaclass=ValidationMetaClass):
0042     defaultReferenceName = "DEFAULT"
0043     mandatories = set()
0044     defaults = {
0045                 "cmssw":        os.environ['CMSSW_BASE'],
0046                 "parallelJobs": "1",
0047                 "jobid":        "",
0048                 "needsproxy":   "false",
0049                }
0050     needpackages = {"Alignment/OfflineValidation"}
0051     optionals = {"jobmode"}
0052 
0053     def __init__(self, valName, alignment, config):
0054         import random
0055         self.name = valName
0056         self.alignmentToValidate = alignment
0057         self.general = config.getGeneral()
0058         self.randomWorkdirPart = "%0i"%random.randint(1,10e9)
0059         self.configFiles = []
0060         self.config = config
0061         self.jobid = ""
0062 
0063         theUpdate = config.getResultingSection(self.valType+":"+self.name,
0064                                                defaultDict = self.defaults,
0065                                                demandPars = self.mandatories)
0066         self.general.update(theUpdate)
0067         self.jobmode = self.general["jobmode"]
0068         self.NJobs = int(self.general["parallelJobs"])
0069         self.needsproxy = boolfromstring(self.general["needsproxy"], "needsproxy")
0070 
0071         # limit maximum number of parallel jobs to 40
0072         # (each output file is approximately 20MB)
0073         maximumNumberJobs = 40
0074         if self.NJobs > maximumNumberJobs:
0075             msg = ("Maximum allowed number of parallel jobs "
0076                    +str(maximumNumberJobs)+" exceeded!!!")
0077             raise AllInOneError(msg)
0078         if self.NJobs > 1 and not isinstance(self, ParallelValidation):
0079             raise AllInOneError("Parallel jobs not implemented for {}!\n"
0080                                 "Please set parallelJobs = 1.".format(type(self).__name__))
0081 
0082         self.jobid = self.general["jobid"]
0083         if self.jobid:
0084             try:  #make sure it's actually a valid jobid
0085                 output = getCommandOutput2("bjobs %(jobid)s 2>&1"%self.general)
0086                 if "is not found" in output: raise RuntimeError
0087             except RuntimeError:
0088                 raise AllInOneError("%s is not a valid jobid.\nMaybe it finished already?"%self.jobid)
0089 
0090         self.cmssw = self.general["cmssw"]
0091         badcharacters = r"\'"
0092         for character in badcharacters:
0093             if character in self.cmssw:
0094                 raise AllInOneError("The bad characters " + badcharacters + " are not allowed in the cmssw\n"
0095                                     "path name.  If you really have it in such a ridiculously named location,\n"
0096                                     "try making a symbolic link somewhere with a decent name.")
0097         try:
0098             os.listdir(self.cmssw)
0099         except OSError:
0100             raise AllInOneError("Your cmssw release " + self.cmssw + ' does not exist')
0101 
0102         if self.cmssw == os.environ["CMSSW_BASE"]:
0103             self.scramarch = os.environ["SCRAM_ARCH"]
0104             self.cmsswreleasebase = os.environ["CMSSW_RELEASE_BASE"]
0105         else:
0106             command = ("cd '" + self.cmssw + "' && eval `scramv1 ru -sh 2> /dev/null`"
0107                        ' && echo "$CMSSW_BASE\n$SCRAM_ARCH\n$CMSSW_RELEASE_BASE"')
0108             commandoutput = getCommandOutput2(command).split('\n')
0109             self.cmssw = commandoutput[0]
0110             self.scramarch = commandoutput[1]
0111             self.cmsswreleasebase = commandoutput[2]
0112 
0113         self.packages = {}
0114         for package in self.needpackages:
0115             for placetolook in self.cmssw, self.cmsswreleasebase:
0116                 pkgpath = os.path.join(placetolook, "src", package)
0117                 if os.path.exists(pkgpath):
0118                     self.packages[package] = pkgpath
0119                     break
0120             else:
0121                 raise AllInOneError("Package {} does not exist in {} or {}!".format(package, self.cmssw, self.cmsswreleasebase))
0122 
0123         self.AutoAlternates = True
0124         if config.has_option("alternateTemplates","AutoAlternates"):
0125             try:
0126                 self.AutoAlternates = json.loads(config.get("alternateTemplates","AutoAlternates").lower())
0127             except ValueError:
0128                 raise AllInOneError("AutoAlternates needs to be true or false, not %s" % config.get("alternateTemplates","AutoAlternates"))
0129 
0130         knownOpts = set(self.defaults.keys())|self.mandatories|self.optionals
0131         ignoreOpts = []
0132         config.checkInput(self.valType+":"+self.name,
0133                           knownSimpleOptions = knownOpts,
0134                           ignoreOptions = ignoreOpts)
0135 
0136     def getRepMap(self, alignment = None):
0137         from .plottingOptions import PlottingOptions
0138         if alignment == None:
0139             alignment = self.alignmentToValidate
0140         try:
0141             result = PlottingOptions(self.config, self.valType)
0142         except KeyError:
0143             result = {}
0144         result.update(alignment.getRepMap())
0145         result.update(self.general)
0146         result.update({
0147                 "workdir": os.path.join(self.general["workdir"],
0148                                         self.randomWorkdirPart),
0149                 "datadir": self.general["datadir"],
0150                 "logdir": self.general["logdir"],
0151                 "CommandLineTemplate": ("#run configfile and post-proccess it\n"
0152                                         "cmsRun %(cfgFile)s\n"
0153                                         "%(postProcess)s "),
0154                 "CMSSW_BASE": self.cmssw,
0155                 "SCRAM_ARCH": self.scramarch,
0156                 "CMSSW_RELEASE_BASE": self.cmsswreleasebase,
0157                 "alignmentName": alignment.name,
0158                 "condLoad": alignment.getConditions(),
0159                 "LoadGlobalTagTemplate": configTemplates.loadGlobalTagTemplate,
0160                 })
0161         result.update(self.packages)
0162         return result
0163 
0164     @abstractproperty
0165     def filesToCompare(self):
0166         pass
0167 
0168     def getCompareStrings( self, requestId = None, plain = False ):
0169         result = {}
0170         repMap = self.getRepMap().copy()
0171         for validationId in self.filesToCompare:
0172             repMap["file"] = self.filesToCompare[ validationId ]
0173             if repMap["file"].startswith( "/castor/" ):
0174                 repMap["file"] = "rfio:%(file)s"%repMap
0175             elif repMap["file"].startswith( "/store/" ):
0176                 repMap["file"] = "root://eoscms.cern.ch//eos/cms%(file)s"%repMap
0177             if plain:
0178                 result[validationId]=repMap["file"]
0179             else:
0180                 result[validationId]= "%(file)s=%(title)s|%(color)s|%(style)s"%repMap
0181         if requestId == None:
0182             return result
0183         else:
0184             if not "." in requestId:
0185                 requestId += ".%s"%self.defaultReferenceName
0186             if not requestId.split(".")[-1] in result:
0187                 msg = ("could not find %s in reference Objects!"
0188                        %requestId.split(".")[-1])
0189                 raise AllInOneError(msg)
0190             return result[ requestId.split(".")[-1] ]
0191 
0192     def createFiles(self, fileContents, path, repMap = None, repMaps = None):
0193         """repMap: single map for all files
0194            repMaps: a dict, with the filenames as the keys"""
0195         if repMap is not None and repMaps is not None:
0196             raise AllInOneError("createFiles can only take repMap or repMaps (or neither), not both")
0197         result = []
0198         for fileName in fileContents:
0199             filePath = os.path.join(path, fileName)
0200             result.append(filePath)
0201 
0202             for (i, filePathi) in enumerate(addIndex(filePath, self.NJobs)):
0203                 theFile = open( filePathi, "w" )
0204                 fileContentsi = fileContents[ fileName ]
0205                 if repMaps is not None:
0206                     repMap = repMaps[fileName]
0207                 if repMap is not None:
0208                     repMap.update({"nIndex": str(i)})
0209                     fileContentsi = replaceByMap(fileContentsi, repMap)
0210                 theFile.write( fileContentsi )
0211                 theFile.close()
0212 
0213         return result
0214 
0215     def createConfiguration(self, fileContents, path, schedule = None, repMap = None, repMaps = None):
0216         self.configFiles = self.createFiles(fileContents,
0217                                             path, repMap = repMap, repMaps = repMaps)
0218         if not schedule == None:
0219             schedule = [os.path.join( path, cfgName) for cfgName in schedule]
0220             for cfgName in schedule:
0221                 if not cfgName in self.configFiles:
0222                     msg = ("scheduled %s missing in generated configfiles: %s"
0223                            %(cfgName, self.configFiles))
0224                     raise AllInOneError(msg)
0225             for cfgName in self.configFiles:
0226                 if not cfgName in schedule:
0227                     msg = ("generated configuration %s not scheduled: %s"
0228                            %(cfgName, schedule))
0229                     raise AllInOneError(msg)
0230             self.configFiles = schedule
0231         return self.configFiles
0232 
0233     def createScript(self, fileContents, path, downloadFiles=[], repMap = None, repMaps = None):
0234         self.scriptFiles = self.createFiles(fileContents,
0235                                             path, repMap = repMap, repMaps = repMaps)
0236         for script in self.scriptFiles:
0237             for scriptwithindex in addIndex(script, self.NJobs):
0238                 os.chmod(scriptwithindex,0o755)
0239         return self.scriptFiles
0240 
0241     def createCrabCfg(self, fileContents, path ):
0242         if self.NJobs > 1:
0243             msg =  ("jobmode 'crab' not supported for parallel validation."
0244                     " Please set parallelJobs = 1.")
0245             raise AllInOneError(msg)
0246         self.crabConfigFiles = self.createFiles(fileContents, path)
0247         return self.crabConfigFiles
0248 
0249 
0250 class GenericValidationData(GenericValidation):
0251     """
0252     Subclass of `GenericValidation` which is the base for validations using
0253     datasets.
0254     """
0255     needParentFiles = False
0256     mandatories = {"dataset", "maxevents"}
0257     defaults = {
0258                 "runRange": "",
0259                 "firstRun": "",
0260                 "lastRun": "",
0261                 "begin": "",
0262                 "end": "",
0263                 "JSON": "",
0264                 "dasinstance": "prod/global",
0265                 "ttrhbuilder":"WithAngleAndTemplate",
0266                 "usepixelqualityflag": "True",
0267                }
0268     optionals = {"magneticfield"}
0269     
0270     def __init__(self, valName, alignment, config):
0271         """
0272         This method adds additional items to the `self.general` dictionary
0273         which are only needed for validations using datasets.
0274         
0275         Arguments:
0276         - `valName`: String which identifies individual validation instances
0277         - `alignment`: `Alignment` instance to validate
0278         - `config`: `BetterConfigParser` instance which includes the
0279                     configuration of the validations
0280         """
0281 
0282         super(GenericValidationData, self).__init__(valName, alignment, config)
0283 
0284         # if maxevents is not specified, cannot calculate number of events for
0285         # each parallel job, and therefore running only a single job
0286         if int( self.general["maxevents"] ) < 0 and self.NJobs > 1:
0287             msg = ("Maximum number of events (maxevents) not specified: "
0288                    "cannot use parallel jobs.")
0289             raise AllInOneError(msg)
0290         if int( self.general["maxevents"] ) / self.NJobs != float( self.general["maxevents"] ) / self.NJobs:
0291             msg = ("maxevents has to be divisible by parallelJobs")
0292             raise AllInOneError(msg)
0293 
0294         tryPredefinedFirst = (not self.jobmode.split( ',' )[0] == "crab" and self.general["JSON"]    == ""
0295                               and self.general["firstRun"] == ""         and self.general["lastRun"] == ""
0296                               and self.general["begin"]    == ""         and self.general["end"]     == "")
0297 
0298         if self.general["dataset"] not in globalDictionaries.usedDatasets:
0299             globalDictionaries.usedDatasets[self.general["dataset"]] = {}
0300 
0301         if self.cmssw not in globalDictionaries.usedDatasets[self.general["dataset"]]:
0302             if globalDictionaries.usedDatasets[self.general["dataset"]] != {}:
0303                 print(("Warning: you use the same dataset '%s' in multiple cmssw releases.\n"
0304                        "This is allowed, but make sure it's not a mistake") % self.general["dataset"])
0305             globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw] = {False: None, True: None}
0306 
0307         Bfield = self.general.get("magneticfield", None)
0308         if globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][tryPredefinedFirst] is None:
0309             dataset = Dataset(
0310                 self.general["dataset"], tryPredefinedFirst = tryPredefinedFirst,
0311                 cmssw = self.cmssw, cmsswrelease = self.cmsswreleasebase, magneticfield = Bfield,
0312                 dasinstance = self.general["dasinstance"])
0313             globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][tryPredefinedFirst] = dataset
0314             if tryPredefinedFirst and not dataset.predefined():                              #No point finding the data twice in that case
0315                 globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][False] = dataset
0316 
0317         self.dataset = globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][tryPredefinedFirst]
0318         self.general["magneticField"] = self.dataset.magneticField()
0319         self.general["defaultMagneticField"] = "MagneticField"
0320         if self.general["magneticField"] == "unknown":
0321             print("Could not get the magnetic field for this dataset.")
0322             print("Using the default: ", self.general["defaultMagneticField"])
0323             self.general["magneticField"] = '.oO[defaultMagneticField]Oo.'
0324         
0325         if not self.jobmode.split( ',' )[0] == "crab":
0326             try:
0327                 self.general["datasetDefinition"] = self.dataset.datasetSnippet(
0328                     jsonPath = self.general["JSON"],
0329                     firstRun = self.general["firstRun"],
0330                     lastRun = self.general["lastRun"],
0331                     begin = self.general["begin"],
0332                     end = self.general["end"],
0333                     parent = self.needParentFiles )
0334             except AllInOneError as e:
0335                 msg = "In section [%s:%s]: "%(self.valType, self.name)
0336                 msg += str(e)
0337                 raise AllInOneError(msg)
0338         else:
0339             if self.dataset.predefined():
0340                 msg = ("For jobmode 'crab' you cannot use predefined datasets "
0341                        "(in your case: '%s')."%( self.dataset.name() ))
0342                 raise AllInOneError( msg )
0343             try:
0344                 theUpdate = config.getResultingSection(self.valType+":"+self.name,
0345                                                        demandPars = ["parallelJobs"])
0346             except AllInOneError as e:
0347                 msg = str(e)[:-1]+" when using 'jobmode: crab'."
0348                 raise AllInOneError(msg)
0349             self.general.update(theUpdate)
0350             if self.general["begin"] or self.general["end"]:
0351                 ( self.general["begin"],
0352                   self.general["end"],
0353                   self.general["firstRun"],
0354                   self.general["lastRun"] ) = self.dataset.convertTimeToRun(
0355                     firstRun = self.general["firstRun"],
0356                     lastRun = self.general["lastRun"],
0357                     begin = self.general["begin"],
0358                     end = self.general["end"],
0359                     shortTuple = False)
0360                 if self.general["begin"] == None:
0361                     self.general["begin"] = ""
0362                 if self.general["end"] == None:
0363                     self.general["end"] = ""
0364                 self.general["firstRun"] = str( self.general["firstRun"] )
0365                 self.general["lastRun"] = str( self.general["lastRun"] )
0366             if ( not self.general["firstRun"] ) and \
0367                    ( self.general["end"] or self.general["lastRun"] ):
0368                 self.general["firstRun"] = str(
0369                     self.dataset.runList()[0]["run_number"])
0370             if ( not self.general["lastRun"] ) and \
0371                    ( self.general["begin"] or self.general["firstRun"] ):
0372                 self.general["lastRun"] = str(
0373                     self.dataset.runList()[-1]["run_number"])
0374             if self.general["firstRun"] and self.general["lastRun"]:
0375                 if int(self.general["firstRun"]) > int(self.general["lastRun"]):
0376                     msg = ( "The lower time/runrange limit ('begin'/'firstRun') "
0377                             "chosen is greater than the upper time/runrange limit "
0378                             "('end'/'lastRun').")
0379                     raise AllInOneError( msg )
0380                 self.general["runRange"] = (self.general["firstRun"]
0381                                             + '-' + self.general["lastRun"])
0382             try:
0383                 self.general["datasetDefinition"] = self.dataset.datasetSnippet(
0384                     jsonPath = self.general["JSON"],
0385                     firstRun = self.general["firstRun"],
0386                     lastRun = self.general["lastRun"],
0387                     begin = self.general["begin"],
0388                     end = self.general["end"],
0389                     crab = True )
0390             except AllInOneError as e:
0391                 msg = "In section [%s:%s]: "%(self.valType, self.name)
0392                 msg += str( e )
0393                 raise AllInOneError( msg )
0394 
0395         self.general["usepixelqualityflag"] = pythonboolstring(self.general["usepixelqualityflag"], "usepixelqualityflag")
0396 
0397     def getRepMap(self, alignment = None):
0398         result = super(GenericValidationData, self).getRepMap(alignment)
0399         outputfile = os.path.expandvars(replaceByMap(
0400                            "%s_%s_.oO[name]Oo..root" % (self.outputBaseName, self.name)
0401                                  , result))
0402         resultfile = os.path.expandvars(replaceByMap(("/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./" +
0403                            "%s_%s_.oO[name]Oo..root" % (self.resultBaseName, self.name))
0404                                  , result))
0405         result.update({
0406                 "resultFile": ".oO[resultFiles[.oO[nIndex]Oo.]]Oo.",
0407                 "resultFiles": addIndex(resultfile, self.NJobs),
0408                 "finalResultFile": resultfile,
0409                 "outputFile": ".oO[outputFiles[.oO[nIndex]Oo.]]Oo.",
0410                 "outputFiles": addIndex(outputfile, self.NJobs),
0411                 "finalOutputFile": outputfile,
0412                 "ProcessName": self.ProcessName,
0413                 "Bookkeeping": self.Bookkeeping,
0414                 "LoadBasicModules": self.LoadBasicModules,
0415                 "TrackSelectionRefitting": self.TrackSelectionRefitting,
0416                 "ValidationConfig": self.ValidationTemplate,
0417                 "FileOutputTemplate": self.FileOutputTemplate,
0418                 "DefinePath": self.DefinePath,
0419                 })
0420         return result
0421 
0422     @property
0423     def cfgName(self):
0424         return "%s.%s.%s_cfg.py"%( self.configBaseName, self.name,
0425                                    self.alignmentToValidate.name )
0426     @abstractproperty
0427     def ProcessName(self):
0428         pass
0429 
0430     @property
0431     def cfgTemplate(self):
0432         return configTemplates.cfgTemplate
0433 
0434     @abstractproperty
0435     def ValidationTemplate(self):
0436         pass
0437 
0438     @property
0439     def filesToCompare(self):
0440         return {self.defaultReferenceName: self.getRepMap()["finalResultFile"]}
0441 
0442     def createConfiguration(self, path ):
0443         repMap = self.getRepMap()
0444         cfgs = {self.cfgName: self.cfgTemplate}
0445         super(GenericValidationData, self).createConfiguration(cfgs, path, repMap=repMap)
0446 
0447     def createScript(self, path, template = configTemplates.scriptTemplate, downloadFiles=[], repMap = None, repMaps = None):
0448         scriptName = "%s.%s.%s.sh"%(self.scriptBaseName, self.name,
0449                                     self.alignmentToValidate.name )
0450         if repMap is None and repMaps is None:
0451             repMap = self.getRepMap()
0452             repMap["CommandLine"]=""
0453             for cfg in self.configFiles:
0454                 repMap["CommandLine"]+= repMap["CommandLineTemplate"]%{"cfgFile":addIndex(cfg, self.NJobs, ".oO[nIndex]Oo."),
0455                                                       "postProcess":""
0456                                                      }
0457         scripts = {scriptName: template}
0458         return super(GenericValidationData, self).createScript(scripts, path, downloadFiles = downloadFiles,
0459                                                                repMap = repMap, repMaps = repMaps)
0460 
0461     def createCrabCfg(self, path, crabCfgBaseName):
0462         """
0463         Method which creates a `crab.cfg` for a validation on datasets.
0464         
0465         Arguments:
0466         - `path`: Path at which the file will be stored.
0467         - `crabCfgBaseName`: String which depends on the actual type of
0468                              validation calling this method.
0469         """
0470         crabCfgName = "crab.%s.%s.%s.cfg"%( crabCfgBaseName, self.name,
0471                                             self.alignmentToValidate.name )
0472         repMap = self.getRepMap()
0473         repMap["script"] = "dummy_script.sh"
0474         # repMap["crabOutputDir"] = os.path.basename( path )
0475         repMap["crabWorkingDir"] = crabCfgName.split( '.cfg' )[0]
0476         self.crabWorkingDir = repMap["crabWorkingDir"]
0477         repMap["numberOfJobs"] = self.general["parallelJobs"]
0478         repMap["cfgFile"] = self.configFiles[0]
0479         repMap["queue"] = self.jobmode.split( ',' )[1].split( '-q' )[1]
0480         if self.dataset.dataType() == "mc":
0481             repMap["McOrData"] = "events = .oO[nEvents]Oo."
0482         elif self.dataset.dataType() == "data":
0483             repMap["McOrData"] = "lumis = -1"
0484             if self.jobmode.split( ',' )[0] == "crab":
0485                 print ("For jobmode 'crab' the parameter 'maxevents' will be "
0486                        "ignored and all events will be processed.")
0487         else:
0488             raise AllInOneError("Unknown data type!  Can't run in crab mode")
0489         crabCfg = {crabCfgName: replaceByMap( configTemplates.crabCfgTemplate,
0490                                               repMap ) }
0491         return super(GenericValidationData, self).createCrabCfg( crabCfg, path )
0492 
0493     @property
0494     def Bookkeeping(self):
0495         return configTemplates.Bookkeeping
0496     @property
0497     def LoadBasicModules(self):
0498         return configTemplates.LoadBasicModules
0499     @abstractproperty
0500     def TrackSelectionRefitting(self):
0501         pass
0502     @property
0503     def FileOutputTemplate(self):
0504         return configTemplates.FileOutputTemplate
0505     @abstractproperty
0506     def DefinePath(self):
0507         pass
0508 
0509 class GenericValidationData_CTSR(GenericValidationData):
0510     #common track selection and refitting
0511     defaults = {
0512         "momentumconstraint": "None",
0513         "openmasswindow": "False",
0514         "cosmicsdecomode": "True",
0515         "removetrackhitfiltercommands": "",
0516         "appendtrackhitfiltercommands": "",
0517     }
0518     def getRepMap(self, alignment=None):
0519         result = super(GenericValidationData_CTSR, self).getRepMap(alignment)
0520 
0521         from .trackSplittingValidation import TrackSplittingValidation
0522         result.update({
0523             "ValidationSequence": self.ValidationSequence,
0524             "istracksplitting": str(isinstance(self, TrackSplittingValidation)),
0525             "cosmics0T": str(self.cosmics0T),
0526             "use_d0cut": str(self.use_d0cut),
0527             "ispvvalidation": str(self.isPVValidation) 
0528         })
0529 
0530         commands = []
0531         for removeorappend in "remove", "append":
0532             optionname = removeorappend + "trackhitfiltercommands"
0533             if result[optionname]:
0534                 for command in result[optionname].split(","):
0535                     command = command.strip()
0536                     commands.append('process.TrackerTrackHitFilter.commands.{}("{}")'.format(removeorappend, command))
0537         result["trackhitfiltercommands"] = "\n".join(commands)
0538 
0539         return result
0540     @property
0541     def use_d0cut(self):
0542         return "Cosmics" not in self.general["trackcollection"]  #use it for collisions only
0543     @property
0544     def isPVValidation(self):
0545         return False  # only for PV Validation sequence
0546     @property
0547     def TrackSelectionRefitting(self):
0548         return configTemplates.CommonTrackSelectionRefitting
0549     @property
0550     def DefinePath(self):
0551         return configTemplates.DefinePath_CommonSelectionRefitting
0552     @abstractproperty
0553     def ValidationSequence(self):
0554         pass
0555     @property
0556     def cosmics0T(self):
0557         if "Cosmics" not in self.general["trackcollection"]: return False
0558         Bfield = self.dataset.magneticFieldForRun()
0559         if Bfield < 0.5: return True
0560         if isinstance(Bfield, str):
0561             if "unknown " in Bfield:
0562                 msg = Bfield.replace("unknown ","",1)
0563             elif Bfield == "unknown":
0564                 msg = "Can't get the B field for %s." % self.dataset.name()
0565             else:
0566                 msg = "B field = {}???".format(Bfield)
0567             raise AllInOneError(msg + "\n"
0568                                 "To use this dataset, specify magneticfield = [value] in your .ini config file.")
0569         return False
0570 
0571 class ParallelValidation(GenericValidation):
0572     @classmethod
0573     def initMerge(cls):
0574         return ""
0575     @abstractmethod
0576     def appendToMerge(self):
0577         pass
0578 
0579     @classmethod
0580     def doInitMerge(cls):
0581         from .plottingOptions import PlottingOptions
0582         result = cls.initMerge()
0583         result = replaceByMap(result, PlottingOptions(None, cls))
0584         if result and result[-1] != "\n": result += "\n"
0585         return result
0586     def doMerge(self):
0587         result = self.appendToMerge()
0588         if result[-1] != "\n": result += "\n"
0589         result += ("if [[ tmpMergeRetCode -eq 0 ]]; then\n"
0590                    "  xrdcp -f .oO[finalOutputFile]Oo. root://eoscms//eos/cms.oO[finalResultFile]Oo.\n"
0591                    "fi\n"
0592                    "if [[ ${tmpMergeRetCode} -gt ${mergeRetCode} ]]; then\n"
0593                    "  mergeRetCode=${tmpMergeRetCode}\n"
0594                    "fi\n")
0595         result = replaceByMap(result, self.getRepMap())
0596         return result
0597 
0598 class ValidationWithPlots(GenericValidation):
0599     @classmethod
0600     def runPlots(cls, validations):
0601         return ("cp .oO[plottingscriptpath]Oo. .\n"
0602                 "root -x -b -q .oO[plottingscriptname]Oo.++")
0603     @abstractmethod
0604     def appendToPlots(self):
0605         pass
0606     @abstractmethod
0607     def plottingscriptname(cls):
0608         """override with a classmethod"""
0609     @abstractmethod
0610     def plottingscripttemplate(cls):
0611         """override with a classmethod"""
0612     @abstractmethod
0613     def plotsdirname(cls):
0614         """override with a classmethod"""
0615 
0616     @classmethod
0617     def doRunPlots(cls, validations):
0618         from .plottingOptions import PlottingOptions
0619         cls.createPlottingScript(validations)
0620         result = cls.runPlots(validations)
0621         result = replaceByMap(result, PlottingOptions(None, cls))
0622         if result and result[-1] != "\n": result += "\n"
0623         return result
0624     @classmethod
0625     def createPlottingScript(cls, validations):
0626         from .plottingOptions import PlottingOptions
0627         repmap = PlottingOptions(None, cls).copy()
0628         filename = replaceByMap(".oO[plottingscriptpath]Oo.", repmap)
0629         repmap["PlottingInstantiation"] = "\n".join(
0630                                                     replaceByMap(v.appendToPlots(), v.getRepMap()).rstrip("\n")
0631                                                          for v in validations
0632                                                    )
0633         plottingscript = replaceByMap(cls.plottingscripttemplate(), repmap)
0634         with open(filename, 'w') as f:
0635             f.write(plottingscript)
0636 
0637 class ValidationWithPlotsSummaryBase(ValidationWithPlots):
0638     class SummaryItem(object):
0639         def __init__(self, name, values, format=None, latexname=None, latexformat=None):
0640             """
0641             name:        name of the summary item, goes on top of the column
0642             values:      value for each alignment (in order of rows)
0643             format:      python format string (default: {:.3g}, meaning up to 3 significant digits)
0644             latexname:   name in latex form, e.g. if name=sigma you might want latexname=\sigma (default: name)
0645             latexformat: format for latex (default: format)
0646             """
0647             if format is None: format = "{:.3g}"
0648             if latexname is None: latexname = name
0649             if latexformat is None: latexformat = format
0650 
0651             self.__name = name
0652             self.__values = values
0653             self.__format = format
0654             self.__latexname = latexname
0655             self.__latexformat = latexformat
0656 
0657         def name(self, latex=False):
0658             if latex:
0659                 return self.__latexname
0660             else:
0661                 return self.__name
0662 
0663         def format(self, value, latex=False):
0664             if latex:
0665                 fmt = self.__latexformat
0666             else:
0667                 fmt = self.__format
0668             if re.match(".*[{][^}]*[fg][}].*", fmt):
0669                 value = float(value)
0670             return fmt.format(value)
0671 
0672         def values(self, latex=False):
0673             result = [self.format(v, latex=latex) for v in self.__values]
0674             return result
0675 
0676         def value(self, i, latex):
0677             return self.values(latex)[i]
0678 
0679     @abstractmethod
0680     def getsummaryitems(cls, folder):
0681         """override with a classmethod that returns a list of SummaryItems
0682            based on the plots saved in folder"""
0683 
0684     __summaryitems = None
0685     __lastfolder = None
0686 
0687     @classmethod
0688     def summaryitemsstring(cls, folder=None, latex=False, transpose=True):
0689         if folder is None: folder = cls.plotsdirname()
0690         if folder.startswith( "/castor/" ):
0691             folder = "rfio:%(file)s"%repMap
0692         elif folder.startswith( "/store/" ):
0693             folder = "root://eoscms.cern.ch//eos/cms%(file)s"%repMap
0694 
0695         if cls.__summaryitems is None or cls.__lastfolder != folder:
0696             cls.__lastfolder = folder
0697             cls.__summaryitems = cls.getsummaryitems(folder)
0698 
0699         summaryitems = cls.__summaryitems
0700 
0701         if not summaryitems:
0702             raise AllInOneError("No summary items!")
0703         size = {len(_.values(latex)) for _ in summaryitems}
0704         if len(size) != 1:
0705             raise AllInOneError("Some summary items have different numbers of values\n{}".format(size))
0706         size = size.pop()
0707 
0708         if transpose:
0709             columnwidths = ([max(len(_.name(latex)) for _ in summaryitems)]
0710                           + [max(len(_.value(i, latex)) for _ in summaryitems) for i in range(size)])
0711         else:
0712             columnwidths = [max(len(entry) for entry in [_.name(latex)] + _.values(latex)) for _ in summaryitems]
0713 
0714         if latex:
0715             join = " & "
0716         else:
0717             join = " "
0718         row = join.join("{{:{}}}".format(width) for width in columnwidths)
0719 
0720         if transpose:
0721             rows = [row.format(*[_.name(latex)]+_.values(latex)) for _ in summaryitems]
0722         else:
0723             rows = []
0724             rows.append(row.format(*(_.name for _ in summaryitems)))
0725             for i in range(size):
0726                 rows.append(row.format(*(_.value(i, latex) for _ in summaryitems)))
0727 
0728         if latex:
0729             join = " \\\\\n"
0730         else:
0731             join = "\n"
0732         result = join.join(rows)
0733         if latex:
0734             result = (r"\begin{{tabular}}{{{}}}".format("|" + "|".join("c"*(len(columnwidths))) + "|") + "\n"
0735                          + result + "\n"
0736                          + r"\end{tabular}")
0737         return result
0738 
0739     @classmethod
0740     def printsummaryitems(cls, *args, **kwargs):
0741         print(cls.summaryitemsstring(*args, **kwargs))
0742     @classmethod
0743     def writesummaryitems(cls, filename, *args, **kwargs):
0744         with open(filename, "w") as f:
0745             f.write(cls.summaryitemsstring(*args, **kwargs)+"\n")
0746 
0747 class ValidationWithPlotsSummary(ValidationWithPlotsSummaryBase):
0748     @classmethod
0749     def getsummaryitems(cls, folder):
0750         result = []
0751         with open(os.path.join(folder, "{}Summary.txt".format(cls.__name__))) as f:
0752             for line in f:
0753                 split = line.rstrip("\n").split("\t")
0754                 kwargs = {}
0755                 for thing in split[:]:
0756                     if thing.startswith("format="):
0757                         kwargs["format"] = thing.replace("format=", "", 1)
0758                         split.remove(thing)
0759                     if thing.startswith("latexname="):
0760                         kwargs["latexname"] = thing.replace("latexname=", "", 1)
0761                         split.remove(thing)
0762                     if thing.startswith("latexformat="):
0763                         kwargs["latexformat"] = thing.replace("latexformat=", "", 1)
0764                         split.remove(thing)
0765 
0766                 name = split[0]
0767                 values = split[1:]
0768                 result.append(cls.SummaryItem(name, values, **kwargs))
0769         return result
0770 
0771 class ValidationWithComparison(GenericValidation):
0772     @classmethod
0773     def doComparison(cls, validations):
0774         from .plottingOptions import PlottingOptions
0775         repmap = PlottingOptions(None, cls).copy()
0776         repmap["compareStrings"] = " , ".join(v.getCompareStrings("OfflineValidation") for v in validations)
0777         repmap["compareStringsPlain"] = " , ".join(v.getCompareStrings("OfflineValidation", True) for v in validations)
0778         comparison = replaceByMap(cls.comparisontemplate(), repmap)
0779         return comparison
0780 
0781     @classmethod
0782     def comparisontemplate(cls):
0783         return configTemplates.compareAlignmentsExecution
0784     @classmethod
0785     def comparealignmentspath(cls):
0786         return ".oO[Alignment/OfflineValidation]Oo./scripts/.oO[compareAlignmentsName]Oo."
0787     @abstractmethod
0788     def comparealignmentsname(cls):
0789         """classmethod"""
0790 
0791 class ValidationForPresentation(ValidationWithPlots):
0792     @abstractmethod
0793     def presentationsubsections(cls):
0794         """classmethod"""