OfflineValidation/scripts/validateAlignments.py

0001 #!/usr/bin/env python3
0002 #test execute: export CMSSW_BASE=/tmp/CMSSW && ./validateAlignments.py -c defaultCRAFTValidation.ini,test.ini -n -N test
0003 from __future__ import print_function
0004 from future.utils import lmap
0005 import subprocess
0006 import os
0007 import sys
0008 import optparse
0009 import datetime
0010 import shutil
0011 import fnmatch
0012 import fileinput
0013 import fileinput
0014 from abc import ABCMeta, abstractmethod
0015 import copy
0016 import itertools
0017 import pprint
0018 import re
0019
0020 import Alignment.OfflineValidation.TkAlAllInOneTool.configTemplates \
0021     as configTemplates
0022 from Alignment.OfflineValidation.TkAlAllInOneTool.TkAlExceptions \
0023     import AllInOneError
0024 from Alignment.OfflineValidation.TkAlAllInOneTool.helperFunctions \
0025     import replaceByMap, getCommandOutput2, addIndex
0026 from Alignment.OfflineValidation.TkAlAllInOneTool.betterConfigParser \
0027     import BetterConfigParser
0028 from Alignment.OfflineValidation.TkAlAllInOneTool.alignment import Alignment
0029
0030 from Alignment.OfflineValidation.TkAlAllInOneTool.genericValidation \
0031     import GenericValidation, ParallelValidation, ValidationWithComparison, ValidationWithPlots
0032 from Alignment.OfflineValidation.TkAlAllInOneTool.geometryComparison \
0033     import GeometryComparison
0034 from Alignment.OfflineValidation.TkAlAllInOneTool.offlineValidation \
0035     import OfflineValidation, OfflineValidationDQM
0036 from Alignment.OfflineValidation.TkAlAllInOneTool.monteCarloValidation \
0037     import MonteCarloValidation
0038 from Alignment.OfflineValidation.TkAlAllInOneTool.trackSplittingValidation \
0039     import TrackSplittingValidation
0040 from Alignment.OfflineValidation.TkAlAllInOneTool.zMuMuValidation \
0041     import ZMuMuValidation
0042 from Alignment.OfflineValidation.TkAlAllInOneTool.primaryVertexValidation \
0043     import PrimaryVertexValidation
0044 from Alignment.OfflineValidation.TkAlAllInOneTool.primaryVertexResolution \
0045     import PrimaryVertexResolution
0046 from Alignment.OfflineValidation.TkAlAllInOneTool.preexistingValidation \
0047     import *
0048 from Alignment.OfflineValidation.TkAlAllInOneTool.plottingOptions \
0049     import PlottingOptions
0050 import Alignment.OfflineValidation.TkAlAllInOneTool.globalDictionaries \
0051     as globalDictionaries
0052 from Alignment.OfflineValidation.TkAlAllInOneTool.overlapValidation \
0053     import OverlapValidation
0054
0055 ####################--- Classes ---############################
0056 #
0057 class ValidationBase(object):
0058
0059     __metaclass__ = ABCMeta
0060
0061     def __init__( self, validation, config, options ):
0062
0063         if validation[1] == "":
0064             # intermediate syntax
0065             valString = validation[0].split( "->" )[0]
0066             self.alignments = validation[0].split( "->" )[1]
0067             # force user to use the normal syntax
0068             if "->" in validation[0]:
0069                 msg = ("Instead of using the intermediate syntax\n'"
0070                        +valString.strip()+"-> "+alignments.strip()
0071                        +":'\nyou have to use the now fully supported syntax \n'"
0072                        +valString.strip()+": "
0073                        +alignments.strip()+"'.")
0074                 raise AllInOneError(msg)
0075         else:
0076             valString = validation[0]
0077             self.alignments = validation[1]
0078         valString = valString.split()
0079         self.valType = valString[0]
0080         self.valName = valString[1]
0081         self.commandLineOptions = options
0082         self.config = config
0083         self.preexisting = ("preexisting" in self.valType)
0084         if self.valType[0] == "*":
0085             self.valType = self.valType[1:]
0086             self.preexisting = True
0087
0088         # workaround for intermediate parallel version
0089         if self.valType == "offlineParallel":
0090             print ("offlineParallel and offline are now the same.  To run an offline parallel validation,\n"
0091                    "just set parallelJobs to something > 1.  There is no reason to call it offlineParallel anymore.")
0092             self.valType = "offline"
0093         self.valSection = self.valType + ":" + self.valName
0094         if not self.config.has_section( self.valSection ):
0095             raise AllInOneError("Validation '%s' of type '%s' is requested in"
0096                                 " '[validation]' section, but is not defined."
0097                                   "\nYou have to add a '[%s]' section."
0098                                   %( self.valName, self.valType, self.valSection ))
0099
0100
0101     @abstractmethod
0102     def createJob( self ):
0103         pass
0104
0105     @abstractmethod
0106     def runJob( self ):
0107         pass
0108
0109     @abstractmethod
0110     def getValidation( self ):
0111         pass
0112
0113     @abstractmethod
0114     def needsproxy(self):
0115         pass
0116
0117
0118 class ValidationJob(ValidationBase):
0119
0120     # these count the jobs of different varieties that are being run
0121     interactCount = 0
0122     jobCount = 0
0123     condorConf = {}
0124
0125     def __init__( self, validation, config, options, *args, **kwargs ):
0126
0127         self.start = 0
0128         self.end = args
0129         self.JobId=[]
0130         super(ValidationJob, self).__init__( validation, config, options )
0131         self.validation = self.__getValidation( self.valType, self.valName,
0132                                                   self.alignments, config,
0133                                                   options )
0134
0135     def __getValidation( self, valType, name, alignments, config, options ):
0136         if valType == "compare":
0137             alignmentsList = alignments.split( "," )
0138             firstAlignList = alignmentsList[0].split()
0139             firstAlignName = firstAlignList[0].strip()
0140             if firstAlignName == "IDEAL":
0141                 raise AllInOneError("'IDEAL' has to be the second (reference)"
0142                                       " alignment in 'compare <val_name>: "
0143                                       "<alignment> <reference>'.")
0144             if len( firstAlignList ) > 1:
0145                 firstRun = firstAlignList[1]
0146             elif config.has_section("IOV"):
0147                 firstRun = config.get("IOV", "iov")
0148             else:
0149                 raise AllInOneError("Have to provide a run number for geometry comparison")
0150             firstAlign = Alignment( firstAlignName, config, firstRun )
0151             firstAlignName = firstAlign.name
0152             secondAlignList = alignmentsList[1].split()
0153             secondAlignName = secondAlignList[0].strip()
0154             if secondAlignName == "IDEAL":
0155                 secondAlign = secondAlignName
0156             else:
0157                 if len( secondAlignList ) > 1:
0158                     secondRun = secondAlignList[1]
0159                 elif config.has_section("IOV"):
0160                     secondRun = config.get("IOV", "iov")
0161                 else:
0162                     raise AllInOneError("Have to provide a run number for geometry comparison")
0163                 secondAlign = Alignment( secondAlignName, config,
0164                                          secondRun )
0165                 secondAlignName = secondAlign.name
0166
0167             validation = GeometryComparison( name, firstAlign, secondAlign,
0168                                              config,
0169                                              self.commandLineOptions.getImages)
0170         elif valType == "offline":
0171             validation = OfflineValidation( name,
0172                 Alignment( alignments.strip(), config ), config )
0173         elif valType == "preexistingoffline":
0174             validation = PreexistingOfflineValidation(name, config)
0175         elif valType == "offlineDQM":
0176             validation = OfflineValidationDQM( name,
0177                 Alignment( alignments.strip(), config ), config )
0178         elif valType == "mcValidate":
0179             validation = MonteCarloValidation( name,
0180                 Alignment( alignments.strip(), config ), config )
0181         elif valType == "preexistingmcValidate":
0182             validation = PreexistingMonteCarloValidation(name, config)
0183         elif valType == "split":
0184             validation = TrackSplittingValidation( name,
0185                 Alignment( alignments.strip(), config ), config )
0186         elif valType == "preexistingsplit":
0187             validation = PreexistingTrackSplittingValidation(name, config)
0188         elif valType == "zmumu":
0189             validation = ZMuMuValidation( name,
0190                 Alignment( alignments.strip(), config ), config )
0191         elif valType == "primaryvertex":
0192             validation = PrimaryVertexValidation( name,
0193                 Alignment( alignments.strip(), config ), config )
0194         elif valType == "pvresolution":
0195             validation = PrimaryVertexResolution( name,
0196                 Alignment( alignments.strip(), config ), config )
0197         elif valType == "preexistingprimaryvertex":
0198             validation = PreexistingPrimaryVertexValidation(name, self.__config)
0199         elif valType == "overlap":
0200             validation = OverlapValidation( name,
0201                 Alignment( alignments.strip(), self.__config ), self.__config )
0202         else:
0203             raise AllInOneError("Unknown validation mode '%s'"%valType)
0204
0205         return validation
0206
0207     def __createJob( self, jobMode, outpath ):
0208         """This private method creates the needed files for the validation job.
0209            """
0210         self.validation.createConfiguration( outpath )
0211         if self.preexisting:
0212             return
0213         self.scripts = sum([addIndex(script, self.validation.NJobs) for script in self.validation.createScript( outpath )], [])
0214         return None
0215
0216     def createJob(self):
0217         """This is the method called to create the job files."""
0218         self.__createJob( self.validation.jobmode,
0219                           os.path.abspath( self.commandLineOptions.Name) )
0220
0221     def runJob( self ):
0222
0223         general = self.config.getGeneral()
0224         log = ""
0225
0226         if self.preexisting:
0227             if self.validation.config.has_section("IOV"):
0228                 iov = self.validation.config.get("IOV", "iov")
0229             else:
0230                 iov = "singleIOV"
0231             preexistingValType = self.valType
0232             originalValType = preexistingValType.replace('preexisting', '')
0233             key = (originalValType, self.validation.originalValName, iov)
0234             if key in ValidationJob.condorConf:
0235                 ValidationJob.condorConf[key].append(("preexisting", "", general["logdir"]))
0236             else:
0237                 ValidationJob.condorConf[key] = [("preexisting", "", general["logdir"])]
0238             log = ">             " + self.validation.name + " is already validated."
0239             return log
0240
0241         for script in self.scripts:
0242             name = os.path.splitext( os.path.basename( script) )[0]
0243             ValidationJob.jobCount += 1
0244             if self.commandLineOptions.dryRun:
0245                 print("%s would run: %s"%( name, os.path.basename( script) ))
0246                 continue
0247             log = ">             Validating "+name
0248             print(">             Validating "+name)
0249             if self.validation.jobmode == "interactive":
0250                 log += getCommandOutput2( script )
0251                 ValidationJob.interactCount += 1
0252             elif self.validation.jobmode.split( "," )[0] == "condor":
0253                 if self.validation.config.has_section("IOV"):
0254                     iov = self.validation.config.get("IOV", "iov")
0255                 else:
0256                     iov = "singleIOV"
0257                 scriptPaths = script.split("/")
0258                 scriptName = scriptPaths[-1]
0259                 scriptName = scriptName.split(".")
0260                 jobName = "%s"%scriptName[0] + "_%s"%scriptName[1]+"_%s"%scriptName[2]
0261                 key = (self.valType, self.valName, iov)
0262                 if key in ValidationJob.condorConf:
0263                     ValidationJob.condorConf[key].append((jobName, script, general["logdir"]))
0264                 else:
0265                     ValidationJob.condorConf[key] = [(jobName, script, general["logdir"])]
0266             else:
0267                 raise AllInOneError("Unknown 'jobmode'!\n"
0268                                       "Please change this parameter either in "
0269                                       "the [general] or in the ["
0270                                       + self.valType + ":" + self.valName
0271                                       + "] section to one of the following "
0272                                       "values:\n"
0273                                       "\tinteractive\n\tcondor, -q <queue>\n")
0274
0275         return log
0276
0277     def getValidation( self ):
0278         return self.validation
0279
0280     def needsproxy(self):
0281         return self.validation.needsproxy and not self.preexisting and not self.commandLineOptions.dryRun
0282
0283     def __iter__(self):
0284         yield self
0285
0286     def __next__(self):
0287         if self.start >= len(self.end):
0288             raise StopIteration
0289         else:
0290             self.start += 1
0291             return self.end[self.start-1]
0292
0293
0294 class ValidationJobMultiIOV(ValidationBase):
0295
0296     def __init__( self, validation, config, options, outPath, *args, **kwargs):
0297         self.start = 0
0298         self.end = args
0299         super(ValidationJobMultiIOV, self).__init__( validation, config, options )
0300         self.optionMultiIOV = self.config.getboolean( self.valSection, "multiIOV" )
0301         if self.optionMultiIOV == True:
0302             self.validation = validation
0303             self.config = config
0304             self.options = options
0305             self.outPath = outPath
0306             self.validations = self.__performMultiIOV(self.validation, self.alignments, self.config,
0307                                                   self.options, self.outPath)
0308
0309
0310     def __performMultiIOV(self, validation, alignments, config, options, outPath):
0311         validations = []
0312         if self.valType == "compare":
0313             alignmentsList = alignments.split( "," )
0314             firstAlignList = alignmentsList[0].split()
0315             firstAlignName = firstAlignList[0].strip()
0316             secondAlignList = alignmentsList[1].split()
0317             secondAlignName = secondAlignList[0].strip()
0318             compareAlignments = "%s"%firstAlignName + "_vs_%s"%secondAlignName
0319             sectionMultiIOV = "multiIOV:compare"
0320             if not self.config.has_section(sectionMultiIOV):
0321                 raise AllInOneError("section'[%s]' not found. Please define the dataset"%sectionMultiIOV)
0322             iovList = self.config.get( sectionMultiIOV, "iovs" )
0323             iovList = re.sub(r"\s+", "", iovList, flags=re.UNICODE).split( "," )
0324             for iov in iovList:
0325                     tmpConfig = BetterConfigParser()
0326                     tmpConfig.read( options.config )
0327                     general = tmpConfig.getGeneral()
0328                     tmpConfig.add_section("IOV")
0329                     tmpConfig.set("IOV", "iov", iov)
0330                     tmpConfig.set("internals","workdir",os.path.join(general["workdir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
0331                     tmpConfig.set("internals","scriptsdir",os.path.join(outPath, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
0332                     tmpConfig.set("general","datadir",os.path.join(general["datadir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
0333                     tmpConfig.set("general","logdir",os.path.join(general["logdir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
0334                     tmpConfig.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
0335                     tmpOptions = copy.deepcopy(options)
0336                     tmpOptions.Name = os.path.join(options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov)
0337                     tmpOptions.config = tmpConfig
0338                     newOutPath = os.path.abspath( tmpOptions.Name )
0339                     if not os.path.exists( newOutPath ):
0340                         os.makedirs( newOutPath )
0341                     elif not os.path.isdir( newOutPath ):
0342                         raise AllInOneError("the file %s is in the way rename the Job or move it away"%newOutPath)
0343                     job = ValidationJob( validation, tmpConfig, tmpOptions, len(iovList) )
0344                     validations.append(job)
0345
0346             return validations
0347
0348         if "preexisting" in self.valType:
0349             preexistingValType = self.valType
0350             preexistingValSection = self.valSection
0351             preexistingEosdir = self.config.get( self.valSection, "eosdirName" )
0352             originalValType = preexistingValType.replace('preexisting', '')
0353             originalValName = self.config.get( self.valSection, "originalValName" )
0354             self.valSection = originalValType + ":" + originalValName
0355             originalAlignment = self.valName
0356
0357         datasetList = self.config.get( self.valSection, "dataset" )
0358         datasetList = re.sub(r"\s+", "", datasetList, flags=re.UNICODE).split( "," )
0359         for dataset in datasetList:
0360             sectionMultiIOV = "multiIOV:%s"%dataset
0361             if not self.config.has_section(sectionMultiIOV):
0362                 raise AllInOneError("section'[%s]' not found. Please define the dataset"%sectionMultiIOV)
0363             else:
0364                 datasetBaseName = self.config.get( sectionMultiIOV, "dataset" )
0365                 iovList = self.config.get( sectionMultiIOV, "iovs" )
0366                 iovList = re.sub(r"\s+", "", iovList, flags=re.UNICODE).split( "," )
0367                 for iov in iovList:
0368                     datasetName = datasetBaseName+"_since%s"%iov
0369                     tmpConfig = BetterConfigParser()
0370                     tmpConfig.read( options.config )
0371                     general = tmpConfig.getGeneral()
0372                     if "preexisting" in self.valType:
0373                         valType = originalValType
0374                         valName = originalValName
0375                     else:
0376                         valType = self.valType
0377                         valName = self.valName
0378                     tmpConfig.add_section("IOV")
0379                     tmpConfig.set("IOV", "iov", iov)
0380                     tmpConfig.set( self.valSection, "dataset", datasetName )
0381                     tmpConfig.set("internals","workdir",os.path.join(general["workdir"], options.Name, valType + "_" + valName + "_%s"%iov) )
0382                     tmpConfig.set("internals","scriptsdir",os.path.join(outPath, valType + "_" + valName + "_%s"%iov) )
0383                     tmpConfig.set("general","datadir",os.path.join(general["datadir"], options.Name, valType + "_" + valName + "_%s"%iov) )
0384                     tmpConfig.set("general","logdir",os.path.join(general["logdir"], options.Name, valType + "_" + valName + "_%s"%iov) )
0385                     tmpConfig.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name, valType + "_" + valName + "_%s"%iov) )
0386                     if "preexisting" in self.valType:
0387                         if self.valType == "preexistingoffline":
0388                             validationClassName = "AlignmentValidation"
0389                         #elif self.valType == "preexistingmcValidate":
0390                         #    validationClassName = "MonteCarloValidation"
0391                         #elif self.valType == "preexistingsplit":
0392                         #    validationClassName = "TrackSplittingValidation"
0393                         #elif self.valType == "preexistingprimaryvertex":
0394                         #    validationClassName = "PrimaryVertexValidation"
0395                         else:
0396                             raise AllInOneError("Unknown validation mode for preexisting option:'%s'"%self.valType)
0397                         preexistingEosdirPath = os.path.join("AlignmentValidation", preexistingEosdir, valType + "_" + valName + "_%s"%iov)
0398                         file = "/eos/cms/store/group/alca_trackeralign/AlignmentValidation/" + "%s"%preexistingEosdirPath + "/%s"%validationClassName + "_%s"%originalValName + "_%s"%originalAlignment + ".root"
0399                         tmpConfig.set(preexistingValSection, "file", file)
0400                     tmpOptions = copy.deepcopy(options)
0401                     tmpOptions.Name = os.path.join(options.Name, valType + "_" + valName + "_%s"%iov)
0402                     tmpOptions.config = tmpConfig
0403                     newOutPath = os.path.abspath( tmpOptions.Name )
0404                     if not os.path.exists( newOutPath ):
0405                         os.makedirs( newOutPath )
0406                     elif not os.path.isdir( newOutPath ):
0407                         raise AllInOneError("the file %s is in the way rename the Job or move it away"%newOutPath)
0408                     job = ValidationJob( validation, tmpConfig, tmpOptions, len(iovList) )
0409                     validations.append(job)
0410
0411         return validations
0412
0413     def createJob( self ):
0414         lmap( lambda validation: validation.createJob(), self.validations )
0415
0416     def runJob( self ):
0417         return [validation.runJob() for validation in self.validations]
0418
0419     @staticmethod
0420     def runCondorJobs(outdir):
0421         dagmanLog = "{}/daglogs".format(outdir)
0422         os.system("mkdir -p {}".format(dagmanLog))
0423
0424
0425         with open("{}/validation.condor".format(outdir), "w") as condor:
0426             condor.write("universe = vanilla" + "\n")
0427             condor.write("executable = $(scriptName).sh" + "\n")
0428             condor.write("log = $(scriptName).log" + "\n")
0429             condor.write("error = $(scriptName).stderr" + "\n")
0430             condor.write("output = $(scriptName).stdout" + "\n")
0431             condor.write('requirements = (OpSysAndVer =?= "CentOS7")' + '\n')
0432             condor.write('+JobFlavour = "tomorrow"' + "\n")
0433             condor.write('+RequestMemory = {}'.format(1540) + "\n")
0434             condor.write('+FileTransferDownloadBytes = {}'.format(1540) + "\n")
0435             condor.write('+AccountingGroup     = "group_u_CMS.CAF.ALCA"' + '\n')
0436             condor.write("queue")
0437
0438         with open("{}/validation.dagman".format(outdir), "w") as dagman:
0439             parents = {}
0440             for (valType, valName, iov), alignments in ValidationJob.condorConf.items():
0441
0442                 parents[(valType, valName, iov)] = []
0443                 for jobInfo in alignments:
0444                     if not "preexisting" in jobInfo[0]:
0445                         dagman.write("JOB {}_{} {}/validation.condor".format(jobInfo[0], iov, outdir) + "\n")
0446                         dagman.write('VARS {}_{} '.format(jobInfo[0], iov) + 'scriptName="{}"'.format('.'.join(jobInfo[1].split('.')[:-1])) + "\n")
0447                         parents[(valType, valName, iov)].append('{}_{}'.format(jobInfo[0], iov))
0448                         dagman.write("\n")
0449
0450                 path =  os.path.join(jobInfo[2], "TkAlMerge.sh")
0451                 if os.path.exists( path ):
0452                     dagman.write("JOB Merge_{}_{}_{} {}/validation.condor".format(valType, valName, iov, outdir) + "\n")
0453                     dagman.write("VARS Merge_{}_{}_{} ".format(valType, valName, iov) + 'scriptName="{}"'.format(os.path.join(jobInfo[2], "TkAlMerge")) + "\n")
0454                     dagman.write("\n")
0455                 else:
0456                     raise AllInOneError("Merge script '[%s]' not found!"%path)
0457
0458             for (valType, valName, iov), alignments in ValidationJob.condorConf.items():
0459                 if len(parents[(valType, valName, iov)]) != 0:
0460                     dagman.write('PARENT {} '.format(" ".join([parent for parent in parents[(valType, valName, iov)]])) + 'CHILD Merge_{}_{}_{}'.format(valType, valName, iov) + "\n")
0461
0462         submitCommands = ["condor_submit_dag -no_submit -outfile_dir {} {}/validation.dagman".format(dagmanLog, outdir), "condor_submit {}/validation.dagman.condor.sub".format(outdir)]
0463
0464         for command in submitCommands:
0465             subprocess.call(command.split(" "))
0466
0467     def getValidation( self ):
0468         return [validation.getValidation() for validation in self.validations]
0469
0470     def needsproxy( self ):
0471         return [validation.needsproxy() for validation in self.validations].join("and") and not self.preexisting and not self.commandLineOptions.dryRun
0472
0473     def __iter__(self):
0474         yield self
0475
0476     def __next__(self):
0477         if self.start >= len(self.end):
0478             raise StopIteration
0479         else:
0480             self.start += 1
0481             return self.end[self.start-1]
0482
0483
0484 ####################--- Functions ---############################
0485 def createMergeScript( path, validations, options ):
0486     if(len(validations) == 0):
0487         raise AllInOneError("Cowardly refusing to merge nothing!")
0488
0489     repMap = {}
0490
0491     comparisonLists = {} # directory of lists containing the validations that are comparable
0492     for validation in validations:
0493         if validation.config.has_section("IOV"):
0494             iov = validation.config.get("IOV", "iov")
0495             validation.defaultReferenceName = iov
0496         for referenceName in validation.filesToCompare:
0497             validationtype = type(validation)
0498             validationName = validation.name
0499             if validation.config.has_section("IOV") and (referenceName == "Tracker_defaultRange" or referenceName == "Tracker_autoRange"):
0500                 referenceName = iov
0501             if issubclass(validationtype, PreexistingValidation):
0502                 validationName = validation.originalValName
0503                 #find the actual validationtype
0504                 for parentclass in validationtype.mro():
0505                     if not issubclass(parentclass, PreexistingValidation):
0506                         validationtype = parentclass
0507                         break
0508             key = (validationtype, validationName, referenceName)
0509
0510
0511             if key in comparisonLists:
0512                 comparisonLists[key].append(validation)
0513             else:
0514                 comparisonLists[key] = [validation]
0515                 repMap[key] = validation.config.getGeneral()
0516                 repMap[key].update({
0517                         "DownloadData":"",
0518                         "CompareAlignments":"",
0519                         "RunValidationPlots":"",
0520                         "CMSSW_BASE": os.environ["CMSSW_BASE"],
0521                         "SCRAM_ARCH": os.environ["SCRAM_ARCH"],
0522                         "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
0523                         })
0524
0525                 # introduced to merge individual validation outputs separately
0526                 #  -> avoids problems with merge script
0527                 repMap[key]["doMerge"] = "mergeRetCode=0\n"
0528                 repMap[key]["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n"
0529                             "    echo -e \\n\"Merging succeeded, removing original files.\"\n")
0530                 repMap[key]["beforeMerge"] = ""
0531                 repMap[key]["mergeParallelFilePrefixes"] = ""
0532                 repMap[key]["createResultsDirectory"]=""
0533
0534     #print("comparisonLists")
0535     #pprint.pprint(comparisonLists)
0536     anythingToMerge = []
0537
0538     for (validationtype, validationName, referenceName), validations in comparisonLists.items():
0539         #pprint.pprint("validations")
0540         #pprint.pprint(validations)
0541         globalDictionaries.plottingOptions = {}
0542         lmap( lambda validation: validation.getRepMap(), validations )
0543         #plotInfo = "plots:offline"
0544         #allPlotInfo = dict(validations[0].config.items(plotInfo))
0545         #repMap[(validationtype, validationName, referenceName)].update(allPlotInfo)
0546
0547         for validation in validations:
0548             validation.getRepMap()
0549             #pprint.pprint("validation in validations")
0550             #pprint.pprint(validation)
0551             #parallel merging
0552             if not (isinstance(validation, PreexistingValidation) or validation.NJobs == 1 or not isinstance(validation, ParallelValidation)):
0553                 if (validationtype, validationName, referenceName) not in anythingToMerge:
0554                     anythingToMerge.append((validationtype, validationName, referenceName))
0555                     repMap[(validationtype, validationName, referenceName)]["doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationtype.valType
0556                     repMap[(validationtype, validationName, referenceName)]["beforeMerge"] += validationtype.doInitMerge()
0557                 repMap[(validationtype, validationName, referenceName)]["doMerge"] += validation.doMerge()
0558                 for f in validation.getRepMap()["outputFiles"]:
0559                     longName = os.path.join("/eos/cms/store/group/alca_trackeralign/AlignmentValidation/",
0560                                             validation.getRepMap()["eosdir"], f)
0561                     repMap[(validationtype, validationName, referenceName)]["rmUnmerged"] += "    rm "+longName+"\n"
0562
0563         repMap[(validationtype, validationName, referenceName)]["rmUnmerged"] += ("else\n"
0564                                                                   "    echo -e \\n\"WARNING: Merging failed, unmerged"
0565                                                                   " files won't be deleted.\\n"
0566                                                                   "(Ignore this warning if merging was done earlier)\"\n"
0567                                                                   "fi\n")
0568
0569
0570         if anythingToMerge:
0571             repMap[(validationtype, validationName, referenceName)]["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap[(validationtype, validationName, referenceName)] )
0572         else:
0573             repMap[(validationtype, validationName, referenceName)]["DownloadData"] = ""
0574
0575         repMap[(validationtype, validationName, referenceName)]["RunValidationPlots"] = ""
0576         repMap[(validationtype, validationName, referenceName)]["plottingscriptpath"] = ""
0577         if issubclass(validationtype, ValidationWithPlots):
0578             repMap[(validationtype, validationName, referenceName)]["RunValidationPlots"] = validationtype.doRunPlots(validations)
0579
0580         repMap[(validationtype, validationName, referenceName)]["CompareAlignments"] = "#run comparisons"
0581         if issubclass(validationtype, ValidationWithComparison):
0582             repMap[(validationtype, validationName, referenceName)]["CompareAlignments"] += validationtype.doComparison(validations)
0583
0584         #if not merging parallel, add code to create results directory and set merge script name accordingly
0585         if validations[0].config.has_section("IOV"):
0586             repMap[(validationtype, validationName, referenceName)]["createResultsDirectory"]=replaceByMap(configTemplates.createResultsDirectoryTemplate, repMap[(validationtype, validationName, referenceName)])
0587             filePath = os.path.join(repMap[(validationtype, validationName, referenceName)]["scriptsdir"], "TkAlMerge.sh")
0588         else:
0589             repMap[(validationtype, validationName, referenceName)]["createResultsDirectory"]=replaceByMap(configTemplates.createResultsDirectoryTemplate, repMap[(validationtype, validationName, referenceName)])
0590             filePath = os.path.join(path, "TkAlMerge.sh")
0591
0592         theFile = open( filePath, "w" )
0593         theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap[(validationtype, validationName, referenceName)]) )
0594         theFile.close()
0595         os.chmod(filePath,0o755)
0596
0597 def loadTemplates( config ):
0598     if config.has_section("alternateTemplates"):
0599         for templateName in config.options("alternateTemplates"):
0600             if templateName == "AutoAlternates":
0601                 continue
0602             newTemplateName = config.get("alternateTemplates", templateName )
0603             #print "replacing default %s template by %s"%( templateName, newTemplateName)
0604             configTemplates.alternateTemplate(templateName, newTemplateName)
0605
0606 def flatten(l):
0607     flattenList = []
0608
0609     for item in l:
0610         if type(item) == list:
0611             flattenList.extend(flatten(item))
0612
0613         else:
0614             flattenList.append(item)
0615
0616     return flattenList
0617
0618
0619 ####################--- Main ---############################
0620 def main(argv = None):
0621     if argv == None:
0622        argv = sys.argv[1:]
0623     optParser = optparse.OptionParser()
0624     optParser.description = """All-in-one Alignment Validation.
0625 This will run various validation procedures either on batch queues or interactively.
0626 If no name is given (-N parameter) a name containing time and date is created automatically.
0627 To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
0628 """
0629     optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
0630                          help="create all scripts and cfg File but do not start jobs (default=False)")
0631     optParser.add_option( "--getImages", dest="getImages", action="store_true", default=True,
0632                          help="get all Images created during the process (default= True)")
0633     defaultConfig = "TkAlConfig.ini"
0634     optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
0635                          help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
0636     optParser.add_option("-N", "--Name", dest="Name",
0637                          help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
0638     optParser.add_option("-r", "--restrictTo", dest="restrictTo",
0639                          help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
0640     optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
0641                          default = False,
0642                          help="run the tool to get full traceback of errors",
0643                          metavar="DEBUG")
0644
0645
0646     (options, args) = optParser.parse_args(argv)
0647
0648     if not options.dryRun:
0649         schedinfo = subprocess.check_output(["myschedd","show"]).decode()
0650         if not 'tzero' in schedinfo:
0651             print("\nAll-In-One Tool: you need to call `module load lxbatch/tzero` before trying to submit jobs. Please do so and try again")
0652             exit(1)
0653
0654
0655     if not options.restrictTo == None:
0656         options.restrictTo = options.restrictTo.split(",")
0657
0658     options.config = [ os.path.abspath( iniFile ) for iniFile in \
0659                        options.config.split( "," )]
0660
0661     config = BetterConfigParser()
0662     outputIniFileSet = set( config.read( options.config ) )
0663     failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
0664
0665     # Check for missing ini file
0666     if options.config == [ os.path.abspath( defaultConfig ) ]:
0667         if ( not os.path.exists( defaultConfig ) ):
0668                 raise AllInOneError( "Default 'ini' file '%s' not found!\n"
0669                                        "You can specify another name with the "
0670                                        "command line option '-c'/'--config'."
0671                                        %( defaultConfig ))
0672     else:
0673         for iniFile in failedIniFiles:
0674             if not os.path.exists( iniFile ):
0675                 raise AllInOneError( "'%s' does not exist. Please check for "
0676                                        "typos in the filename passed to the "
0677                                        "'-c'/'--config' option!"
0678                                        %( iniFile ))
0679             else:
0680                 raise AllInOneError(( "'%s' does exist, but parsing of the "
0681                                        "content failed!" ) % iniFile)
0682
0683     # get the job name
0684     if options.Name == None:
0685         existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
0686                                               "alignmentValidation_*" )
0687         if len( existingValDirs ) > 0:
0688             options.Name = existingValDirs[-1]
0689         else:
0690             print("Cannot guess last working directory!")
0691             print ( "Please use the parameter '-N' or '--Name' to specify "
0692                     "the task for which you want a status report." )
0693             return 1
0694
0695     # set output path
0696     outPath = os.path.abspath( options.Name )
0697
0698     general = config.getGeneral()
0699     config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
0700     config.set("internals","scriptsdir",outPath)
0701     config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
0702     config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
0703     config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
0704
0705     if not os.path.exists( outPath ):
0706         os.makedirs( outPath )
0707     elif not os.path.isdir( outPath ):
0708         raise AllInOneError("the file %s is in the way rename the Job or move it away"%outPath)
0709
0710     # replace default templates by the ones specified in the "alternateTemplates" section
0711     loadTemplates( config )
0712
0713     #save backup configuration file
0714     backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w"  )
0715     config.write( backupConfigFile )
0716
0717     #copy proxy, if there is one
0718     try:
0719         proxyexists = int(getCommandOutput2("voms-proxy-info --timeleft")) > 10
0720     except RuntimeError:
0721         proxyexists = False
0722
0723     if proxyexists:
0724         shutil.copyfile(getCommandOutput2("voms-proxy-info --path").strip(), os.path.join(outPath, ".user_proxy"))
0725
0726     validations = []
0727     jobs = []
0728     for validation in config.items("validation"):
0729         validation = validation[0].split("-")
0730         alignmentList = [validation[1]]
0731         validationsToAdd = [(validation[0],alignment) \
0732                                 for alignment in alignmentList]
0733         validations.extend(validationsToAdd)
0734
0735     for validation in validations:
0736
0737         job = ValidationJobMultiIOV(validation, config, options, outPath, len(validations))
0738         if (job.optionMultiIOV == True):
0739             jobs.extend(job)
0740         else:
0741             jobs.extend( ValidationJob(validation, config, options, 1) )
0742
0743     for job in jobs:
0744         if job.needsproxy and not proxyexists:
0745             raise AllInOneError("At least one job needs a grid proxy, please init one.")
0746
0747     lmap( lambda job: job.createJob(), jobs )
0748
0749     validations = [ job.getValidation() for job in jobs ]
0750     validations = flatten(validations)
0751
0752     createMergeScript(outPath, validations, options)
0753
0754     lmap( lambda job: job.runJob(), jobs )
0755
0756     if options.dryRun:
0757         pass
0758     else:
0759         ValidationJobMultiIOV.runCondorJobs(outPath)
0760
0761
0762 if __name__ == "__main__":
0763     # main(["-n","-N","test","-c","defaultCRAFTValidation.ini,latestObjects.ini","--getImages"])
0764     if "-d" in sys.argv[1:] or "--debug" in sys.argv[1:]:
0765         main()
0766     else:
0767         try:
0768             main()
0769         except AllInOneError as e:
0770             print("\nAll-In-One Tool:", str(e))
0771             exit(1)