File indexing completed on 2021-10-25 04:54:51
0001
0002
0003 from __future__ import print_function
0004 from future.utils import lmap
0005 import subprocess
0006 import os
0007 import sys
0008 import optparse
0009 import datetime
0010 import shutil
0011 import fnmatch
0012 import fileinput
0013 import fileinput
0014 from abc import ABCMeta, abstractmethod
0015 import copy
0016 import itertools
0017 import pprint
0018 import re
0019
0020 import Alignment.OfflineValidation.TkAlAllInOneTool.configTemplates \
0021 as configTemplates
0022 from Alignment.OfflineValidation.TkAlAllInOneTool.TkAlExceptions \
0023 import AllInOneError
0024 from Alignment.OfflineValidation.TkAlAllInOneTool.helperFunctions \
0025 import replaceByMap, getCommandOutput2, addIndex
0026 from Alignment.OfflineValidation.TkAlAllInOneTool.betterConfigParser \
0027 import BetterConfigParser
0028 from Alignment.OfflineValidation.TkAlAllInOneTool.alignment import Alignment
0029
0030 from Alignment.OfflineValidation.TkAlAllInOneTool.genericValidation \
0031 import GenericValidation, ParallelValidation, ValidationWithComparison, ValidationWithPlots
0032 from Alignment.OfflineValidation.TkAlAllInOneTool.geometryComparison \
0033 import GeometryComparison
0034 from Alignment.OfflineValidation.TkAlAllInOneTool.offlineValidation \
0035 import OfflineValidation, OfflineValidationDQM
0036 from Alignment.OfflineValidation.TkAlAllInOneTool.monteCarloValidation \
0037 import MonteCarloValidation
0038 from Alignment.OfflineValidation.TkAlAllInOneTool.trackSplittingValidation \
0039 import TrackSplittingValidation
0040 from Alignment.OfflineValidation.TkAlAllInOneTool.zMuMuValidation \
0041 import ZMuMuValidation
0042 from Alignment.OfflineValidation.TkAlAllInOneTool.primaryVertexValidation \
0043 import PrimaryVertexValidation
0044 from Alignment.OfflineValidation.TkAlAllInOneTool.primaryVertexResolution \
0045 import PrimaryVertexResolution
0046 from Alignment.OfflineValidation.TkAlAllInOneTool.preexistingValidation \
0047 import *
0048 from Alignment.OfflineValidation.TkAlAllInOneTool.plottingOptions \
0049 import PlottingOptions
0050 import Alignment.OfflineValidation.TkAlAllInOneTool.globalDictionaries \
0051 as globalDictionaries
0052 from Alignment.OfflineValidation.TkAlAllInOneTool.overlapValidation \
0053 import OverlapValidation
0054
0055
0056
0057 class ValidationBase(object):
0058
0059 __metaclass__ = ABCMeta
0060
0061 def __init__( self, validation, config, options ):
0062
0063 if validation[1] == "":
0064
0065 valString = validation[0].split( "->" )[0]
0066 self.alignments = validation[0].split( "->" )[1]
0067
0068 if "->" in validation[0]:
0069 msg = ("Instead of using the intermediate syntax\n'"
0070 +valString.strip()+"-> "+alignments.strip()
0071 +":'\nyou have to use the now fully supported syntax \n'"
0072 +valString.strip()+": "
0073 +alignments.strip()+"'.")
0074 raise AllInOneError(msg)
0075 else:
0076 valString = validation[0]
0077 self.alignments = validation[1]
0078 valString = valString.split()
0079 self.valType = valString[0]
0080 self.valName = valString[1]
0081 self.commandLineOptions = options
0082 self.config = config
0083 self.preexisting = ("preexisting" in self.valType)
0084 if self.valType[0] == "*":
0085 self.valType = self.valType[1:]
0086 self.preexisting = True
0087
0088
0089 if self.valType == "offlineParallel":
0090 print ("offlineParallel and offline are now the same. To run an offline parallel validation,\n"
0091 "just set parallelJobs to something > 1. There is no reason to call it offlineParallel anymore.")
0092 self.valType = "offline"
0093 self.valSection = self.valType + ":" + self.valName
0094 if not self.config.has_section( self.valSection ):
0095 raise AllInOneError("Validation '%s' of type '%s' is requested in"
0096 " '[validation]' section, but is not defined."
0097 "\nYou have to add a '[%s]' section."
0098 %( self.valName, self.valType, self.valSection ))
0099
0100
0101 @abstractmethod
0102 def createJob( self ):
0103 pass
0104
0105 @abstractmethod
0106 def runJob( self ):
0107 pass
0108
0109 @abstractmethod
0110 def getValidation( self ):
0111 pass
0112
0113 @abstractmethod
0114 def needsproxy(self):
0115 pass
0116
0117
0118 class ValidationJob(ValidationBase):
0119
0120
0121 interactCount = 0
0122 jobCount = 0
0123 condorConf = {}
0124
0125 def __init__( self, validation, config, options, *args, **kwargs ):
0126
0127 self.start = 0
0128 self.end = args
0129 self.JobId=[]
0130 super(ValidationJob, self).__init__( validation, config, options )
0131 self.validation = self.__getValidation( self.valType, self.valName,
0132 self.alignments, config,
0133 options )
0134
0135 def __getValidation( self, valType, name, alignments, config, options ):
0136 if valType == "compare":
0137 alignmentsList = alignments.split( "," )
0138 firstAlignList = alignmentsList[0].split()
0139 firstAlignName = firstAlignList[0].strip()
0140 if firstAlignName == "IDEAL":
0141 raise AllInOneError("'IDEAL' has to be the second (reference)"
0142 " alignment in 'compare <val_name>: "
0143 "<alignment> <reference>'.")
0144 if len( firstAlignList ) > 1:
0145 firstRun = firstAlignList[1]
0146 elif config.has_section("IOV"):
0147 firstRun = config.get("IOV", "iov")
0148 else:
0149 raise AllInOneError("Have to provide a run number for geometry comparison")
0150 firstAlign = Alignment( firstAlignName, config, firstRun )
0151 firstAlignName = firstAlign.name
0152 secondAlignList = alignmentsList[1].split()
0153 secondAlignName = secondAlignList[0].strip()
0154 if secondAlignName == "IDEAL":
0155 secondAlign = secondAlignName
0156 else:
0157 if len( secondAlignList ) > 1:
0158 secondRun = secondAlignList[1]
0159 elif config.has_section("IOV"):
0160 secondRun = config.get("IOV", "iov")
0161 else:
0162 raise AllInOneError("Have to provide a run number for geometry comparison")
0163 secondAlign = Alignment( secondAlignName, config,
0164 secondRun )
0165 secondAlignName = secondAlign.name
0166
0167 validation = GeometryComparison( name, firstAlign, secondAlign,
0168 config,
0169 self.commandLineOptions.getImages)
0170 elif valType == "offline":
0171 validation = OfflineValidation( name,
0172 Alignment( alignments.strip(), config ), config )
0173 elif valType == "preexistingoffline":
0174 validation = PreexistingOfflineValidation(name, config)
0175 elif valType == "offlineDQM":
0176 validation = OfflineValidationDQM( name,
0177 Alignment( alignments.strip(), config ), config )
0178 elif valType == "mcValidate":
0179 validation = MonteCarloValidation( name,
0180 Alignment( alignments.strip(), config ), config )
0181 elif valType == "preexistingmcValidate":
0182 validation = PreexistingMonteCarloValidation(name, config)
0183 elif valType == "split":
0184 validation = TrackSplittingValidation( name,
0185 Alignment( alignments.strip(), config ), config )
0186 elif valType == "preexistingsplit":
0187 validation = PreexistingTrackSplittingValidation(name, config)
0188 elif valType == "zmumu":
0189 validation = ZMuMuValidation( name,
0190 Alignment( alignments.strip(), config ), config )
0191 elif valType == "primaryvertex":
0192 validation = PrimaryVertexValidation( name,
0193 Alignment( alignments.strip(), config ), config )
0194 elif valType == "pvresolution":
0195 validation = PrimaryVertexResolution( name,
0196 Alignment( alignments.strip(), config ), config )
0197 elif valType == "preexistingprimaryvertex":
0198 validation = PreexistingPrimaryVertexValidation(name, self.__config)
0199 elif valType == "overlap":
0200 validation = OverlapValidation( name,
0201 Alignment( alignments.strip(), self.__config ), self.__config )
0202 else:
0203 raise AllInOneError("Unknown validation mode '%s'"%valType)
0204
0205 return validation
0206
0207 def __createJob( self, jobMode, outpath ):
0208 """This private method creates the needed files for the validation job.
0209 """
0210 self.validation.createConfiguration( outpath )
0211 if self.preexisting:
0212 return
0213 self.scripts = sum([addIndex(script, self.validation.NJobs) for script in self.validation.createScript( outpath )], [])
0214 return None
0215
0216 def createJob(self):
0217 """This is the method called to create the job files."""
0218 self.__createJob( self.validation.jobmode,
0219 os.path.abspath( self.commandLineOptions.Name) )
0220
0221 def runJob( self ):
0222
0223 general = self.config.getGeneral()
0224 log = ""
0225
0226 if self.preexisting:
0227 if self.validation.config.has_section("IOV"):
0228 iov = self.validation.config.get("IOV", "iov")
0229 else:
0230 iov = "singleIOV"
0231 preexistingValType = self.valType
0232 originalValType = preexistingValType.replace('preexisting', '')
0233 key = (originalValType, self.validation.originalValName, iov)
0234 if key in ValidationJob.condorConf:
0235 ValidationJob.condorConf[key].append(("preexisting", "", general["logdir"]))
0236 else:
0237 ValidationJob.condorConf[key] = [("preexisting", "", general["logdir"])]
0238 log = "> " + self.validation.name + " is already validated."
0239 return log
0240
0241 for script in self.scripts:
0242 name = os.path.splitext( os.path.basename( script) )[0]
0243 ValidationJob.jobCount += 1
0244 if self.commandLineOptions.dryRun:
0245 print("%s would run: %s"%( name, os.path.basename( script) ))
0246 continue
0247 log = "> Validating "+name
0248 print("> Validating "+name)
0249 if self.validation.jobmode == "interactive":
0250 log += getCommandOutput2( script )
0251 ValidationJob.interactCount += 1
0252 elif self.validation.jobmode.split( "," )[0] == "condor":
0253 if self.validation.config.has_section("IOV"):
0254 iov = self.validation.config.get("IOV", "iov")
0255 else:
0256 iov = "singleIOV"
0257 scriptPaths = script.split("/")
0258 scriptName = scriptPaths[-1]
0259 scriptName = scriptName.split(".")
0260 jobName = "%s"%scriptName[0] + "_%s"%scriptName[1]+"_%s"%scriptName[2]
0261 key = (self.valType, self.valName, iov)
0262 if key in ValidationJob.condorConf:
0263 ValidationJob.condorConf[key].append((jobName, script, general["logdir"]))
0264 else:
0265 ValidationJob.condorConf[key] = [(jobName, script, general["logdir"])]
0266 else:
0267 raise AllInOneError("Unknown 'jobmode'!\n"
0268 "Please change this parameter either in "
0269 "the [general] or in the ["
0270 + self.valType + ":" + self.valName
0271 + "] section to one of the following "
0272 "values:\n"
0273 "\tinteractive\n\tcondor, -q <queue>\n")
0274
0275 return log
0276
0277 def getValidation( self ):
0278 return self.validation
0279
0280 def needsproxy(self):
0281 return self.validation.needsproxy and not self.preexisting and not self.commandLineOptions.dryRun
0282
0283 def __iter__(self):
0284 yield self
0285
0286 def __next__(self):
0287 if self.start >= len(self.end):
0288 raise StopIteration
0289 else:
0290 self.start += 1
0291 return self.end[self.start-1]
0292
0293
0294 class ValidationJobMultiIOV(ValidationBase):
0295
0296 def __init__( self, validation, config, options, outPath, *args, **kwargs):
0297 self.start = 0
0298 self.end = args
0299 super(ValidationJobMultiIOV, self).__init__( validation, config, options )
0300 self.optionMultiIOV = self.config.getboolean( self.valSection, "multiIOV" )
0301 if self.optionMultiIOV == True:
0302 self.validation = validation
0303 self.config = config
0304 self.options = options
0305 self.outPath = outPath
0306 self.validations = self.__performMultiIOV(self.validation, self.alignments, self.config,
0307 self.options, self.outPath)
0308
0309
0310 def __performMultiIOV(self, validation, alignments, config, options, outPath):
0311 validations = []
0312 if self.valType == "compare":
0313 alignmentsList = alignments.split( "," )
0314 firstAlignList = alignmentsList[0].split()
0315 firstAlignName = firstAlignList[0].strip()
0316 secondAlignList = alignmentsList[1].split()
0317 secondAlignName = secondAlignList[0].strip()
0318 compareAlignments = "%s"%firstAlignName + "_vs_%s"%secondAlignName
0319 sectionMultiIOV = "multiIOV:compare"
0320 if not self.config.has_section(sectionMultiIOV):
0321 raise AllInOneError("section'[%s]' not found. Please define the dataset"%sectionMultiIOV)
0322 iovList = self.config.get( sectionMultiIOV, "iovs" )
0323 iovList = re.sub(r"\s+", "", iovList, flags=re.UNICODE).split( "," )
0324 for iov in iovList:
0325 tmpConfig = BetterConfigParser()
0326 tmpConfig.read( options.config )
0327 general = tmpConfig.getGeneral()
0328 tmpConfig.add_section("IOV")
0329 tmpConfig.set("IOV", "iov", iov)
0330 tmpConfig.set("internals","workdir",os.path.join(general["workdir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
0331 tmpConfig.set("internals","scriptsdir",os.path.join(outPath, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
0332 tmpConfig.set("general","datadir",os.path.join(general["datadir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
0333 tmpConfig.set("general","logdir",os.path.join(general["logdir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
0334 tmpConfig.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
0335 tmpOptions = copy.deepcopy(options)
0336 tmpOptions.Name = os.path.join(options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov)
0337 tmpOptions.config = tmpConfig
0338 newOutPath = os.path.abspath( tmpOptions.Name )
0339 if not os.path.exists( newOutPath ):
0340 os.makedirs( newOutPath )
0341 elif not os.path.isdir( newOutPath ):
0342 raise AllInOneError("the file %s is in the way rename the Job or move it away"%newOutPath)
0343 job = ValidationJob( validation, tmpConfig, tmpOptions, len(iovList) )
0344 validations.append(job)
0345
0346 return validations
0347
0348 if "preexisting" in self.valType:
0349 preexistingValType = self.valType
0350 preexistingValSection = self.valSection
0351 preexistingEosdir = self.config.get( self.valSection, "eosdirName" )
0352 originalValType = preexistingValType.replace('preexisting', '')
0353 originalValName = self.config.get( self.valSection, "originalValName" )
0354 self.valSection = originalValType + ":" + originalValName
0355 originalAlignment = self.valName
0356
0357 datasetList = self.config.get( self.valSection, "dataset" )
0358 datasetList = re.sub(r"\s+", "", datasetList, flags=re.UNICODE).split( "," )
0359 for dataset in datasetList:
0360 sectionMultiIOV = "multiIOV:%s"%dataset
0361 if not self.config.has_section(sectionMultiIOV):
0362 raise AllInOneError("section'[%s]' not found. Please define the dataset"%sectionMultiIOV)
0363 else:
0364 datasetBaseName = self.config.get( sectionMultiIOV, "dataset" )
0365 iovList = self.config.get( sectionMultiIOV, "iovs" )
0366 iovList = re.sub(r"\s+", "", iovList, flags=re.UNICODE).split( "," )
0367 for iov in iovList:
0368 datasetName = datasetBaseName+"_since%s"%iov
0369 tmpConfig = BetterConfigParser()
0370 tmpConfig.read( options.config )
0371 general = tmpConfig.getGeneral()
0372 if "preexisting" in self.valType:
0373 valType = originalValType
0374 valName = originalValName
0375 else:
0376 valType = self.valType
0377 valName = self.valName
0378 tmpConfig.add_section("IOV")
0379 tmpConfig.set("IOV", "iov", iov)
0380 tmpConfig.set( self.valSection, "dataset", datasetName )
0381 tmpConfig.set("internals","workdir",os.path.join(general["workdir"], options.Name, valType + "_" + valName + "_%s"%iov) )
0382 tmpConfig.set("internals","scriptsdir",os.path.join(outPath, valType + "_" + valName + "_%s"%iov) )
0383 tmpConfig.set("general","datadir",os.path.join(general["datadir"], options.Name, valType + "_" + valName + "_%s"%iov) )
0384 tmpConfig.set("general","logdir",os.path.join(general["logdir"], options.Name, valType + "_" + valName + "_%s"%iov) )
0385 tmpConfig.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name, valType + "_" + valName + "_%s"%iov) )
0386 if "preexisting" in self.valType:
0387 if self.valType == "preexistingoffline":
0388 validationClassName = "AlignmentValidation"
0389
0390
0391
0392
0393
0394
0395 else:
0396 raise AllInOneError("Unknown validation mode for preexisting option:'%s'"%self.valType)
0397 preexistingEosdirPath = os.path.join("AlignmentValidation", preexistingEosdir, valType + "_" + valName + "_%s"%iov)
0398 file = "/eos/cms/store/group/alca_trackeralign/AlignmentValidation/" + "%s"%preexistingEosdirPath + "/%s"%validationClassName + "_%s"%originalValName + "_%s"%originalAlignment + ".root"
0399 tmpConfig.set(preexistingValSection, "file", file)
0400 tmpOptions = copy.deepcopy(options)
0401 tmpOptions.Name = os.path.join(options.Name, valType + "_" + valName + "_%s"%iov)
0402 tmpOptions.config = tmpConfig
0403 newOutPath = os.path.abspath( tmpOptions.Name )
0404 if not os.path.exists( newOutPath ):
0405 os.makedirs( newOutPath )
0406 elif not os.path.isdir( newOutPath ):
0407 raise AllInOneError("the file %s is in the way rename the Job or move it away"%newOutPath)
0408 job = ValidationJob( validation, tmpConfig, tmpOptions, len(iovList) )
0409 validations.append(job)
0410
0411 return validations
0412
0413 def createJob( self ):
0414 lmap( lambda validation: validation.createJob(), self.validations )
0415
0416 def runJob( self ):
0417 return [validation.runJob() for validation in self.validations]
0418
0419 @staticmethod
0420 def runCondorJobs(outdir):
0421 dagmanLog = "{}/daglogs".format(outdir)
0422 os.system("mkdir -p {}".format(dagmanLog))
0423
0424
0425 with open("{}/validation.condor".format(outdir), "w") as condor:
0426 condor.write("universe = vanilla" + "\n")
0427 condor.write("executable = $(scriptName).sh" + "\n")
0428 condor.write("log = $(scriptName).log" + "\n")
0429 condor.write("error = $(scriptName).stderr" + "\n")
0430 condor.write("output = $(scriptName).stdout" + "\n")
0431 condor.write('requirements = (OpSysAndVer =?= "CentOS7")' + '\n')
0432 condor.write('+JobFlavour = "tomorrow"' + "\n")
0433 condor.write('+RequestMemory = {}'.format(1540) + "\n")
0434 condor.write('+FileTransferDownloadBytes = {}'.format(1540) + "\n")
0435 condor.write('+AccountingGroup = "group_u_CMS.CAF.ALCA"' + '\n')
0436 condor.write("queue")
0437
0438 with open("{}/validation.dagman".format(outdir), "w") as dagman:
0439 parents = {}
0440 for (valType, valName, iov), alignments in ValidationJob.condorConf.items():
0441
0442 parents[(valType, valName, iov)] = []
0443 for jobInfo in alignments:
0444 if not "preexisting" in jobInfo[0]:
0445 dagman.write("JOB {}_{} {}/validation.condor".format(jobInfo[0], iov, outdir) + "\n")
0446 dagman.write('VARS {}_{} '.format(jobInfo[0], iov) + 'scriptName="{}"'.format('.'.join(jobInfo[1].split('.')[:-1])) + "\n")
0447 parents[(valType, valName, iov)].append('{}_{}'.format(jobInfo[0], iov))
0448 dagman.write("\n")
0449
0450 path = os.path.join(jobInfo[2], "TkAlMerge.sh")
0451 if os.path.exists( path ):
0452 dagman.write("JOB Merge_{}_{}_{} {}/validation.condor".format(valType, valName, iov, outdir) + "\n")
0453 dagman.write("VARS Merge_{}_{}_{} ".format(valType, valName, iov) + 'scriptName="{}"'.format(os.path.join(jobInfo[2], "TkAlMerge")) + "\n")
0454 dagman.write("\n")
0455 else:
0456 raise AllInOneError("Merge script '[%s]' not found!"%path)
0457
0458 for (valType, valName, iov), alignments in ValidationJob.condorConf.items():
0459 if len(parents[(valType, valName, iov)]) != 0:
0460 dagman.write('PARENT {} '.format(" ".join([parent for parent in parents[(valType, valName, iov)]])) + 'CHILD Merge_{}_{}_{}'.format(valType, valName, iov) + "\n")
0461
0462 submitCommands = ["condor_submit_dag -no_submit -outfile_dir {} {}/validation.dagman".format(dagmanLog, outdir), "condor_submit {}/validation.dagman.condor.sub".format(outdir)]
0463
0464 for command in submitCommands:
0465 subprocess.call(command.split(" "))
0466
0467 def getValidation( self ):
0468 return [validation.getValidation() for validation in self.validations]
0469
0470 def needsproxy( self ):
0471 return [validation.needsproxy() for validation in self.validations].join("and") and not self.preexisting and not self.commandLineOptions.dryRun
0472
0473 def __iter__(self):
0474 yield self
0475
0476 def __next__(self):
0477 if self.start >= len(self.end):
0478 raise StopIteration
0479 else:
0480 self.start += 1
0481 return self.end[self.start-1]
0482
0483
0484
0485 def createMergeScript( path, validations, options ):
0486 if(len(validations) == 0):
0487 raise AllInOneError("Cowardly refusing to merge nothing!")
0488
0489 repMap = {}
0490
0491 comparisonLists = {}
0492 for validation in validations:
0493 if validation.config.has_section("IOV"):
0494 iov = validation.config.get("IOV", "iov")
0495 validation.defaultReferenceName = iov
0496 for referenceName in validation.filesToCompare:
0497 validationtype = type(validation)
0498 validationName = validation.name
0499 if validation.config.has_section("IOV") and (referenceName == "Tracker_defaultRange" or referenceName == "Tracker_autoRange"):
0500 referenceName = iov
0501 if issubclass(validationtype, PreexistingValidation):
0502 validationName = validation.originalValName
0503
0504 for parentclass in validationtype.mro():
0505 if not issubclass(parentclass, PreexistingValidation):
0506 validationtype = parentclass
0507 break
0508 key = (validationtype, validationName, referenceName)
0509
0510
0511 if key in comparisonLists:
0512 comparisonLists[key].append(validation)
0513 else:
0514 comparisonLists[key] = [validation]
0515 repMap[key] = validation.config.getGeneral()
0516 repMap[key].update({
0517 "DownloadData":"",
0518 "CompareAlignments":"",
0519 "RunValidationPlots":"",
0520 "CMSSW_BASE": os.environ["CMSSW_BASE"],
0521 "SCRAM_ARCH": os.environ["SCRAM_ARCH"],
0522 "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
0523 })
0524
0525
0526
0527 repMap[key]["doMerge"] = "mergeRetCode=0\n"
0528 repMap[key]["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n"
0529 " echo -e \\n\"Merging succeeded, removing original files.\"\n")
0530 repMap[key]["beforeMerge"] = ""
0531 repMap[key]["mergeParallelFilePrefixes"] = ""
0532 repMap[key]["createResultsDirectory"]=""
0533
0534
0535
0536 anythingToMerge = []
0537
0538 for (validationtype, validationName, referenceName), validations in comparisonLists.items():
0539
0540
0541 globalDictionaries.plottingOptions = {}
0542 lmap( lambda validation: validation.getRepMap(), validations )
0543
0544
0545
0546
0547 for validation in validations:
0548 validation.getRepMap()
0549
0550
0551
0552 if not (isinstance(validation, PreexistingValidation) or validation.NJobs == 1 or not isinstance(validation, ParallelValidation)):
0553 if (validationtype, validationName, referenceName) not in anythingToMerge:
0554 anythingToMerge.append((validationtype, validationName, referenceName))
0555 repMap[(validationtype, validationName, referenceName)]["doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationtype.valType
0556 repMap[(validationtype, validationName, referenceName)]["beforeMerge"] += validationtype.doInitMerge()
0557 repMap[(validationtype, validationName, referenceName)]["doMerge"] += validation.doMerge()
0558 for f in validation.getRepMap()["outputFiles"]:
0559 longName = os.path.join("/eos/cms/store/group/alca_trackeralign/AlignmentValidation/",
0560 validation.getRepMap()["eosdir"], f)
0561 repMap[(validationtype, validationName, referenceName)]["rmUnmerged"] += " rm "+longName+"\n"
0562
0563 repMap[(validationtype, validationName, referenceName)]["rmUnmerged"] += ("else\n"
0564 " echo -e \\n\"WARNING: Merging failed, unmerged"
0565 " files won't be deleted.\\n"
0566 "(Ignore this warning if merging was done earlier)\"\n"
0567 "fi\n")
0568
0569
0570 if anythingToMerge:
0571 repMap[(validationtype, validationName, referenceName)]["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap[(validationtype, validationName, referenceName)] )
0572 else:
0573 repMap[(validationtype, validationName, referenceName)]["DownloadData"] = ""
0574
0575 repMap[(validationtype, validationName, referenceName)]["RunValidationPlots"] = ""
0576 repMap[(validationtype, validationName, referenceName)]["plottingscriptpath"] = ""
0577 if issubclass(validationtype, ValidationWithPlots):
0578 repMap[(validationtype, validationName, referenceName)]["RunValidationPlots"] = validationtype.doRunPlots(validations)
0579
0580 repMap[(validationtype, validationName, referenceName)]["CompareAlignments"] = "#run comparisons"
0581 if issubclass(validationtype, ValidationWithComparison):
0582 repMap[(validationtype, validationName, referenceName)]["CompareAlignments"] += validationtype.doComparison(validations)
0583
0584
0585 if validations[0].config.has_section("IOV"):
0586 repMap[(validationtype, validationName, referenceName)]["createResultsDirectory"]=replaceByMap(configTemplates.createResultsDirectoryTemplate, repMap[(validationtype, validationName, referenceName)])
0587 filePath = os.path.join(repMap[(validationtype, validationName, referenceName)]["scriptsdir"], "TkAlMerge.sh")
0588 else:
0589 repMap[(validationtype, validationName, referenceName)]["createResultsDirectory"]=replaceByMap(configTemplates.createResultsDirectoryTemplate, repMap[(validationtype, validationName, referenceName)])
0590 filePath = os.path.join(path, "TkAlMerge.sh")
0591
0592 theFile = open( filePath, "w" )
0593 theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap[(validationtype, validationName, referenceName)]) )
0594 theFile.close()
0595 os.chmod(filePath,0o755)
0596
0597 def loadTemplates( config ):
0598 if config.has_section("alternateTemplates"):
0599 for templateName in config.options("alternateTemplates"):
0600 if templateName == "AutoAlternates":
0601 continue
0602 newTemplateName = config.get("alternateTemplates", templateName )
0603
0604 configTemplates.alternateTemplate(templateName, newTemplateName)
0605
0606 def flatten(l):
0607 flattenList = []
0608
0609 for item in l:
0610 if type(item) == list:
0611 flattenList.extend(flatten(item))
0612
0613 else:
0614 flattenList.append(item)
0615
0616 return flattenList
0617
0618
0619
0620 def main(argv = None):
0621 if argv == None:
0622 argv = sys.argv[1:]
0623 optParser = optparse.OptionParser()
0624 optParser.description = """All-in-one Alignment Validation.
0625 This will run various validation procedures either on batch queues or interactively.
0626 If no name is given (-N parameter) a name containing time and date is created automatically.
0627 To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
0628 """
0629 optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
0630 help="create all scripts and cfg File but do not start jobs (default=False)")
0631 optParser.add_option( "--getImages", dest="getImages", action="store_true", default=True,
0632 help="get all Images created during the process (default= True)")
0633 defaultConfig = "TkAlConfig.ini"
0634 optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
0635 help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
0636 optParser.add_option("-N", "--Name", dest="Name",
0637 help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
0638 optParser.add_option("-r", "--restrictTo", dest="restrictTo",
0639 help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
0640 optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
0641 default = False,
0642 help="run the tool to get full traceback of errors",
0643 metavar="DEBUG")
0644
0645
0646 (options, args) = optParser.parse_args(argv)
0647
0648 if not options.dryRun:
0649 schedinfo = subprocess.check_output(["myschedd","show"]).decode()
0650 if not 'tzero' in schedinfo:
0651 print("\nAll-In-One Tool: you need to call `module load lxbatch/tzero` before trying to submit jobs. Please do so and try again")
0652 exit(1)
0653
0654
0655 if not options.restrictTo == None:
0656 options.restrictTo = options.restrictTo.split(",")
0657
0658 options.config = [ os.path.abspath( iniFile ) for iniFile in \
0659 options.config.split( "," )]
0660
0661 config = BetterConfigParser()
0662 outputIniFileSet = set( config.read( options.config ) )
0663 failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
0664
0665
0666 if options.config == [ os.path.abspath( defaultConfig ) ]:
0667 if ( not os.path.exists( defaultConfig ) ):
0668 raise AllInOneError( "Default 'ini' file '%s' not found!\n"
0669 "You can specify another name with the "
0670 "command line option '-c'/'--config'."
0671 %( defaultConfig ))
0672 else:
0673 for iniFile in failedIniFiles:
0674 if not os.path.exists( iniFile ):
0675 raise AllInOneError( "'%s' does not exist. Please check for "
0676 "typos in the filename passed to the "
0677 "'-c'/'--config' option!"
0678 %( iniFile ))
0679 else:
0680 raise AllInOneError(( "'%s' does exist, but parsing of the "
0681 "content failed!" ) % iniFile)
0682
0683
0684 if options.Name == None:
0685 existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
0686 "alignmentValidation_*" )
0687 if len( existingValDirs ) > 0:
0688 options.Name = existingValDirs[-1]
0689 else:
0690 print("Cannot guess last working directory!")
0691 print ( "Please use the parameter '-N' or '--Name' to specify "
0692 "the task for which you want a status report." )
0693 return 1
0694
0695
0696 outPath = os.path.abspath( options.Name )
0697
0698 general = config.getGeneral()
0699 config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
0700 config.set("internals","scriptsdir",outPath)
0701 config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
0702 config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
0703 config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
0704
0705 if not os.path.exists( outPath ):
0706 os.makedirs( outPath )
0707 elif not os.path.isdir( outPath ):
0708 raise AllInOneError("the file %s is in the way rename the Job or move it away"%outPath)
0709
0710
0711 loadTemplates( config )
0712
0713
0714 backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" )
0715 config.write( backupConfigFile )
0716
0717
0718 try:
0719 proxyexists = int(getCommandOutput2("voms-proxy-info --timeleft")) > 10
0720 except RuntimeError:
0721 proxyexists = False
0722
0723 if proxyexists:
0724 shutil.copyfile(getCommandOutput2("voms-proxy-info --path").strip(), os.path.join(outPath, ".user_proxy"))
0725
0726 validations = []
0727 jobs = []
0728 for validation in config.items("validation"):
0729 validation = validation[0].split("-")
0730 alignmentList = [validation[1]]
0731 validationsToAdd = [(validation[0],alignment) \
0732 for alignment in alignmentList]
0733 validations.extend(validationsToAdd)
0734
0735 for validation in validations:
0736
0737 job = ValidationJobMultiIOV(validation, config, options, outPath, len(validations))
0738 if (job.optionMultiIOV == True):
0739 jobs.extend(job)
0740 else:
0741 jobs.extend( ValidationJob(validation, config, options, 1) )
0742
0743 for job in jobs:
0744 if job.needsproxy and not proxyexists:
0745 raise AllInOneError("At least one job needs a grid proxy, please init one.")
0746
0747 lmap( lambda job: job.createJob(), jobs )
0748
0749 validations = [ job.getValidation() for job in jobs ]
0750 validations = flatten(validations)
0751
0752 createMergeScript(outPath, validations, options)
0753
0754 lmap( lambda job: job.runJob(), jobs )
0755
0756 if options.dryRun:
0757 pass
0758 else:
0759 ValidationJobMultiIOV.runCondorJobs(outPath)
0760
0761
0762 if __name__ == "__main__":
0763
0764 if "-d" in sys.argv[1:] or "--debug" in sys.argv[1:]:
0765 main()
0766 else:
0767 try:
0768 main()
0769 except AllInOneError as e:
0770 print("\nAll-In-One Tool:", str(e))
0771 exit(1)