File indexing completed on 2023-03-17 10:40:23
0001 from __future__ import absolute_import
0002 from .alternateValidationTemplates import *
0003 from .offlineValidationTemplates import *
0004 from .primaryVertexValidationTemplates import *
0005 from .primaryVertexResolutionTemplates import *
0006 from .geometryComparisonTemplates import *
0007 from .monteCarloValidationTemplates import *
0008 from .trackSplittingValidationTemplates import *
0009 from .zMuMuValidationTemplates import *
0010 from .TkAlExceptions import AllInOneError
0011 from .overlapValidationTemplates import *
0012
0013
0014
0015
0016
0017
0018
0019
0020
0021
0022
0023 loadGlobalTagTemplate="""
0024 #Global tag
0025 process.load("Configuration.StandardSequences.FrontierConditions_GlobalTag_cff")
0026 from Configuration.AlCa.GlobalTag import GlobalTag
0027 process.GlobalTag = GlobalTag(process.GlobalTag,".oO[GlobalTag]Oo.")
0028 """
0029
0030
0031
0032
0033 conditionsTemplate="""
0034 process.conditionsIn.oO[rcdName]Oo. = CalibTracker.Configuration.Common.PoolDBESSource_cfi.poolDBESSource.clone(
0035 connect = cms.string('.oO[connectString]Oo.'),
0036 toGet = cms.VPSet(cms.PSet(record = cms.string('.oO[rcdName]Oo.'),
0037 tag = cms.string('.oO[tagName]Oo.')
0038 )
0039 )
0040 )
0041 process.prefer_conditionsIn.oO[rcdName]Oo. = cms.ESPrefer("PoolDBESSource", "conditionsIn.oO[rcdName]Oo.")
0042 """
0043
0044
0045
0046
0047
0048 scriptTemplate="""#!/bin/bash
0049 #init
0050 #ulimit -v 3072000
0051 #export STAGE_SVCCLASS=cmscafuser
0052 #save path to the condor batch working directory (/pool/condor)
0053
0054 export CONDORWORKDIR=`pwd -P`
0055 echo CONDOR working directory is $CONDORWORKDIR
0056 source /afs/cern.ch/cms/caf/setup.sh
0057 export X509_USER_PROXY=.oO[scriptsdir]Oo./.user_proxy
0058 cd .oO[CMSSW_BASE]Oo./src
0059 export SCRAM_ARCH=.oO[SCRAM_ARCH]Oo.
0060 eval `scramv1 ru -sh`
0061 #mkdir -p .oO[datadir]Oo. &>! /dev/null
0062
0063 #remove possible result file from previous runs
0064 previous_results=$(ls /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo.)
0065 for file in ${previous_results}
0066 do
0067 if [ ${file} = /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./.oO[outputFile]Oo. ]
0068 then
0069 xrdcp -f root://eoscms//eos/cms${file} root://eoscms//eos/cms${file}.bak
0070 fi
0071 done
0072
0073 if [[ $HOSTNAME = lxplus[0-9]*[.a-z0-9]* ]] # check for interactive mode
0074 then
0075 mkdir -p .oO[workdir]Oo.
0076 rm -f .oO[workdir]Oo./*
0077 cd .oO[workdir]Oo.
0078 else
0079 mkdir -p $CONDORWORKDIR/TkAllInOneTool
0080 cd $CONDORWORKDIR/TkAllInOneTool
0081 fi
0082
0083 # rm -f .oO[workdir]Oo./*
0084 # cd .oO[workdir]Oo.
0085
0086 #run
0087 pwd
0088 df -h .
0089 which cmsRun
0090 .oO[CommandLine]Oo.
0091 echo "----"
0092 echo "List of files in $(pwd):"
0093 ls -ltr
0094 echo "----"
0095 echo ""
0096
0097
0098 #retrieve
0099 mkdir -p .oO[logdir]Oo. >&! /dev/null
0100 gzip -f LOGFILE_*_.oO[name]Oo..log
0101 find . -maxdepth 1 -name "LOGFILE*.oO[alignmentName]Oo.*" -print | xargs -I {} bash -c "cp {} .oO[logdir]Oo."
0102
0103 #copy root files to eos
0104 mkdir -p /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo.
0105 if [ .oO[parallelJobs]Oo. -eq 1 ]
0106 then
0107 root_files=$(ls --color=never -d *.oO[alignmentName]Oo.*.root)
0108 else
0109 root_files=$(ls --color=never -d *.oO[alignmentName]Oo._.oO[nIndex]Oo.*.root)
0110 fi
0111 echo ${root_files}
0112
0113 for file in ${root_files}
0114 do
0115 xrdcp -f ${file} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo.
0116 echo ${file}
0117 done
0118
0119 #cleanup
0120 if [[ $HOSTNAME = lxplus[0-9]*[.a-z0-9]* ]] # check for interactive mode
0121 then
0122 rm -rf .oO[workdir]Oo.
0123 fi
0124 echo "done."
0125 """
0126
0127
0128
0129
0130 cfgTemplate="""
0131 import FWCore.ParameterSet.Config as cms
0132
0133 process = cms.Process(".oO[ProcessName]Oo.")
0134
0135 .oO[datasetDefinition]Oo.
0136 .oO[Bookkeeping]Oo.
0137 .oO[LoadBasicModules]Oo.
0138 .oO[TrackSelectionRefitting]Oo.
0139 .oO[LoadGlobalTagTemplate]Oo.
0140 .oO[condLoad]Oo.
0141 .oO[ValidationConfig]Oo.
0142 .oO[FileOutputTemplate]Oo.
0143
0144 .oO[DefinePath]Oo.
0145
0146 print("Done")
0147 """
0148
0149
0150
0151
0152 Bookkeeping = """
0153 process.options = cms.untracked.PSet(
0154 wantSummary = cms.untracked.bool(False),
0155 Rethrow = cms.untracked.vstring("ProductNotFound"), # make this exception fatal
0156 fileMode = cms.untracked.string('NOMERGE') # no ordering needed, but calls endRun/beginRun etc. at file boundaries
0157 )
0158
0159 process.load("FWCore.MessageLogger.MessageLogger_cfi")
0160 process.MessageLogger.cerr.FwkReport.reportEvery = 1000
0161 process.MessageLogger.cout.enableStatistics = cms.untracked.bool(True)
0162 """
0163
0164
0165
0166
0167 CommonTrackSelectionRefitting = """
0168 import Alignment.CommonAlignment.tools.trackselectionRefitting as trackselRefit
0169 process.seqTrackselRefit = trackselRefit.getSequence(process, '.oO[trackcollection]Oo.',
0170 isPVValidation=.oO[ispvvalidation]Oo.,
0171 TTRHBuilder='.oO[ttrhbuilder]Oo.',
0172 usePixelQualityFlag=.oO[usepixelqualityflag]Oo.,
0173 openMassWindow=.oO[openmasswindow]Oo.,
0174 cosmicsDecoMode=.oO[cosmicsdecomode]Oo.,
0175 cosmicsZeroTesla=.oO[cosmics0T]Oo.,
0176 momentumConstraint=.oO[momentumconstraint]Oo.,
0177 cosmicTrackSplitting=.oO[istracksplitting]Oo.,
0178 use_d0cut=.oO[use_d0cut]Oo.,
0179 )
0180
0181 .oO[trackhitfiltercommands]Oo.
0182 """
0183
0184
0185
0186
0187 SingleTrackRefitter = """
0188 process.load("RecoTracker.TrackProducer.TrackRefitters_cff")
0189 process.TrackRefitter.src = ".oO[TrackCollection]Oo."
0190 process.TrackRefitter.TTRHBuilder = ".oO[ttrhbuilder]Oo."
0191 process.TrackRefitter.NavigationSchool = ""
0192 """
0193
0194
0195
0196
0197 LoadBasicModules = """
0198 process.load("RecoVertex.BeamSpotProducer.BeamSpot_cff")
0199 process.load("Configuration.Geometry.GeometryDB_cff")
0200 process.load('Configuration.StandardSequences.Services_cff')
0201 process.load("Configuration.StandardSequences..oO[magneticField]Oo._cff")
0202 """
0203
0204
0205
0206
0207 FileOutputTemplate = """
0208 process.TFileService = cms.Service("TFileService",
0209 fileName = cms.string('.oO[outputFile]Oo.')
0210 )
0211 """
0212
0213
0214
0215
0216 DefinePath_CommonSelectionRefitting = """
0217 process.p = cms.Path(
0218 process.seqTrackselRefit*.oO[ValidationSequence]Oo.)
0219 """
0220
0221
0222
0223 mergeTemplate="""#!/bin/bash
0224 CWD=`pwd -P`
0225 cd .oO[CMSSW_BASE]Oo./src
0226 export SCRAM_ARCH=.oO[SCRAM_ARCH]Oo.
0227 eval `scramv1 ru -sh`
0228
0229
0230 .oO[createResultsDirectory]Oo.
0231
0232 if [[ $HOSTNAME = lxplus[0-9]*[.a-z0-9]* ]] # check for interactive mode
0233 then
0234 mkdir -p .oO[workdir]Oo.
0235 cd .oO[workdir]Oo.
0236 else
0237 cd $CWD
0238 fi
0239 echo "Working directory: $(pwd -P)"
0240
0241 ###############################################################################
0242 # download root files from eos
0243 root_files=$(ls /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo. \
0244 | grep ".root$" | grep -v "result.root$")
0245 #for file in ${root_files}
0246 #do
0247 # xrdcp -f root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./${file} .
0248 # echo ${file}
0249 #done
0250
0251
0252 #run
0253 .oO[DownloadData]Oo.
0254 .oO[CompareAlignments]Oo.
0255
0256 .oO[RunValidationPlots]Oo.
0257
0258 # clean-up
0259 # ls -l *.root
0260 rm -f *.root
0261
0262 #zip stdout and stderr from the farm jobs
0263 cd .oO[logdir]Oo.
0264 find . -name "*.stderr" -exec gzip -f {} \;
0265 find . -name "*.stdout" -exec gzip -f {} \;
0266 """
0267
0268
0269
0270
0271
0272 mergeParallelOfflineTemplate="""#!/bin/bash
0273 CWD=`pwd -P`
0274 cd .oO[CMSSW_BASE]Oo./src
0275 export SCRAM_ARCH=.oO[SCRAM_ARCH]Oo.
0276 eval `scramv1 ru -sh`
0277
0278 if [[ $HOSTNAME = lxplus[0-9]*[.a-z0-9]* ]] # check for interactive mode
0279 then
0280 mkdir -p .oO[workdir]Oo.
0281 cd .oO[workdir]Oo.
0282 else
0283 cd $CWD
0284 fi
0285 echo "Working directory: $(pwd -P)"
0286
0287 ###############################################################################
0288 # download root files from eos
0289 root_files=$(ls /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo. \
0290 | grep ".root$" | grep -v "result.root$")
0291 #for file in ${root_files}
0292 #do
0293 # xrdcp -f root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./${file} .
0294 # echo ${file}
0295 #done
0296
0297
0298 #run
0299 .oO[DownloadData]Oo.
0300 """
0301
0302
0303
0304 createResultsDirectoryTemplate="""
0305 #create results-directory and copy used configuration there
0306 mkdir -p .oO[datadir]Oo.
0307 cp .oO[logdir]Oo./usedConfiguration.ini .oO[datadir]Oo.
0308 """
0309
0310
0311
0312
0313 mergeParallelResults="""
0314
0315 .oO[beforeMerge]Oo.
0316 .oO[doMerge]Oo.
0317
0318 # create log file
0319 ls -al .oO[mergeParallelFilePrefixes]Oo. > .oO[datadir]Oo./log_rootfilelist.txt
0320
0321 # Remove parallel job files
0322 .oO[rmUnmerged]Oo.
0323 """
0324
0325
0326
0327
0328 compareAlignmentsExecution="""
0329 #merge for .oO[validationId]Oo. if it does not exist or is not up-to-date
0330 echo -e "\n\nComparing validations"
0331 mkdir -p /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./
0332 cp .oO[Alignment/OfflineValidation]Oo./scripts/compareFileAges.C .
0333 root -x -q -b -l "compareFileAges.C(\\\"root://eoscms.cern.ch//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./.oO[validationId]Oo._result.root\\\", \\\".oO[compareStringsPlain]Oo.\\\")"
0334 comparisonNeeded=${?}
0335
0336 if [[ ${comparisonNeeded} -eq 1 ]]
0337 then
0338 cp .oO[compareAlignmentsPath]Oo. .
0339 root -x -q -b -l '.oO[compareAlignmentsName]Oo.++(\".oO[compareStrings]Oo.\", ".oO[legendheader]Oo.", ".oO[customtitle]Oo.", ".oO[customrighttitle]Oo.", .oO[bigtext]Oo.)'
0340 mv result.root .oO[validationId]Oo._result.root
0341 xrdcp -f .oO[validationId]Oo._result.root root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo.
0342 else
0343 echo ".oO[validationId]Oo._result.root is up-to-date, no need to compare again."
0344 xrdcp -f root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./.oO[validationId]Oo._result.root .
0345 fi
0346 """
0347
0348
0349
0350
0351 crabCfgTemplate="""
0352 [CRAB]
0353 jobtype = cmssw
0354 scheduler = caf
0355 use_server = 0
0356
0357 [CMSSW]
0358 datasetpath = .oO[dataset]Oo.
0359 pset = .oO[cfgFile]Oo.
0360 total_number_of_.oO[McOrData]Oo.
0361 number_of_jobs = .oO[numberOfJobs]Oo.
0362 output_file = .oO[outputFile]Oo.
0363 runselection = .oO[runRange]Oo.
0364 lumi_mask = .oO[JSON]Oo.
0365
0366 [USER]
0367 return_data = 0
0368 copy_data = 1
0369 storage_element = T2_CH_CERN
0370 user_remote_dir = .oO[eosdir]Oo.
0371 ui_working_dir = .oO[crabWorkingDir]Oo.
0372 # script_exe = .oO[script]Oo.
0373 # .oO[email]Oo.
0374
0375 [CAF]
0376 queue = .oO[queue]Oo.
0377 """
0378
0379
0380
0381
0382
0383
0384
0385
0386
0387
0388
0389
0390
0391 def alternateTemplate( templateName, alternateTemplateName ):
0392
0393 if not templateName in globals().keys():
0394 msg = "unknown template to replace %s"%templateName
0395 raise AllInOneError(msg)
0396 if not alternateTemplateName in globals().keys():
0397 msg = "unknown template to replace %s"%alternateTemplateName
0398 raise AllInOneError(msg)
0399 globals()[ templateName ] = globals()[ alternateTemplateName ]
0400
