File indexing completed on 2023-10-25 09:43:59
0001
0002
0003 from __future__ import print_function
0004 import os
0005 import sys
0006 import configparser as ConfigParser
0007 import optparse
0008 import datetime
0009
0010 scriptTemplate = """
0011 #!/bin/bash
0012 #init
0013 export STAGE_SVCCLASS=cmscaf
0014 source /afs/cern.ch/cms/sw/cmsset_default.sh
0015 cd %(CMSSW_BASE)s
0016 eval `scramv1 ru -sh`
0017 rfmkdir -p %(workDir)s
0018 rm -f %(workDir)s/*
0019 cd %(workDir)s
0020
0021 #run
0022 pwd
0023 df -h .
0024 #run configfile and post-proccess it
0025 %(commandLine)s &> cmsRunOut.%(name)s.log
0026
0027 echo "----"
0028 echo "List of files in $(pwd):"
0029 ls -ltr
0030 echo "----"
0031 echo ""
0032
0033 #retrive
0034 rfmkdir %(outDir)s
0035 rfmkdir %(outDir)s/%(name)s
0036 gzip cmsRunOut.%(name)s.log
0037 rfmkdir %(logDir)s
0038 rfcp cmsRunOut.%(name)s%(isn)s.log.gz %(logDir)s
0039 for i in *.root
0040 do
0041 rfcp $i %(outDir)s/%(name)s/${i/.root}%(isn)s.root
0042 done
0043
0044 #cleanup
0045 #rm -rf %(workDir)s
0046 echo "done."
0047 """
0048
0049 extractDQMtemplate = """
0050 import FWCore.ParameterSet.Config as cms
0051
0052 process = cms.Process("EDMtoMEConvert")
0053 process.load("DQMServices.Examples.test.MessageLogger_cfi")
0054
0055 process.load("DQMServices.Components.EDMtoMEConverter_cff")
0056
0057 process.maxEvents = cms.untracked.PSet(
0058 input = cms.untracked.int32(-1)
0059 )
0060 process.source = cms.Source("PoolSource",
0061 fileNames = cms.untracked.vstring(%(AlCARecos)s)
0062 )
0063
0064 process.p1 = cms.Path(process.EDMtoMEConverter*process.dqmSaver)
0065 process.dqmSaver.convention = 'Offline'
0066 process.dqmSaver.workflow = '/ConverterTester/Test/RECO'
0067 """
0068
0069
0070 class BetterConfigParser(ConfigParser.ConfigParser):
0071 def optionxform(self, optionstr):
0072 return optionstr
0073
0074 def exists( self, section, option):
0075 try:
0076 items = self.items(section)
0077 except ConfigParser.NoSectionError:
0078 return False
0079 for item in items:
0080 if item[0] == option:
0081 return True
0082 return False
0083
0084 class Sample:
0085 def __init__(self, name, config):
0086 self.name = name
0087 self.files = config.get("sample:%s"%name, "files").split("\n")
0088 self.globalTag = config.get("sample:%s"%name, "globalTag")
0089 self.maxEvents = config.get("sample:%s"%name, "maxEvents")
0090 self.AlCaRecos = config.get("sample:%s"%name, "AlCaRecos")
0091 self.AlcaRecoScripts = None
0092 self.DQMScript = None
0093 self.DQMConfig = None
0094 self.__config = config
0095
0096 def createScript(self, path, style, commandLine, isn = None):
0097 repMap = {
0098 "name": self.name,
0099 "workDir": self.__config.get("general", "workDir"),
0100 "outDir": self.__config.get("general", "outDir"),
0101 "logDir": self.__config.get("general", "logDir"),
0102 "CMSSW_BASE": os.environ["CMSSW_BASE"],
0103 "commandLine":commandLine
0104 }
0105 fileName = "%s_%s"%(style,self.name)
0106 if not isn == None:
0107 repMap["isn"] = "_%03i"%isn
0108 fileName += repMap["isn"]
0109 else:
0110 repMap["isn"] = ""
0111
0112 fileName += ".sh"
0113 scriptPath = os.path.join(path, fileName)
0114 scriptFile = open( scriptPath, "w" )
0115 scriptFile.write( scriptTemplate%repMap )
0116 scriptFile.close()
0117 os.chmod(scriptPath, 0o755)
0118 return scriptPath
0119
0120 def createDQMExtract(self, path):
0121 filePrefix = "ALCARECO%s"%self.AlCaRecos.split("+")[0]
0122
0123 alcaFiles = []
0124 for i in range( 0,len(self.files) ):
0125 alcaFilePath = os.path.join( self.__config.get("general","outDir"), self.name,"%s_%03i.root"%(filePrefix,i+1) )
0126 alcaFilePath = os.path.expandvars(alcaFilePath)
0127 alcaFilePath = os.path.abspath(alcaFilePath)
0128 if alcaFilePath.startswith("/castor"):
0129 alcaFilePath = "'rfio:"+alcaFilePath+"'"
0130 else:
0131 alcaFilePath = "'file:"+alcaFilePath+"'"
0132 alcaFiles.append(alcaFilePath)
0133 repMap = { "AlCARecos":", ".join(alcaFiles)}
0134 fileName = "extractDQM_%s_cfg.py"%self.name
0135 cfgPath = os.path.join(path, fileName)
0136 cfgFile = open( cfgPath, "w" )
0137 cfgFile.write( extractDQMtemplate%repMap )
0138 cfgFile.close()
0139 return cfgPath
0140
0141 def createScripts(self, path):
0142 result = []
0143 repMap = {"maxEvents":self.maxEvents,
0144 "globalTag":self.globalTag,
0145 "AlCaRecos":self.AlCaRecos,
0146 "sampleFile":"" }
0147 i = 1
0148 for sampleFile in self.files:
0149 repMap["sampleFile"] = sampleFile
0150 result.append(self.createScript(path, "AlCaReco", """
0151 cmsDriver.py step3 -s ALCA:%(AlCaRecos)s+DQM \\
0152 -n %(maxEvents)s \\
0153 --filein %(sampleFile)s\\
0154 --conditions FrontierConditions_GlobalTag,%(globalTag)s \\
0155 --eventcontent RECO\\
0156 --mc"""%repMap, i))
0157 i += 1
0158 self.AlcaRecoScripts = result
0159 self.DQMConfig = self.createDQMExtract(path)
0160 self.DQMScript = self.createScript(path, "extractDQM", "cmsRun %s"%self.DQMConfig)
0161 return result
0162
0163
0164
0165 def getCommandOutput2(command):
0166 child = os.popen(command)
0167 data = child.read()
0168 err = child.close()
0169 if err:
0170 raise RuntimeError('%s failed w/ exit code %d' % (command, err))
0171 return data
0172
0173 def runJob(jobName, script, config):
0174 jobMode = config.get("general","jobMode")
0175 print("> Testing "+jobName)
0176 if jobMode == "interactive":
0177 getCommandOutput2( script )
0178 if jobMode.split(",")[0] == "lxBatch":
0179 commands = jobMode.split(",")[1]
0180 getCommandOutput2("bsub "+commands+" -J "+jobName+" "+ script)
0181
0182 def readSamples( config):
0183 result = []
0184 for section in config.sections():
0185 if "sample:" in section:
0186 name = section.split("sample:")[1]
0187 result.append( Sample(name, config) )
0188 return result
0189
0190 def main(argv = None):
0191 if argv == None:
0192 argv = sys.argv[1:]
0193 optParser = optparse.OptionParser()
0194 optParser.description = "test the tracker AlCaReco production and DQM"
0195 optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
0196 help="create all scripts and cfg File but do not start jobs (default=False)")
0197 optParser.add_option("-c", "--config", dest="config",
0198 help="configuration to use (default testConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
0199 optParser.add_option("-N", "--Name", dest="Name",
0200 help="Name of this test (default: test_DATE_TIME)", metavar="NAME")
0201
0202 (options, args) = optParser.parse_args(argv)
0203
0204 if options.config == None:
0205 options.config = "testConfig.ini"
0206 else:
0207 options.config = options.config.split(",")
0208 result = []
0209 for iniFile in options.config:
0210 result.append( os.path.abspath(iniFile) )
0211 options.config = result
0212
0213 config = BetterConfigParser()
0214 config.read( options.config )
0215
0216 if options.Name == None:
0217 options.Name = "test_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
0218
0219 outPath = os.path.abspath( options.Name )
0220 for dir in [ "workDir","dataDir","logDir" ]:
0221 if not config.has_option("general", dir):
0222 config.set("general", dir, os.path.join( os.getcwd(), options.Name ))
0223 else:
0224 config.set("general", dir, os.path.join( config.get("general",dir), options.Name ))
0225
0226 if not os.path.exists( outPath ):
0227 os.makedirs( outPath )
0228 elif not os.path.isdir( outPath ):
0229 raise "the file %s is in the way rename the Job or move it away"%outPath
0230
0231 samples = readSamples( config )
0232 for sample in samples:
0233 sample.createScripts( outPath )
0234 for scriptPath in sample.AlcaRecoScripts:
0235 if not options.dryRun:
0236 runJob( sample.name, scriptPath, config )
0237
0238 if __name__ == "__main__":
0239 main()