File indexing completed on 2024-11-26 02:34:11
0001
0002
0003 import os
0004 import sys
0005 import configparser as ConfigParser
0006 import optparse
0007 import datetime
0008
0009 scriptTemplate = """
0010 #!/bin/bash
0011 #init
0012 export STAGE_SVCCLASS=cmscaf
0013 source /afs/cern.ch/cms/sw/cmsset_default.sh
0014 cd %(CMSSW_BASE)s
0015 eval `scramv1 ru -sh`
0016 rfmkdir -p %(workDir)s
0017 rm -f %(workDir)s/*
0018 cd %(workDir)s
0019
0020 #run
0021 pwd
0022 df -h .
0023 #run configfile and post-proccess it
0024 %(commandLine)s &> cmsRunOut.%(name)s.log
0025
0026 echo "----"
0027 echo "List of files in $(pwd):"
0028 ls -ltr
0029 echo "----"
0030 echo ""
0031
0032 #retrive
0033 rfmkdir %(outDir)s
0034 rfmkdir %(outDir)s/%(name)s
0035 gzip cmsRunOut.%(name)s.log
0036 rfmkdir %(logDir)s
0037 rfcp cmsRunOut.%(name)s%(isn)s.log.gz %(logDir)s
0038 for i in *.root
0039 do
0040 rfcp $i %(outDir)s/%(name)s/${i/.root}%(isn)s.root
0041 done
0042
0043 #cleanup
0044 #rm -rf %(workDir)s
0045 echo "done."
0046 """
0047
0048 extractDQMtemplate = """
0049 import FWCore.ParameterSet.Config as cms
0050
0051 process = cms.Process("EDMtoMEConvert")
0052 process.load("DQMServices.Examples.test.MessageLogger_cfi")
0053
0054 process.load("DQMServices.Components.EDMtoMEConverter_cff")
0055
0056 process.maxEvents = cms.untracked.PSet(
0057 input = cms.untracked.int32(-1)
0058 )
0059 process.source = cms.Source("PoolSource",
0060 fileNames = cms.untracked.vstring(%(AlCARecos)s)
0061 )
0062
0063 process.p1 = cms.Path(process.EDMtoMEConverter*process.dqmSaver)
0064 process.dqmSaver.convention = 'Offline'
0065 process.dqmSaver.workflow = '/ConverterTester/Test/RECO'
0066 """
0067
0068
0069 class BetterConfigParser(ConfigParser.ConfigParser):
0070 def optionxform(self, optionstr):
0071 return optionstr
0072
0073 def exists( self, section, option):
0074 try:
0075 items = self.items(section)
0076 except ConfigParser.NoSectionError:
0077 return False
0078 for item in items:
0079 if item[0] == option:
0080 return True
0081 return False
0082
0083 class Sample:
0084 def __init__(self, name, config):
0085 self.name = name
0086 self.files = config.get("sample:%s"%name, "files").split("\n")
0087 self.globalTag = config.get("sample:%s"%name, "globalTag")
0088 self.maxEvents = config.get("sample:%s"%name, "maxEvents")
0089 self.AlCaRecos = config.get("sample:%s"%name, "AlCaRecos")
0090 self.AlcaRecoScripts = None
0091 self.DQMScript = None
0092 self.DQMConfig = None
0093 self.__config = config
0094
0095 def createScript(self, path, style, commandLine, isn = None):
0096 repMap = {
0097 "name": self.name,
0098 "workDir": self.__config.get("general", "workDir"),
0099 "outDir": self.__config.get("general", "outDir"),
0100 "logDir": self.__config.get("general", "logDir"),
0101 "CMSSW_BASE": os.environ["CMSSW_BASE"],
0102 "commandLine":commandLine
0103 }
0104 fileName = "%s_%s"%(style,self.name)
0105 if not isn == None:
0106 repMap["isn"] = "_%03i"%isn
0107 fileName += repMap["isn"]
0108 else:
0109 repMap["isn"] = ""
0110
0111 fileName += ".sh"
0112 scriptPath = os.path.join(path, fileName)
0113 scriptFile = open( scriptPath, "w" )
0114 scriptFile.write( scriptTemplate%repMap )
0115 scriptFile.close()
0116 os.chmod(scriptPath, 0o755)
0117 return scriptPath
0118
0119 def createDQMExtract(self, path):
0120 filePrefix = "ALCARECO%s"%self.AlCaRecos.split("+")[0]
0121
0122 alcaFiles = []
0123 for i in range( 0,len(self.files) ):
0124 alcaFilePath = os.path.join( self.__config.get("general","outDir"), self.name,"%s_%03i.root"%(filePrefix,i+1) )
0125 alcaFilePath = os.path.expandvars(alcaFilePath)
0126 alcaFilePath = os.path.abspath(alcaFilePath)
0127 if alcaFilePath.startswith("/castor"):
0128 alcaFilePath = "'rfio:"+alcaFilePath+"'"
0129 else:
0130 alcaFilePath = "'file:"+alcaFilePath+"'"
0131 alcaFiles.append(alcaFilePath)
0132 repMap = { "AlCARecos":", ".join(alcaFiles)}
0133 fileName = "extractDQM_%s_cfg.py"%self.name
0134 cfgPath = os.path.join(path, fileName)
0135 cfgFile = open( cfgPath, "w" )
0136 cfgFile.write( extractDQMtemplate%repMap )
0137 cfgFile.close()
0138 return cfgPath
0139
0140 def createScripts(self, path):
0141 result = []
0142 repMap = {"maxEvents":self.maxEvents,
0143 "globalTag":self.globalTag,
0144 "AlCaRecos":self.AlCaRecos,
0145 "sampleFile":"" }
0146 i = 1
0147 for sampleFile in self.files:
0148 repMap["sampleFile"] = sampleFile
0149 result.append(self.createScript(path, "AlCaReco", """
0150 cmsDriver.py step3 -s ALCA:%(AlCaRecos)s+DQM \\
0151 -n %(maxEvents)s \\
0152 --filein %(sampleFile)s\\
0153 --conditions FrontierConditions_GlobalTag,%(globalTag)s \\
0154 --eventcontent RECO\\
0155 --mc"""%repMap, i))
0156 i += 1
0157 self.AlcaRecoScripts = result
0158 self.DQMConfig = self.createDQMExtract(path)
0159 self.DQMScript = self.createScript(path, "extractDQM", "cmsRun %s"%self.DQMConfig)
0160 return result
0161
0162
0163
0164 def getCommandOutput2(command):
0165 child = os.popen(command)
0166 data = child.read()
0167 err = child.close()
0168 if err:
0169 raise RuntimeError('%s failed w/ exit code %d' % (command, err))
0170 return data
0171
0172 def runJob(jobName, script, config):
0173 jobMode = config.get("general","jobMode")
0174 print("> Testing "+jobName)
0175 if jobMode == "interactive":
0176 getCommandOutput2( script )
0177 if jobMode.split(",")[0] == "lxBatch":
0178 commands = jobMode.split(",")[1]
0179 getCommandOutput2("bsub "+commands+" -J "+jobName+" "+ script)
0180
0181 def readSamples( config):
0182 result = []
0183 for section in config.sections():
0184 if "sample:" in section:
0185 name = section.split("sample:")[1]
0186 result.append( Sample(name, config) )
0187 return result
0188
0189 def main(argv = None):
0190 if argv == None:
0191 argv = sys.argv[1:]
0192 optParser = optparse.OptionParser()
0193 optParser.description = "test the tracker AlCaReco production and DQM"
0194 optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
0195 help="create all scripts and cfg File but do not start jobs (default=False)")
0196 optParser.add_option("-c", "--config", dest="config",
0197 help="configuration to use (default testConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
0198 optParser.add_option("-N", "--Name", dest="Name",
0199 help="Name of this test (default: test_DATE_TIME)", metavar="NAME")
0200
0201 (options, args) = optParser.parse_args(argv)
0202
0203 if options.config == None:
0204 options.config = "testConfig.ini"
0205 else:
0206 options.config = options.config.split(",")
0207 result = []
0208 for iniFile in options.config:
0209 result.append( os.path.abspath(iniFile) )
0210 options.config = result
0211
0212 config = BetterConfigParser()
0213 config.read( options.config )
0214
0215 if options.Name == None:
0216 options.Name = "test_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
0217
0218 outPath = os.path.abspath( options.Name )
0219 for dir in [ "workDir","dataDir","logDir" ]:
0220 if not config.has_option("general", dir):
0221 config.set("general", dir, os.path.join( os.getcwd(), options.Name ))
0222 else:
0223 config.set("general", dir, os.path.join( config.get("general",dir), options.Name ))
0224
0225 if not os.path.exists( outPath ):
0226 os.makedirs( outPath )
0227 elif not os.path.isdir( outPath ):
0228 raise "the file %s is in the way rename the Job or move it away"%outPath
0229
0230 samples = readSamples( config )
0231 for sample in samples:
0232 sample.createScripts( outPath )
0233 for scriptPath in sample.AlcaRecoScripts:
0234 if not options.dryRun:
0235 runJob( sample.name, scriptPath, config )
0236
0237 if __name__ == "__main__":
0238 main()