Alignment/test/alcarecoTester.py

0001 #!/usr/bin/env python3
0002
0003 import os
0004 import sys
0005 import configparser as ConfigParser
0006 import optparse
0007 import datetime
0008
0009 scriptTemplate = """
0010 #!/bin/bash
0011 #init
0012 export STAGE_SVCCLASS=cmscaf
0013 source /afs/cern.ch/cms/sw/cmsset_default.sh
0014 cd %(CMSSW_BASE)s
0015 eval `scramv1 ru -sh`
0016 rfmkdir -p %(workDir)s
0017 rm -f %(workDir)s/*
0018 cd %(workDir)s
0019
0020 #run
0021 pwd
0022 df -h .
0023 #run configfile and post-proccess it
0024 %(commandLine)s &> cmsRunOut.%(name)s.log
0025
0026 echo "----"
0027 echo "List of files in $(pwd):"
0028 ls -ltr
0029 echo "----"
0030 echo ""
0031
0032 #retrive
0033 rfmkdir %(outDir)s
0034 rfmkdir %(outDir)s/%(name)s
0035 gzip cmsRunOut.%(name)s.log
0036 rfmkdir %(logDir)s
0037 rfcp cmsRunOut.%(name)s%(isn)s.log.gz %(logDir)s
0038 for i in *.root
0039 do
0040    rfcp $i %(outDir)s/%(name)s/${i/.root}%(isn)s.root
0041 done
0042
0043 #cleanup
0044 #rm -rf %(workDir)s
0045 echo "done."
0046 """
0047
0048 extractDQMtemplate = """
0049 import FWCore.ParameterSet.Config as cms
0050
0051 process = cms.Process("EDMtoMEConvert")
0052 process.load("DQMServices.Examples.test.MessageLogger_cfi")
0053
0054 process.load("DQMServices.Components.EDMtoMEConverter_cff")
0055
0056 process.maxEvents = cms.untracked.PSet(
0057     input = cms.untracked.int32(-1)
0058 )
0059 process.source = cms.Source("PoolSource",
0060     fileNames = cms.untracked.vstring(%(AlCARecos)s)
0061 )
0062
0063 process.p1 = cms.Path(process.EDMtoMEConverter*process.dqmSaver)
0064 process.dqmSaver.convention = 'Offline'
0065 process.dqmSaver.workflow = '/ConverterTester/Test/RECO'
0066 """
0067
0068 ####################--- Classes ---############################
0069 class BetterConfigParser(ConfigParser.ConfigParser):
0070     def optionxform(self, optionstr):
0071         return optionstr
0072
0073     def exists( self, section, option):
0074         try:
0075             items = self.items(section)
0076         except ConfigParser.NoSectionError:
0077             return False
0078         for item in items:
0079             if item[0] == option:
0080                 return True
0081         return False
0082
0083 class Sample:
0084    def __init__(self, name, config):
0085       self.name = name
0086       self.files = config.get("sample:%s"%name, "files").split("\n")
0087       self.globalTag = config.get("sample:%s"%name, "globalTag")
0088       self.maxEvents = config.get("sample:%s"%name, "maxEvents")
0089       self.AlCaRecos = config.get("sample:%s"%name, "AlCaRecos")
0090       self.AlcaRecoScripts = None
0091       self.DQMScript = None
0092       self.DQMConfig = None
0093       self.__config = config
0094
0095    def createScript(self, path, style, commandLine, isn = None):
0096       repMap = {
0097                    "name": self.name,
0098                    "workDir": self.__config.get("general", "workDir"),
0099                    "outDir": self.__config.get("general", "outDir"),
0100                    "logDir": self.__config.get("general", "logDir"),
0101                    "CMSSW_BASE": os.environ["CMSSW_BASE"],
0102                    "commandLine":commandLine
0103                    }
0104       fileName = "%s_%s"%(style,self.name)
0105       if not isn == None:
0106          repMap["isn"] = "_%03i"%isn
0107          fileName += repMap["isn"]
0108       else:
0109          repMap["isn"] = ""
0110
0111       fileName += ".sh"
0112       scriptPath = os.path.join(path, fileName)
0113       scriptFile = open( scriptPath, "w" )
0114       scriptFile.write( scriptTemplate%repMap )
0115       scriptFile.close()
0116       os.chmod(scriptPath, 0o755)
0117       return scriptPath
0118
0119    def createDQMExtract(self, path):
0120       filePrefix = "ALCARECO%s"%self.AlCaRecos.split("+")[0]
0121
0122       alcaFiles = []
0123       for i in range( 0,len(self.files) ):
0124          alcaFilePath = os.path.join( self.__config.get("general","outDir"), self.name,"%s_%03i.root"%(filePrefix,i+1) )
0125          alcaFilePath = os.path.expandvars(alcaFilePath)
0126          alcaFilePath = os.path.abspath(alcaFilePath)
0127          if alcaFilePath.startswith("/castor"):
0128             alcaFilePath = "'rfio:"+alcaFilePath+"'"
0129          else:
0130             alcaFilePath = "'file:"+alcaFilePath+"'"
0131          alcaFiles.append(alcaFilePath)
0132       repMap = { "AlCARecos":", ".join(alcaFiles)}
0133       fileName = "extractDQM_%s_cfg.py"%self.name
0134       cfgPath = os.path.join(path, fileName)
0135       cfgFile = open( cfgPath, "w" )
0136       cfgFile.write( extractDQMtemplate%repMap )
0137       cfgFile.close()
0138       return cfgPath
0139
0140    def createScripts(self, path):
0141       result = []
0142       repMap = {"maxEvents":self.maxEvents,
0143                 "globalTag":self.globalTag,
0144                 "AlCaRecos":self.AlCaRecos,
0145                 "sampleFile":"" }
0146       i = 1
0147       for sampleFile in self.files:
0148          repMap["sampleFile"] = sampleFile
0149          result.append(self.createScript(path, "AlCaReco", """
0150 cmsDriver.py step3 -s ALCA:%(AlCaRecos)s+DQM \\
0151     -n %(maxEvents)s \\
0152     --filein %(sampleFile)s\\
0153     --conditions FrontierConditions_GlobalTag,%(globalTag)s \\
0154     --eventcontent RECO\\
0155     --mc"""%repMap, i))
0156          i += 1
0157       self.AlcaRecoScripts = result
0158       self.DQMConfig = self.createDQMExtract(path)
0159       self.DQMScript = self.createScript(path, "extractDQM", "cmsRun %s"%self.DQMConfig)
0160       return result
0161
0162 ######################### Helpers ####################
0163 #excute [command] and return output
0164 def getCommandOutput2(command):
0165     child = os.popen(command)
0166     data = child.read()
0167     err = child.close()
0168     if err:
0169         raise RuntimeError('%s failed w/ exit code %d' % (command, err))
0170     return data
0171
0172 def runJob(jobName, script, config):
0173    jobMode = config.get("general","jobMode")
0174    print("> Testing "+jobName)
0175    if jobMode == "interactive":
0176        getCommandOutput2( script )
0177    if jobMode.split(",")[0] == "lxBatch":
0178        commands = jobMode.split(",")[1]
0179        getCommandOutput2("bsub "+commands+" -J "+jobName+" "+ script)
0180
0181 def readSamples( config):
0182    result = []
0183    for section in config.sections():
0184       if "sample:" in section:
0185          name = section.split("sample:")[1]
0186          result.append( Sample(name, config) )
0187    return result
0188
0189 def main(argv = None):
0190     if argv == None:
0191        argv = sys.argv[1:]
0192     optParser = optparse.OptionParser()
0193     optParser.description = "test the tracker AlCaReco production and DQM"
0194     optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
0195                          help="create all scripts and cfg File but do not start jobs (default=False)")
0196     optParser.add_option("-c", "--config", dest="config",
0197                          help="configuration to use (default testConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
0198     optParser.add_option("-N", "--Name", dest="Name",
0199                          help="Name of this test (default: test_DATE_TIME)", metavar="NAME")
0200
0201     (options, args) = optParser.parse_args(argv)
0202
0203     if options.config == None:
0204         options.config = "testConfig.ini"
0205     else:
0206         options.config = options.config.split(",")
0207         result = []
0208         for iniFile in options.config:
0209             result.append( os.path.abspath(iniFile) )
0210         options.config = result
0211
0212     config = BetterConfigParser()
0213     config.read( options.config )
0214
0215     if options.Name == None:
0216         options.Name = "test_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
0217
0218     outPath = os.path.abspath( options.Name )
0219     for dir in [ "workDir","dataDir","logDir" ]:
0220        if not config.has_option("general", dir):
0221           config.set("general", dir, os.path.join( os.getcwd(), options.Name ))
0222        else:
0223           config.set("general", dir, os.path.join( config.get("general",dir), options.Name ))
0224
0225     if not os.path.exists( outPath ):
0226         os.makedirs( outPath )
0227     elif not os.path.isdir( outPath ):
0228         raise "the file %s is in the way rename the Job or move it away"%outPath
0229
0230     samples = readSamples( config )
0231     for sample in samples:
0232        sample.createScripts( outPath )
0233        for scriptPath in sample.AlcaRecoScripts:
0234           if not options.dryRun:
0235              runJob( sample.name, scriptPath, config )
0236
0237 if __name__ == "__main__":
0238     main()