Alignment/test/alcarecoTester.py

0001 #!/usr/bin/env python3
0002
0003 from __future__ import print_function
0004 import os
0005 import sys
0006 import configparser as ConfigParser
0007 import optparse
0008 import datetime
0009
0010 scriptTemplate = """
0011 #!/bin/bash
0012 #init
0013 export STAGE_SVCCLASS=cmscaf
0014 source /afs/cern.ch/cms/sw/cmsset_default.sh
0015 cd %(CMSSW_BASE)s
0016 eval `scramv1 ru -sh`
0017 rfmkdir -p %(workDir)s
0018 rm -f %(workDir)s/*
0019 cd %(workDir)s
0020
0021 #run
0022 pwd
0023 df -h .
0024 #run configfile and post-proccess it
0025 %(commandLine)s &> cmsRunOut.%(name)s.log
0026
0027 echo "----"
0028 echo "List of files in $(pwd):"
0029 ls -ltr
0030 echo "----"
0031 echo ""
0032
0033 #retrive
0034 rfmkdir %(outDir)s
0035 rfmkdir %(outDir)s/%(name)s
0036 gzip cmsRunOut.%(name)s.log
0037 rfmkdir %(logDir)s
0038 rfcp cmsRunOut.%(name)s%(isn)s.log.gz %(logDir)s
0039 for i in *.root
0040 do
0041    rfcp $i %(outDir)s/%(name)s/${i/.root}%(isn)s.root
0042 done
0043
0044 #cleanup
0045 #rm -rf %(workDir)s
0046 echo "done."
0047 """
0048
0049 extractDQMtemplate = """
0050 import FWCore.ParameterSet.Config as cms
0051
0052 process = cms.Process("EDMtoMEConvert")
0053 process.load("DQMServices.Examples.test.MessageLogger_cfi")
0054
0055 process.load("DQMServices.Components.EDMtoMEConverter_cff")
0056
0057 process.maxEvents = cms.untracked.PSet(
0058     input = cms.untracked.int32(-1)
0059 )
0060 process.source = cms.Source("PoolSource",
0061     fileNames = cms.untracked.vstring(%(AlCARecos)s)
0062 )
0063
0064 process.p1 = cms.Path(process.EDMtoMEConverter*process.dqmSaver)
0065 process.dqmSaver.convention = 'Offline'
0066 process.dqmSaver.workflow = '/ConverterTester/Test/RECO'
0067 """
0068
0069 ####################--- Classes ---############################
0070 class BetterConfigParser(ConfigParser.ConfigParser):
0071     def optionxform(self, optionstr):
0072         return optionstr
0073
0074     def exists( self, section, option):
0075         try:
0076             items = self.items(section)
0077         except ConfigParser.NoSectionError:
0078             return False
0079         for item in items:
0080             if item[0] == option:
0081                 return True
0082         return False
0083
0084 class Sample:
0085    def __init__(self, name, config):
0086       self.name = name
0087       self.files = config.get("sample:%s"%name, "files").split("\n")
0088       self.globalTag = config.get("sample:%s"%name, "globalTag")
0089       self.maxEvents = config.get("sample:%s"%name, "maxEvents")
0090       self.AlCaRecos = config.get("sample:%s"%name, "AlCaRecos")
0091       self.AlcaRecoScripts = None
0092       self.DQMScript = None
0093       self.DQMConfig = None
0094       self.__config = config
0095
0096    def createScript(self, path, style, commandLine, isn = None):
0097       repMap = {
0098                    "name": self.name,
0099                    "workDir": self.__config.get("general", "workDir"),
0100                    "outDir": self.__config.get("general", "outDir"),
0101                    "logDir": self.__config.get("general", "logDir"),
0102                    "CMSSW_BASE": os.environ["CMSSW_BASE"],
0103                    "commandLine":commandLine
0104                    }
0105       fileName = "%s_%s"%(style,self.name)
0106       if not isn == None:
0107          repMap["isn"] = "_%03i"%isn
0108          fileName += repMap["isn"]
0109       else:
0110          repMap["isn"] = ""
0111
0112       fileName += ".sh"
0113       scriptPath = os.path.join(path, fileName)
0114       scriptFile = open( scriptPath, "w" )
0115       scriptFile.write( scriptTemplate%repMap )
0116       scriptFile.close()
0117       os.chmod(scriptPath, 0o755)
0118       return scriptPath
0119
0120    def createDQMExtract(self, path):
0121       filePrefix = "ALCARECO%s"%self.AlCaRecos.split("+")[0]
0122
0123       alcaFiles = []
0124       for i in range( 0,len(self.files) ):
0125          alcaFilePath = os.path.join( self.__config.get("general","outDir"), self.name,"%s_%03i.root"%(filePrefix,i+1) )
0126          alcaFilePath = os.path.expandvars(alcaFilePath)
0127          alcaFilePath = os.path.abspath(alcaFilePath)
0128          if alcaFilePath.startswith("/castor"):
0129             alcaFilePath = "'rfio:"+alcaFilePath+"'"
0130          else:
0131             alcaFilePath = "'file:"+alcaFilePath+"'"
0132          alcaFiles.append(alcaFilePath)
0133       repMap = { "AlCARecos":", ".join(alcaFiles)}
0134       fileName = "extractDQM_%s_cfg.py"%self.name
0135       cfgPath = os.path.join(path, fileName)
0136       cfgFile = open( cfgPath, "w" )
0137       cfgFile.write( extractDQMtemplate%repMap )
0138       cfgFile.close()
0139       return cfgPath
0140
0141    def createScripts(self, path):
0142       result = []
0143       repMap = {"maxEvents":self.maxEvents,
0144                 "globalTag":self.globalTag,
0145                 "AlCaRecos":self.AlCaRecos,
0146                 "sampleFile":"" }
0147       i = 1
0148       for sampleFile in self.files:
0149          repMap["sampleFile"] = sampleFile
0150          result.append(self.createScript(path, "AlCaReco", """
0151 cmsDriver.py step3 -s ALCA:%(AlCaRecos)s+DQM \\
0152     -n %(maxEvents)s \\
0153     --filein %(sampleFile)s\\
0154     --conditions FrontierConditions_GlobalTag,%(globalTag)s \\
0155     --eventcontent RECO\\
0156     --mc"""%repMap, i))
0157          i += 1
0158       self.AlcaRecoScripts = result
0159       self.DQMConfig = self.createDQMExtract(path)
0160       self.DQMScript = self.createScript(path, "extractDQM", "cmsRun %s"%self.DQMConfig)
0161       return result
0162
0163 ######################### Helpers ####################
0164 #excute [command] and return output
0165 def getCommandOutput2(command):
0166     child = os.popen(command)
0167     data = child.read()
0168     err = child.close()
0169     if err:
0170         raise RuntimeError('%s failed w/ exit code %d' % (command, err))
0171     return data
0172
0173 def runJob(jobName, script, config):
0174    jobMode = config.get("general","jobMode")
0175    print("> Testing "+jobName)
0176    if jobMode == "interactive":
0177        getCommandOutput2( script )
0178    if jobMode.split(",")[0] == "lxBatch":
0179        commands = jobMode.split(",")[1]
0180        getCommandOutput2("bsub "+commands+" -J "+jobName+" "+ script)
0181
0182 def readSamples( config):
0183    result = []
0184    for section in config.sections():
0185       if "sample:" in section:
0186          name = section.split("sample:")[1]
0187          result.append( Sample(name, config) )
0188    return result
0189
0190 def main(argv = None):
0191     if argv == None:
0192        argv = sys.argv[1:]
0193     optParser = optparse.OptionParser()
0194     optParser.description = "test the tracker AlCaReco production and DQM"
0195     optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
0196                          help="create all scripts and cfg File but do not start jobs (default=False)")
0197     optParser.add_option("-c", "--config", dest="config",
0198                          help="configuration to use (default testConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
0199     optParser.add_option("-N", "--Name", dest="Name",
0200                          help="Name of this test (default: test_DATE_TIME)", metavar="NAME")
0201
0202     (options, args) = optParser.parse_args(argv)
0203
0204     if options.config == None:
0205         options.config = "testConfig.ini"
0206     else:
0207         options.config = options.config.split(",")
0208         result = []
0209         for iniFile in options.config:
0210             result.append( os.path.abspath(iniFile) )
0211         options.config = result
0212
0213     config = BetterConfigParser()
0214     config.read( options.config )
0215
0216     if options.Name == None:
0217         options.Name = "test_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
0218
0219     outPath = os.path.abspath( options.Name )
0220     for dir in [ "workDir","dataDir","logDir" ]:
0221        if not config.has_option("general", dir):
0222           config.set("general", dir, os.path.join( os.getcwd(), options.Name ))
0223        else:
0224           config.set("general", dir, os.path.join( config.get("general",dir), options.Name ))
0225
0226     if not os.path.exists( outPath ):
0227         os.makedirs( outPath )
0228     elif not os.path.isdir( outPath ):
0229         raise "the file %s is in the way rename the Job or move it away"%outPath
0230
0231     samples = readSamples( config )
0232     for sample in samples:
0233        sample.createScripts( outPath )
0234        for scriptPath in sample.AlcaRecoScripts:
0235           if not options.dryRun:
0236              runJob( sample.name, scriptPath, config )
0237
0238 if __name__ == "__main__":
0239     main()