ReleaseScripts/scripts/addOnTests.py

0001 #! /usr/bin/env python3
0002
0003 from __future__ import print_function
0004 from builtins import range
0005 import os
0006 import time
0007 import sys
0008 import re
0009 import random
0010 from threading import Thread
0011
0012 scriptPath = os.path.dirname( os.path.abspath(sys.argv[0]) )
0013 if scriptPath not in sys.path:
0014     sys.path.append(scriptPath)
0015
0016
0017 class testit(Thread):
0018     def __init__(self,dirName, commandList):
0019         Thread.__init__(self)
0020         self.dirName = dirName
0021         self.commandList = commandList
0022         self.status=-1
0023         self.report=''
0024         self.nfail=[]
0025         self.npass=[]
0026
0027         return
0028
0029     def run(self):
0030
0031         startime='date %s' %time.asctime()
0032         exitCodes = []
0033
0034         for command in self.commandList:
0035
0036             if not os.path.exists(self.dirName):
0037                 os.makedirs(self.dirName)
0038
0039             commandbase = command.replace(' ','_').replace('/','_')
0040             logfile='%s.log' % commandbase[:150].replace("'",'').replace('"','').replace('../','')
0041
0042             executable = 'cd '+self.dirName+'; '+command+' > '+logfile+' 2>&1'
0043
0044             ret = os.system(executable)
0045             exitCodes.append( ret )
0046
0047         endtime='date %s' %time.asctime()
0048         tottime='%s-%s'%(endtime,startime)
0049
0050         for i in range(len(self.commandList)):
0051             command = self.commandList[i]
0052             exitcode = exitCodes[i]
0053             if exitcode != 0:
0054                 log='%s : FAILED - time: %s s - exit: %s\n' %(command,tottime,exitcode)
0055                 self.report+='%s\n'%log
0056                 self.nfail.append(1)
0057                 self.npass.append(0)
0058             else:
0059                 log='%s : PASSED - time: %s s - exit: %s\n' %(command,tottime,exitcode)
0060                 self.report+='%s\n'%log
0061                 self.nfail.append(0)
0062                 self.npass.append(1)
0063
0064         return
0065
0066 class StandardTester(object):
0067
0068     def __init__(self, nThrMax=4):
0069
0070         self.threadList = []
0071         self.maxThreads = nThrMax
0072         self.prepare()
0073
0074         return
0075
0076     def activeThreads(self):
0077
0078         nActive = 0
0079         for t in self.threadList:
0080             if t.is_alive() : nActive += 1
0081
0082         return nActive
0083
0084     def prepare(self):
0085
0086         self.devPath = os.environ['LOCALRT'] + '/src/'
0087         self.relPath = self.devPath
0088         if 'CMSSW_RELEASE_BASE' in os.environ and (os.environ['CMSSW_RELEASE_BASE'] != ""): self.relPath = os.environ['CMSSW_RELEASE_BASE'] + '/src/'
0089
0090         lines = { 'read312RV' : ['cmsRun '+self.file2Path('Utilities/ReleaseScripts/scripts/read312RV_cfg.py')],
0091                   'fastsim'   : ["cmsDriver.py TTbar_8TeV_TuneCUETP8M1_cfi  --conditions auto:run1_mc --fast  -n 100 --eventcontent AODSIM,DQM --relval 100000,1000 -s GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid,L1,DIGI2RAW,L1Reco,RECO,VALIDATION  --customise=HLTrigger/Configuration/CustomConfigs.L1THLT --datatier GEN-SIM-DIGI-RECO,DQMIO --beamspot Realistic8TeVCollision"],
0092                   'fastsim1'  : ["cmsDriver.py TTbar_13TeV_TuneCUETP8M1_cfi --conditions auto:run2_mc_l1stage1 --fast  -n 100 --eventcontent AODSIM,DQM --relval 100000,1000 -s GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid,L1,DIGI2RAW,L1Reco,RECO,VALIDATION  --customise=HLTrigger/Configuration/CustomConfigs.L1THLT --datatier GEN-SIM-DIGI-RECO,DQMIO --beamspot NominalCollision2015 --era Run2_25ns"],
0093                   'fastsim2'  : ["cmsDriver.py TTbar_13TeV_TuneCUETP8M1_cfi --conditions auto:run2_mc --fast  -n 100 --eventcontent AODSIM,DQM --relval 100000,1000 -s GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid,L1,DIGI2RAW,L1Reco,RECO,VALIDATION  --customise=HLTrigger/Configuration/CustomConfigs.L1THLT --datatier GEN-SIM-DIGI-RECO,DQMIO --beamspot NominalCollision2015 --era Run2_2016"],
0094                   'pat1'      : ['cmsRun '+self.file2Path('PhysicsTools/PatAlgos/test/IntegrationTest_cfg.py')],
0095                 }
0096
0097         hltTests = {}
0098         hltFlag_data = ' realData=True  globalTag=@ inputFiles=@ '
0099         hltFlag_mc   = ' realData=False globalTag=@ inputFiles=@ '
0100         from Configuration.HLT.addOnTestsHLT import addOnTestsHLT
0101         hltTestsToAdd = addOnTestsHLT()
0102         for key in hltTestsToAdd:
0103             if '_data_' in key:
0104                 hltTests[key] = [hltTestsToAdd[key][0],
0105                                  'cmsRun '+self.file2Path(hltTestsToAdd[key][1])+hltFlag_data,
0106                                  hltTestsToAdd[key][2]]
0107             elif '_mc_' in key:
0108                 hltTests[key] = [hltTestsToAdd[key][0],
0109                                  'cmsRun '+self.file2Path(hltTestsToAdd[key][1])+hltFlag_mc,
0110                                  hltTestsToAdd[key][2]]
0111             else:
0112                 hltTests[key] = [hltTestsToAdd[key][0],
0113                                  'cmsRun '+self.file2Path(hltTestsToAdd[key][1]),
0114                                  hltTestsToAdd[key][2]]
0115
0116         self.commands={}
0117         for dirName, command in lines.items():
0118             self.commands[dirName] = command
0119
0120         for dirName, commandList in hltTests.items():
0121             self.commands[dirName] = commandList
0122         return
0123
0124     def dumpTest(self):
0125         print(",".join(self.commands.keys()))
0126         return
0127
0128     def file2Path(self,rFile):
0129
0130         fullPath = self.relPath + rFile
0131         if os.path.exists(self.devPath + rFile): fullPath = self.devPath + rFile
0132         return fullPath
0133
0134     def runTests(self, testList = None):
0135
0136         actDir = os.getcwd()
0137
0138         if not os.path.exists('addOnTests'):
0139             os.makedirs('addOnTests')
0140         os.chdir('addOnTests')
0141
0142         nfail=0
0143         npass=0
0144         report=''
0145
0146         print('Running in %s thread(s)' % self.maxThreads)
0147
0148         if testList:
0149             self.commands = {d:c for d,c in self.commands.items() if d in testList}
0150         for dirName, command in self.commands.items():
0151
0152             # make sure we don't run more than the allowed number of threads:
0153             while self.activeThreads() >= self.maxThreads:
0154                 time.sleep(10)
0155                 continue
0156
0157             print('Preparing to run %s' % str(command))
0158             current = testit(dirName, command)
0159             self.threadList.append(current)
0160             current.start()
0161             time.sleep(random.randint(1,5)) # try to avoid race cond by sleeping random amount of time [1,5] sec
0162
0163         # wait until all threads are finished
0164         while self.activeThreads() > 0:
0165             time.sleep(5)
0166
0167         # all threads are done now, check status ...
0168         for pingle in self.threadList:
0169             pingle.join()
0170             for f in pingle.nfail: nfail  += f
0171             for p in pingle.npass: npass  += p
0172             report += pingle.report
0173             print(pingle.report)
0174             sys.stdout.flush()
0175
0176         reportSumm = '\n %s tests passed, %s failed \n' %(npass,nfail)
0177         print(reportSumm)
0178
0179         runall_report_name='runall-report.log'
0180         runall_report=open(runall_report_name,'w')
0181         runall_report.write(report+reportSumm)
0182         runall_report.close()
0183
0184         # get the logs to the logs dir:
0185         print('==> in :', os.getcwd())
0186         print('    going to copy log files to logs dir ...')
0187         if not os.path.exists('logs'):
0188             os.makedirs('logs')
0189         for dirName in self.commands:
0190             cmd = "for L in `ls "+dirName+"/*.log`; do cp $L logs/cmsDriver-`dirname $L`_`basename $L` ; done"
0191             print("going to ",cmd)
0192             os.system(cmd)
0193
0194         import pickle
0195         pickle.dump(self.commands, open('logs/addOnTests.pkl', 'wb'), protocol=2)
0196
0197         os.chdir(actDir)
0198
0199         return
0200
0201     def upload(self, tgtDir):
0202
0203         print("in ", os.getcwd())
0204
0205         if not os.path.exists(tgtDir):
0206             os.makedirs(tgtDir)
0207
0208         cmd = 'tar cf - addOnTests.log addOnTests/logs | (cd '+tgtDir+' ; tar xf - ) '
0209         try:
0210             print('executing: ',cmd)
0211             ret = os.system(cmd)
0212             if ret != 0:
0213                 print("ERROR uploading logs:", ret, cmd)
0214         except Exception as e:
0215             print("EXCEPTION while uploading addOnTest-logs : ", str(e))
0216
0217         return
0218
0219
0220 def main(argv) :
0221
0222     import getopt
0223
0224     try:
0225         opts, args = getopt.getopt(argv, "dj:t:", ["nproc=", 'uploadDir=', 'tests=','noRun','dump'])
0226     except getopt.GetoptError as e:
0227         print("unknown option", str(e))
0228         sys.exit(2)
0229
0230     np        = 4
0231     uploadDir = None
0232     runTests  = True
0233     testList  = None
0234     dump      = False
0235     for opt, arg in opts :
0236         if opt in ('-j', "--nproc" ):
0237             np=int(arg)
0238         if opt in ("--uploadDir", ):
0239             uploadDir = arg
0240         if opt in ('--noRun', ):
0241             runTests = False
0242         if opt in ('-d','--dump', ):
0243             dump = True
0244         if opt in ('-t','--tests', ):
0245             testList = arg.split(",")
0246
0247     tester = StandardTester(np)
0248     if dump:
0249         tester.dumpTest()
0250     else:
0251         if runTests:
0252             tester.runTests(testList)
0253         if uploadDir:
0254             tester.upload(uploadDir)
0255     return
0256
0257 if __name__ == '__main__' :
0258     main(sys.argv[1:])