ReleaseScripts/scripts/addOnTests.py

0001 #! /usr/bin/env python3
0002
0003 from builtins import range
0004 import os
0005 import time
0006 import sys
0007 import re
0008 import random
0009 from threading import Thread
0010
0011 scriptPath = os.path.dirname( os.path.abspath(sys.argv[0]) )
0012 if scriptPath not in sys.path:
0013     sys.path.append(scriptPath)
0014
0015 class testit(Thread):
0016     def __init__(self,dirName, commandList):
0017         Thread.__init__(self)
0018         self.dirName = dirName
0019         self.commandList = commandList
0020         self.status=-1
0021         self.report=''
0022         self.nfail=[]
0023         self.npass=[]
0024
0025     def run(self):
0026         try:
0027             os.makedirs(self.dirName)
0028         except:
0029             pass
0030
0031         with open(self.dirName+'/cmdLog', 'w') as clf:
0032             clf.write(f'# {self.dirName}\n')
0033
0034             for cmdIdx, command in enumerate(self.commandList):
0035                 clf.write(f'\n{command}\n')
0036
0037                 time_start = time.time()
0038                 exitcode = os.system(f'cd {self.dirName} && {command} > step{cmdIdx+1}.log 2>&1')
0039                 time_elapsed_sec = round(time.time() - time_start)
0040
0041                 timelog = f'elapsed time: {time_elapsed_sec} sec (ended on {time.asctime()})'
0042                 logline = f'[{self.dirName}:{cmdIdx+1}] {command} : '
0043                 if exitcode != 0:
0044                     logline += 'FAILED'
0045                     self.nfail.append(1)
0046                     self.npass.append(0)
0047                 else:
0048                     logline += 'PASSED'
0049                     self.nfail.append(0)
0050                     self.npass.append(1)
0051                 logline += f' - {timelog} - exit: {exitcode}'
0052                 self.report += logline+'\n\n'
0053
0054 class StandardTester(object):
0055
0056     def __init__(self, nThrMax=4):
0057
0058         self.threadList = []
0059         self.maxThreads = nThrMax
0060         self.prepare()
0061
0062         return
0063
0064     def activeThreads(self):
0065
0066         nActive = 0
0067         for t in self.threadList:
0068             if t.is_alive() : nActive += 1
0069
0070         return nActive
0071
0072     def prepare(self):
0073
0074         self.devPath = os.environ['LOCALRT'] + '/src/'
0075         self.relPath = self.devPath
0076         if 'CMSSW_RELEASE_BASE' in os.environ and (os.environ['CMSSW_RELEASE_BASE'] != ""): self.relPath = os.environ['CMSSW_RELEASE_BASE'] + '/src/'
0077
0078         lines = { 'read312RV' : ['cmsRun '+self.file2Path('Utilities/ReleaseScripts/scripts/read312RV_cfg.py')],
0079                   'fastsim'   : ["cmsDriver.py TTbar_8TeV_TuneCUETP8M1_cfi  --conditions auto:run1_mc --fast  -n 100 --eventcontent AODSIM,DQM --relval 100000,1000 -s GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid,L1,DIGI2RAW,L1Reco,RECO,VALIDATION  --customise=HLTrigger/Configuration/CustomConfigs.L1THLT --datatier GEN-SIM-DIGI-RECO,DQMIO --beamspot Realistic8TeVCollision"],
0080                   'fastsim1'  : ["cmsDriver.py TTbar_13TeV_TuneCUETP8M1_cfi --conditions auto:run2_mc_l1stage1 --fast  -n 100 --eventcontent AODSIM,DQM --relval 100000,1000 -s GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid,L1,DIGI2RAW,L1Reco,RECO,VALIDATION  --customise=HLTrigger/Configuration/CustomConfigs.L1THLT --datatier GEN-SIM-DIGI-RECO,DQMIO --beamspot NominalCollision2015 --era Run2_25ns"],
0081                   'fastsim2'  : ["cmsDriver.py TTbar_13TeV_TuneCUETP8M1_cfi --conditions auto:run2_mc --fast  -n 100 --eventcontent AODSIM,DQM --relval 100000,1000 -s GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid,L1,DIGI2RAW,L1Reco,RECO,VALIDATION  --customise=HLTrigger/Configuration/CustomConfigs.L1THLT --datatier GEN-SIM-DIGI-RECO,DQMIO --beamspot NominalCollision2015 --era Run2_2016"],
0082                   'pat1'      : ['cmsRun '+self.file2Path('PhysicsTools/PatAlgos/test/IntegrationTest_cfg.py')],
0083                 }
0084
0085         hltTests = {}
0086         hltFlag_data = 'realData=True globalTag=@ inputFiles=@'
0087         hltFlag_mc = 'realData=False globalTag=@ inputFiles=@'
0088         from Configuration.HLT.addOnTestsHLT import addOnTestsHLT
0089         hltTestsToAdd = addOnTestsHLT()
0090         for key in hltTestsToAdd:
0091             if '_data_' in key:
0092                 hltTests[key] = [hltTestsToAdd[key][0],
0093                                  'cmsRun '+self.file2Path(hltTestsToAdd[key][1])+' '+hltFlag_data,
0094                                  hltTestsToAdd[key][2]]
0095             elif '_mc_' in key:
0096                 hltTests[key] = [hltTestsToAdd[key][0],
0097                                  'cmsRun '+self.file2Path(hltTestsToAdd[key][1])+' '+hltFlag_mc,
0098                                  hltTestsToAdd[key][2]]
0099             else:
0100                 hltTests[key] = [hltTestsToAdd[key][0],
0101                                  'cmsRun '+self.file2Path(hltTestsToAdd[key][1]),
0102                                  hltTestsToAdd[key][2]]
0103
0104         self.commands = {}
0105         for dirName, command in lines.items():
0106             self.commands[dirName] = command
0107
0108         for dirName, commandList in hltTests.items():
0109             self.commands[dirName] = commandList
0110         return
0111
0112     def dumpTest(self):
0113         print(",".join(self.commands.keys()))
0114         return
0115
0116     def file2Path(self,rFile):
0117
0118         fullPath = self.relPath + rFile
0119         if os.path.exists(self.devPath + rFile): fullPath = self.devPath + rFile
0120         return fullPath
0121
0122     def runTests(self, testList = None):
0123
0124         actDir = os.getcwd()
0125
0126         if not os.path.exists('addOnTests'):
0127             os.makedirs('addOnTests')
0128         os.chdir('addOnTests')
0129
0130         nfail=0
0131         npass=0
0132         report=''
0133
0134         print('Running in %s thread(s)' % self.maxThreads)
0135
0136         if testList:
0137             self.commands = {d:c for d,c in self.commands.items() if d in testList}
0138         for dirName, command in self.commands.items():
0139
0140             # make sure we don't run more than the allowed number of threads:
0141             while self.activeThreads() >= self.maxThreads:
0142                 time.sleep(10)
0143                 continue
0144
0145             print('Preparing to run %s' % str(command))
0146             current = testit(dirName, command)
0147             self.threadList.append(current)
0148             current.start()
0149             time.sleep(random.randint(1,5)) # try to avoid race cond by sleeping random amount of time [1,5] sec
0150
0151         # wait until all threads are finished
0152         while self.activeThreads() > 0:
0153             time.sleep(5)
0154
0155         # all threads are done now, check status ...
0156         for pingle in self.threadList:
0157             pingle.join()
0158             for f in pingle.nfail: nfail  += f
0159             for p in pingle.npass: npass  += p
0160             report += pingle.report
0161             print(pingle.report)
0162             sys.stdout.flush()
0163
0164         reportSumm = '\n %s tests passed, %s failed \n' %(npass,nfail)
0165         print(reportSumm)
0166
0167         runall_report_name='runall-report.log'
0168         runall_report=open(runall_report_name,'w')
0169         runall_report.write(report+reportSumm)
0170         runall_report.close()
0171
0172         # get the logs to the logs dir:
0173         print('==> in :', os.getcwd())
0174         print('    going to copy log files to logs dir ...')
0175         if not os.path.exists('logs'):
0176             os.makedirs('logs')
0177         for dirName in self.commands:
0178             cmd = "for L in `ls "+dirName+"/*.log`; do cp $L logs/cmsDriver-`dirname $L`_`basename $L` ; done"
0179             print("going to ",cmd)
0180             os.system(cmd)
0181
0182         import pickle
0183         pickle.dump(self.commands, open('logs/addOnTests.pkl', 'wb'), protocol=2)
0184
0185         os.chdir(actDir)
0186
0187         return
0188
0189     def upload(self, tgtDir):
0190
0191         print("in ", os.getcwd())
0192
0193         if not os.path.exists(tgtDir):
0194             os.makedirs(tgtDir)
0195
0196         cmd = 'tar cf - addOnTests.log addOnTests/logs | (cd '+tgtDir+' ; tar xf - ) '
0197         try:
0198             print('executing: ',cmd)
0199             ret = os.system(cmd)
0200             if ret != 0:
0201                 print("ERROR uploading logs:", ret, cmd)
0202         except Exception as e:
0203             print("EXCEPTION while uploading addOnTest-logs : ", str(e))
0204
0205         return
0206
0207
0208 def main(argv) :
0209
0210     import getopt
0211
0212     try:
0213         opts, args = getopt.getopt(argv, "dj:t:", ["nproc=", 'uploadDir=', 'tests=','noRun','dump'])
0214     except getopt.GetoptError as e:
0215         print("unknown option", str(e))
0216         sys.exit(2)
0217
0218     np        = 4
0219     uploadDir = None
0220     runTests  = True
0221     testList  = None
0222     dump      = False
0223     for opt, arg in opts :
0224         if opt in ('-j', "--nproc" ):
0225             np=int(arg)
0226         if opt in ("--uploadDir", ):
0227             uploadDir = arg
0228         if opt in ('--noRun', ):
0229             runTests = False
0230         if opt in ('-d','--dump', ):
0231             dump = True
0232         if opt in ('-t','--tests', ):
0233             testList = arg.split(",")
0234
0235     tester = StandardTester(np)
0236     if dump:
0237         tester.dumpTest()
0238     else:
0239         if runTests:
0240             tester.runTests(testList)
0241         if uploadDir:
0242             tester.upload(uploadDir)
0243     return
0244
0245 if __name__ == '__main__' :
0246     main(sys.argv[1:])