ReleaseScripts/scripts/addOnTests.py

0001 #! /usr/bin/env python3
0002
0003 from __future__ import print_function
0004 from builtins import range
0005 import os
0006 import time
0007 import sys
0008 import re
0009 import random
0010 from threading import Thread
0011
0012 scriptPath = os.path.dirname( os.path.abspath(sys.argv[0]) )
0013 if scriptPath not in sys.path:
0014     sys.path.append(scriptPath)
0015
0016 class testit(Thread):
0017     def __init__(self,dirName, commandList):
0018         Thread.__init__(self)
0019         self.dirName = dirName
0020         self.commandList = commandList
0021         self.status=-1
0022         self.report=''
0023         self.nfail=[]
0024         self.npass=[]
0025
0026     def run(self):
0027         try:
0028             os.makedirs(self.dirName)
0029         except:
0030             pass
0031
0032         with open(self.dirName+'/cmdLog', 'w') as clf:
0033             clf.write(f'# {self.dirName}\n')
0034
0035             for cmdIdx, command in enumerate(self.commandList):
0036                 clf.write(f'\n{command}\n')
0037
0038                 time_start = time.time()
0039                 exitcode = os.system(f'cd {self.dirName} && {command} > step{cmdIdx+1}.log 2>&1')
0040                 time_elapsed_sec = round(time.time() - time_start)
0041
0042                 timelog = f'elapsed time: {time_elapsed_sec} sec (ended on {time.asctime()})'
0043                 logline = f'[{self.dirName}:{cmdIdx+1}] {command} : '
0044                 if exitcode != 0:
0045                     logline += 'FAILED'
0046                     self.nfail.append(1)
0047                     self.npass.append(0)
0048                 else:
0049                     logline += 'PASSED'
0050                     self.nfail.append(0)
0051                     self.npass.append(1)
0052                 logline += f' - {timelog} - exit: {exitcode}'
0053                 self.report += logline+'\n\n'
0054
0055 class StandardTester(object):
0056
0057     def __init__(self, nThrMax=4):
0058
0059         self.threadList = []
0060         self.maxThreads = nThrMax
0061         self.prepare()
0062
0063         return
0064
0065     def activeThreads(self):
0066
0067         nActive = 0
0068         for t in self.threadList:
0069             if t.is_alive() : nActive += 1
0070
0071         return nActive
0072
0073     def prepare(self):
0074
0075         self.devPath = os.environ['LOCALRT'] + '/src/'
0076         self.relPath = self.devPath
0077         if 'CMSSW_RELEASE_BASE' in os.environ and (os.environ['CMSSW_RELEASE_BASE'] != ""): self.relPath = os.environ['CMSSW_RELEASE_BASE'] + '/src/'
0078
0079         lines = { 'read312RV' : ['cmsRun '+self.file2Path('Utilities/ReleaseScripts/scripts/read312RV_cfg.py')],
0080                   'fastsim'   : ["cmsDriver.py TTbar_8TeV_TuneCUETP8M1_cfi  --conditions auto:run1_mc --fast  -n 100 --eventcontent AODSIM,DQM --relval 100000,1000 -s GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid,L1,DIGI2RAW,L1Reco,RECO,VALIDATION  --customise=HLTrigger/Configuration/CustomConfigs.L1THLT --datatier GEN-SIM-DIGI-RECO,DQMIO --beamspot Realistic8TeVCollision"],
0081                   'fastsim1'  : ["cmsDriver.py TTbar_13TeV_TuneCUETP8M1_cfi --conditions auto:run2_mc_l1stage1 --fast  -n 100 --eventcontent AODSIM,DQM --relval 100000,1000 -s GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid,L1,DIGI2RAW,L1Reco,RECO,VALIDATION  --customise=HLTrigger/Configuration/CustomConfigs.L1THLT --datatier GEN-SIM-DIGI-RECO,DQMIO --beamspot NominalCollision2015 --era Run2_25ns"],
0082                   'fastsim2'  : ["cmsDriver.py TTbar_13TeV_TuneCUETP8M1_cfi --conditions auto:run2_mc --fast  -n 100 --eventcontent AODSIM,DQM --relval 100000,1000 -s GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid,L1,DIGI2RAW,L1Reco,RECO,VALIDATION  --customise=HLTrigger/Configuration/CustomConfigs.L1THLT --datatier GEN-SIM-DIGI-RECO,DQMIO --beamspot NominalCollision2015 --era Run2_2016"],
0083                   'pat1'      : ['cmsRun '+self.file2Path('PhysicsTools/PatAlgos/test/IntegrationTest_cfg.py')],
0084                 }
0085
0086         hltTests = {}
0087         hltFlag_data = 'realData=True globalTag=@ inputFiles=@'
0088         hltFlag_mc = 'realData=False globalTag=@ inputFiles=@'
0089         from Configuration.HLT.addOnTestsHLT import addOnTestsHLT
0090         hltTestsToAdd = addOnTestsHLT()
0091         for key in hltTestsToAdd:
0092             if '_data_' in key:
0093                 hltTests[key] = [hltTestsToAdd[key][0],
0094                                  'cmsRun '+self.file2Path(hltTestsToAdd[key][1])+' '+hltFlag_data,
0095                                  hltTestsToAdd[key][2]]
0096             elif '_mc_' in key:
0097                 hltTests[key] = [hltTestsToAdd[key][0],
0098                                  'cmsRun '+self.file2Path(hltTestsToAdd[key][1])+' '+hltFlag_mc,
0099                                  hltTestsToAdd[key][2]]
0100             else:
0101                 hltTests[key] = [hltTestsToAdd[key][0],
0102                                  'cmsRun '+self.file2Path(hltTestsToAdd[key][1]),
0103                                  hltTestsToAdd[key][2]]
0104
0105         self.commands = {}
0106         for dirName, command in lines.items():
0107             self.commands[dirName] = command
0108
0109         for dirName, commandList in hltTests.items():
0110             self.commands[dirName] = commandList
0111         return
0112
0113     def dumpTest(self):
0114         print(",".join(self.commands.keys()))
0115         return
0116
0117     def file2Path(self,rFile):
0118
0119         fullPath = self.relPath + rFile
0120         if os.path.exists(self.devPath + rFile): fullPath = self.devPath + rFile
0121         return fullPath
0122
0123     def runTests(self, testList = None):
0124
0125         actDir = os.getcwd()
0126
0127         if not os.path.exists('addOnTests'):
0128             os.makedirs('addOnTests')
0129         os.chdir('addOnTests')
0130
0131         nfail=0
0132         npass=0
0133         report=''
0134
0135         print('Running in %s thread(s)' % self.maxThreads)
0136
0137         if testList:
0138             self.commands = {d:c for d,c in self.commands.items() if d in testList}
0139         for dirName, command in self.commands.items():
0140
0141             # make sure we don't run more than the allowed number of threads:
0142             while self.activeThreads() >= self.maxThreads:
0143                 time.sleep(10)
0144                 continue
0145
0146             print('Preparing to run %s' % str(command))
0147             current = testit(dirName, command)
0148             self.threadList.append(current)
0149             current.start()
0150             time.sleep(random.randint(1,5)) # try to avoid race cond by sleeping random amount of time [1,5] sec
0151
0152         # wait until all threads are finished
0153         while self.activeThreads() > 0:
0154             time.sleep(5)
0155
0156         # all threads are done now, check status ...
0157         for pingle in self.threadList:
0158             pingle.join()
0159             for f in pingle.nfail: nfail  += f
0160             for p in pingle.npass: npass  += p
0161             report += pingle.report
0162             print(pingle.report)
0163             sys.stdout.flush()
0164
0165         reportSumm = '\n %s tests passed, %s failed \n' %(npass,nfail)
0166         print(reportSumm)
0167
0168         runall_report_name='runall-report.log'
0169         runall_report=open(runall_report_name,'w')
0170         runall_report.write(report+reportSumm)
0171         runall_report.close()
0172
0173         # get the logs to the logs dir:
0174         print('==> in :', os.getcwd())
0175         print('    going to copy log files to logs dir ...')
0176         if not os.path.exists('logs'):
0177             os.makedirs('logs')
0178         for dirName in self.commands:
0179             cmd = "for L in `ls "+dirName+"/*.log`; do cp $L logs/cmsDriver-`dirname $L`_`basename $L` ; done"
0180             print("going to ",cmd)
0181             os.system(cmd)
0182
0183         import pickle
0184         pickle.dump(self.commands, open('logs/addOnTests.pkl', 'wb'), protocol=2)
0185
0186         os.chdir(actDir)
0187
0188         return
0189
0190     def upload(self, tgtDir):
0191
0192         print("in ", os.getcwd())
0193
0194         if not os.path.exists(tgtDir):
0195             os.makedirs(tgtDir)
0196
0197         cmd = 'tar cf - addOnTests.log addOnTests/logs | (cd '+tgtDir+' ; tar xf - ) '
0198         try:
0199             print('executing: ',cmd)
0200             ret = os.system(cmd)
0201             if ret != 0:
0202                 print("ERROR uploading logs:", ret, cmd)
0203         except Exception as e:
0204             print("EXCEPTION while uploading addOnTest-logs : ", str(e))
0205
0206         return
0207
0208
0209 def main(argv) :
0210
0211     import getopt
0212
0213     try:
0214         opts, args = getopt.getopt(argv, "dj:t:", ["nproc=", 'uploadDir=', 'tests=','noRun','dump'])
0215     except getopt.GetoptError as e:
0216         print("unknown option", str(e))
0217         sys.exit(2)
0218
0219     np        = 4
0220     uploadDir = None
0221     runTests  = True
0222     testList  = None
0223     dump      = False
0224     for opt, arg in opts :
0225         if opt in ('-j', "--nproc" ):
0226             np=int(arg)
0227         if opt in ("--uploadDir", ):
0228             uploadDir = arg
0229         if opt in ('--noRun', ):
0230             runTests = False
0231         if opt in ('-d','--dump', ):
0232             dump = True
0233         if opt in ('-t','--tests', ):
0234             testList = arg.split(",")
0235
0236     tester = StandardTester(np)
0237     if dump:
0238         tester.dumpTest()
0239     else:
0240         if runTests:
0241             tester.runTests(testList)
0242         if uploadDir:
0243             tester.upload(uploadDir)
0244     return
0245
0246 if __name__ == '__main__' :
0247     main(sys.argv[1:])