CondDB/test/condTestRegression.py

0001 #!/usr/bin/env python3
0002
0003 from __future__ import print_function
0004 import sys, os
0005 import glob
0006 import time
0007 import subprocess
0008 import shutil
0009 import re
0010 import json
0011
0012 readers = {
0013           }
0014
0015 writers = { 'CMSSW_12_3_0'       : [ ('slc7_amd64_gcc10',  'ref1230-s710.db')],
0016             'CMSSW_11_3_0'       : [ ('slc7_amd64_gcc900', 'ref1130-s7900.db')],
0017             'CMSSW_11_1_0'       : [ ('slc7_amd64_gcc820', 'ref1110-s7820.db')],
0018             'CMSSW_10_4_0'       : [ ('slc7_amd64_gcc700', 'ref1040-s7700.db')],
0019             'CMSSW_9_0_1'        : [ ('slc6_amd64_gcc630', 'ref901-s6630.db')],
0020             'CMSSW_8_1_0'        : [ ('slc6_amd64_gcc530', 'ref750-s6530.db'),('slc6_amd64_gcc600', 'ref750-s600.db')],
0021             'CMSSW_7_6_6'        : [ ('slc6_amd64_gcc493', 'ref750-s6493.db')]
0022           }
0023
0024 os2image_overrides = {"slc7": "cc7"}
0025
0026 def check_output(*popenargs, **kwargs):
0027     '''Mimics subprocess.check_output() in Python 2.6
0028     '''
0029
0030     process = subprocess.Popen(*popenargs, **kwargs)
0031     stdout, stderr = process.communicate()
0032     returnCode = process.returncode
0033
0034     if returnCode:
0035         msg = '\nERROR from process (ret=%s): \n' %(str(returnCode),)
0036         msg += '      stderr: %s\n' % (str(stderr),)
0037         msg += '      stdout: %s\n' % (str(stdout),)
0038         cmd = kwargs.get("args")
0039         if cmd is None:
0040             cmd = popenargs[0]
0041         raise subprocess.CalledProcessError(returnCode, cmd+msg)
0042
0043     return stdout
0044
0045
0046 # nice one from:
0047 # https://www.daniweb.com/software-development/python/code/216610/timing-a-function-python
0048 def print_timing(func):
0049     def wrapper(*arg):
0050         t1 = time.time()
0051         res = func(*arg)
0052         t2 = time.time()
0053         print('\n%s(%s) took %0.3f ms\n' % (func.__name__, ','.join([str(x) for x in arg[1:]]), (t2-t1)*1000.0))
0054         return res
0055     return wrapper
0056
0057
0058 class CondRegressionTester(object):
0059
0060     @print_timing
0061     def __init__(self):
0062
0063         tmpBase = '/tmp'
0064         if 'CMSSW_BASE' in os.environ:
0065             tmpBase = os.path.join(os.environ['CMSSW_BASE'],'tmp')
0066         self.topDir = os.path.join( tmpBase, 'cmsCondRegTst-'+time.strftime('%Y-%m-%d-%H-%M'))
0067         if not os.path.exists(self.topDir): os.makedirs(self.topDir)
0068
0069         self.dbDir = os.path.join( self.topDir, 'dbDir' )
0070         if not os.path.exists(self.dbDir): os.makedirs(self.dbDir)
0071
0072         self.logDir = os.path.join( self.topDir, 'logs' )
0073         if not os.path.exists(self.logDir):
0074             os.makedirs(self.logDir)
0075         else:  # if it exists, remove the logDir and re-create it
0076             shutil.rmtree(self.logDir, ignore_errors=True)
0077             os.makedirs(self.logDir)
0078
0079         # add the IB/release itself:
0080         self.regTestSrcDir = os.path.join( os.environ['LOCALRT'], 'src', 'CondCore', 'CondDB', 'test' )
0081         self.rel = os.environ['CMSSW_VERSION']
0082         self.arch = os.environ['SCRAM_ARCH']
0083         self.dbName = 'self-%s-%s.db' % (self.rel, self.arch)
0084
0085         self.dbList = {}
0086
0087         self.status = {}
0088
0089         return
0090
0091     def summary(self, verbose=False, jsonOut=False):
0092         if verbose:
0093             allReaders = dict(readers)
0094             allReaders['SELF']=['%s' %(self.arch)]
0095             dbNames = []
0096             header = ( 'Write', )
0097             reslen = len(header[0])
0098             for result in sorted(self.status.keys()):
0099                 if len(result)>reslen:
0100                     reslen = len(result)
0101
0102             fmt = ' %' + '%s' %(reslen) + 's '
0103             for reader in sorted(allReaders.keys()):
0104                 for arch in allReaders[reader]:
0105                     if reader == 'SELF':
0106                         readerArch = 'Read: %s [%s]' %(self.rel,self.arch)
0107                     else:
0108                         readerArch = 'Read: %s [%s]' %(reader,arch)
0109                     fmt += '| %' + '%s' %len(readerArch) + 's '
0110                     header += tuple([readerArch])
0111             fmt += '|'
0112             print('SELF: %s [%s]\n' %(self.rel,self.arch))
0113             print(fmt %header)
0114             for result in sorted(self.status.keys()):
0115                 params = (result,)+tuple([self.status[result][key] for key in sorted(self.status[result].keys())])
0116                 print(fmt %params)
0117
0118         if jsonOut:
0119             print(json.dumps( self.status, sort_keys=True, indent=4 ))
0120
0121         overall = True
0122         for result in sorted(self.status.keys()):
0123             for result_arch in sorted(self.status[result].keys()):
0124                 if not self.status[result][result_arch]: overall = False
0125
0126         return overall
0127
0128     @print_timing
0129     def run(self, rel, arch, readOrWrite, dbName):
0130
0131         if readOrWrite == 'write':
0132             self.dbList['%s [%s]'%(rel,arch)] = dbName
0133
0134         cmd ="scram -a %s list -c %s | grep '\\b%s\\b' | head -1 | sed 's|.* ||'" %(arch,rel,rel)
0135         out =check_output(cmd, shell=True, universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
0136         ind = out.find( arch )
0137         if ind == -1:
0138             raise Exception('Could not locate the reference release %s with "%s" [ got %s ]' %(rel,cmd,out))
0139
0140         cmsPath = out[:ind-1]
0141         # using wildcard to support the path for normal ( BASE/ARCH/cms/cmssw/RELEASE ) and patch releases ( BASE-PATCH/ARCH/cms/cmssw-patch/RELEASE )
0142         releaseDir = '%s/%s/cms/*/%s' %(cmsPath,arch,rel)
0143
0144         cmd =  'source %s/cmsset_default.sh; export SCRAM_ARCH=%s; cd %s/src ; eval `scram runtime -sh`; cd - ; ' %(cmsPath,arch,releaseDir)
0145         cmd += "echo 'CMSSW_BASE='$CMSSW_BASE; echo 'RELEASE_BASE='$RELEASE_BASE; echo 'PATH='$PATH; echo 'LD_LIBRARY_PATH='$LD_LIBRARY_PATH;"
0146         cmd += '$LOCALRT/test/%s/testReadWritePayloads %s sqlite_file:///%s/%s ' % (arch, readOrWrite, self.dbDir, dbName)
0147
0148         cur_os = os.environ['SCRAM_ARCH'].split("_")[0]
0149         rel_os = arch.split("_")[0]
0150         if cur_os in os2image_overrides: cur_os = os2image_overrides[cur_os]
0151         if rel_os in os2image_overrides: rel_os = os2image_overrides[rel_os]
0152         if rel_os != cur_os:
0153           run_script = "%s/run_condTestRegression.sh" % self.topDir
0154           check_output("echo '%s' > %s; chmod +x %s" % (cmd, run_script, run_script), shell=True, universal_newlines=True, env={}, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
0155           cmd = "%s/common/cmssw-env --cmsos %s -- %s" % (cmsPath, rel_os, run_script)
0156         print("Running:",cmd)
0157         try:
0158             #opening a process with a clean environment ( to avoid to inherit scram variables )
0159             res = check_output(cmd, shell=True, universal_newlines=True, env={}, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
0160         except Exception as e:
0161             self.log( rel, arch, readOrWrite, str(e) )
0162             raise e
0163
0164         self.log( rel, arch, readOrWrite, ''.join(res) )
0165
0166     @print_timing
0167     def runSelf(self, readOrWrite, dbNameIn=None):
0168
0169         dbName = self.dbName # set the default
0170         if dbNameIn : dbName = dbNameIn
0171
0172         if readOrWrite == 'write':
0173             self.dbList['%s [%s]'%(self.rel,self.arch)] = dbName
0174
0175         execName = 'test/%s/testReadWritePayloads' %self.arch
0176         executable = '%s/%s' %(os.environ['LOCALRT'],execName)
0177         if not os.path.exists(executable):
0178             print('Executable %s not found in local release.' % executable)
0179             executable = None
0180             for rel_base_env in ['CMSSW_BASE', 'CMSSW_RELEASE_BASE', 'CMSSW_FULL_RELEASE_BASE' ]:
0181                 if os.getenv(rel_base_env) and os.path.exists(str(os.environ[rel_base_env])+'/%s' %execName):
0182                     executable = str(os.environ[rel_base_env])+'/%s' %execName
0183                     break
0184
0185         if executable is None:
0186             raise Exception("Can't find the %s executable." %execName )
0187
0188         # we run in the local environment, but need to make sure that we start "top-level" of the devel area
0189         # and we assume that the test was already built
0190         cmd = 'export SCRAM_ARCH=%s; cd %s/src; eval `scram runtime -sh 2>/dev/null` ; ' % (os.environ['SCRAM_ARCH'],os.environ['CMSSW_BASE'], )
0191         cmd += "echo 'CMSSW_BASE='$CMSSW_BASE; echo 'RELEASE_BASE='$RELEASE_BASE; echo 'PATH='$PATH; echo 'LD_LIBRARY_PATH='$LD_LIBRARY_PATH; echo 'LOCALRT='$LOCALRT;"
0192         cmd += '%s %s sqlite_file:///%s/%s ' % (executable, readOrWrite, self.dbDir, dbName)
0193
0194         try:
0195             res = check_output(cmd, shell=True, universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
0196         except Exception as e:
0197             self.log( self.rel, self.arch, readOrWrite, str(e) )
0198             raise e
0199
0200         self.log( self.rel, self.arch, readOrWrite, ''.join(res) )
0201
0202     def log(self, rel, arch, readOrWrite, msg):
0203
0204         with open(os.path.join(self.logDir, readOrWrite+'-'+rel+'-'+arch+'.log'), 'a') as logFile:
0205             logFile.write( '\n'+'='*80+'\n' )
0206             logFile.write( str(msg) )
0207
0208     @print_timing
0209     def runAll(self):
0210
0211         # write all DBs (including the one from this IB/devArea)
0212         print('='*80)
0213         print("going to write DBs ...")
0214         self.runSelf('write')
0215         for rel in writers.keys():
0216             for arch,dbName in writers[rel]:
0217                 self.run(rel, arch, 'write', dbName)
0218
0219         # now try to read back with all reference releases all the DBs written before ...
0220         print('='*80)
0221         print("going to read back DBs ...")
0222         for rel in readers.keys():
0223             for arch in readers[rel]:
0224                 for writer in self.dbList.keys(): # for any given rel/arch we check all written DBs
0225                     dbName = self.dbList[writer]
0226                     try:
0227                         self.run(rel, arch, 'read', dbName)
0228                         status = True
0229                         print("rel %s reading %s was OK." % (rel, writer))
0230                     except:
0231                         status = False
0232                         print("rel %s reading %s FAILED." % (rel, writer))
0233                     if writer not in self.status.keys():
0234                         key = '%s [%s]' %(rel,arch)
0235                         self.status[writer] = { key : status }
0236                     else:
0237                         self.status[writer]['%s [%s]' %(rel,arch)] = status
0238
0239         # ... and also with this IB/devArea
0240         for writer in self.dbList.keys(): # for any given rel/arch we check all written DBs
0241             dbName = self.dbList[writer]
0242             try:
0243                 self.runSelf('read', dbName)
0244                 status = True
0245                 print("rel %s reading %s was OK." % (self.rel, writer))
0246             except:
0247                 status = False
0248                 print("rel %s reading %s FAILED." % (self.rel, writer))
0249             if writer not in self.status.keys():
0250                 self.status[writer] = { 'SELF': status }
0251             else:
0252                 self.status[writer]['SELF'] = status
0253         print('='*80)
0254
0255 crt = CondRegressionTester()
0256 crt.runAll()
0257 status = crt.summary(verbose=True)
0258 print("\n==> overall status: ", status)
0259
0260 # return the overall result to the caller:
0261 if status:
0262     sys.exit(0)
0263 else:
0264     sys.exit(-1)
0265