CondDB/test/condTestRegression.py

0001 #!/usr/bin/env python3
0002
0003 import sys, os
0004 import glob
0005 import time
0006 import subprocess
0007 import shutil
0008 import re
0009 import json
0010
0011 readers = {
0012           }
0013
0014 writers = { 'CMSSW_12_3_0'       : [ ('slc7_amd64_gcc10',  'ref1230-s710.db')],
0015             'CMSSW_11_3_0'       : [ ('slc7_amd64_gcc900', 'ref1130-s7900.db')],
0016             'CMSSW_11_1_0'       : [ ('slc7_amd64_gcc820', 'ref1110-s7820.db')],
0017             'CMSSW_10_4_0'       : [ ('slc7_amd64_gcc700', 'ref1040-s7700.db')],
0018             'CMSSW_9_0_1'        : [ ('slc6_amd64_gcc630', 'ref901-s6630.db')],
0019             'CMSSW_8_1_0'        : [ ('slc6_amd64_gcc530', 'ref750-s6530.db'),('slc6_amd64_gcc600', 'ref750-s600.db')],
0020             'CMSSW_7_6_6'        : [ ('slc6_amd64_gcc493', 'ref750-s6493.db')]
0021           }
0022
0023 os2image_overrides = {"slc7": "cc7"}
0024
0025 def check_output(*popenargs, **kwargs):
0026     '''Mimics subprocess.check_output() in Python 2.6
0027     '''
0028
0029     process = subprocess.Popen(*popenargs, **kwargs)
0030     stdout, stderr = process.communicate()
0031     returnCode = process.returncode
0032
0033     if returnCode:
0034         msg = '\nERROR from process (ret=%s): \n' %(str(returnCode),)
0035         msg += '      stderr: %s\n' % (str(stderr),)
0036         msg += '      stdout: %s\n' % (str(stdout),)
0037         cmd = kwargs.get("args")
0038         if cmd is None:
0039             cmd = popenargs[0]
0040         raise subprocess.CalledProcessError(returnCode, cmd+msg)
0041
0042     return stdout
0043
0044
0045 # nice one from:
0046 # https://www.daniweb.com/software-development/python/code/216610/timing-a-function-python
0047 def print_timing(func):
0048     def wrapper(*arg):
0049         t1 = time.time()
0050         res = func(*arg)
0051         t2 = time.time()
0052         print('\n%s(%s) took %0.3f ms\n' % (func.__name__, ','.join([str(x) for x in arg[1:]]), (t2-t1)*1000.0))
0053         return res
0054     return wrapper
0055
0056
0057 class CondRegressionTester(object):
0058
0059     @print_timing
0060     def __init__(self):
0061
0062         tmpBase = '/tmp'
0063         if 'CMSSW_BASE' in os.environ:
0064             tmpBase = os.path.join(os.environ['CMSSW_BASE'],'tmp')
0065         self.topDir = os.path.join( tmpBase, 'cmsCondRegTst-'+time.strftime('%Y-%m-%d-%H-%M'))
0066         if not os.path.exists(self.topDir): os.makedirs(self.topDir)
0067
0068         self.dbDir = os.path.join( self.topDir, 'dbDir' )
0069         if not os.path.exists(self.dbDir): os.makedirs(self.dbDir)
0070
0071         self.logDir = os.path.join( self.topDir, 'logs' )
0072         if not os.path.exists(self.logDir):
0073             os.makedirs(self.logDir)
0074         else:  # if it exists, remove the logDir and re-create it
0075             shutil.rmtree(self.logDir, ignore_errors=True)
0076             os.makedirs(self.logDir)
0077
0078         # add the IB/release itself:
0079         self.regTestSrcDir = os.path.join( os.environ['LOCALRT'], 'src', 'CondCore', 'CondDB', 'test' )
0080         self.rel = os.environ['CMSSW_VERSION']
0081         self.arch = os.environ['SCRAM_ARCH']
0082         self.dbName = 'self-%s-%s.db' % (self.rel, self.arch)
0083
0084         self.dbList = {}
0085
0086         self.status = {}
0087
0088         return
0089
0090     def summary(self, verbose=False, jsonOut=False):
0091         if verbose:
0092             allReaders = dict(readers)
0093             allReaders['SELF']=['%s' %(self.arch)]
0094             dbNames = []
0095             header = ( 'Write', )
0096             reslen = len(header[0])
0097             for result in sorted(self.status.keys()):
0098                 if len(result)>reslen:
0099                     reslen = len(result)
0100
0101             fmt = ' %' + '%s' %(reslen) + 's '
0102             for reader in sorted(allReaders.keys()):
0103                 for arch in allReaders[reader]:
0104                     if reader == 'SELF':
0105                         readerArch = 'Read: %s [%s]' %(self.rel,self.arch)
0106                     else:
0107                         readerArch = 'Read: %s [%s]' %(reader,arch)
0108                     fmt += '| %' + '%s' %len(readerArch) + 's '
0109                     header += tuple([readerArch])
0110             fmt += '|'
0111             print('SELF: %s [%s]\n' %(self.rel,self.arch))
0112             print(fmt %header)
0113             for result in sorted(self.status.keys()):
0114                 params = (result,)+tuple([self.status[result][key] for key in sorted(self.status[result].keys())])
0115                 print(fmt %params)
0116
0117         if jsonOut:
0118             print(json.dumps( self.status, sort_keys=True, indent=4 ))
0119
0120         overall = True
0121         for result in sorted(self.status.keys()):
0122             for result_arch in sorted(self.status[result].keys()):
0123                 if not self.status[result][result_arch]: overall = False
0124
0125         return overall
0126
0127     @print_timing
0128     def run(self, rel, arch, readOrWrite, dbName):
0129
0130         if readOrWrite == 'write':
0131             self.dbList['%s [%s]'%(rel,arch)] = dbName
0132
0133         cmd ="scram -a %s list -c %s | grep '\\b%s\\b' | head -1 | sed 's|.* ||'" %(arch,rel,rel)
0134         out =check_output(cmd, shell=True, universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
0135         ind = out.find( arch )
0136         if ind == -1:
0137             raise Exception('Could not locate the reference release %s with "%s" [ got %s ]' %(rel,cmd,out))
0138
0139         cmsPath = out[:ind-1]
0140         # using wildcard to support the path for normal ( BASE/ARCH/cms/cmssw/RELEASE ) and patch releases ( BASE-PATCH/ARCH/cms/cmssw-patch/RELEASE )
0141         releaseDir = '%s/%s/cms/*/%s' %(cmsPath,arch,rel)
0142
0143         cmd =  'source %s/cmsset_default.sh; export SCRAM_ARCH=%s; cd %s/src ; eval `scram runtime -sh`; cd - ; ' %(cmsPath,arch,releaseDir)
0144         cmd += "echo 'CMSSW_BASE='$CMSSW_BASE; echo 'RELEASE_BASE='$RELEASE_BASE; echo 'PATH='$PATH; echo 'LD_LIBRARY_PATH='$LD_LIBRARY_PATH;"
0145         cmd += '$LOCALRT/test/%s/testReadWritePayloads %s sqlite_file:///%s/%s ' % (arch, readOrWrite, self.dbDir, dbName)
0146
0147         cur_os = os.environ['SCRAM_ARCH'].split("_")[0]
0148         rel_os = arch.split("_")[0]
0149         if cur_os in os2image_overrides: cur_os = os2image_overrides[cur_os]
0150         if rel_os in os2image_overrides: rel_os = os2image_overrides[rel_os]
0151         if rel_os != cur_os:
0152           run_script = "%s/run_condTestRegression.sh" % self.topDir
0153           check_output("echo '%s' > %s; chmod +x %s" % (cmd, run_script, run_script), shell=True, universal_newlines=True, env={}, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
0154           cmd = "%s/common/cmssw-env --cmsos %s -- %s" % (cmsPath, rel_os, run_script)
0155         print("Running:",cmd)
0156         try:
0157             #opening a process with a clean environment ( to avoid to inherit scram variables )
0158             res = check_output(cmd, shell=True, universal_newlines=True, env={}, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
0159         except Exception as e:
0160             self.log( rel, arch, readOrWrite, str(e) )
0161             raise e
0162
0163         self.log( rel, arch, readOrWrite, ''.join(res) )
0164
0165     @print_timing
0166     def runSelf(self, readOrWrite, dbNameIn=None):
0167
0168         dbName = self.dbName # set the default
0169         if dbNameIn : dbName = dbNameIn
0170
0171         if readOrWrite == 'write':
0172             self.dbList['%s [%s]'%(self.rel,self.arch)] = dbName
0173
0174         execName = 'test/%s/testReadWritePayloads' %self.arch
0175         executable = '%s/%s' %(os.environ['LOCALRT'],execName)
0176         if not os.path.exists(executable):
0177             print('Executable %s not found in local release.' % executable)
0178             executable = None
0179             for rel_base_env in ['CMSSW_BASE', 'CMSSW_RELEASE_BASE', 'CMSSW_FULL_RELEASE_BASE' ]:
0180                 if os.getenv(rel_base_env) and os.path.exists(str(os.environ[rel_base_env])+'/%s' %execName):
0181                     executable = str(os.environ[rel_base_env])+'/%s' %execName
0182                     break
0183
0184         if executable is None:
0185             raise Exception("Can't find the %s executable." %execName )
0186
0187         # we run in the local environment, but need to make sure that we start "top-level" of the devel area
0188         # and we assume that the test was already built
0189         cmd = 'export SCRAM_ARCH=%s; cd %s/src; eval `scram runtime -sh 2>/dev/null` ; ' % (os.environ['SCRAM_ARCH'],os.environ['CMSSW_BASE'], )
0190         cmd += "echo 'CMSSW_BASE='$CMSSW_BASE; echo 'RELEASE_BASE='$RELEASE_BASE; echo 'PATH='$PATH; echo 'LD_LIBRARY_PATH='$LD_LIBRARY_PATH; echo 'LOCALRT='$LOCALRT;"
0191         cmd += '%s %s sqlite_file:///%s/%s ' % (executable, readOrWrite, self.dbDir, dbName)
0192
0193         try:
0194             res = check_output(cmd, shell=True, universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
0195         except Exception as e:
0196             self.log( self.rel, self.arch, readOrWrite, str(e) )
0197             raise e
0198
0199         self.log( self.rel, self.arch, readOrWrite, ''.join(res) )
0200
0201     def log(self, rel, arch, readOrWrite, msg):
0202
0203         with open(os.path.join(self.logDir, readOrWrite+'-'+rel+'-'+arch+'.log'), 'a') as logFile:
0204             logFile.write( '\n'+'='*80+'\n' )
0205             logFile.write( str(msg) )
0206
0207     @print_timing
0208     def runAll(self):
0209
0210         # write all DBs (including the one from this IB/devArea)
0211         print('='*80)
0212         print("going to write DBs ...")
0213         self.runSelf('write')
0214         for rel in writers.keys():
0215             for arch,dbName in writers[rel]:
0216                 self.run(rel, arch, 'write', dbName)
0217
0218         # now try to read back with all reference releases all the DBs written before ...
0219         print('='*80)
0220         print("going to read back DBs ...")
0221         for rel in readers.keys():
0222             for arch in readers[rel]:
0223                 for writer in self.dbList.keys(): # for any given rel/arch we check all written DBs
0224                     dbName = self.dbList[writer]
0225                     try:
0226                         self.run(rel, arch, 'read', dbName)
0227                         status = True
0228                         print("rel %s reading %s was OK." % (rel, writer))
0229                     except:
0230                         status = False
0231                         print("rel %s reading %s FAILED." % (rel, writer))
0232                     if writer not in self.status.keys():
0233                         key = '%s [%s]' %(rel,arch)
0234                         self.status[writer] = { key : status }
0235                     else:
0236                         self.status[writer]['%s [%s]' %(rel,arch)] = status
0237
0238         # ... and also with this IB/devArea
0239         for writer in self.dbList.keys(): # for any given rel/arch we check all written DBs
0240             dbName = self.dbList[writer]
0241             try:
0242                 self.runSelf('read', dbName)
0243                 status = True
0244                 print("rel %s reading %s was OK." % (self.rel, writer))
0245             except:
0246                 status = False
0247                 print("rel %s reading %s FAILED." % (self.rel, writer))
0248             if writer not in self.status.keys():
0249                 self.status[writer] = { 'SELF': status }
0250             else:
0251                 self.status[writer]['SELF'] = status
0252         print('='*80)
0253
0254 crt = CondRegressionTester()
0255 crt.runAll()
0256 status = crt.summary(verbose=True)
0257 print("\n==> overall status: ", status)
0258
0259 # return the overall result to the caller:
0260 if status:
0261     sys.exit(0)
0262 else:
0263     sys.exit(-1)
0264