Utilities/scripts/conddbCopyTest.py

0001 #!/usr/bin/env python3
0002
0003 import sqlite3
0004 import subprocess
0005 import json
0006 import os
0007 import shutil
0008 import datetime
0009
0010 fileName = 'copy_test.db'
0011
0012 class DB:
0013     def __init__(self):
0014         pass
0015
0016     def setSynchronizationType( self, tag, synchType ):
0017         db = sqlite3.connect(fileName)
0018         cursor = db.cursor()
0019         cursor.execute('UPDATE TAG SET SYNCHRONIZATION =? WHERE NAME =?',(synchType,tag,))
0020         db.commit()
0021
0022     def getLastInsertedSince( self, tag, snapshot ):
0023         db = sqlite3.connect(fileName)
0024         cursor = db.cursor()
0025         cursor.execute('SELECT SINCE, INSERTION_TIME FROM IOV WHERE TAG_NAME =? AND INSERTION_TIME >? ORDER BY INSERTION_TIME DESC',(tag,snapshot))
0026         row = cursor.fetchone()
0027         return row
0028
0029 def prepareFile( inputTag, sourceTag, startingSince ):
0030     command = "conddb --yes copy %s %s --destdb %s -f %s" %(inputTag,sourceTag,fileName,startingSince)
0031     pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
0032     out = pipe.communicate()[0]
0033
0034 def copy( sourceTag, destTag, since, logFileName ):
0035     command = "conddb --yes --db %s copy %s %s -f %s --synchronize" %(fileName,sourceTag,destTag,since)
0036     pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
0037     out = pipe.communicate()[0]
0038     lines = out.split('\n')
0039     ret = pipe.returncode
0040     for line in lines:
0041         print(line)
0042     with open(logFileName,'a') as logFile:
0043         logFile.write(out)
0044     return ret==0
0045
0046 class CopyTest:
0047     def __init__(self, db):
0048         self.db = db
0049         self.errors = 0
0050         self.logFileName = 'conddbCopyTest.log'
0051
0052     def log( self, msg ):
0053         print(msg)
0054         with open(self.logFileName,'a') as logFile:
0055             logFile.write(msg)
0056             logFile.write('\n')
0057
0058     def execute( self, sourceTag, baseFile, destTag, synchro, destSince, success, expectedAction ):
0059         insertedSince = None
0060         metaDestFile = '%s.txt' %destTag
0061         #shutil.copyfile( baseFile, destFile )
0062         self.log( '# ---------------------------------------------------------------------------')
0063         self.log( '# Testing tag %s with synch=%s, destSince=%s - expecting ret=%s action=%s' %(destTag,synchro,destSince,success,expectedAction))
0064
0065         descr = 'Testing conditionsUpload with synch:%s - expected action: %s' %(synchro,expectedAction)
0066         beforeUpload = datetime.datetime.utcnow()
0067         ret = copy( sourceTag, destTag, destSince, self.logFileName )
0068         if ret != success:
0069             self.log( 'ERROR: the return value for the copy of tag %s with sychro %s was %s, while the expected result is %s' %(destTag,synchro,ret,success))
0070             self.errors += 1
0071         else:
0072             row = self.db.getLastInsertedSince( destTag, beforeUpload )
0073             if ret == True:
0074                 if expectedAction == 'CREATE' or expectedAction == 'INSERT' or expectedAction == 'APPEND':
0075                     if destSince != row[0]:
0076                         self.log( 'ERROR: the since inserted is %s, expected value is %s - expected action: %s' %(row[0],destSince,expectedAction))
0077                         self.errors += 1
0078                     else:
0079                         self.log( '# OK: Found expected value for last since inserted: %s timestamp: %s' %(row[0],row[1]))
0080                         insertedSince = row[0]
0081                 elif expectedAction == 'SYNCHRONIZE':
0082                     if destSince == row[0]:
0083                         self.log( 'ERROR: the since inserted %s has not been synchronized with the FCSR - expected action: %s' %(row[0],expectedAction))
0084                         self.errors += 1
0085                     else:
0086                         self.log( '# OK: Found synchronized value for the last since inserted: %s timestamp: %s' %(row[0],row[1]))
0087                         insertedSince = row[0]
0088                 else:
0089                     self.log( 'ERROR: found an appended since %s - expected action: %s' %(row[0],expectedAction))
0090                     self.errors += 1
0091             else:
0092                 if not row is None:
0093                     self.log( 'ERROR: found new insered since: %s timestamp: %s' %(row[0],row[1]))
0094                     self.errors += 1
0095                 if expectedAction != 'FAIL':
0096                     self.log( 'ERROR: Upload failed. Expected value: %s' %(destSince))
0097                     self.errors += 1
0098                 else:
0099                     self.log( '# OK: Upload failed as expected.')
0100         return insertedSince
0101
0102
0103 def main():
0104     print('Testing...')
0105     bfile0 = fileName
0106     bfile1 = fileName
0107     db = DB()
0108     inputTag = 'runinfo_31X_mc'
0109     inputTag0  ='runinfo_0'
0110     inputTag1 = 'runinfo_1'
0111     prepareFile( inputTag,inputTag0,1)
0112     prepareFile( inputTag,inputTag1,100)
0113     test = CopyTest( db )
0114     # test with synch=any
0115     tag = 'test_CondUpload_any'
0116     test.execute( inputTag0, bfile0, tag, 'any', 1, True, 'CREATE' )
0117     test.execute( inputTag0, bfile0, tag, 'any', 200, True, 'APPEND' )
0118     test.execute( inputTag0, bfile0, tag, 'any', 100, True, 'INSERT')
0119     test.execute( inputTag0, bfile0, tag, 'any', 200, True, 'INSERT')
0120     # test with synch=validation
0121     tag = 'test_CondUpload_validation'
0122     test.execute( inputTag0, bfile0, tag, 'validation', 1, True, 'CREATE')
0123     db.setSynchronizationType( tag, 'validation' )
0124     test.execute( inputTag0, bfile0, tag, 'validation', 1, True, 'INSERT')
0125     test.execute( inputTag0, bfile0, tag, 'validation', 200, True, 'APPEND')
0126     test.execute( inputTag0, bfile0, tag, 'validation', 100, True, 'INSERT')
0127     # test with synch=mc
0128     tag = 'test_CondUpload_mc'
0129     test.execute( inputTag0, bfile0, tag, 'mc', 1, True, 'CREATE')
0130     db.setSynchronizationType( tag, 'mc' )
0131     test.execute( inputTag1, bfile1, tag, 'mc', 1, False, 'FAIL')
0132     test.execute( inputTag0, bfile0, tag, 'mc', 1, False, 'FAIL')
0133     test.execute( inputTag0, bfile0, tag, 'mc', 200, False, 'FAIL')
0134     # test with synch=hlt
0135     tag = 'test_CondUpload_hlt'
0136     test.execute( inputTag0, bfile0, tag, 'hlt', 1, True, 'CREATE')
0137     db.setSynchronizationType( tag, 'hlt' )
0138     test.execute( inputTag0, bfile0, tag, 'hlt', 200, True, 'SYNCHRONIZE')
0139     fcsr = test.execute( inputTag0, bfile0, tag, 'hlt', 100, True, 'SYNCHRONIZE')
0140     if not fcsr is None:
0141         since = fcsr + 200
0142         test.execute( inputTag0, bfile0, tag, 'hlt', since, True, 'APPEND')
0143         since = fcsr + 100
0144         test.execute( inputTag0, bfile0, tag, 'hlt', since, True, 'INSERT')
0145     # test with synch=express
0146     tag = 'test_CondUpload_express'
0147     test.execute( inputTag0, bfile0, tag, 'express', 1, True, 'CREATE')
0148     db.setSynchronizationType( tag, 'express' )
0149     test.execute( inputTag0, bfile0, tag, 'express', 200, True, 'SYNCHRONIZE')
0150     fcsr = test.execute( inputTag0, bfile0, tag, 'express', 100, True, 'SYNCHRONIZE')
0151     if not fcsr is None:
0152         since = fcsr + 200
0153         test.execute( inputTag0, bfile0, tag, 'express', since, True, 'APPEND')
0154         since = fcsr + 100
0155         test.execute( inputTag0, bfile0, tag, 'express', since, True, 'INSERT')
0156     # test with synch=prompt
0157     tag = 'test_CondUpload_prompt'
0158     test.execute( inputTag0, bfile0, tag, 'prompt', 1, True, 'CREATE')
0159     db.setSynchronizationType( tag, 'prompt' )
0160     test.execute( inputTag0, bfile0, tag, 'prompt', 200, True, 'SYNCHRONIZE')
0161     fcsr = test.execute( inputTag0, bfile0, tag, 'prompt', 100, True, 'SYNCHRONIZE')
0162     if not fcsr is None:
0163         since = fcsr + 200
0164         test.execute( inputTag0, bfile0, tag, 'prompt', since, True, 'APPEND')
0165         since = fcsr + 100
0166         test.execute( inputTag0, bfile0, tag, 'prompt', since, True, 'INSERT')
0167     # test with synch=pcl
0168     tag = 'test_CondUpload_pcl'
0169     test.execute( inputTag0, bfile0, tag, 'pcl', 1, True, 'CREATE')
0170     db.setSynchronizationType( tag, 'pcl' )
0171     test.execute( inputTag0, bfile0, tag, 'pcl', 200, False, 'FAIL')
0172     if not fcsr is None:
0173         since = fcsr + 200
0174         test.execute( inputTag0, bfile0, tag, 'pcl', since, True, 'APPEND')
0175         since = fcsr + 100
0176         test.execute( inputTag0, bfile0, tag, 'pcl', since, True, 'INSERT')
0177     # test with synch=offline
0178     tag = 'test_CondUpload_offline'
0179     test.execute( inputTag0, bfile0, tag, 'offline', 1, True, 'CREATE')
0180     db.setSynchronizationType( tag, 'offline' )
0181     test.execute( inputTag0, bfile0, tag, 'offline', 1000, True, 'APPEND')
0182     test.execute( inputTag0, bfile0, tag, 'offline', 500, False, 'FAIL' )
0183     test.execute( inputTag0, bfile0, tag, 'offline', 1000, False, 'FAIL' )
0184     test.execute( inputTag0, bfile0, tag, 'offline', 2000, True, 'APPEND' )
0185     # test with synch=runmc
0186     tag = 'test_CondUpload_runmc'
0187     test.execute( inputTag0, bfile0, tag, 'runmc', 1, True, 'CREATE')
0188     db.setSynchronizationType( tag, 'runmc' )
0189     test.execute( inputTag0, bfile0, tag, 'runmc', 1000, True, 'APPEND')
0190     test.execute( inputTag0, bfile0, tag, 'runmc', 500, False, 'FAIL' )
0191     test.execute( inputTag0, bfile0, tag, 'runmc', 1000, False, 'FAIL' )
0192     test.execute( inputTag0, bfile0, tag, 'runmc', 2000, True, 'APPEND' )
0193     os.remove( fileName )
0194     print('Done. Errors: %s' %test.errors)
0195
0196
0197 if __name__ == '__main__':
0198     main()