Utilities/scripts/conddbCopyTest.py

0001 #!/usr/bin/env python3
0002
0003 from __future__ import print_function
0004 import sqlite3
0005 import subprocess
0006 import json
0007 import os
0008 import shutil
0009 import datetime
0010
0011 fileName = 'copy_test.db'
0012
0013 class DB:
0014     def __init__(self):
0015         pass
0016
0017     def setSynchronizationType( self, tag, synchType ):
0018         db = sqlite3.connect(fileName)
0019         cursor = db.cursor()
0020         cursor.execute('UPDATE TAG SET SYNCHRONIZATION =? WHERE NAME =?',(synchType,tag,))
0021         db.commit()
0022
0023     def getLastInsertedSince( self, tag, snapshot ):
0024         db = sqlite3.connect(fileName)
0025         cursor = db.cursor()
0026         cursor.execute('SELECT SINCE, INSERTION_TIME FROM IOV WHERE TAG_NAME =? AND INSERTION_TIME >? ORDER BY INSERTION_TIME DESC',(tag,snapshot))
0027         row = cursor.fetchone()
0028         return row
0029
0030 def prepareFile( inputTag, sourceTag, startingSince ):
0031     command = "conddb --yes copy %s %s --destdb %s -f %s" %(inputTag,sourceTag,fileName,startingSince)
0032     pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
0033     out = pipe.communicate()[0]
0034
0035 def copy( sourceTag, destTag, since, logFileName ):
0036     command = "conddb --yes --db %s copy %s %s -f %s --synchronize" %(fileName,sourceTag,destTag,since)
0037     pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
0038     out = pipe.communicate()[0]
0039     lines = out.split('\n')
0040     ret = pipe.returncode
0041     for line in lines:
0042         print(line)
0043     with open(logFileName,'a') as logFile:
0044         logFile.write(out)
0045     return ret==0
0046
0047 class CopyTest:
0048     def __init__(self, db):
0049         self.db = db
0050         self.errors = 0
0051         self.logFileName = 'conddbCopyTest.log'
0052
0053     def log( self, msg ):
0054         print(msg)
0055         with open(self.logFileName,'a') as logFile:
0056             logFile.write(msg)
0057             logFile.write('\n')
0058
0059     def execute( self, sourceTag, baseFile, destTag, synchro, destSince, success, expectedAction ):
0060         insertedSince = None
0061         metaDestFile = '%s.txt' %destTag
0062         #shutil.copyfile( baseFile, destFile )
0063         self.log( '# ---------------------------------------------------------------------------')
0064         self.log( '# Testing tag %s with synch=%s, destSince=%s - expecting ret=%s action=%s' %(destTag,synchro,destSince,success,expectedAction))
0065
0066         descr = 'Testing conditionsUpload with synch:%s - expected action: %s' %(synchro,expectedAction)
0067         beforeUpload = datetime.datetime.utcnow()
0068         ret = copy( sourceTag, destTag, destSince, self.logFileName )
0069         if ret != success:
0070             self.log( 'ERROR: the return value for the copy of tag %s with sychro %s was %s, while the expected result is %s' %(destTag,synchro,ret,success))
0071             self.errors += 1
0072         else:
0073             row = self.db.getLastInsertedSince( destTag, beforeUpload )
0074             if ret == True:
0075                 if expectedAction == 'CREATE' or expectedAction == 'INSERT' or expectedAction == 'APPEND':
0076                     if destSince != row[0]:
0077                         self.log( 'ERROR: the since inserted is %s, expected value is %s - expected action: %s' %(row[0],destSince,expectedAction))
0078                         self.errors += 1
0079                     else:
0080                         self.log( '# OK: Found expected value for last since inserted: %s timestamp: %s' %(row[0],row[1]))
0081                         insertedSince = row[0]
0082                 elif expectedAction == 'SYNCHRONIZE':
0083                     if destSince == row[0]:
0084                         self.log( 'ERROR: the since inserted %s has not been synchronized with the FCSR - expected action: %s' %(row[0],expectedAction))
0085                         self.errors += 1
0086                     else:
0087                         self.log( '# OK: Found synchronized value for the last since inserted: %s timestamp: %s' %(row[0],row[1]))
0088                         insertedSince = row[0]
0089                 else:
0090                     self.log( 'ERROR: found an appended since %s - expected action: %s' %(row[0],expectedAction))
0091                     self.errors += 1
0092             else:
0093                 if not row is None:
0094                     self.log( 'ERROR: found new insered since: %s timestamp: %s' %(row[0],row[1]))
0095                     self.errors += 1
0096                 if expectedAction != 'FAIL':
0097                     self.log( 'ERROR: Upload failed. Expected value: %s' %(destSince))
0098                     self.errors += 1
0099                 else:
0100                     self.log( '# OK: Upload failed as expected.')
0101         return insertedSince
0102
0103
0104 def main():
0105     print('Testing...')
0106     bfile0 = fileName
0107     bfile1 = fileName
0108     db = DB()
0109     inputTag = 'runinfo_31X_mc'
0110     inputTag0  ='runinfo_0'
0111     inputTag1 = 'runinfo_1'
0112     prepareFile( inputTag,inputTag0,1)
0113     prepareFile( inputTag,inputTag1,100)
0114     test = CopyTest( db )
0115     # test with synch=any
0116     tag = 'test_CondUpload_any'
0117     test.execute( inputTag0, bfile0, tag, 'any', 1, True, 'CREATE' )
0118     test.execute( inputTag0, bfile0, tag, 'any', 200, True, 'APPEND' )
0119     test.execute( inputTag0, bfile0, tag, 'any', 100, True, 'INSERT')
0120     test.execute( inputTag0, bfile0, tag, 'any', 200, True, 'INSERT')
0121     # test with synch=validation
0122     tag = 'test_CondUpload_validation'
0123     test.execute( inputTag0, bfile0, tag, 'validation', 1, True, 'CREATE')
0124     db.setSynchronizationType( tag, 'validation' )
0125     test.execute( inputTag0, bfile0, tag, 'validation', 1, True, 'INSERT')
0126     test.execute( inputTag0, bfile0, tag, 'validation', 200, True, 'APPEND')
0127     test.execute( inputTag0, bfile0, tag, 'validation', 100, True, 'INSERT')
0128     # test with synch=mc
0129     tag = 'test_CondUpload_mc'
0130     test.execute( inputTag0, bfile0, tag, 'mc', 1, True, 'CREATE')
0131     db.setSynchronizationType( tag, 'mc' )
0132     test.execute( inputTag1, bfile1, tag, 'mc', 1, False, 'FAIL')
0133     test.execute( inputTag0, bfile0, tag, 'mc', 1, False, 'FAIL')
0134     test.execute( inputTag0, bfile0, tag, 'mc', 200, False, 'FAIL')
0135     # test with synch=hlt
0136     tag = 'test_CondUpload_hlt'
0137     test.execute( inputTag0, bfile0, tag, 'hlt', 1, True, 'CREATE')
0138     db.setSynchronizationType( tag, 'hlt' )
0139     test.execute( inputTag0, bfile0, tag, 'hlt', 200, True, 'SYNCHRONIZE')
0140     fcsr = test.execute( inputTag0, bfile0, tag, 'hlt', 100, True, 'SYNCHRONIZE')
0141     if not fcsr is None:
0142         since = fcsr + 200
0143         test.execute( inputTag0, bfile0, tag, 'hlt', since, True, 'APPEND')
0144         since = fcsr + 100
0145         test.execute( inputTag0, bfile0, tag, 'hlt', since, True, 'INSERT')
0146     # test with synch=express
0147     tag = 'test_CondUpload_express'
0148     test.execute( inputTag0, bfile0, tag, 'express', 1, True, 'CREATE')
0149     db.setSynchronizationType( tag, 'express' )
0150     test.execute( inputTag0, bfile0, tag, 'express', 200, True, 'SYNCHRONIZE')
0151     fcsr = test.execute( inputTag0, bfile0, tag, 'express', 100, True, 'SYNCHRONIZE')
0152     if not fcsr is None:
0153         since = fcsr + 200
0154         test.execute( inputTag0, bfile0, tag, 'express', since, True, 'APPEND')
0155         since = fcsr + 100
0156         test.execute( inputTag0, bfile0, tag, 'express', since, True, 'INSERT')
0157     # test with synch=prompt
0158     tag = 'test_CondUpload_prompt'
0159     test.execute( inputTag0, bfile0, tag, 'prompt', 1, True, 'CREATE')
0160     db.setSynchronizationType( tag, 'prompt' )
0161     test.execute( inputTag0, bfile0, tag, 'prompt', 200, True, 'SYNCHRONIZE')
0162     fcsr = test.execute( inputTag0, bfile0, tag, 'prompt', 100, True, 'SYNCHRONIZE')
0163     if not fcsr is None:
0164         since = fcsr + 200
0165         test.execute( inputTag0, bfile0, tag, 'prompt', since, True, 'APPEND')
0166         since = fcsr + 100
0167         test.execute( inputTag0, bfile0, tag, 'prompt', since, True, 'INSERT')
0168     # test with synch=pcl
0169     tag = 'test_CondUpload_pcl'
0170     test.execute( inputTag0, bfile0, tag, 'pcl', 1, True, 'CREATE')
0171     db.setSynchronizationType( tag, 'pcl' )
0172     test.execute( inputTag0, bfile0, tag, 'pcl', 200, False, 'FAIL')
0173     if not fcsr is None:
0174         since = fcsr + 200
0175         test.execute( inputTag0, bfile0, tag, 'pcl', since, True, 'APPEND')
0176         since = fcsr + 100
0177         test.execute( inputTag0, bfile0, tag, 'pcl', since, True, 'INSERT')
0178     # test with synch=offline
0179     tag = 'test_CondUpload_offline'
0180     test.execute( inputTag0, bfile0, tag, 'offline', 1, True, 'CREATE')
0181     db.setSynchronizationType( tag, 'offline' )
0182     test.execute( inputTag0, bfile0, tag, 'offline', 1000, True, 'APPEND')
0183     test.execute( inputTag0, bfile0, tag, 'offline', 500, False, 'FAIL' )
0184     test.execute( inputTag0, bfile0, tag, 'offline', 1000, False, 'FAIL' )
0185     test.execute( inputTag0, bfile0, tag, 'offline', 2000, True, 'APPEND' )
0186     # test with synch=runmc
0187     tag = 'test_CondUpload_runmc'
0188     test.execute( inputTag0, bfile0, tag, 'runmc', 1, True, 'CREATE')
0189     db.setSynchronizationType( tag, 'runmc' )
0190     test.execute( inputTag0, bfile0, tag, 'runmc', 1000, True, 'APPEND')
0191     test.execute( inputTag0, bfile0, tag, 'runmc', 500, False, 'FAIL' )
0192     test.execute( inputTag0, bfile0, tag, 'runmc', 1000, False, 'FAIL' )
0193     test.execute( inputTag0, bfile0, tag, 'runmc', 2000, True, 'APPEND' )
0194     os.remove( fileName )
0195     print('Done. Errors: %s' %test.errors)
0196
0197
0198 if __name__ == '__main__':
0199     main()