Back to home page

Project CMSSW displayed by LXR
 
 

    

File indexing completed on 2024-11-25 02:29:20 

0001 #!/usr/bin/env python3
0002 
0003 import cx_Oracle
0004 import subprocess
0005 import json
0006 import os
0007 import shutil
0008 import datetime
0009 
0010 # Requirement 1: a conddb key for the authentication with valid permission on writing on prep CMS_CONDITIONS account 
0011 #                this could be dropped introducing a specific entry in the .netrc 
0012 # Requirement 2: an entry "Dropbox" in the .netrc for the authentication
0013 
0014 class DB:
0015     def __init__(self, serviceName, schemaName ):
0016         self.serviceName = serviceName
0017         self.schemaName = schemaName
0018         self.connStr = None
0019 
0020     def connect( self ):
0021         command = "cmscond_authentication_manager -s %s --list_conn | grep '%s@%s'" %(self.serviceName,self.schemaName,self.serviceName)
0022         pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
0023         out = pipe.communicate()[0]
0024         srvconn = '%s@%s' %(self.schemaName,self.serviceName)
0025         rowpwd = out.split(srvconn)[1].split(self.schemaName)[1]
0026         pwd = ''
0027         for c in rowpwd:
0028             if c != ' ' and c != '\n':
0029                 pwd += c
0030         self.connStr =  '%s/%s@%s' %(self.schemaName,pwd,self.serviceName)
0031 
0032     def setSynchronizationType( self, tag, synchType ):
0033         db = cx_Oracle.connect(self.connStr)
0034         cursor = db.cursor()
0035         db.begin()
0036         cursor.execute('UPDATE TAG SET SYNCHRONIZATION =:SYNCH WHERE NAME =:NAME',(synchType,tag,))
0037         db.commit()
0038 
0039     def getLastInsertedSince( self, tag, snapshot ):
0040         db = cx_Oracle.connect(self.connStr)
0041         cursor = db.cursor()
0042         cursor.execute('SELECT SINCE, INSERTION_TIME FROM IOV WHERE TAG_NAME =:TAG_NAME AND INSERTION_TIME >:TIME ORDER BY INSERTION_TIME DESC',(tag,snapshot))
0043         row = cursor.fetchone()
0044         return row
0045 
0046     def removeTag( self, tag ):
0047         db = cx_Oracle.connect(self.connStr)
0048         cursor = db.cursor()
0049         db.begin()
0050         cursor.execute('DELETE FROM IOV WHERE TAG_NAME =:TAG_NAME',(tag,))
0051         cursor.execute('DELETE FROM TAG_LOG WHERE TAG_NAME=:TAG_NAME',(tag,))
0052         cursor.execute('DELETE FROM TAG WHERE NAME=:NAME',(tag,))
0053         db.commit()
0054 
0055 def makeBaseFile( inputTag, startingSince ):
0056     cwd = os.getcwd()
0057     baseFile = '%s_%s.db' %(inputTag,startingSince)
0058     baseFilePath = os.path.join(cwd,baseFile)
0059     if os.path.exists( baseFile ):
0060         os.remove( baseFile )
0061     command = "conddb_import -c sqlite_file:%s -f oracle://cms_orcon_adg/CMS_CONDITIONS -i %s -t %s -b %s" %(baseFile,inputTag,inputTag,startingSince)
0062     pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
0063     out = pipe.communicate()[0]
0064     if not os.path.exists( baseFile ):
0065         msg = 'ERROR: base file has not been created: %s' %out
0066         raise Exception( msg )
0067     return baseFile
0068         
0069 
0070 def makeMetadataFile( inputTag, destTag, since, description ):
0071     cwd = os.getcwd()
0072     metadataFile = os.path.join(cwd,'%s.txt') %destTag
0073     if os.path.exists( metadataFile ):
0074         os.remove( metadataFile )
0075     metadata = {}
0076     metadata[ "destinationDatabase" ] = "oracle://cms_orcoff_prep/CMS_CONDITIONS"
0077     tagList = {}
0078     tagList[ destTag ] = { "dependencies": {}, "synchronizeTo": "any" }
0079     metadata[ "destinationTags" ] = tagList
0080     metadata[ "inputTag" ] = inputTag
0081     metadata[ "since" ] = since
0082     metadata[ "userText" ] = description
0083     fileName = destTag+".txt"
0084     with open( fileName, "w" ) as file:
0085         file.write(json.dumps(metadata,file,indent=4,sort_keys=True))
0086 
0087 def uploadFile( fileName, logFileName ):
0088     command = "uploadConditions.py %s" %fileName
0089     pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
0090     out = pipe.communicate()[0]
0091     lines = out.split('\n')
0092     ret = False
0093     for line in lines:
0094         if line.startswith('upload ended with code:'):
0095             returnCode = line.split('upload ended with code:')[1].strip()
0096             if returnCode == '0':
0097                 ret = True
0098             break
0099     with open(logFileName,'a') as logFile:
0100         logFile.write(out)
0101     return ret
0102 
0103 class UploadTest:
0104     def __init__(self, db):
0105         self.db = db
0106         self.errors = 0
0107         self.logFileName = 'conditionUploadTest.log'
0108 
0109     def log( self, msg ):
0110         print(msg)
0111         with open(self.logFileName,'a') as logFile:
0112             logFile.write(msg)
0113             logFile.write('\n')
0114 
0115     def upload( self, inputTag, baseFile, destTag, synchro, destSince, success, expectedAction ):
0116         insertedSince = None
0117         destFile = '%s.db' %destTag
0118         metaDestFile = '%s.txt' %destTag
0119         shutil.copyfile( baseFile, destFile )
0120         self.log( '# ---------------------------------------------------------------------------')
0121         self.log( '# Testing tag %s with synch=%s, destSince=%s - expecting ret=%s action=%s' %(destTag,synchro,destSince,success,expectedAction))
0122     
0123         descr = 'Testing conditionsUpload with synch:%s - expected action: %s' %(synchro,expectedAction)
0124         makeMetadataFile( inputTag, destTag, destSince, descr )
0125         beforeUpload = datetime.datetime.utcnow()
0126         ret = uploadFile( destFile, self.logFileName )
0127         if ret != success:
0128             self.log( 'ERROR: the return value for the upload of tag %s with sychro %s was %s, while the expected result is %s' %(destTag,synchro,ret,success))
0129             self.errors += 1
0130         else:
0131             row = self.db.getLastInsertedSince( destTag, beforeUpload )
0132             if ret == True:
0133                 if expectedAction == 'CREATE' or expectedAction == 'INSERT' or expectedAction == 'APPEND':
0134                     if destSince != row[0]:
0135                         self.log( 'ERROR: the since inserted is %s, expected value is %s - expected action: %s' %(row[0],destSince,expectedAction))
0136                         self.errors += 1
0137                     else:
0138                         self.log( '# OK: Found expected value for last since inserted: %s timestamp: %s' %(row[0],row[1]))
0139                         insertedSince = row[0]
0140                 elif expectedAction == 'SYNCHRONIZE':
0141                     if destSince == row[0]:
0142                         self.log( 'ERROR: the since inserted %s has not been synchronized with the FCSR - expected action: %s' %(row[0],expectedAction))
0143                         self.errors += 1
0144                     else:
0145                         self.log( '# OK: Found synchronized value for the last since inserted: %s timestamp: %s' %(row[0],row[1]))
0146                         insertedSince = row[0]
0147                 else:
0148                     self.log( 'ERROR: found an appended since %s - expected action: %s' %(row[0],expectedAction))
0149                     self.errors += 1
0150             else:
0151                 if not row is None:
0152                     self.log( 'ERROR: found new insered since: %s timestamp: %s' %(row[0],row[1]))
0153                     self.errors += 1
0154                 if expectedAction != 'FAIL':
0155                     self.log( 'ERROR: Upload failed. Expected value: %s' %(destSince))
0156                     self.errors += 1
0157                 else:
0158                     self.log( '# OK: Upload failed as expected.')
0159         os.remove( destFile )
0160         os.remove( metaDestFile )
0161         return insertedSince
0162 
0163 
0164 def main():
0165     print('Testing...')
0166     serviceName = 'cms_orcoff_prep'
0167     schemaName = 'CMS_CONDITIONS'
0168     db = DB(serviceName,schemaName)
0169     db.connect()
0170     inputTag = 'runinfo_31X_mc'
0171     bfile0 = makeBaseFile( inputTag,1)
0172     bfile1 = makeBaseFile( inputTag,100)
0173     test = UploadTest( db )
0174     # test with synch=any
0175     tag = 'test_CondUpload_any'
0176     test.upload( inputTag, bfile0, tag, 'any', 1, True, 'CREATE' )
0177     test.upload( inputTag, bfile1, tag, 'any', 1, False, 'FAIL' )  
0178     test.upload( inputTag, bfile0, tag, 'any', 200, True, 'APPEND' )  
0179     test.upload( inputTag, bfile0, tag, 'any', 100, True, 'INSERT')  
0180     test.upload( inputTag, bfile0, tag, 'any', 200, True, 'INSERT')  
0181     db.removeTag( tag )
0182     # test with synch=validation
0183     tag = 'test_CondUpload_validation'
0184     test.upload( inputTag, bfile0, tag, 'validation', 1, True, 'CREATE')  
0185     db.setSynchronizationType( tag, 'validation' ) 
0186     test.upload( inputTag, bfile0, tag, 'validation', 1, True, 'INSERT')  
0187     test.upload( inputTag, bfile0, tag, 'validation', 200, True, 'APPEND')  
0188     test.upload( inputTag, bfile0, tag, 'validation', 100, True, 'INSERT')  
0189     db.removeTag( tag )
0190     # test with synch=mc
0191     tag = 'test_CondUpload_mc'
0192     test.upload( inputTag, bfile1, tag, 'mc', 1, False, 'FAIL')  
0193     test.upload( inputTag, bfile0, tag, 'mc', 1, True, 'CREATE')  
0194     db.setSynchronizationType( tag, 'mc' ) 
0195     test.upload( inputTag, bfile0, tag, 'mc', 1, False, 'FAIL')  
0196     test.upload( inputTag, bfile0, tag, 'mc', 200, False, 'FAIL') 
0197     db.removeTag( tag )
0198     # test with synch=hlt
0199     tag = 'test_CondUpload_hlt'
0200     test.upload( inputTag, bfile0, tag, 'hlt', 1, True, 'CREATE')  
0201     db.setSynchronizationType( tag, 'hlt' ) 
0202     test.upload( inputTag, bfile0, tag, 'hlt', 200, True, 'SYNCHRONIZE')  
0203     fcsr = test.upload( inputTag, bfile0, tag, 'hlt', 100, True, 'SYNCHRONIZE')  
0204     if not fcsr is None:
0205         since = fcsr + 200
0206         test.upload( inputTag, bfile0, tag, 'hlt', since, True, 'APPEND')  
0207         since = fcsr + 100
0208         test.upload( inputTag, bfile0, tag, 'hlt', since, True, 'INSERT')  
0209     db.removeTag( tag )
0210     # test with synch=express
0211     tag = 'test_CondUpload_express'
0212     test.upload( inputTag, bfile0, tag, 'express', 1, True, 'CREATE')  
0213     db.setSynchronizationType( tag, 'express' ) 
0214     test.upload( inputTag, bfile0, tag, 'express', 200, True, 'SYNCHRONIZE')  
0215     fcsr = test.upload( inputTag, bfile0, tag, 'express', 100, True, 'SYNCHRONIZE')  
0216     if not fcsr is None:
0217         since = fcsr + 200
0218         test.upload( inputTag, bfile0, tag, 'express', since, True, 'APPEND')  
0219         since = fcsr + 100
0220         test.upload( inputTag, bfile0, tag, 'express', since, True, 'INSERT')  
0221     db.removeTag( tag )
0222     # test with synch=prompt
0223     tag = 'test_CondUpload_prompt'
0224     test.upload( inputTag, bfile0, tag, 'prompt', 1, True, 'CREATE')  
0225     db.setSynchronizationType( tag, 'prompt' ) 
0226     test.upload( inputTag, bfile0, tag, 'prompt', 200, True, 'SYNCHRONIZE')  
0227     fcsr = test.upload( inputTag, bfile0, tag, 'prompt', 100, True, 'SYNCHRONIZE')  
0228     if not fcsr is None:
0229         since = fcsr + 200
0230         test.upload( inputTag, bfile0, tag, 'prompt', since, True, 'APPEND')  
0231         since = fcsr + 100
0232         test.upload( inputTag, bfile0, tag, 'prompt', since, True, 'INSERT')  
0233     db.removeTag( tag )
0234     # test with synch=pcl
0235     tag = 'test_CondUpload_pcl'
0236     test.upload( inputTag, bfile0, tag, 'pcl', 1, True, 'CREATE')  
0237     db.setSynchronizationType( tag, 'pcl' ) 
0238     test.upload( inputTag, bfile0, tag, 'pcl', 200, False, 'FAIL')  
0239     if not fcsr is None:
0240         since = fcsr + 200
0241         test.upload( inputTag, bfile0, tag, 'pcl', since, True, 'APPEND')  
0242         since = fcsr + 100
0243         test.upload( inputTag, bfile0, tag, 'pcl', since, True, 'INSERT')  
0244     db.removeTag( tag )
0245     # test with synch=offline
0246     tag = 'test_CondUpload_offline'
0247     test.upload( inputTag, bfile0, tag, 'offline', 1, True, 'CREATE')  
0248     db.setSynchronizationType( tag, 'offline' ) 
0249     test.upload( inputTag, bfile0, tag, 'offline', 1000, True, 'APPEND')
0250     test.upload( inputTag, bfile0, tag, 'offline', 500, False, 'FAIL' ) 
0251     test.upload( inputTag, bfile0, tag, 'offline', 1000, False, 'FAIL' ) 
0252     test.upload( inputTag, bfile0, tag, 'offline', 2000, True, 'APPEND' ) 
0253     db.removeTag( tag )
0254     # test with synch=runmc
0255     tag = 'test_CondUpload_runmc'
0256     test.upload( inputTag, bfile0, tag, 'runmc', 1, True, 'CREATE')  
0257     db.setSynchronizationType( tag, 'runmc' ) 
0258     test.upload( inputTag, bfile0, tag, 'runmc', 1000, True, 'APPEND')
0259     test.upload( inputTag, bfile0, tag, 'runmc', 500, False, 'FAIL' ) 
0260     test.upload( inputTag, bfile0, tag, 'runmc', 1000, False, 'FAIL' ) 
0261     test.upload( inputTag, bfile0, tag, 'runmc', 2000, True, 'APPEND' )
0262     db.removeTag( tag )
0263     os.remove( bfile0 )
0264     os.remove( bfile1 )
0265     print('Done. Errors: %s' %test.errors)
0266     
0267 if __name__ == '__main__':
0268     main()
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.2.1 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!