Back to home page

Project CMSSW displayed by LXR
 
 

    

File indexing completed on 2024-04-06 12:01:54 

0001 #!/usr/bin/env python3
0002 
0003 from __future__ import print_function
0004 import cx_Oracle
0005 import subprocess
0006 import json
0007 import os
0008 import shutil
0009 import datetime
0010 
0011 # Requirement 1: a conddb key for the authentication with valid permission on writing on prep CMS_CONDITIONS account 
0012 #                this could be dropped introducing a specific entry in the .netrc 
0013 # Requirement 2: an entry "Dropbox" in the .netrc for the authentication
0014 
0015 class DB:
0016     def __init__(self, serviceName, schemaName ):
0017         self.serviceName = serviceName
0018         self.schemaName = schemaName
0019         self.connStr = None
0020 
0021     def connect( self ):
0022         command = "cmscond_authentication_manager -s %s --list_conn | grep '%s@%s'" %(self.serviceName,self.schemaName,self.serviceName)
0023         pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
0024         out = pipe.communicate()[0]
0025         srvconn = '%s@%s' %(self.schemaName,self.serviceName)
0026         rowpwd = out.split(srvconn)[1].split(self.schemaName)[1]
0027         pwd = ''
0028         for c in rowpwd:
0029             if c != ' ' and c != '\n':
0030                 pwd += c
0031         self.connStr =  '%s/%s@%s' %(self.schemaName,pwd,self.serviceName)
0032 
0033     def setSynchronizationType( self, tag, synchType ):
0034         db = cx_Oracle.connect(self.connStr)
0035         cursor = db.cursor()
0036         db.begin()
0037         cursor.execute('UPDATE TAG SET SYNCHRONIZATION =:SYNCH WHERE NAME =:NAME',(synchType,tag,))
0038         db.commit()
0039 
0040     def getLastInsertedSince( self, tag, snapshot ):
0041         db = cx_Oracle.connect(self.connStr)
0042         cursor = db.cursor()
0043         cursor.execute('SELECT SINCE, INSERTION_TIME FROM IOV WHERE TAG_NAME =:TAG_NAME AND INSERTION_TIME >:TIME ORDER BY INSERTION_TIME DESC',(tag,snapshot))
0044         row = cursor.fetchone()
0045         return row
0046 
0047     def removeTag( self, tag ):
0048         db = cx_Oracle.connect(self.connStr)
0049         cursor = db.cursor()
0050         db.begin()
0051         cursor.execute('DELETE FROM IOV WHERE TAG_NAME =:TAG_NAME',(tag,))
0052         cursor.execute('DELETE FROM TAG_LOG WHERE TAG_NAME=:TAG_NAME',(tag,))
0053         cursor.execute('DELETE FROM TAG WHERE NAME=:NAME',(tag,))
0054         db.commit()
0055 
0056 def makeBaseFile( inputTag, startingSince ):
0057     cwd = os.getcwd()
0058     baseFile = '%s_%s.db' %(inputTag,startingSince)
0059     baseFilePath = os.path.join(cwd,baseFile)
0060     if os.path.exists( baseFile ):
0061         os.remove( baseFile )
0062     command = "conddb_import -c sqlite_file:%s -f oracle://cms_orcon_adg/CMS_CONDITIONS -i %s -t %s -b %s" %(baseFile,inputTag,inputTag,startingSince)
0063     pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
0064     out = pipe.communicate()[0]
0065     if not os.path.exists( baseFile ):
0066         msg = 'ERROR: base file has not been created: %s' %out
0067         raise Exception( msg )
0068     return baseFile
0069         
0070 
0071 def makeMetadataFile( inputTag, destTag, since, description ):
0072     cwd = os.getcwd()
0073     metadataFile = os.path.join(cwd,'%s.txt') %destTag
0074     if os.path.exists( metadataFile ):
0075         os.remove( metadataFile )
0076     metadata = {}
0077     metadata[ "destinationDatabase" ] = "oracle://cms_orcoff_prep/CMS_CONDITIONS"
0078     tagList = {}
0079     tagList[ destTag ] = { "dependencies": {}, "synchronizeTo": "any" }
0080     metadata[ "destinationTags" ] = tagList
0081     metadata[ "inputTag" ] = inputTag
0082     metadata[ "since" ] = since
0083     metadata[ "userText" ] = description
0084     fileName = destTag+".txt"
0085     with open( fileName, "w" ) as file:
0086         file.write(json.dumps(metadata,file,indent=4,sort_keys=True))
0087 
0088 def uploadFile( fileName, logFileName ):
0089     command = "uploadConditions.py %s" %fileName
0090     pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
0091     out = pipe.communicate()[0]
0092     lines = out.split('\n')
0093     ret = False
0094     for line in lines:
0095         if line.startswith('upload ended with code:'):
0096             returnCode = line.split('upload ended with code:')[1].strip()
0097             if returnCode == '0':
0098                 ret = True
0099             break
0100     with open(logFileName,'a') as logFile:
0101         logFile.write(out)
0102     return ret
0103 
0104 class UploadTest:
0105     def __init__(self, db):
0106         self.db = db
0107         self.errors = 0
0108         self.logFileName = 'conditionUploadTest.log'
0109 
0110     def log( self, msg ):
0111         print(msg)
0112         with open(self.logFileName,'a') as logFile:
0113             logFile.write(msg)
0114             logFile.write('\n')
0115 
0116     def upload( self, inputTag, baseFile, destTag, synchro, destSince, success, expectedAction ):
0117         insertedSince = None
0118         destFile = '%s.db' %destTag
0119         metaDestFile = '%s.txt' %destTag
0120         shutil.copyfile( baseFile, destFile )
0121         self.log( '# ---------------------------------------------------------------------------')
0122         self.log( '# Testing tag %s with synch=%s, destSince=%s - expecting ret=%s action=%s' %(destTag,synchro,destSince,success,expectedAction))
0123     
0124         descr = 'Testing conditionsUpload with synch:%s - expected action: %s' %(synchro,expectedAction)
0125         makeMetadataFile( inputTag, destTag, destSince, descr )
0126         beforeUpload = datetime.datetime.utcnow()
0127         ret = uploadFile( destFile, self.logFileName )
0128         if ret != success:
0129             self.log( 'ERROR: the return value for the upload of tag %s with sychro %s was %s, while the expected result is %s' %(destTag,synchro,ret,success))
0130             self.errors += 1
0131         else:
0132             row = self.db.getLastInsertedSince( destTag, beforeUpload )
0133             if ret == True:
0134                 if expectedAction == 'CREATE' or expectedAction == 'INSERT' or expectedAction == 'APPEND':
0135                     if destSince != row[0]:
0136                         self.log( 'ERROR: the since inserted is %s, expected value is %s - expected action: %s' %(row[0],destSince,expectedAction))
0137                         self.errors += 1
0138                     else:
0139                         self.log( '# OK: Found expected value for last since inserted: %s timestamp: %s' %(row[0],row[1]))
0140                         insertedSince = row[0]
0141                 elif expectedAction == 'SYNCHRONIZE':
0142                     if destSince == row[0]:
0143                         self.log( 'ERROR: the since inserted %s has not been synchronized with the FCSR - expected action: %s' %(row[0],expectedAction))
0144                         self.errors += 1
0145                     else:
0146                         self.log( '# OK: Found synchronized value for the last since inserted: %s timestamp: %s' %(row[0],row[1]))
0147                         insertedSince = row[0]
0148                 else:
0149                     self.log( 'ERROR: found an appended since %s - expected action: %s' %(row[0],expectedAction))
0150                     self.errors += 1
0151             else:
0152                 if not row is None:
0153                     self.log( 'ERROR: found new insered since: %s timestamp: %s' %(row[0],row[1]))
0154                     self.errors += 1
0155                 if expectedAction != 'FAIL':
0156                     self.log( 'ERROR: Upload failed. Expected value: %s' %(destSince))
0157                     self.errors += 1
0158                 else:
0159                     self.log( '# OK: Upload failed as expected.')
0160         os.remove( destFile )
0161         os.remove( metaDestFile )
0162         return insertedSince
0163 
0164 
0165 def main():
0166     print('Testing...')
0167     serviceName = 'cms_orcoff_prep'
0168     schemaName = 'CMS_CONDITIONS'
0169     db = DB(serviceName,schemaName)
0170     db.connect()
0171     inputTag = 'runinfo_31X_mc'
0172     bfile0 = makeBaseFile( inputTag,1)
0173     bfile1 = makeBaseFile( inputTag,100)
0174     test = UploadTest( db )
0175     # test with synch=any
0176     tag = 'test_CondUpload_any'
0177     test.upload( inputTag, bfile0, tag, 'any', 1, True, 'CREATE' )
0178     test.upload( inputTag, bfile1, tag, 'any', 1, False, 'FAIL' )  
0179     test.upload( inputTag, bfile0, tag, 'any', 200, True, 'APPEND' )  
0180     test.upload( inputTag, bfile0, tag, 'any', 100, True, 'INSERT')  
0181     test.upload( inputTag, bfile0, tag, 'any', 200, True, 'INSERT')  
0182     db.removeTag( tag )
0183     # test with synch=validation
0184     tag = 'test_CondUpload_validation'
0185     test.upload( inputTag, bfile0, tag, 'validation', 1, True, 'CREATE')  
0186     db.setSynchronizationType( tag, 'validation' ) 
0187     test.upload( inputTag, bfile0, tag, 'validation', 1, True, 'INSERT')  
0188     test.upload( inputTag, bfile0, tag, 'validation', 200, True, 'APPEND')  
0189     test.upload( inputTag, bfile0, tag, 'validation', 100, True, 'INSERT')  
0190     db.removeTag( tag )
0191     # test with synch=mc
0192     tag = 'test_CondUpload_mc'
0193     test.upload( inputTag, bfile1, tag, 'mc', 1, False, 'FAIL')  
0194     test.upload( inputTag, bfile0, tag, 'mc', 1, True, 'CREATE')  
0195     db.setSynchronizationType( tag, 'mc' ) 
0196     test.upload( inputTag, bfile0, tag, 'mc', 1, False, 'FAIL')  
0197     test.upload( inputTag, bfile0, tag, 'mc', 200, False, 'FAIL') 
0198     db.removeTag( tag )
0199     # test with synch=hlt
0200     tag = 'test_CondUpload_hlt'
0201     test.upload( inputTag, bfile0, tag, 'hlt', 1, True, 'CREATE')  
0202     db.setSynchronizationType( tag, 'hlt' ) 
0203     test.upload( inputTag, bfile0, tag, 'hlt', 200, True, 'SYNCHRONIZE')  
0204     fcsr = test.upload( inputTag, bfile0, tag, 'hlt', 100, True, 'SYNCHRONIZE')  
0205     if not fcsr is None:
0206         since = fcsr + 200
0207         test.upload( inputTag, bfile0, tag, 'hlt', since, True, 'APPEND')  
0208         since = fcsr + 100
0209         test.upload( inputTag, bfile0, tag, 'hlt', since, True, 'INSERT')  
0210     db.removeTag( tag )
0211     # test with synch=express
0212     tag = 'test_CondUpload_express'
0213     test.upload( inputTag, bfile0, tag, 'express', 1, True, 'CREATE')  
0214     db.setSynchronizationType( tag, 'express' ) 
0215     test.upload( inputTag, bfile0, tag, 'express', 200, True, 'SYNCHRONIZE')  
0216     fcsr = test.upload( inputTag, bfile0, tag, 'express', 100, True, 'SYNCHRONIZE')  
0217     if not fcsr is None:
0218         since = fcsr + 200
0219         test.upload( inputTag, bfile0, tag, 'express', since, True, 'APPEND')  
0220         since = fcsr + 100
0221         test.upload( inputTag, bfile0, tag, 'express', since, True, 'INSERT')  
0222     db.removeTag( tag )
0223     # test with synch=prompt
0224     tag = 'test_CondUpload_prompt'
0225     test.upload( inputTag, bfile0, tag, 'prompt', 1, True, 'CREATE')  
0226     db.setSynchronizationType( tag, 'prompt' ) 
0227     test.upload( inputTag, bfile0, tag, 'prompt', 200, True, 'SYNCHRONIZE')  
0228     fcsr = test.upload( inputTag, bfile0, tag, 'prompt', 100, True, 'SYNCHRONIZE')  
0229     if not fcsr is None:
0230         since = fcsr + 200
0231         test.upload( inputTag, bfile0, tag, 'prompt', since, True, 'APPEND')  
0232         since = fcsr + 100
0233         test.upload( inputTag, bfile0, tag, 'prompt', since, True, 'INSERT')  
0234     db.removeTag( tag )
0235     # test with synch=pcl
0236     tag = 'test_CondUpload_pcl'
0237     test.upload( inputTag, bfile0, tag, 'pcl', 1, True, 'CREATE')  
0238     db.setSynchronizationType( tag, 'pcl' ) 
0239     test.upload( inputTag, bfile0, tag, 'pcl', 200, False, 'FAIL')  
0240     if not fcsr is None:
0241         since = fcsr + 200
0242         test.upload( inputTag, bfile0, tag, 'pcl', since, True, 'APPEND')  
0243         since = fcsr + 100
0244         test.upload( inputTag, bfile0, tag, 'pcl', since, True, 'INSERT')  
0245     db.removeTag( tag )
0246     # test with synch=offline
0247     tag = 'test_CondUpload_offline'
0248     test.upload( inputTag, bfile0, tag, 'offline', 1, True, 'CREATE')  
0249     db.setSynchronizationType( tag, 'offline' ) 
0250     test.upload( inputTag, bfile0, tag, 'offline', 1000, True, 'APPEND')
0251     test.upload( inputTag, bfile0, tag, 'offline', 500, False, 'FAIL' ) 
0252     test.upload( inputTag, bfile0, tag, 'offline', 1000, False, 'FAIL' ) 
0253     test.upload( inputTag, bfile0, tag, 'offline', 2000, True, 'APPEND' ) 
0254     db.removeTag( tag )
0255     # test with synch=runmc
0256     tag = 'test_CondUpload_runmc'
0257     test.upload( inputTag, bfile0, tag, 'runmc', 1, True, 'CREATE')  
0258     db.setSynchronizationType( tag, 'runmc' ) 
0259     test.upload( inputTag, bfile0, tag, 'runmc', 1000, True, 'APPEND')
0260     test.upload( inputTag, bfile0, tag, 'runmc', 500, False, 'FAIL' ) 
0261     test.upload( inputTag, bfile0, tag, 'runmc', 1000, False, 'FAIL' ) 
0262     test.upload( inputTag, bfile0, tag, 'runmc', 2000, True, 'APPEND' )
0263     db.removeTag( tag )
0264     os.remove( bfile0 )
0265     os.remove( bfile1 )
0266     print('Done. Errors: %s' %test.errors)
0267     
0268 if __name__ == '__main__':
0269     main()
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.2.1 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!