HLT/scripts/run_AlCaRecoTriggerBitsUpdateWorkflow.py

0001 #!/usr/bin/env python3
0002
0003 """
0004 Example script to test reading from local sqlite db.
0005 """
0006 from __future__ import print_function
0007 import os
0008 import sys
0009 import ast
0010 import optparse
0011 import multiprocessing
0012 import CondCore.Utilities.conddblib as conddb
0013
0014 officialdbs = {
0015     # frontier connections
0016     'frontier://PromptProd/CMS_CONDITIONS'   :'pro',
0017     'frontier://FrontierProd/CMS_CONDITIONS' :'pro',
0018     'frontier://FrontierArc/CMS_CONDITIONS'  :'arc',
0019     'frontier://FrontierInt/CMS_CONDITIONS'  :'int',
0020     'frontier://FrontierPrep/CMS_CONDITIONS' :'dev'
0021 }
0022
0023 ##############################################
0024 def getCommandOutput(command):
0025 ##############################################
0026     """This function executes `command` and returns it output.
0027     Arguments:
0028     - `command`: Shell command to be invoked by this function.
0029     """
0030     print(command)
0031     child = os.popen(command)
0032     data = child.read()
0033     err = child.close()
0034     if err:
0035         raise Exception('%s failed w/ exit code %d' % (command, err))
0036     return data
0037
0038 ##############################################
0039 def get_iovs(db, tag):
0040 ##############################################
0041     """Retrieve the list of IOVs from `db` for `tag`.
0042
0043     Arguments:
0044     - `db`: database connection string
0045     - `tag`: tag of database record
0046     """
0047
0048     ### unfortunately some gymnastics is needed here
0049     ### to make use of the conddb library
0050     if db in officialdbs.keys():
0051         db = officialdbs[db]
0052
0053     ## allow to use input sqlite files as well
0054     db = db.replace("sqlite_file:", "").replace("sqlite:", "")
0055
0056     con = conddb.connect(url = conddb.make_url(db))
0057     session = con.session()
0058     IOV = session.get_dbtype(conddb.IOV)
0059
0060     iovs = set(session.query(IOV.since).filter(IOV.tag_name == tag).all())
0061     if len(iovs) == 0:
0062         print("No IOVs found for tag '"+tag+"' in database '"+db+"'.")
0063         sys.exit(1)
0064
0065     session.close()
0066
0067     return sorted([int(item[0]) for item in iovs])
0068
0069 ##############################################
0070 def updateBits(blob):
0071 ##############################################
0072     if(blob[0]<blob[1]):
0073         command = 'conddb --yes --db %s copy %s --destdb %s --from %s --to %s' % (blob[4],
0074                                                                                   blob[5],
0075                                                                                   blob[5]+"_IOV_"+str(blob[2])+".db",
0076                                                                                   str(blob[2]) ,
0077                                                                                   str(blob[3]))
0078         getCommandOutput(command)
0079     else:
0080         # last IOV needs special command
0081         command = 'conddb --yes --db %s copy %s --destdb %s --from %s' % (blob[4],
0082                                                                           blob[5],
0083                                                                           blob[5]+"_IOV_"+str(blob[2])+".db",
0084                                                                           str(blob[2]))
0085         getCommandOutput(command)
0086
0087     # update the trigger bits
0088     cmsRunCommand='cmsRun $CMSSW_BASE/src/CondTools/HLT/test/AlCaRecoTriggerBitsRcdUpdate_TEMPL_cfg.py \
0089     inputDB=%s inputTag=%s outputDB=%s outputTag=%s firstRun=%s' % ("sqlite_file:"+blob[5]+"_IOV_"+str(blob[2])+".db",
0090                                                                     blob[5],
0091                                                                     "sqlite_file:"+blob[5]+"_IOV_"+str(blob[2])+"_updated.db",
0092                                                                     blob[6],
0093                                                                     str(blob[2]))
0094     getCommandOutput(cmsRunCommand)
0095
0096 ##############################################
0097 def main():
0098 ##############################################
0099
0100     defaultDB     = 'sqlite_file:mySqlite.db'
0101     defaultInTag  = 'myInTag'
0102     defaultOutTag = 'myOutTag'
0103     defaultProc   = 20
0104
0105     parser = optparse.OptionParser(usage = 'Usage: %prog [options] <file> [<file> ...]\n')
0106     parser.add_option('-f', '--inputDB',
0107                       dest = 'inputDB',
0108                       default = defaultDB,
0109                       help = 'file to inspect')
0110     parser.add_option('-i', '--inputTag',
0111                       dest = 'InputTag',
0112                       default = defaultInTag,
0113                       help = 'tag to be inspected')
0114     parser.add_option('-d', '--destTag',
0115                       dest = 'destTag',
0116                       default = defaultOutTag,
0117                       help = 'tag to be written')
0118     parser.add_option('-p', '--processes',
0119                       dest = 'nproc',
0120                       default = defaultProc,
0121                       help = 'multiprocesses to run')
0122     parser.add_option("-C", '--clean',
0123                       dest="doTheCleaning",
0124                       action="store_true",
0125                       default = True,
0126                       help = 'if true remove the transient files')
0127
0128     (options, arguments) = parser.parse_args()
0129
0130     db_url = options.inputDB
0131     if db_url in officialdbs.keys():
0132         db_url = officialdbs[db_url]
0133
0134     ## allow to use input sqlite files as well
0135     db_url = db_url.replace("sqlite_file:", "").replace("sqlite:", "")
0136
0137     sinces = get_iovs(options.inputDB,options.InputTag)
0138
0139     print("List of sinces: %s" % sinces)
0140
0141     myInputTuple=[]
0142
0143     for i,since in enumerate(sinces):
0144         if(i<len(sinces)-1):
0145             #                   0             1        2               3      4                5               6
0146             myInputTuple.append((i,len(sinces)-1,sinces[i],sinces[i+1]-1,db_url,options.InputTag,options.destTag))
0147         else:
0148             myInputTuple.append((i,len(sinces)-1,sinces[i],-1,db_url,options.InputTag,options.destTag))
0149
0150     pool = multiprocessing.Pool(processes=options.nproc)  # start nproc worker processes
0151     count = pool.map(updateBits,myInputTuple)
0152
0153     # merge the output
0154     for i,since in enumerate(sinces):
0155         mergeCommand='conddb --yes --db %s copy %s --destdb %s --from %s' % (options.InputTag+"_IOV_"+str(sinces[i])+"_updated.db",
0156                                                                              options.destTag,
0157                                                                              options.destTag+".db",
0158                                                                              str(sinces[i]))
0159         getCommandOutput(mergeCommand)
0160
0161     # clean the house (after all is done)
0162     if(options.doTheCleaning):
0163         cleanCommand = 'rm -fr *updated*.db *IOV_*.db'
0164         getCommandOutput(cleanCommand)
0165     else:
0166         print("======> keeping the transient files")
0167
0168 if __name__ == "__main__":
0169      main()