SiStrip/scripts/SiStripDAQPopCon.py

0001 #!/usr/bin/env python3
0002 '''Script that runs a single O2O for SiStrip DAQ.
0003 @author: Huilin Qu
0004 '''
0005
0006 import os
0007 import atexit
0008 import logging
0009 import socket
0010 import argparse
0011 import subprocess
0012 from functools import partial
0013 import CondTools.SiStrip.o2o_helper as helper
0014 from CondTools.SiStrip.o2o_db_cfgmap import DbManagerDAQ
0015 from CondTools.SiStrip.o2o_db_gain import DbManagerGain
0016
0017 jobDirVar = 'JOBDIR'
0018 cfg_template = 'CondTools/SiStrip/python/SiStripO2O_cfg_template.py'
0019
0020 def runjob(args):
0021     if args.debug:
0022         logging.debug(str(args))
0023
0024     # read cfglines from input cfgfile
0025     with open(args.cfgfile) as cfgfile:
0026         cfglines = cfgfile.read()
0027         logging.debug(cfglines)
0028
0029     # create config from template
0030     job_file = 'cfg_{type}_{run}.py'.format(type=args.analyzer, run=args.since)
0031     output_db = '{type}_{run}.db'.format(type=args.analyzer, run=args.since)
0032     if os.path.exists(output_db):
0033         logging.info('Output sqlite file %s already exists! Deleting...' % output_db)
0034         os.remove(output_db)
0035     hashmap_db = 'hashmap_{type}_{run}.db'.format(type=args.analyzer, run=args.since)
0036     if os.path.exists(hashmap_db):
0037         logging.info('Hashmap sqlite file %s already exists! Deleting...' % hashmap_db)
0038         os.remove(hashmap_db)
0039     replace_dict = {'_CFGLINES_' : cfglines.replace('\\', ''),
0040                     '_ANALYZER_' : args.analyzer,
0041                     '_USEANALYSIS_':'False',
0042                     '_CONDDB_'   : args.condDbRead,
0043                     '_DBFILE_'   : 'sqlite:///%s' % output_db,
0044                     '_TARGETTAG_': args.inputTag,
0045                     '_RUNNUMBER_': args.since,
0046                     '_HASHMAPDB_': args.hashmapDb,
0047                     '_MAPDBFILE_': 'sqlite:///%s' % hashmap_db,
0048                     '_SKIPPED_'  : '',
0049                     '_WHITELISTED_': '',
0050                     }
0051     if args.analyzer == 'SiStripO2OApvGain':
0052         # special treatment for G1 O2O
0053         skipped = ''
0054         if args.skiplistFile:
0055             with open(args.skiplistFile) as skipfile:
0056                 skipped = skipfile.read()
0057         else:
0058             logging.warning('Skipped module list not provided! No module will be skipped...')
0059         whitelisted = ''
0060         if args.whitelistFile:
0061             with open(args.whitelistFile) as wfile:
0062                 whitelisted = wfile.read()
0063         else:
0064             logging.warning('Module whitelist not provided!')
0065         replace_dict['_USEANALYSIS_'] = 'True'
0066         replace_dict['_SKIPPED_'] = skipped
0067         replace_dict['_WHITELISTED_'] = whitelisted
0068         replace_dict['_HASHMAPDB_'] = ''
0069         replace_dict['_MAPDBFILE_'] = ''
0070
0071     # find template cfg file
0072     for basedir in os.environ['CMSSW_SEARCH_PATH'].split(':'):
0073         templatefile = os.path.join(basedir, cfg_template)
0074         if os.path.exists(templatefile):
0075             logging.info('Use template config file %s' % templatefile)
0076             break
0077     config = helper.insert_to_file(templatefile, job_file, replace_dict)
0078     logging.info('Start running O2O...')
0079     logging.debug(' ... config:\n%s\n' % config)
0080
0081     # run cmssw job: raise error if failed
0082     command = 'cmsRun %s' % job_file
0083     pipe = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
0084     atexit.register(partial(helper.kill_subproc_noexcept, pipe))
0085     out = pipe.communicate()[0]
0086     if isinstance(out, bytes):
0087         out = out.decode("utf8", "replace")
0088     logging.info('\n%s\n' % out)
0089     logging.info('@@@CMSSW job return code = %d@@@' % pipe.returncode)
0090     if pipe.returncode != 0:
0091         raise RuntimeError('O2O job FAILED!')
0092
0093     # upload: raise error if failed
0094     if args.no_upload:
0095         logging.info('Will not run uploading as requested!')
0096         return
0097     if args.use_uploader:
0098         f = helper.upload_payload
0099     else:
0100         f = helper.copy_payload
0101     f(dbFile=output_db, inputTag=args.inputTag, destTags=args.destTags, destDb=args.destDb, since=args.since,
0102       userText='{type}, run: {run}'.format(type=args.analyzer, run=args.since))
0103
0104     # post O2O tasks: bookkeeping for fast O2O or G1 O2O
0105     if args.analyzer == 'SiStripO2OApvGain':
0106         logging.info('Writting bookkeeping info to database.')
0107         dbmgr = DbManagerGain(args.bookkeeping_db)
0108         dbmgr.update_gain_logs(args.since, job_file)
0109     else:
0110         logging.info('Updating config-to-payload hash map to database.')
0111         dbmgr = DbManagerDAQ(args.bookkeeping_db)
0112         dbmgr.update_hashmap(hashmap_db)
0113
0114     # clean up
0115     try:
0116         os.remove(output_db)
0117         os.remove(output_db.replace('.db', '.txt'))
0118         os.remove(hashmap_db)  # may not exist
0119     except OSError:
0120         pass
0121
0122 def main():
0123     parser = argparse.ArgumentParser(description='Run a single O2O job for SiStrip DAQ and upload the payloads to condition database.')
0124     parser.add_argument('analyzer', metavar='ANALYZER', help='Which EDAnalyzer to use to create the payload.')
0125     parser.add_argument('since', metavar='SINCE', type=str, help='Run number.')
0126     parser.add_argument('cfgfile', metavar='CFGLINES', help='File containing configuration lines.')
0127     parser.add_argument('--destTags', required=True, help='Destination tag name(s) for upload. Use comma to separate multiple values.')
0128     parser.add_argument('--destDb', required=True, help='Destination DB to upload.')
0129     parser.add_argument('--inputTag', required=True, help='Tag name to be used in the sqlite file.')
0130     parser.add_argument('--condDbRead', default='oracle://cms_orcon_prod/CMS_CONDITIONS', help='Connection string for the DB from which the fast O2O retrives payloads.')
0131     parser.add_argument('--hashmapDb', default='', help='DB to read and write config-to-payload hash (for fast O2O).')
0132     parser.add_argument('--skiplistFile', default='', help='File containing the devices to be skipped in G1 O2O.')
0133     parser.add_argument('--whitelistFile', default='', help='File of the whitelisted devices in G1 O2O.')
0134
0135     parser.add_argument('--no-upload', action="store_true", default=False, help='Do not upload payload. Default: %(default)s.')
0136     parser.add_argument('--use-uploader', action="store_true", default=False, help='Use conditionUploader instead of conddb copy. Default: %(default)s.')
0137     parser.add_argument('--bookkeeping-db', default='prod', choices=['prod', 'dev', 'private'], help='Bookkeeping database for fast O2O and G1 O2O. Default: %(default)s.')
0138     parser.add_argument('--debug', action="store_true", default=False, help='Switch on debug mode. Default: %(default)s.')
0139     args = parser.parse_args()
0140
0141     loglevel = logging.INFO
0142     if args.debug:
0143         loglevel = logging.DEBUG
0144         if args.bookkeeping_db == 'prod':
0145             args.bookkeeping_db = 'dev'
0146     logging.basicConfig(level=loglevel, format='[%(asctime)s] %(levelname)s: %(message)s')
0147
0148     if not args.since.isdigit():
0149         raise RuntimeError('Since (=%s) must be a valid run number!'%(args.since))
0150
0151     args.destTags = args.destTags.strip().split(',')
0152
0153     logging.info('Running O2O %s on machine [%s]' % (args.analyzer, socket.gethostname()))
0154
0155     try:
0156         jobdirbase = os.environ[jobDirVar]
0157     except KeyError:
0158         jobdirbase = '/tmp'
0159         logging.warning('%s not set in env, will use %s' % (jobDirVar, jobdirbase))
0160
0161     # change filepaths in args to abs path
0162     args.cfgfile = os.path.abspath(args.cfgfile)
0163     if args.skiplistFile:
0164         args.skiplistFile = os.path.abspath(args.skiplistFile)
0165     if args.whitelistFile:
0166         args.whitelistFile = os.path.abspath(args.whitelistFile)
0167
0168     # change to O2O working directory
0169     jobdir = os.path.join(jobdirbase, args.since, args.analyzer)
0170     if not os.path.exists(jobdir):
0171         os.makedirs(jobdir)
0172     os.chdir(jobdir)
0173     logging.info('Running O2O in %s' % jobdir)
0174
0175     # run job and upload
0176     runjob(args)
0177
0178     logging.info('Done!')
0179
0180
0181
0182 if __name__ == '__main__':
0183     main()