SiStrip/scripts/o2oRun_SiStripDAQ.py

0001 #!/usr/bin/env python3
0002 '''
0003 Main Script to run all SiStrip DAQ O2Os at the same time.
0004 @author: Huilin Qu
0005 '''
0006
0007 import os
0008 import sys
0009 import atexit
0010 import logging
0011 import argparse
0012 import subprocess
0013 import traceback
0014 import json
0015 from functools import partial
0016
0017 import CondTools.SiStrip.o2o_helper as helper
0018
0019 logDirVar = 'O2O_LOG_FOLDER'
0020
0021 def run(args):
0022     logging.debug(args)
0023
0024     is_ok = True
0025     status = {}
0026     processes = {}
0027
0028     for analyzer in args.analyzers:
0029         o2ocmd = 'SiStripDAQPopCon.py {analyzer} {since} {cfgfile}'.format(
0030             analyzer=analyzer, since=args.since, cfgfile=args.cfgfile)
0031         o2ocmd += ' --destTags {destTags}'
0032         o2ocmd += ' --destDb {destDb}'
0033         o2ocmd += ' --inputTag {inputTag}'
0034         o2ocmd += ' --condDbRead {condDbRead}'
0035         o2ocmd += ' --hashmapDb {hashmapDb}'
0036         if args.skiplistFile:
0037             o2ocmd += ' --skiplistFile %s' % args.skiplistFile
0038         if args.whitelistFile:
0039             o2ocmd += ' --whitelistFile %s' % args.whitelistFile
0040         if args.debug:
0041             o2ocmd += ' --debug'
0042
0043         jobname = analyzer.replace('O2O', '')
0044         cmd = 'o2o --db {db} -v run -n {jobname} "{o2ocmd}"'.format(db=args.db, jobname=jobname, o2ocmd=o2ocmd)
0045         logging.info('Start running command:\n %s' % cmd)
0046
0047         processes[jobname] = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
0048         atexit.register(partial(helper.kill_subproc_noexcept, processes[jobname]))
0049
0050     for jobname in processes:
0051         status[jobname] = {'job':None, 'upload':None, 'fast':None, 'changed':None}
0052         p = processes[jobname]
0053         log = p.communicate()[0].decode()
0054         logging.debug('=== log from o2o run ===\n%s' % log)
0055         if p.returncode == 0:
0056             logging.info('Job for %s finished successfully!' % jobname)
0057             status[jobname]['job'] = True
0058             status[jobname]['upload'] = True
0059             for line in log.split('\n'):
0060                 if '@@@' not in line:
0061                     continue
0062                 if 'FastO2O' in line:
0063                     status[jobname]['fast'] = ('true' in line)
0064                 if 'PayloadChange' in line:
0065                     status[jobname]['changed'] = ('true' in line)
0066         else:
0067             logging.error('Job %s FAILED!' % jobname)
0068             status[jobname]['job'] = '@@@CMSSW job return code = 0@@@' in log
0069             status[jobname]['upload'] = '@@@Upload return code = 0@@@' in log
0070             is_ok = False
0071
0072     return is_ok, status
0073
0074 def summary(args, is_ok, status, logfile):
0075     summary = json.dumps(status, sort_keys=True, indent=2)
0076     if is_ok:
0077         logging.info('O2O finished successfully! Summary: %s' % summary)
0078     else:
0079         logging.error('O2O FAILED! Summary: %s' % summary)
0080
0081     debugLabel = '[TEST] ' if args.debug else ''
0082
0083     # send the summary email
0084     helper.send_mail(subject='%sNew O2O, IOV: %s' % (debugLabel, args.since),
0085              message=summary,
0086              send_to=args.mail_to,
0087              send_from=args.mail_from)
0088     # send the detailed log
0089     with open(logfile, 'r') as log:
0090         helper.send_mail(subject='%sNew O2O Log, IOV: %s' % (debugLabel, args.since),
0091                  message=log.read(),
0092                  send_to=args.mail_log_to,
0093                  send_from=args.mail_from)
0094
0095
0096 def main():
0097     parser = argparse.ArgumentParser(description='Run all SiStrip DAQ O2Os at the same time.')
0098     parser.add_argument('since', metavar='SINCE', type=str, help='Run number.')
0099     parser.add_argument('cfgfile', metavar='CFGLINES', help='File containing configuration lines.')
0100     parser.add_argument('--skiplistFile', default='', help='File containing the devices to be skipped in G1 O2O.')
0101     parser.add_argument('--whitelistFile', default='', help='File of the whitelisted devices in G1 O2O.')
0102
0103     parser.add_argument('--analyzers',
0104                         default='SiStripO2OBadStrip,SiStripO2OFedCabling,SiStripO2OLatency,SiStripO2ONoises,SiStripO2OPedestals,SiStripO2OThreshold',
0105                         help='Which EDAnalyzers to run.')
0106     parser.add_argument('--mail-from', default='trk.o2o@cern.ch', help='Account to send email notification.')
0107     parser.add_argument('--mail-to', default='cms-tracker-o2o-notification@cern.ch', help='List of O2O notification recipients.')
0108     parser.add_argument('--mail-log-to', default='trk.o2o@cern.ch', help='List of O2O log recipients.')
0109     parser.add_argument('--db', default='pro', help='The database for o2o job management: pro ( for prod ) or dev ( for prep ). Default: %(default)s.')
0110     parser.add_argument('--debug', action="store_true", default=False, help='Switch on debug mode. Default: %(default)s.')
0111
0112     args = parser.parse_args()
0113     if args.debug:
0114         args.mail_to = args.mail_log_to
0115
0116     args.analyzers = args.analyzers.strip().split(',')
0117     args.mail_to = args.mail_to.strip().split(',')
0118     args.mail_log_to = args.mail_log_to.strip().split(',')
0119
0120     # Should NOT use logging before it's set up
0121     try:
0122         logdir = os.environ[logDirVar] if logDirVar in os.environ else '/tmp'
0123         if not os.path.exists(logdir):
0124             os.makedirs(logdir)
0125         logfile = os.path.join(logdir, 'SiStripsO2O_Run%s.log' % str(args.since))
0126         loglevel = logging.DEBUG if args.debug else logging.INFO
0127         helper.configLogger(logfile, loglevel)
0128     except Exception:
0129         # in case we failed before logging is set up
0130         # print the error, send an email, and exit
0131         helper.send_mail('O2O Failure, IOV: %s' % args.since, traceback.format_exc(), args.mail_to, args.mail_from)
0132         raise
0133
0134     try:
0135         is_ok, status = run(args)
0136         summary(args, is_ok, status, logfile)
0137     except Exception:
0138         # in case we failed before logging is set up
0139         # print the error, send an email, and exit
0140         helper.send_mail('O2O Failure, IOV: %s' % args.since, traceback.format_exc(), args.mail_to, args.mail_from)
0141         raise
0142
0143     if not is_ok:
0144         return ' --- O2O FAILED! ---'
0145
0146 if __name__ == '__main__':
0147     sys.exit(main())