SiStrip/python/o2o_helper.py

0001 '''
0002 Helper Script for StripO2O
0003 @author: Huilin Qu
0004 '''
0005
0006 import os
0007 import subprocess
0008 import logging
0009 import json
0010 import smtplib
0011 from email.mime.multipart import MIMEMultipart
0012 from email.mime.text import MIMEText
0013 import sqlite3
0014
0015 def kill_subproc_noexcept(p):
0016     '''Kill a subprocess without throwing OSError.
0017     Used for cleaning up subprocesses when the main script crashes.'''
0018     try:
0019         p.terminate()
0020     except OSError:
0021         pass
0022
0023
0024 def configLogger(logfile,loglevel=logging.INFO):
0025     '''Setting up logging to both file and console.
0026     @see: https://docs.python.org/2/howto/logging-cookbook.html
0027     '''
0028     # set up logging to file
0029     logging.basicConfig(level=loglevel,
0030                         format='[%(asctime)s] %(levelname)s: %(message)s',
0031                         filename=logfile,
0032                         filemode='w')
0033     # define a Handler which writes INFO messages or higher to the sys.stderr
0034     console = logging.StreamHandler()
0035     console.setLevel(loglevel)
0036     # set a format which is simpler for console use
0037     formatter = logging.Formatter('[%(levelname)s] %(message)s')
0038     # tell the handler to use this format
0039     console.setFormatter(formatter)
0040     # add the handler to the root logger
0041     logging.getLogger('').addHandler(console)
0042
0043 def insert_to_file(template, target, replace_dict):
0044     '''Update the template file based on the replace_dict, and write to the target.'''
0045     logging.debug('Creating "%s" from template "%s" using dictionary:'%(target, template))
0046     logging.debug(replace_dict)
0047     with open(template, 'r') as input_file:
0048         config=input_file.read()
0049     with open(target, 'w') as output_file:
0050         for key, value in replace_dict.items():
0051             config = config.replace(key, value)
0052         output_file.write(config)
0053     return config
0054
0055 def create_metadata(metadataFilename, inputTag, destTags, destDb, since, userText):
0056     '''Create metadata file for the conditionsUpload service.
0057     @see: uploadConditions.runWizard()
0058     @see: https://twiki.cern.ch/twiki/bin/view/CMS/DropBox
0059
0060     Keyword arguments:
0061     metadataFilename -- output metadata filename
0062     inputTag -- input tag name
0063     destTags -- a list of destination tags
0064     destDb -- [destinationDatabase] in metadata
0065     since -- [since] in metadata
0066     userText -- [userText] in metadata
0067     '''
0068     if isinstance(destTags, str):
0069         destTags = [destTags]
0070     if since:
0071         since = int(since)  # convert to int if since is not None (input since can be a str)
0072     destinationTags = {}
0073     for destinationTag in destTags:
0074         destinationTags[destinationTag] = {}
0075     metadata = {
0076         'destinationDatabase': destDb,
0077         'destinationTags': destinationTags,
0078         'inputTag': inputTag,
0079         'since': since,
0080         'userText': userText,
0081     }
0082     logging.info('Writing metadata in %s', metadataFilename)
0083     logging.debug(metadata)
0084     with open(metadataFilename, 'wb') as metadataFile:
0085         metadataFile.write(json.dumps(metadata, sort_keys=True, indent=4))
0086
0087 def upload_payload(dbFile, inputTag, destTags, destDb, since, userText):
0088     '''Upload payload using conditionUploader. '''
0089     if isinstance(destTags, str):
0090         destTags = [destTags]
0091     metadataFilename = dbFile.replace('.db', '.txt')
0092     create_metadata(metadataFilename, inputTag, destTags, destDb, since, userText)
0093     logging.info('Uploading tag [%s] from %s to [%s] in %s:' % (inputTag, dbFile, ','.join(destTags), destDb))
0094     command = "uploadConditions.py %s" % dbFile
0095     pipe = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
0096     out = pipe.communicate()[0]
0097     logging.info(out)
0098     logging.info('@@@Upload return code = %d@@@' % pipe.returncode)
0099     if pipe.returncode != 0:
0100         raise RuntimeError('Upload FAILED!')
0101
0102
0103 def copy_payload(dbFile, inputTag, destTags, destDb, since, userText):
0104     '''Upload payload using conddb copy.'''
0105     if isinstance(destTags, str):
0106         destTags = [destTags]
0107     if destDb.lower() == 'oracle://cms_orcon_prod/cms_conditions':
0108         copyDestDb = 'onlineorapro'
0109     elif destDb.lower() == 'oracle://cms_orcoff_prep/cms_conditions':
0110         copyDestDb = 'oradev'
0111     else:
0112         copyDestDb = destDb
0113     success = 0
0114     def copy(dest):
0115         command = 'conddb --force --yes --db {db} copy {inputTag} {destTag} --destdb {destDb} --synchronize --note "{note}"'.format(
0116             db=dbFile, inputTag=inputTag, destTag=dest, destDb=copyDestDb, note=userText)
0117         logging.info('Copy tag [%s] from %s to [%s] in %s:' % (inputTag, dbFile, dest, destDb))
0118         logging.debug(command)
0119         pipe = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
0120         out = pipe.communicate()[0]
0121         logging.info(out)
0122         return pipe.returncode
0123     for dest in destTags:
0124         returncode = copy(dest)
0125         if returncode == 0: success += 1
0126     logging.info('@@@Upload return code = %d@@@' % (success - len(destTags)))
0127     if success != len(destTags):
0128         raise RuntimeError('Upload FAILED!')
0129
0130
0131 def send_mail(subject, message, send_to, send_from, text_attachments=[]):
0132     '''Send an email. [send_to] needs to be a list.'''
0133     msg = MIMEMultipart()
0134     msg['Subject'] = subject
0135     msg['From'] = send_from
0136     msg['To'] = ','.join(send_to)
0137     msg.attach(MIMEText(message))
0138
0139     for fn in text_attachments:
0140         with open(fn, 'rb') as txtfile:
0141             attachment = MIMEText(txtfile.read())
0142             attachment.add_header('Content-Disposition', 'attachment', filename=os.path.basename(fn))
0143             msg.attach(attachment)
0144
0145     s = smtplib.SMTP('localhost')
0146     s.sendmail(send_from, send_to, msg.as_string())
0147     s.quit()
0148
0149 def exists_iov(dbFile, tag):
0150     '''Check if there exists any IOV for a specific tag in the given sqlite file.'''
0151     dataConnection = sqlite3.connect(dbFile)
0152     dataCursor = dataConnection.cursor()
0153     dataCursor.execute('select SINCE from IOV where TAG_NAME=:tag_name', {'tag_name' : tag})
0154     return len(dataCursor.fetchall()) > 0