Utilities/python/popcon2dropbox.py

0001 from __future__ import print_function
0002 import subprocess
0003 import json
0004 import netrc
0005 import sqlite3
0006 import os
0007 import sys
0008 import shutil
0009 import logging
0010 from datetime import datetime
0011
0012 errorInImportFileFolder = 'import_errors'
0013 dateformatForFolder = "%Y-%m-%d-%H-%M-%S"
0014 dateformatForLabel = "%Y-%m-%d %H:%M:%S"
0015
0016 auth_path_key = 'COND_AUTH_PATH'
0017
0018 messageLevelEnvVar = 'POPCON_LOG_LEVEL'
0019 fmt_str = "[%(asctime)s] %(levelname)s: %(message)s"
0020 logLevel = logging.INFO
0021 if messageLevelEnvVar in os.environ:
0022     levStr = os.environ[messageLevelEnvVar]
0023     if levStr == 'DEBUG':
0024         logLevel = logging.DEBUG
0025 logFormatter = logging.Formatter(fmt_str)
0026 logger = logging.getLogger()
0027 logger.setLevel(logLevel)
0028 consoleHandler = logging.StreamHandler(sys.stdout)
0029 consoleHandler.setFormatter(logFormatter)
0030 logger.addHandler(consoleHandler)
0031
0032 def checkFile( dbName ):
0033     dbFileName = '%s.db' %dbName
0034     # check if the expected input file is there...
0035     # exit code < 0 => error
0036     # exit code = 0 => skip
0037     # exit code = 1 => import
0038     if not os.path.exists( dbFileName ):
0039        logger.error('The file generated by PopCon with the data to be imported %s has not been found.'%dbFileName )
0040        return -1
0041
0042     empty = True
0043     try:
0044        dbcon = sqlite3.connect( dbFileName )
0045        dbcur = dbcon.cursor()
0046        dbcur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='IOV'");
0047        if dbcur.fetchone() is None:
0048            logger.error('The condition database with the data to be imported has not been found in the file generated by PopCon: %s'%dbFileName )
0049            return -1
0050        dbcur.execute('SELECT * FROM IOV')
0051        rows = dbcur.fetchall()
0052        for r in rows:
0053           empty = False
0054        dbcon.close()
0055        if empty:
0056            logger.warning('The data set generated by PopCon contains no data to be imported. The import will be skipped.')
0057            return 0
0058        return 1
0059     except Exception as e:
0060        logger.error('Check on input data failed: %s' %str(e))
0061        return -2
0062
0063 def saveFileForImportErrors( datef, dbName, withMetadata=False ):
0064     # save a copy of the files in case of upload failure...
0065     leafFolderName = datef.strftime(dateformatForFolder)
0066     fileFolder = os.path.join( errorInImportFileFolder, leafFolderName)
0067     if not os.path.exists(fileFolder):
0068         os.makedirs(fileFolder)
0069     df= '%s.db' %dbName
0070     dataDestFile = os.path.join( fileFolder, df)
0071     if not os.path.exists(dataDestFile):
0072         shutil.copy2(df, dataDestFile)
0073     if withMetadata:
0074         mf= '%s.txt' %dbName
0075         metadataDestFile = os.path.join( fileFolder, mf )
0076         if not os.path.exists(metadataDestFile):
0077             shutil.copy2(df, metadataDestFile)
0078     logger.error("Upload failed. Data file and metadata saved in folder '%s'" %os.path.abspath(fileFolder))
0079
0080 def upload( args, dbName ):
0081     destDb = args.destDb
0082     destTag = args.destTag
0083     comment = args.comment
0084
0085     datef = datetime.now()
0086
0087     # first remove any existing metadata file...
0088     if os.path.exists( '%s.txt' %dbName ):
0089        logger.debug('Removing already existing file %s' %dbName)
0090        os.remove( '%s.txt' %dbName )
0091
0092     # dump Metadata for the Upload
0093     uploadMd = {}
0094     uploadMd['destinationDatabase'] = destDb
0095     tags = {}
0096     tagInfo = {}
0097     tags[ destTag ] = tagInfo
0098     uploadMd['destinationTags'] = tags
0099     uploadMd['inputTag'] = destTag
0100     uploadMd['since'] = None
0101     datelabel = datef.strftime(dateformatForLabel)
0102     commentStr = ''
0103     if not comment is None:
0104        commentStr = comment
0105     uploadMd['userText'] = '%s : %s' %(datelabel,commentStr)
0106     with open( '%s.txt' %dbName, 'wb') as jf:
0107        jf.write( json.dumps( uploadMd, sort_keys=True, indent = 2 ) )
0108        jf.write('\n')
0109
0110     # run the upload
0111     uploadCommand = 'uploadConditions.py %s' %dbName
0112     ret = 0
0113     try:
0114        pipe = subprocess.Popen( uploadCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
0115        stdout = pipe.communicate()[0]
0116        print(stdout)
0117        retCode = pipe.returncode
0118        if retCode != 0:
0119            saveFileForImportErrors( datef, dbName, True )
0120        ret |= retCode
0121     except Exception as e:
0122        ret |= 1
0123        logger.error(str(e))
0124     return ret
0125
0126 def copy( args, dbName ):
0127     dbFileName = '%s.db' %dbName
0128     destDb = args.destDb
0129     destTag = args.destTag
0130     comment = args.comment
0131
0132     datef = datetime.now()
0133     destMap = { "oracle://cms_orcoff_prep/cms_conditions": "oradev", "oracle://cms_orcon_prod/cms_conditions": "onlineorapro"  }
0134     if destDb.lower() in destMap.keys():
0135         destDb = destMap[destDb.lower()]
0136     else:
0137         if destDb.startswith('sqlite'):
0138             destDb = destDb.split(':')[1]
0139         else:
0140             logger.error( 'Destination connection %s is not supported.' %destDb )
0141             return
0142     # run the copy
0143     note = '"Importing data with O2O execution"'
0144     commandOptions = '--force --yes --db %s copy %s %s --destdb %s --synchronize --note %s' %(dbFileName,destTag,destTag,destDb,note)
0145     copyCommand = 'conddb %s' %commandOptions
0146     logger.info( 'Executing command: %s' %copyCommand )
0147     try:
0148         pipe = subprocess.Popen( copyCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
0149         stdout = pipe.communicate()[0]
0150         print(stdout)
0151         retCode = pipe.returncode
0152         if retCode != 0:
0153             saveFileForImportErrors( datef, dbName )
0154         ret = retCode
0155     except Exception as e:
0156         ret = 1
0157         logger.error( str(e) )
0158     return ret
0159
0160 def run( args ):
0161
0162     dbName = datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S-%f')
0163     dbFileName = '%s.db' %dbName
0164
0165     if args.auth is not None and not args.auth=='':
0166         if auth_path_key in os.environ:
0167             logger.warning("Cannot set authentication path to %s in the environment, since it is already set." %args.auth)
0168         else:
0169             logger.info("Setting the authentication path to %s in the environment." %args.auth)
0170             os.environ[auth_path_key]=args.auth
0171     if os.path.exists( '%s.db' %dbName ):
0172        logger.info("Removing files with name %s" %dbName )
0173        os.remove( '%s.db' %dbName )
0174     if os.path.exists( '%s.txt' %dbName ):
0175        os.remove( '%s.txt' %dbName )
0176     command = 'cmsRun %s ' %args.job_file
0177     command += ' targetFile=%s' %dbFileName
0178     command += ' destinationDatabase=%s' %args.destDb
0179     command += ' destinationTag=%s' %args.destTag
0180     command += ' 2>&1'
0181     logger.info( 'Executing command: %s' %command )
0182     pipe = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
0183     stdout = pipe.communicate()[0]
0184     retCode = pipe.returncode
0185     print(stdout)
0186     logger.info('PopCon Analyzer return code is: %s' %retCode )
0187     if retCode!=0:
0188        logger.error( 'O2O job failed. Skipping upload.' )
0189        return retCode
0190
0191     ret = checkFile( dbName )
0192     if ret > 0:
0193         if args.copy:
0194             ret = copy( args, dbName )
0195         else:
0196             ret = upload( args, dbName )
0197     if ret >=0:
0198         logger.info('Deleting local file %s.db' %dbName )
0199         os.remove( '%s.db' %dbName )
0200     return ret