Utilities/scripts/uploadConditions.py

0001 #!/usr/bin/env python3
0002 '''Script that uploads to the new CMS conditions uploader.
0003 Adapted to the new infrastructure from v6 of the upload.py script for the DropBox from Miguel Ojeda.
0004 '''
0005 from __future__ import print_function
0006
0007 __author__ = 'Andreas Pfeiffer'
0008 __copyright__ = 'Copyright 2015, CERN CMS'
0009 __credits__ = ['Giacomo Govi', 'Salvatore Di Guida', 'Miguel Ojeda', 'Andreas Pfeiffer']
0010 __license__ = 'Unknown'
0011 __maintainer__ = 'Giacomo Govi'
0012 __email__ = 'giacomo.govi@cern.ch'
0013 __version__ = 1
0014
0015
0016 import os
0017 import sys
0018 import optparse
0019 import hashlib
0020 import tarfile
0021 import netrc
0022 import getpass
0023 import errno
0024 import sqlite3
0025 import cx_Oracle
0026 import json
0027 import tempfile
0028 from datetime import datetime
0029
0030 defaultBackend = 'online'
0031 defaultHostname = 'cms-conddb-prod.cern.ch'
0032 defaultDevHostname = 'cms-conddb-dev.cern.ch'
0033 defaultUrlTemplate = 'https://%s/cmsDbUpload/'
0034 defaultTemporaryFile = 'upload.tar.bz2'
0035 defaultNetrcHost = 'ConditionUploader'
0036 defaultWorkflow = 'offline'
0037 prodLogDbSrv = 'cms_orcoff_prod'
0038 devLogDbSrv = 'cms_orcoff_prep'
0039 logDbSchema = 'CMS_COND_DROPBOX'
0040 authPathEnvVar = 'COND_AUTH_PATH'
0041 waitForRetry = 15
0042
0043 # common/http.py start (plus the "# Try to extract..." section bit)
0044 import time
0045 import logging
0046 import io
0047
0048 import pycurl
0049 import socket
0050 import copy
0051
0052 def getInput(default, prompt = ''):
0053     '''Like input() but with a default and automatic strip().
0054     '''
0055
0056     answer = input(prompt)
0057     if answer:
0058         return answer.strip()
0059
0060     return default.strip()
0061
0062
0063 def getInputWorkflow(prompt = ''):
0064     '''Like getInput() but tailored to get target workflows (synchronization options).
0065     '''
0066
0067     while True:
0068         workflow = getInput(defaultWorkflow, prompt)
0069
0070         if workflow in frozenset(['offline', 'hlt', 'express', 'prompt', 'pcl']):
0071             return workflow
0072
0073         logging.error('Please specify one of the allowed workflows. See above for the explanation on each of them.')
0074
0075
0076 def getInputChoose(optionsList, default, prompt = ''):
0077     '''Makes the user choose from a list of options.
0078     '''
0079
0080     while True:
0081         index = getInput(default, prompt)
0082
0083         try:
0084             return optionsList[int(index)]
0085         except ValueError:
0086             logging.error('Please specify an index of the list (i.e. integer).')
0087         except IndexError:
0088             logging.error('The index you provided is not in the given list.')
0089
0090
0091 def getInputRepeat(prompt = ''):
0092     '''Like input() but repeats if nothing is provided and automatic strip().
0093     '''
0094
0095     while True:
0096         answer = input(prompt)
0097         if answer:
0098             return answer.strip()
0099
0100         logging.error('You need to provide a value.')
0101
0102
0103 def runWizard(basename, dataFilename, metadataFilename):
0104     while True:
0105         print('''\nWizard for metadata for %s
0106
0107 I will ask you some questions to fill the metadata file. For some of the questions there are defaults between square brackets (i.e. []), leave empty (i.e. hit Enter) to use them.''' % basename)
0108
0109         # Try to get the available inputTags
0110         dataConnection = sqlite3.connect(dataFilename)
0111         dataCursor = dataConnection.cursor()
0112
0113         dataCursor.execute('select NAME from TAG')
0114         records = dataCursor.fetchall()
0115         inputTags = []
0116         for rec in records:
0117             inputTags.append(rec[0])
0118
0119         if len(inputTags) == 0:
0120             raise Exception("Could not find any input tag in the data file.")
0121
0122         else:
0123             print('\nI found the following input tags in your SQLite data file:')
0124             for (index, inputTag) in enumerate(inputTags):
0125                 print('   %s) %s' % (index, inputTag))
0126
0127             inputTag = getInputChoose(inputTags, '0',
0128                                       '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. 0 (you select the first in the list)\ninputTag [0]: ')
0129
0130         destinationDatabase = ''
0131         ntry = 0
0132         while ( destinationDatabase != 'oracle://cms_orcon_prod/CMS_CONDITIONS' and destinationDatabase != 'oracle://cms_orcoff_prep/CMS_CONDITIONS' ):
0133             if ntry==0:
0134                 inputMessage = \
0135                 '\nWhich is the destination database where the tags should be exported? \nPossible choices: oracle://cms_orcon_prod/CMS_CONDITIONS (or prod); oracle://cms_orcoff_prep/CMS_CONDITIONS (or prep) \ndestinationDatabase: '
0136             elif ntry==1:
0137                 inputMessage = \
0138                 '\nPlease choose one of the two valid destinations: \noracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \
0139 \ndestinationDatabase: '
0140             else:
0141                 raise Exception('No valid destination chosen. Bailing out...')
0142             destinationDatabase = getInputRepeat(inputMessage)
0143             if destinationDatabase == 'prod':
0144                 destinationDatabase = 'oracle://cms_orcon_prod/CMS_CONDITIONS'
0145             if destinationDatabase == 'prep':
0146                 destinationDatabase = 'oracle://cms_orcoff_prep/CMS_CONDITIONS'
0147             ntry += 1
0148
0149         while True:
0150             since = getInput('',
0151                              '\nWhich is the given since? (if not specified, the one from the SQLite data file will be taken -- note that even if specified, still this may not be the final since, depending on the synchronization options you select later: if the synchronization target is not offline, and the since you give is smaller than the next possible one (i.e. you give a run number earlier than the one which will be started/processed next in prompt/hlt/express), the DropBox will move the since ahead to go to the first safe run instead of the value you gave)\ne.g. 1234\nsince []: ')
0152             if not since:
0153                 since = None
0154                 break
0155             else:
0156                 try:
0157                     since = int(since)
0158                     break
0159                 except ValueError:
0160                     logging.error('The since value has to be an integer or empty (null).')
0161
0162         userText = getInput('',
0163                             '\nWrite any comments/text you may want to describe your request\ne.g. Muon alignment scenario for...\nuserText []: ')
0164
0165         destinationTags = {}
0166         while True:
0167             destinationTag = getInput('',
0168                                       '\nWhich is the next destination tag to be added (leave empty to stop)?\ne.g. BeamSpotObjects_PCL_byRun_v0_offline\ndestinationTag []: ')
0169             if not destinationTag:
0170                 if len(destinationTags) == 0:
0171                     logging.error('There must be at least one destination tag.')
0172                     continue
0173                 break
0174
0175             if destinationTag in destinationTags:
0176                 logging.warning(
0177                     'You already added this destination tag. Overwriting the previous one with this new one.')
0178
0179             destinationTags[destinationTag] = {
0180             }
0181
0182         metadata = {
0183             'destinationDatabase': destinationDatabase,
0184             'destinationTags': destinationTags,
0185             'inputTag': inputTag,
0186             'since': since,
0187             'userText': userText,
0188         }
0189
0190         metadata = json.dumps(metadata, sort_keys=True, indent=4)
0191         print('\nThis is the generated metadata:\n%s' % metadata)
0192
0193         if getInput('n',
0194                     '\nIs it fine (i.e. save in %s and *upload* the conditions if this is the latest file)?\nAnswer [n]: ' % metadataFilename).lower() == 'y':
0195             break
0196     logging.info('Saving generated metadata in %s...', metadataFilename)
0197     with open(metadataFilename, 'w') as metadataFile:
0198         metadataFile.write(metadata)
0199
0200 class HTTPError(Exception):
0201     '''A common HTTP exception.
0202
0203     self.code is the response HTTP code as an integer.
0204     self.response is the response body (i.e. page).
0205     '''
0206
0207     def __init__(self, code, response):
0208         self.code = code
0209         self.response = response
0210
0211         # Try to extract the error message if possible (i.e. known error page format)
0212         try:
0213             self.args = (response.split('<p>')[1].split('</p>')[0], )
0214         except Exception:
0215             self.args = (self.response, )
0216
0217
0218 CERN_SSO_CURL_CAPATH = '/etc/pki/tls/certs'
0219
0220 class HTTP(object):
0221     '''Class used for querying URLs using the HTTP protocol.
0222     '''
0223
0224     retryCodes = frozenset([502, 503])
0225
0226     def __init__(self):
0227         self.setBaseUrl()
0228         self.setRetries()
0229
0230         self.curl = pycurl.Curl()
0231         self.curl.setopt(self.curl.COOKIEFILE, '')      # in memory
0232
0233         #-toDo: make sure we have the right options set here to use ssl
0234         #-review(2015-09-25): check and see - action: AP
0235         # self.curl.setopt(self.curl.SSL_VERIFYPEER, 1)
0236         self.curl.setopt(self.curl.SSL_VERIFYPEER, 0)
0237         self.curl.setopt(self.curl.SSL_VERIFYHOST, 2)
0238
0239         self.baseUrl = None
0240
0241         self.token = None
0242
0243     def getCookies(self):
0244         '''Returns the list of cookies.
0245         '''
0246         return self.curl.getinfo(self.curl.INFO_COOKIELIST)
0247
0248     def discardCookies(self):
0249         '''Discards cookies.
0250         '''
0251         self.curl.setopt(self.curl.COOKIELIST, 'ALL')
0252
0253
0254     def setBaseUrl(self, baseUrl = ''):
0255         '''Allows to set a base URL which will be prefixed to all the URLs
0256         that will be queried later.
0257         '''
0258         self.baseUrl = baseUrl
0259
0260
0261     def setProxy(self, proxy = ''):
0262         '''Allows to set a proxy.
0263         '''
0264         self.curl.setopt(self.curl.PROXY, proxy)
0265
0266
0267     def setTimeout(self, timeout = 0):
0268         '''Allows to set a timeout.
0269         '''
0270         self.curl.setopt(self.curl.TIMEOUT, timeout)
0271
0272
0273     def setRetries(self, retries = ()):
0274         '''Allows to set retries.
0275
0276         The retries are a sequence of the seconds to wait per retry.
0277
0278         The retries are done on:
0279             * PyCurl errors (includes network problems, e.g. not being able
0280               to connect to the host).
0281             * 502 Bad Gateway (for the moment, to avoid temporary
0282               Apache-CherryPy issues).
0283             * 503 Service Temporarily Unavailable (for when we update
0284               the frontends).
0285         '''
0286         self.retries = retries
0287
0288     def getToken(self, username, password):
0289
0290         url = self.baseUrl + 'token'
0291
0292         self.curl.setopt(pycurl.URL, url)
0293         self.curl.setopt(pycurl.VERBOSE, 0)
0294
0295         #-toDo: check if/why these are needed ...
0296         #-ap: hmm ...
0297         # self.curl.setopt(pycurl.DNS_CACHE_TIMEOUT, 0)
0298         # self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
0299         #-end hmmm ...
0300         #-review(2015-09-25): check and see - action: AP
0301
0302         self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
0303         # self.curl.setopt( self.curl.POST, {})
0304         self.curl.setopt(self.curl.HTTPGET, 0)
0305
0306         response = io.BytesIO()
0307         self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
0308         self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
0309         logging.debug('going to connect to server at: %s' % url )
0310
0311         self.curl.perform()
0312         code = self.curl.getinfo(pycurl.RESPONSE_CODE)
0313         logging.debug('got: %s ', str(code))
0314         if code in ( 502,503,504 ):
0315             logging.debug('Trying again after %d seconds...', waitForRetry)
0316             time.sleep( waitForRetry )
0317             response = io.StringIO()
0318             self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
0319             self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
0320             self.curl.perform()
0321             code = self.curl.getinfo(pycurl.RESPONSE_CODE)
0322         resp = response.getvalue().decode('UTF-8')
0323         errorMsg = None
0324         if code==500 and not resp.find("INVALID_CREDENTIALS")==-1:
0325             logging.error("Invalid credentials provided.")
0326             return None
0327         if code==403 and not resp.find("Unauthorized access")==-1:
0328             logging.error("Unauthorized access. Please check the membership of group 'cms-cond-dropbox'")
0329             return None
0330         if code==200:
0331             try:
0332                 self.token = json.loads( resp )['token']
0333             except Exception as e:
0334                 errorMsg = 'Error while decoding returned json string'
0335                 logging.debug('http::getToken> error while decoding json: %s ', str(resp) )
0336                 logging.debug("error getting token: %s", str(e))
0337                 resp = None
0338         else:
0339             errorMsg = 'HTTP Error code %s ' %code
0340             logging.debug('got: %s ', str(code))
0341             logging.debug('http::getToken> got error from server: %s ', str(resp) )
0342             resp = None
0343         if resp is None:
0344             raise Exception(errorMsg)
0345
0346         logging.debug('token: %s', self.token)
0347         logging.debug('returning: %s', response.getvalue().decode('UTF-8'))
0348
0349         return response.getvalue()
0350
0351     def query(self, url, data = None, files = None, keepCookies = True):
0352         '''Queries a URL, optionally with some data (dictionary).
0353
0354         If no data is specified, a GET request will be used.
0355         If some data is specified, a POST request will be used.
0356
0357         If files is specified, it must be a dictionary like data but
0358         the values are filenames.
0359
0360         By default, cookies are kept in-between requests.
0361
0362         A HTTPError exception is raised if the response's HTTP code is not 200.
0363         '''
0364
0365         if not keepCookies:
0366             self.discardCookies()
0367
0368         url = self.baseUrl + url
0369
0370         # make sure the logs are safe ... at least somewhat :)
0371         data4log = copy.copy(data)
0372         if data4log:
0373             if 'password' in data4log.keys():
0374                 data4log['password'] = '*'
0375
0376         retries = [0] + list(self.retries)
0377
0378         while True:
0379             logging.debug('Querying %s with data %s and files %s (retries left: %s, current sleep: %s)...', url, data4log, files, len(retries), retries[0])
0380
0381             time.sleep(retries.pop(0))
0382
0383             try:
0384                 self.curl.setopt(self.curl.URL, url)
0385                 self.curl.setopt(self.curl.HTTPGET, 1)
0386
0387                 # from now on we use the token we got from the login
0388                 self.curl.setopt(pycurl.USERPWD, '%s:""' % ( str(self.token), ) )
0389                 self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
0390
0391                 if data is not None or files is not None:
0392                     # If there is data or files to send, use a POST request
0393
0394                     finalData = {}
0395
0396                     if data is not None:
0397                         finalData.update(data)
0398
0399                     if files is not None:
0400                         for (key, fileName) in files.items():
0401                             finalData[key] = (self.curl.FORM_FILE, fileName)
0402                     self.curl.setopt( self.curl.HTTPPOST, list(finalData.items()) )
0403
0404                 self.curl.setopt(pycurl.VERBOSE, 0)
0405
0406                 response = io.BytesIO()
0407                 self.curl.setopt(self.curl.WRITEFUNCTION, response.write)
0408                 self.curl.perform()
0409
0410                 code = self.curl.getinfo(self.curl.RESPONSE_CODE)
0411
0412                 if code in self.retryCodes and len(retries) > 0:
0413                     logging.debug('Retrying since we got the %s error code...', code)
0414                     continue
0415
0416                 if code != 200:
0417                     raise HTTPError(code, response.getvalue())
0418
0419                 return response.getvalue()
0420
0421             except pycurl.error as e:
0422                 if len(retries) == 0:
0423                     raise e
0424                 logging.debug('Retrying since we got the %s pycurl exception...', str(e))
0425
0426 # common/http.py end
0427
0428 def addToTarFile(tarFile, fileobj, arcname):
0429     tarInfo = tarFile.gettarinfo(fileobj = fileobj, arcname = arcname)
0430     tarInfo.mode = 0o400
0431     tarInfo.uid = tarInfo.gid = tarInfo.mtime = 0
0432     tarInfo.uname = tarInfo.gname = 'root'
0433     tarFile.addfile(tarInfo, fileobj)
0434
0435 class ConditionsUploader(object):
0436     '''Upload conditions to the CMS conditions uploader service.
0437     '''
0438
0439     def __init__(self, hostname = defaultHostname, urlTemplate = defaultUrlTemplate):
0440         self.hostname = hostname
0441         self.urlTemplate = urlTemplate
0442         self.userName = None
0443         self.http = None
0444         self.password = None
0445         self.token = None
0446
0447     def setHost( self, hostname ):
0448         if not hostname==self.hostname:
0449             self.token = None
0450             self.hostname = hostname
0451
0452     def signIn(self, username, password ):
0453         if self.token is None:
0454             logging.debug("Initializing connection with server %s",self.hostname)
0455             ''' init the server.
0456             '''
0457             self.http = HTTP()
0458             if socket.getfqdn().strip().endswith('.cms'):
0459                 self.http.setProxy('https://cmsproxy.cms:3128/')
0460             self.http.setBaseUrl(self.urlTemplate % self.hostname)
0461             '''Signs in the server.
0462             '''
0463             logging.info('%s: Signing in user %s ...', self.hostname, username)
0464             try:
0465                 self.token = self.http.getToken(username, password)
0466             except Exception as e:
0467                 ret = -1
0468                 # optionally, we may want to have a different return for network related errors:
0469                 #code = self.http.curl.getinfo(pycurl.RESPONSE_CODE)
0470                 #if code in ( 502,503,504 ):
0471                 #    ret = -10
0472                 logging.error("Caught exception when trying to connect to %s: %s" % (self.hostname, str(e)) )
0473                 return ret
0474
0475             if not self.token:
0476                 logging.error("could not get token for user %s from %s" % (username, self.hostname) )
0477                 return -2
0478             logging.debug( "got: '%s'", str(self.token) )
0479             self.userName = username
0480             self.password = password
0481         else:
0482             logging.debug("User %s has been already authenticated." %username)
0483         return 0
0484
0485     def signOut(self):
0486         '''Signs out the server.
0487         '''
0488
0489         logging.info('%s: Signing out...', self.hostname)
0490         # self.http.query('logout')
0491         self.token = None
0492
0493
0494     def _checkForUpdates(self):
0495         '''Updates this script, if a new version is found.
0496         '''
0497
0498         logging.debug('%s: Checking if a newer version of this script is available ...', self.hostname)
0499         version = int(self.http.query('getUploadScriptVersion'))
0500
0501         if version <= __version__:
0502             logging.debug('%s: Script is up-to-date.', self.hostname)
0503             return
0504
0505         logging.info('%s: Updating to a newer version (%s) than the current one (%s): downloading ...', self.hostname, version, __version__)
0506
0507         uploadScript = self.http.query('getUploadScript')
0508
0509         self.signOut()
0510
0511         logging.info('%s: ... saving the new version ...', self.hostname)
0512         with open(sys.argv[0], 'wb') as f:
0513             f.write(uploadScript)
0514
0515         logging.info('%s: ... executing the new version...', self.hostname)
0516         os.execl(sys.executable, *([sys.executable] + sys.argv))
0517
0518
0519     def uploadFile(self, filename, backend = defaultBackend, temporaryFile = defaultTemporaryFile):
0520         '''Uploads a file to the dropBox.
0521
0522         The filename can be without extension, with .db or with .txt extension.
0523         It will be stripped and then both .db and .txt files are used.
0524         '''
0525
0526         basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
0527         basename = os.path.basename(basepath)
0528
0529         logging.debug('%s: %s: Creating tar file for upload ...', self.hostname, basename)
0530
0531         try:
0532             tarFile = tarfile.open(temporaryFile, 'w:bz2')
0533
0534             with open('%s.db' % basepath, 'rb') as data:
0535                 addToTarFile(tarFile, data, 'data.db')
0536         except Exception as e:
0537             msg = 'Error when creating tar file. \n'
0538             msg += 'Please check that you have write access to the directory you are running,\n'
0539             msg += 'and that you have enough space on this disk (df -h .)\n'
0540             logging.error(msg)
0541             raise Exception(msg)
0542
0543         with tempfile.NamedTemporaryFile(mode='rb+') as metadata:
0544             with open('%s.txt' % basepath, 'r') as originalMetadata:
0545                 metadata.write(json.dumps(json.load(originalMetadata), sort_keys = True, indent = 4).encode())
0546
0547             metadata.seek(0)
0548             addToTarFile(tarFile, metadata, 'metadata.txt')
0549
0550         tarFile.close()
0551
0552         logging.debug('%s: %s: Calculating hash...', self.hostname, basename)
0553
0554         fileHash = hashlib.sha1()
0555         with open(temporaryFile, 'rb') as f:
0556             while True:
0557                 data = f.read(4 * 1024 * 1024)
0558                 if not data:
0559                     break
0560                 fileHash.update(data)
0561
0562         fileHash = fileHash.hexdigest()
0563         fileInfo = os.stat(temporaryFile)
0564         fileSize = fileInfo.st_size
0565
0566         logging.debug('%s: %s: Hash: %s', self.hostname, basename, fileHash)
0567
0568         logging.info('%s: %s: Uploading file (%s, size %s) to the %s backend...', self.hostname, basename, fileHash, fileSize, backend)
0569         os.rename(temporaryFile, fileHash)
0570         try:
0571             ret = self.http.query('uploadFile',
0572                               {
0573                                 'backend': backend,
0574                                 'fileName': basename,
0575                                 'userName': self.userName,
0576                               },
0577                               files = {
0578                                         'uploadedFile': fileHash,
0579                                       }
0580                               )
0581         except Exception as e:
0582             logging.error('Error from uploading: %s' % str(e))
0583             ret = json.dumps( { "status": -1, "upload" : { 'itemStatus' : { basename : {'status':'failed', 'info':str(e)}}}, "error" : str(e)} )
0584
0585         os.unlink(fileHash)
0586
0587         statusInfo = json.loads(ret)['upload']
0588         logging.debug( 'upload returned: %s', statusInfo )
0589
0590         okTags      = []
0591         skippedTags = []
0592         failedTags  = []
0593         for tag, info in statusInfo['itemStatus'].items():
0594             logging.debug('checking tag %s, info %s', tag, str(json.dumps(info, indent=4,sort_keys=True)) )
0595             if 'ok'   in info['status'].lower() :
0596                 okTags.append( tag )
0597                 logging.info('tag %s successfully uploaded', tag)
0598             if 'skip' in info['status'].lower() :
0599                 skippedTags.append( tag )
0600                 logging.warning('found tag %s to be skipped. reason:  \n ... \t%s ', tag, info['info'])
0601             if 'fail' in info['status'].lower() :
0602                 failedTags.append( tag )
0603                 logging.error('found tag %s failed to upload. reason: \n ... \t%s ', tag, info['info'])
0604
0605         if len(okTags)      > 0: logging.info   ("tags sucessfully uploaded: %s ", str(okTags) )
0606         if len(skippedTags) > 0: logging.warning("tags SKIPped to upload   : %s ", str(skippedTags) )
0607         if len(failedTags)  > 0: logging.error  ("tags FAILed  to upload   : %s ", str(failedTags) )
0608
0609         fileLogURL = 'https://cms-conddb.cern.ch/cmsDbBrowser/logs/show_cond_uploader_log/%s/%s'
0610         backend = 'Prod'
0611         if self.hostname=='cms-conddb-dev.cern.ch':
0612             backend = 'Prep'
0613         logging.info('file log at: %s', fileLogURL % (backend,fileHash))
0614
0615         return len(okTags)>0
0616
0617 def getCredentials( options ):
0618
0619     username = None
0620     password = None
0621     netrcPath = None
0622     if authPathEnvVar in os.environ:
0623         authPath = os.environ[authPathEnvVar]
0624         netrcPath = os.path.join(authPath,'.netrc')
0625     if options.authPath is not None:
0626         netrcPath = os.path.join( options.authPath,'.netrc' )
0627     try:
0628         # Try to find the netrc entry
0629         (username, account, password) = netrc.netrc( netrcPath ).authenticators(options.netrcHost)
0630     except Exception:
0631         # netrc entry not found, ask for the username and password
0632         logging.info(
0633             'netrc entry "%s" not found: if you wish not to have to retype your password, you can add an entry in your .netrc file. However, beware of the risks of having your password stored as plaintext. Instead.',
0634             options.netrcHost)
0635
0636         # Try to get a default username
0637         defaultUsername = getpass.getuser()
0638         if defaultUsername is None:
0639             defaultUsername = '(not found)'
0640
0641         username = getInput(defaultUsername, '\nUsername [%s]: ' % defaultUsername)
0642         password = getpass.getpass('Password: ')
0643
0644     return username, password
0645
0646
0647 def uploadAllFiles(options, arguments):
0648
0649     ret = {}
0650     ret['status'] = 0
0651
0652     # Check that we can read the data and metadata files
0653     # If the metadata file does not exist, start the wizard
0654     for filename in arguments:
0655         basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
0656         basename = os.path.basename(basepath)
0657         dataFilename = '%s.db' % basepath
0658         metadataFilename = '%s.txt' % basepath
0659
0660         logging.info('Checking %s...', basename)
0661
0662         # Data file
0663         try:
0664             with open(dataFilename, 'rb') as dataFile:
0665                 pass
0666         except IOError as e:
0667             errMsg = 'Impossible to open SQLite data file %s' %dataFilename
0668             logging.error( errMsg )
0669             ret['status'] = -3
0670             ret['error'] = errMsg
0671             return ret
0672
0673         # Check the data file
0674         empty = True
0675         try:
0676             dbcon = sqlite3.connect( dataFilename )
0677             dbcur = dbcon.cursor()
0678             dbcur.execute('SELECT * FROM IOV')
0679             rows = dbcur.fetchall()
0680             for r in rows:
0681                 empty = False
0682             dbcon.close()
0683             if empty:
0684                 errMsg = 'The input SQLite data file %s contains no data.' %dataFilename
0685                 logging.error( errMsg )
0686                 ret['status'] = -4
0687                 ret['error'] = errMsg
0688                 return ret
0689         except Exception as e:
0690             errMsg = 'Check on input SQLite data file %s failed: %s' %(dataFilename,str(e))
0691             logging.error( errMsg )
0692             ret['status'] = -5
0693             ret['error'] = errMsg
0694             return ret
0695
0696         # Metadata file
0697         try:
0698             with open(metadataFilename, 'rb') as metadataFile:
0699                 pass
0700         except IOError as e:
0701             if e.errno != errno.ENOENT:
0702                 errMsg = 'Impossible to open file %s (for other reason than not existing)' %metadataFilename
0703                 logging.error( errMsg )
0704                 ret['status'] = -4
0705                 ret['error'] = errMsg
0706                 return ret
0707
0708             if getInput('y', '\nIt looks like the metadata file %s does not exist. Do you want me to create it and help you fill it?\nAnswer [y]: ' % metadataFilename).lower() != 'y':
0709                 errMsg = 'Metadata file %s does not exist' %metadataFilename
0710                 logging.error( errMsg )
0711                 ret['status'] = -5
0712                 ret['error'] = errMsg
0713                 return ret
0714             # Wizard
0715             runWizard(basename, dataFilename, metadataFilename)
0716
0717     # Upload files
0718     try:
0719         dropBox = ConditionsUploader(options.hostname, options.urlTemplate)
0720
0721         # Authentication
0722         username, password = getCredentials(options)
0723
0724         results = {}
0725         for filename in arguments:
0726             backend = options.backend
0727             basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
0728             metadataFilename = '%s.txt' % basepath
0729             with open(metadataFilename, 'rb') as metadataFile:
0730                 metadata = json.load( metadataFile )
0731             # When dest db = prep the hostname has to be set to dev.
0732             forceHost = False
0733             destDb = metadata['destinationDatabase']
0734             if destDb.startswith('oracle://cms_orcon_prod') or destDb.startswith('oracle://cms_orcoff_prep'):
0735                 hostName = defaultHostname
0736                 if destDb.startswith('oracle://cms_orcoff_prep'):
0737                      hostName = defaultDevHostname
0738                 dropBox.setHost( hostName )
0739                 authRet = dropBox.signIn( username, password )
0740                 if not authRet==0:
0741                     msg = "Error trying to connect to the server. Aborting."
0742                     if authRet==-2:
0743                         msg = "Error while signin in. Aborting."
0744                     logging.error(msg)
0745                     return { 'status' : authRet, 'error' : msg }
0746                 results[filename] = dropBox.uploadFile(filename, options.backend, options.temporaryFile)
0747             else:
0748                 results[filename] = False
0749                 logging.error("DestinationDatabase %s is not valid. Skipping the upload." %destDb)
0750             if not results[filename]:
0751                 if ret['status']<0:
0752                     ret['status'] = 0
0753                 ret['status'] += 1
0754         ret['files'] = results
0755         logging.debug("all files processed, logging out now.")
0756
0757         dropBox.signOut()
0758
0759     except HTTPError as e:
0760         logging.error('got HTTP error: %s', str(e))
0761         return { 'status' : -1, 'error' : str(e) }
0762
0763     return ret
0764
0765 def uploadTier0Files(filenames, username, password, cookieFileName = None):
0766     '''Uploads a bunch of files coming from Tier0.
0767     This has the following requirements:
0768         * Username/Password based authentication.
0769         * Uses the online backend.
0770         * Ignores errors related to the upload/content (e.g. duplicated file).
0771     '''
0772
0773     dropBox = ConditionsUploader()
0774
0775     dropBox.signIn(username, password)
0776
0777     for filename in filenames:
0778         try:
0779             result = dropBox.uploadFile(filename, backend = 'test')
0780         except HTTPError as e:
0781             if e.code == 400:
0782                 # 400 Bad Request: This is an exception related to the upload
0783                 # being wrong for some reason (e.g. duplicated file).
0784                 # Since for Tier0 this is not an issue, continue
0785                 logging.error('HTTP Exception 400 Bad Request: Upload-related, skipping. Message: %s', e)
0786                 continue
0787
0788             # In any other case, re-raise.
0789             raise
0790
0791         #-toDo: add a flag to say if we should retry or not. So far, all retries are done server-side (Tier-0),
0792         #       if we flag as failed any retry would not help and would result in the same error (e.g.
0793         #       when a file with an identical hash is uploaded again)
0794         #-review(2015-09-25): get feedback from tests at Tier-0 (action: AP)
0795
0796         if not result: # dropbox reported an error when uploading, do not retry.
0797             logging.error('Error from dropbox, upload-related, skipping.')
0798             continue
0799
0800     dropBox.signOut()
0801
0802 def re_upload( options ):
0803     netrcPath = None
0804     logDbSrv = prodLogDbSrv
0805     if options.hostname == defaultDevHostname:
0806         logDbSrv = devLogDbSrv
0807     if options.authPath is not None:
0808         netrcPath = os.path.join( options.authPath,'.netrc' )
0809     try:
0810         netrcKey = '%s/%s' %(logDbSrv,logDbSchema)
0811         # Try to find the netrc entry
0812         (username, account, password) = netrc.netrc( netrcPath ).authenticators( netrcKey )
0813     except IOError as e:
0814         logging.error('Cannot access netrc file.')
0815         return 1
0816     except Exception as e:
0817         logging.error('Netrc file is invalid: %s' %str(e))
0818         return 1
0819     conStr = '%s/%s@%s' %(username,password,logDbSrv)
0820     con = cx_Oracle.connect( conStr )
0821     cur = con.cursor()
0822     fh = options.reUpload
0823     cur.execute('SELECT FILECONTENT, STATE FROM FILES WHERE FILEHASH = :HASH',{'HASH':fh})
0824     res = cur.fetchall()
0825     found = False
0826     fdata = None
0827     for r in res:
0828         found = True
0829         logging.info("Found file %s in state '%s;" %(fh,r[1]))
0830         fdata = r[0].read().decode('bz2')
0831     con.close()
0832     if not found:
0833         logging.error("No file uploaded found with hash %s" %fh)
0834         return 1
0835     # writing as a tar file and open it ( is there a why to open it in memory?)
0836     fname = '%s.tar' %fh
0837     with open(fname, "wb" ) as f:
0838         f.write(fdata)
0839     rname = 'reupload_%s' %fh
0840     with tarfile.open(fname) as tar:
0841         tar.extractall()
0842     os.remove(fname)
0843     dfile = 'data.db'
0844     mdfile = 'metadata.txt'
0845     if os.path.exists(dfile):
0846         os.utime(dfile,None)
0847         os.chmod(dfile,0o755)
0848         os.rename(dfile,'%s.db' %rname)
0849     else:
0850         logging.error('Tar file does not contain the data file')
0851         return 1
0852     if os.path.exists(mdfile):
0853         os.utime(mdfile,None)
0854         os.chmod(mdfile,0o755)
0855         mdata = None
0856         with open(mdfile) as md:
0857             mdata = json.load(md)
0858         datelabel = datetime.now().strftime("%y-%m-%d %H:%M:%S")
0859         if mdata is None:
0860             logging.error('Metadata file is empty.')
0861             return 1
0862         logging.debug('Preparing new metadata file...')
0863         mdata['userText'] = 'reupload %s : %s' %(datelabel,mdata['userText'])
0864         with open( '%s.txt' %rname, 'wb') as jf:
0865             jf.write( json.dumps( mdata, sort_keys=True, indent = 2 ) )
0866             jf.write('\n')
0867         os.remove(mdfile)
0868     else:
0869         logging.error('Tar file does not contain the metadata file')
0870         return 1
0871     logging.info('Files %s prepared for the upload.' %rname)
0872     arguments = [rname]
0873     return upload(options, arguments)
0874
0875 def upload(options, arguments):
0876     results = uploadAllFiles(options, arguments)
0877
0878     if 'status' not in results:
0879         print('Unexpected error.')
0880         return -1
0881     ret = results['status']
0882     print(results)
0883     print("upload ended with code: %s" %ret)
0884     return ret
0885
0886 def main():
0887     '''Entry point.
0888     '''
0889
0890     parser = optparse.OptionParser(usage =
0891         'Usage: %prog [options] <file> [<file> ...]\n'
0892     )
0893
0894     parser.add_option('-d', '--debug',
0895         dest = 'debug',
0896         action="store_true",
0897         default = False,
0898         help = 'Switch on printing debug information. Default: %default',
0899     )
0900
0901     parser.add_option('-b', '--backend',
0902         dest = 'backend',
0903         default = defaultBackend,
0904         help = 'dropBox\'s backend to upload to. Default: %default',
0905     )
0906
0907     parser.add_option('-H', '--hostname',
0908         dest = 'hostname',
0909         default = defaultHostname,
0910         help = 'dropBox\'s hostname. Default: %default',
0911     )
0912
0913     parser.add_option('-u', '--urlTemplate',
0914         dest = 'urlTemplate',
0915         default = defaultUrlTemplate,
0916         help = 'dropBox\'s URL template. Default: %default',
0917     )
0918
0919     parser.add_option('-f', '--temporaryFile',
0920         dest = 'temporaryFile',
0921         default = defaultTemporaryFile,
0922         help = 'Temporary file that will be used to store the first tar file. Note that it then will be moved to a file with the hash of the file as its name, so there will be two temporary files created in fact. Default: %default',
0923     )
0924
0925     parser.add_option('-n', '--netrcHost',
0926         dest = 'netrcHost',
0927         default = defaultNetrcHost,
0928         help = 'The netrc host (machine) from where the username and password will be read. Default: %default',
0929     )
0930
0931     parser.add_option('-a', '--authPath',
0932         dest = 'authPath',
0933         default = None,
0934         help = 'The path of the .netrc file for the authentication. Default: $HOME',
0935     )
0936
0937     parser.add_option('-r', '--reUpload',
0938         dest = 'reUpload',
0939         default = None,
0940         help = 'The hash of the file to upload again.',
0941     )
0942
0943     (options, arguments) = parser.parse_args()
0944
0945     logLevel = logging.INFO
0946     if options.debug:
0947         logLevel = logging.DEBUG
0948     logging.basicConfig(
0949         format = '[%(asctime)s] %(levelname)s: %(message)s',
0950         level = logLevel,
0951     )
0952
0953     if len(arguments) < 1:
0954         if options.reUpload is None:
0955             parser.print_help()
0956             return -2
0957         else:
0958             return re_upload(options)
0959     if options.reUpload is not None:
0960         print("ERROR: options -r can't be specified on a new file upload.")
0961         return -2
0962
0963     return upload(options, arguments)
0964
0965 def testTier0Upload():
0966
0967     global defaultNetrcHost
0968
0969     (username, account, password) = netrc.netrc().authenticators(defaultNetrcHost)
0970
0971     filenames = ['testFiles/localSqlite-top2']
0972
0973     uploadTier0Files(filenames, username, password, cookieFileName = None)
0974
0975
0976 if __name__ == '__main__':
0977
0978     sys.exit(main())
0979     # testTier0Upload()