Utilities/scripts/uploadConditions.py

0001 #!/usr/bin/env python3
0002 '''Script that uploads to the new CMS conditions uploader.
0003 Adapted to the new infrastructure from v6 of the upload.py script for the DropBox from Miguel Ojeda.
0004 '''
0005
0006 __author__ = 'Andreas Pfeiffer'
0007 __copyright__ = 'Copyright 2015, CERN CMS'
0008 __credits__ = ['Giacomo Govi', 'Salvatore Di Guida', 'Miguel Ojeda', 'Andreas Pfeiffer']
0009 __license__ = 'Unknown'
0010 __maintainer__ = 'Giacomo Govi'
0011 __email__ = 'giacomo.govi@cern.ch'
0012 __version__ = 1
0013
0014
0015 import os
0016 import sys
0017 import optparse
0018 import hashlib
0019 import tarfile
0020 import netrc
0021 import getpass
0022 import errno
0023 import sqlite3
0024 import cx_Oracle
0025 import json
0026 import tempfile
0027 from datetime import datetime
0028
0029 defaultBackend = 'online'
0030 defaultHostname = 'cms-conddb-prod.cern.ch'
0031 defaultDevHostname = 'cms-conddb-dev.cern.ch'
0032 defaultUrlTemplate = 'https://%s/cmsDbUpload/'
0033 defaultTemporaryFile = 'upload.tar.bz2'
0034 defaultNetrcHost = 'ConditionUploader'
0035 defaultWorkflow = 'offline'
0036 prodLogDbSrv = 'cms_orcoff_prod'
0037 devLogDbSrv = 'cms_orcoff_prep'
0038 logDbSchema = 'CMS_COND_DROPBOX'
0039 authPathEnvVar = 'COND_AUTH_PATH'
0040 waitForRetry = 15
0041
0042 # common/http.py start (plus the "# Try to extract..." section bit)
0043 import time
0044 import logging
0045 import io
0046
0047 import pycurl
0048 import socket
0049 import copy
0050
0051 def getInput(default, prompt = ''):
0052     '''Like input() but with a default and automatic strip().
0053     '''
0054
0055     answer = input(prompt)
0056     if answer:
0057         return answer.strip()
0058
0059     return default.strip()
0060
0061
0062 def getInputWorkflow(prompt = ''):
0063     '''Like getInput() but tailored to get target workflows (synchronization options).
0064     '''
0065
0066     while True:
0067         workflow = getInput(defaultWorkflow, prompt)
0068
0069         if workflow in frozenset(['offline', 'hlt', 'express', 'prompt', 'pcl']):
0070             return workflow
0071
0072         logging.error('Please specify one of the allowed workflows. See above for the explanation on each of them.')
0073
0074
0075 def getInputChoose(optionsList, default, prompt = ''):
0076     '''Makes the user choose from a list of options.
0077     '''
0078
0079     while True:
0080         index = getInput(default, prompt)
0081
0082         try:
0083             return optionsList[int(index)]
0084         except ValueError:
0085             logging.error('Please specify an index of the list (i.e. integer).')
0086         except IndexError:
0087             logging.error('The index you provided is not in the given list.')
0088
0089
0090 def getInputRepeat(prompt = ''):
0091     '''Like input() but repeats if nothing is provided and automatic strip().
0092     '''
0093
0094     while True:
0095         answer = input(prompt)
0096         if answer:
0097             return answer.strip()
0098
0099         logging.error('You need to provide a value.')
0100
0101
0102 def runWizard(basename, dataFilename, metadataFilename):
0103     while True:
0104         print('''\nWizard for metadata for %s
0105
0106 I will ask you some questions to fill the metadata file. For some of the questions there are defaults between square brackets (i.e. []), leave empty (i.e. hit Enter) to use them.''' % basename)
0107
0108         # Try to get the available inputTags
0109         dataConnection = sqlite3.connect(dataFilename)
0110         dataCursor = dataConnection.cursor()
0111
0112         dataCursor.execute('select NAME from TAG')
0113         records = dataCursor.fetchall()
0114         inputTags = []
0115         for rec in records:
0116             inputTags.append(rec[0])
0117
0118         if len(inputTags) == 0:
0119             raise Exception("Could not find any input tag in the data file.")
0120
0121         else:
0122             print('\nI found the following input tags in your SQLite data file:')
0123             for (index, inputTag) in enumerate(inputTags):
0124                 print('   %s) %s' % (index, inputTag))
0125
0126             inputTag = getInputChoose(inputTags, '0',
0127                                       '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. 0 (you select the first in the list)\ninputTag [0]: ')
0128
0129         destinationDatabase = ''
0130         ntry = 0
0131         while ( destinationDatabase != 'oracle://cms_orcon_prod/CMS_CONDITIONS' and destinationDatabase != 'oracle://cms_orcoff_prep/CMS_CONDITIONS' ):
0132             if ntry==0:
0133                 inputMessage = \
0134                 '\nWhich is the destination database where the tags should be exported? \nPossible choices: oracle://cms_orcon_prod/CMS_CONDITIONS (or prod); oracle://cms_orcoff_prep/CMS_CONDITIONS (or prep) \ndestinationDatabase: '
0135             elif ntry==1:
0136                 inputMessage = \
0137                 '\nPlease choose one of the two valid destinations: \noracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \
0138 \ndestinationDatabase: '
0139             else:
0140                 raise Exception('No valid destination chosen. Bailing out...')
0141             destinationDatabase = getInputRepeat(inputMessage)
0142             if destinationDatabase == 'prod':
0143                 destinationDatabase = 'oracle://cms_orcon_prod/CMS_CONDITIONS'
0144             if destinationDatabase == 'prep':
0145                 destinationDatabase = 'oracle://cms_orcoff_prep/CMS_CONDITIONS'
0146             ntry += 1
0147
0148         while True:
0149             since = getInput('',
0150                              '\nWhich is the given since? (if not specified, the one from the SQLite data file will be taken -- note that even if specified, still this may not be the final since, depending on the synchronization options you select later: if the synchronization target is not offline, and the since you give is smaller than the next possible one (i.e. you give a run number earlier than the one which will be started/processed next in prompt/hlt/express), the DropBox will move the since ahead to go to the first safe run instead of the value you gave)\ne.g. 1234\nsince []: ')
0151             if not since:
0152                 since = None
0153                 break
0154             else:
0155                 try:
0156                     since = int(since)
0157                     break
0158                 except ValueError:
0159                     logging.error('The since value has to be an integer or empty (null).')
0160
0161         userText = getInput('',
0162                             '\nWrite any comments/text you may want to describe your request\ne.g. Muon alignment scenario for...\nuserText []: ')
0163
0164         destinationTags = {}
0165         while True:
0166             destinationTag = getInput('',
0167                                       '\nWhich is the next destination tag to be added (leave empty to stop)?\ne.g. BeamSpotObjects_PCL_byRun_v0_offline\ndestinationTag []: ')
0168             if not destinationTag:
0169                 if len(destinationTags) == 0:
0170                     logging.error('There must be at least one destination tag.')
0171                     continue
0172                 break
0173
0174             if destinationTag in destinationTags:
0175                 logging.warning(
0176                     'You already added this destination tag. Overwriting the previous one with this new one.')
0177
0178             destinationTags[destinationTag] = {
0179             }
0180
0181         metadata = {
0182             'destinationDatabase': destinationDatabase,
0183             'destinationTags': destinationTags,
0184             'inputTag': inputTag,
0185             'since': since,
0186             'userText': userText,
0187         }
0188
0189         metadata = json.dumps(metadata, sort_keys=True, indent=4)
0190         print('\nThis is the generated metadata:\n%s' % metadata)
0191
0192         if getInput('n',
0193                     '\nIs it fine (i.e. save in %s and *upload* the conditions if this is the latest file)?\nAnswer [n]: ' % metadataFilename).lower() == 'y':
0194             break
0195     logging.info('Saving generated metadata in %s...', metadataFilename)
0196     with open(metadataFilename, 'w') as metadataFile:
0197         metadataFile.write(metadata)
0198
0199 class HTTPError(Exception):
0200     '''A common HTTP exception.
0201
0202     self.code is the response HTTP code as an integer.
0203     self.response is the response body (i.e. page).
0204     '''
0205
0206     def __init__(self, code, response):
0207         self.code = code
0208         self.response = response
0209
0210         # Try to extract the error message if possible (i.e. known error page format)
0211         try:
0212             self.args = (response.split('<p>')[1].split('</p>')[0], )
0213         except Exception:
0214             self.args = (self.response, )
0215
0216
0217 CERN_SSO_CURL_CAPATH = '/etc/pki/tls/certs'
0218
0219 class HTTP(object):
0220     '''Class used for querying URLs using the HTTP protocol.
0221     '''
0222
0223     retryCodes = frozenset([502, 503])
0224
0225     def __init__(self):
0226         self.setBaseUrl()
0227         self.setRetries()
0228
0229         self.curl = pycurl.Curl()
0230         self.curl.setopt(self.curl.COOKIEFILE, '')      # in memory
0231
0232         #-toDo: make sure we have the right options set here to use ssl
0233         #-review(2015-09-25): check and see - action: AP
0234         # self.curl.setopt(self.curl.SSL_VERIFYPEER, 1)
0235         self.curl.setopt(self.curl.SSL_VERIFYPEER, 0)
0236         self.curl.setopt(self.curl.SSL_VERIFYHOST, 2)
0237
0238         self.baseUrl = None
0239
0240         self.token = None
0241
0242     def getCookies(self):
0243         '''Returns the list of cookies.
0244         '''
0245         return self.curl.getinfo(self.curl.INFO_COOKIELIST)
0246
0247     def discardCookies(self):
0248         '''Discards cookies.
0249         '''
0250         self.curl.setopt(self.curl.COOKIELIST, 'ALL')
0251
0252
0253     def setBaseUrl(self, baseUrl = ''):
0254         '''Allows to set a base URL which will be prefixed to all the URLs
0255         that will be queried later.
0256         '''
0257         self.baseUrl = baseUrl
0258
0259
0260     def setProxy(self, proxy = ''):
0261         '''Allows to set a proxy.
0262         '''
0263         self.curl.setopt(self.curl.PROXY, proxy)
0264
0265
0266     def setTimeout(self, timeout = 0):
0267         '''Allows to set a timeout.
0268         '''
0269         self.curl.setopt(self.curl.TIMEOUT, timeout)
0270
0271
0272     def setRetries(self, retries = ()):
0273         '''Allows to set retries.
0274
0275         The retries are a sequence of the seconds to wait per retry.
0276
0277         The retries are done on:
0278             * PyCurl errors (includes network problems, e.g. not being able
0279               to connect to the host).
0280             * 502 Bad Gateway (for the moment, to avoid temporary
0281               Apache-CherryPy issues).
0282             * 503 Service Temporarily Unavailable (for when we update
0283               the frontends).
0284         '''
0285         self.retries = retries
0286
0287     def getToken(self, username, password):
0288
0289         url = self.baseUrl + 'token'
0290
0291         self.curl.setopt(pycurl.URL, url)
0292         self.curl.setopt(pycurl.VERBOSE, 0)
0293
0294         #-toDo: check if/why these are needed ...
0295         #-ap: hmm ...
0296         # self.curl.setopt(pycurl.DNS_CACHE_TIMEOUT, 0)
0297         # self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
0298         #-end hmmm ...
0299         #-review(2015-09-25): check and see - action: AP
0300
0301         self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
0302         # self.curl.setopt( self.curl.POST, {})
0303         self.curl.setopt(self.curl.HTTPGET, 0)
0304
0305         response = io.BytesIO()
0306         self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
0307         self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
0308         logging.debug('going to connect to server at: %s' % url )
0309
0310         self.curl.perform()
0311         code = self.curl.getinfo(pycurl.RESPONSE_CODE)
0312         logging.debug('got: %s ', str(code))
0313         if code in ( 502,503,504 ):
0314             logging.debug('Trying again after %d seconds...', waitForRetry)
0315             time.sleep( waitForRetry )
0316             response = io.StringIO()
0317             self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
0318             self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
0319             self.curl.perform()
0320             code = self.curl.getinfo(pycurl.RESPONSE_CODE)
0321         resp = response.getvalue().decode('UTF-8')
0322         errorMsg = None
0323         if code==500 and not resp.find("INVALID_CREDENTIALS")==-1:
0324             logging.error("Invalid credentials provided.")
0325             return None
0326         if code==403 and not resp.find("Unauthorized access")==-1:
0327             logging.error("Unauthorized access. Please check the membership of group 'cms-cond-dropbox'")
0328             return None
0329         if code==200:
0330             try:
0331                 self.token = json.loads( resp )['token']
0332             except Exception as e:
0333                 errorMsg = 'Error while decoding returned json string'
0334                 logging.debug('http::getToken> error while decoding json: %s ', str(resp) )
0335                 logging.debug("error getting token: %s", str(e))
0336                 resp = None
0337         else:
0338             errorMsg = 'HTTP Error code %s ' %code
0339             logging.debug('got: %s ', str(code))
0340             logging.debug('http::getToken> got error from server: %s ', str(resp) )
0341             resp = None
0342         if resp is None:
0343             raise Exception(errorMsg)
0344
0345         logging.debug('token: %s', self.token)
0346         logging.debug('returning: %s', response.getvalue().decode('UTF-8'))
0347
0348         return response.getvalue()
0349
0350     def query(self, url, data = None, files = None, keepCookies = True):
0351         '''Queries a URL, optionally with some data (dictionary).
0352
0353         If no data is specified, a GET request will be used.
0354         If some data is specified, a POST request will be used.
0355
0356         If files is specified, it must be a dictionary like data but
0357         the values are filenames.
0358
0359         By default, cookies are kept in-between requests.
0360
0361         A HTTPError exception is raised if the response's HTTP code is not 200.
0362         '''
0363
0364         if not keepCookies:
0365             self.discardCookies()
0366
0367         url = self.baseUrl + url
0368
0369         # make sure the logs are safe ... at least somewhat :)
0370         data4log = copy.copy(data)
0371         if data4log:
0372             if 'password' in data4log.keys():
0373                 data4log['password'] = '*'
0374
0375         retries = [0] + list(self.retries)
0376
0377         while True:
0378             logging.debug('Querying %s with data %s and files %s (retries left: %s, current sleep: %s)...', url, data4log, files, len(retries), retries[0])
0379
0380             time.sleep(retries.pop(0))
0381
0382             try:
0383                 self.curl.setopt(self.curl.URL, url)
0384                 self.curl.setopt(self.curl.HTTPGET, 1)
0385
0386                 # from now on we use the token we got from the login
0387                 self.curl.setopt(pycurl.USERPWD, '%s:""' % ( str(self.token), ) )
0388                 self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
0389
0390                 if data is not None or files is not None:
0391                     # If there is data or files to send, use a POST request
0392
0393                     finalData = {}
0394
0395                     if data is not None:
0396                         finalData.update(data)
0397
0398                     if files is not None:
0399                         for (key, fileName) in files.items():
0400                             finalData[key] = (self.curl.FORM_FILE, fileName)
0401                     self.curl.setopt( self.curl.HTTPPOST, list(finalData.items()) )
0402
0403                 self.curl.setopt(pycurl.VERBOSE, 0)
0404
0405                 response = io.BytesIO()
0406                 self.curl.setopt(self.curl.WRITEFUNCTION, response.write)
0407                 self.curl.perform()
0408
0409                 code = self.curl.getinfo(self.curl.RESPONSE_CODE)
0410
0411                 if code in self.retryCodes and len(retries) > 0:
0412                     logging.debug('Retrying since we got the %s error code...', code)
0413                     continue
0414
0415                 if code != 200:
0416                     raise HTTPError(code, response.getvalue())
0417
0418                 return response.getvalue()
0419
0420             except pycurl.error as e:
0421                 if len(retries) == 0:
0422                     raise e
0423                 logging.debug('Retrying since we got the %s pycurl exception...', str(e))
0424
0425 # common/http.py end
0426
0427 def addToTarFile(tarFile, fileobj, arcname):
0428     tarInfo = tarFile.gettarinfo(fileobj = fileobj, arcname = arcname)
0429     tarInfo.mode = 0o400
0430     tarInfo.uid = tarInfo.gid = tarInfo.mtime = 0
0431     tarInfo.uname = tarInfo.gname = 'root'
0432     tarFile.addfile(tarInfo, fileobj)
0433
0434 class ConditionsUploader(object):
0435     '''Upload conditions to the CMS conditions uploader service.
0436     '''
0437
0438     def __init__(self, hostname = defaultHostname, urlTemplate = defaultUrlTemplate):
0439         self.hostname = hostname
0440         self.urlTemplate = urlTemplate
0441         self.userName = None
0442         self.http = None
0443         self.password = None
0444         self.token = None
0445
0446     def setHost( self, hostname ):
0447         if not hostname==self.hostname:
0448             self.token = None
0449             self.hostname = hostname
0450
0451     def signIn(self, username, password ):
0452         if self.token is None:
0453             logging.debug("Initializing connection with server %s",self.hostname)
0454             ''' init the server.
0455             '''
0456             self.http = HTTP()
0457             if socket.getfqdn().strip().endswith('.cms'):
0458                 self.http.setProxy('https://cmsproxy.cms:3128/')
0459             self.http.setBaseUrl(self.urlTemplate % self.hostname)
0460             '''Signs in the server.
0461             '''
0462             logging.info('%s: Signing in user %s ...', self.hostname, username)
0463             try:
0464                 self.token = self.http.getToken(username, password)
0465             except Exception as e:
0466                 ret = -1
0467                 # optionally, we may want to have a different return for network related errors:
0468                 #code = self.http.curl.getinfo(pycurl.RESPONSE_CODE)
0469                 #if code in ( 502,503,504 ):
0470                 #    ret = -10
0471                 logging.error("Caught exception when trying to connect to %s: %s" % (self.hostname, str(e)) )
0472                 return ret
0473
0474             if not self.token:
0475                 logging.error("could not get token for user %s from %s" % (username, self.hostname) )
0476                 return -2
0477             logging.debug( "got: '%s'", str(self.token) )
0478             self.userName = username
0479             self.password = password
0480         else:
0481             logging.debug("User %s has been already authenticated." %username)
0482         return 0
0483
0484     def signOut(self):
0485         '''Signs out the server.
0486         '''
0487
0488         logging.info('%s: Signing out...', self.hostname)
0489         # self.http.query('logout')
0490         self.token = None
0491
0492
0493     def _checkForUpdates(self):
0494         '''Updates this script, if a new version is found.
0495         '''
0496
0497         logging.debug('%s: Checking if a newer version of this script is available ...', self.hostname)
0498         version = int(self.http.query('getUploadScriptVersion'))
0499
0500         if version <= __version__:
0501             logging.debug('%s: Script is up-to-date.', self.hostname)
0502             return
0503
0504         logging.info('%s: Updating to a newer version (%s) than the current one (%s): downloading ...', self.hostname, version, __version__)
0505
0506         uploadScript = self.http.query('getUploadScript')
0507
0508         self.signOut()
0509
0510         logging.info('%s: ... saving the new version ...', self.hostname)
0511         with open(sys.argv[0], 'wb') as f:
0512             f.write(uploadScript)
0513
0514         logging.info('%s: ... executing the new version...', self.hostname)
0515         os.execl(sys.executable, *([sys.executable] + sys.argv))
0516
0517
0518     def uploadFile(self, filename, backend = defaultBackend, temporaryFile = defaultTemporaryFile):
0519         '''Uploads a file to the dropBox.
0520
0521         The filename can be without extension, with .db or with .txt extension.
0522         It will be stripped and then both .db and .txt files are used.
0523         '''
0524
0525         basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
0526         basename = os.path.basename(basepath)
0527
0528         logging.debug('%s: %s: Creating tar file for upload ...', self.hostname, basename)
0529
0530         try:
0531             tarFile = tarfile.open(temporaryFile, 'w:bz2')
0532
0533             with open('%s.db' % basepath, 'rb') as data:
0534                 addToTarFile(tarFile, data, 'data.db')
0535         except Exception as e:
0536             msg = 'Error when creating tar file. \n'
0537             msg += 'Please check that you have write access to the directory you are running,\n'
0538             msg += 'and that you have enough space on this disk (df -h .)\n'
0539             logging.error(msg)
0540             raise Exception(msg)
0541
0542         with tempfile.NamedTemporaryFile(mode='rb+') as metadata:
0543             with open('%s.txt' % basepath, 'r') as originalMetadata:
0544                 metadata.write(json.dumps(json.load(originalMetadata), sort_keys = True, indent = 4).encode())
0545
0546             metadata.seek(0)
0547             addToTarFile(tarFile, metadata, 'metadata.txt')
0548
0549         tarFile.close()
0550
0551         logging.debug('%s: %s: Calculating hash...', self.hostname, basename)
0552
0553         fileHash = hashlib.sha1()
0554         with open(temporaryFile, 'rb') as f:
0555             while True:
0556                 data = f.read(4 * 1024 * 1024)
0557                 if not data:
0558                     break
0559                 fileHash.update(data)
0560
0561         fileHash = fileHash.hexdigest()
0562         fileInfo = os.stat(temporaryFile)
0563         fileSize = fileInfo.st_size
0564
0565         logging.debug('%s: %s: Hash: %s', self.hostname, basename, fileHash)
0566
0567         logging.info('%s: %s: Uploading file (%s, size %s) to the %s backend...', self.hostname, basename, fileHash, fileSize, backend)
0568         os.rename(temporaryFile, fileHash)
0569         try:
0570             ret = self.http.query('uploadFile',
0571                               {
0572                                 'backend': backend,
0573                                 'fileName': basename,
0574                                 'userName': self.userName,
0575                               },
0576                               files = {
0577                                         'uploadedFile': fileHash,
0578                                       }
0579                               )
0580         except Exception as e:
0581             logging.error('Error from uploading: %s' % str(e))
0582             ret = json.dumps( { "status": -1, "upload" : { 'itemStatus' : { basename : {'status':'failed', 'info':str(e)}}}, "error" : str(e)} )
0583
0584         os.unlink(fileHash)
0585
0586         statusInfo = json.loads(ret)['upload']
0587         logging.debug( 'upload returned: %s', statusInfo )
0588
0589         okTags      = []
0590         skippedTags = []
0591         failedTags  = []
0592         for tag, info in statusInfo['itemStatus'].items():
0593             logging.debug('checking tag %s, info %s', tag, str(json.dumps(info, indent=4,sort_keys=True)) )
0594             if 'ok'   in info['status'].lower() :
0595                 okTags.append( tag )
0596                 logging.info('tag %s successfully uploaded', tag)
0597             if 'skip' in info['status'].lower() :
0598                 skippedTags.append( tag )
0599                 logging.warning('found tag %s to be skipped. reason:  \n ... \t%s ', tag, info['info'])
0600             if 'fail' in info['status'].lower() :
0601                 failedTags.append( tag )
0602                 logging.error('found tag %s failed to upload. reason: \n ... \t%s ', tag, info['info'])
0603
0604         if len(okTags)      > 0: logging.info   ("tags sucessfully uploaded: %s ", str(okTags) )
0605         if len(skippedTags) > 0: logging.warning("tags SKIPped to upload   : %s ", str(skippedTags) )
0606         if len(failedTags)  > 0: logging.error  ("tags FAILed  to upload   : %s ", str(failedTags) )
0607
0608         fileLogURL = 'https://cms-conddb.cern.ch/cmsDbBrowser/logs/show_cond_uploader_log/%s/%s'
0609         backend = 'Prod'
0610         if self.hostname=='cms-conddb-dev.cern.ch':
0611             backend = 'Prep'
0612         logging.info('file log at: %s', fileLogURL % (backend,fileHash))
0613
0614         return len(okTags)>0
0615
0616 def getCredentials( options ):
0617
0618     username = None
0619     password = None
0620     netrcPath = None
0621     if authPathEnvVar in os.environ:
0622         authPath = os.environ[authPathEnvVar]
0623         netrcPath = os.path.join(authPath,'.netrc')
0624     if options.authPath is not None:
0625         netrcPath = os.path.join( options.authPath,'.netrc' )
0626     try:
0627         # Try to find the netrc entry
0628         (username, account, password) = netrc.netrc( netrcPath ).authenticators(options.netrcHost)
0629     except Exception:
0630         # netrc entry not found, ask for the username and password
0631         logging.info(
0632             'netrc entry "%s" not found: if you wish not to have to retype your password, you can add an entry in your .netrc file. However, beware of the risks of having your password stored as plaintext. Instead.',
0633             options.netrcHost)
0634
0635         # Try to get a default username
0636         defaultUsername = getpass.getuser()
0637         if defaultUsername is None:
0638             defaultUsername = '(not found)'
0639
0640         username = getInput(defaultUsername, '\nUsername [%s]: ' % defaultUsername)
0641         password = getpass.getpass('Password: ')
0642
0643     return username, password
0644
0645
0646 def uploadAllFiles(options, arguments):
0647
0648     ret = {}
0649     ret['status'] = 0
0650
0651     # Check that we can read the data and metadata files
0652     # If the metadata file does not exist, start the wizard
0653     for filename in arguments:
0654         basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
0655         basename = os.path.basename(basepath)
0656         dataFilename = '%s.db' % basepath
0657         metadataFilename = '%s.txt' % basepath
0658
0659         logging.info('Checking %s...', basename)
0660
0661         # Data file
0662         try:
0663             with open(dataFilename, 'rb') as dataFile:
0664                 pass
0665         except IOError as e:
0666             errMsg = 'Impossible to open SQLite data file %s' %dataFilename
0667             logging.error( errMsg )
0668             ret['status'] = -3
0669             ret['error'] = errMsg
0670             return ret
0671
0672         # Check the data file
0673         empty = True
0674         try:
0675             dbcon = sqlite3.connect( dataFilename )
0676             dbcur = dbcon.cursor()
0677             dbcur.execute('SELECT * FROM IOV')
0678             rows = dbcur.fetchall()
0679             for r in rows:
0680                 empty = False
0681             dbcon.close()
0682             if empty:
0683                 errMsg = 'The input SQLite data file %s contains no data.' %dataFilename
0684                 logging.error( errMsg )
0685                 ret['status'] = -4
0686                 ret['error'] = errMsg
0687                 return ret
0688         except Exception as e:
0689             errMsg = 'Check on input SQLite data file %s failed: %s' %(dataFilename,str(e))
0690             logging.error( errMsg )
0691             ret['status'] = -5
0692             ret['error'] = errMsg
0693             return ret
0694
0695         # Metadata file
0696         try:
0697             with open(metadataFilename, 'rb') as metadataFile:
0698                 pass
0699         except IOError as e:
0700             if e.errno != errno.ENOENT:
0701                 errMsg = 'Impossible to open file %s (for other reason than not existing)' %metadataFilename
0702                 logging.error( errMsg )
0703                 ret['status'] = -4
0704                 ret['error'] = errMsg
0705                 return ret
0706
0707             if getInput('y', '\nIt looks like the metadata file %s does not exist. Do you want me to create it and help you fill it?\nAnswer [y]: ' % metadataFilename).lower() != 'y':
0708                 errMsg = 'Metadata file %s does not exist' %metadataFilename
0709                 logging.error( errMsg )
0710                 ret['status'] = -5
0711                 ret['error'] = errMsg
0712                 return ret
0713             # Wizard
0714             runWizard(basename, dataFilename, metadataFilename)
0715
0716     # Upload files
0717     try:
0718         dropBox = ConditionsUploader(options.hostname, options.urlTemplate)
0719
0720         # Authentication
0721         username, password = getCredentials(options)
0722
0723         results = {}
0724         for filename in arguments:
0725             backend = options.backend
0726             basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
0727             metadataFilename = '%s.txt' % basepath
0728             with open(metadataFilename, 'rb') as metadataFile:
0729                 metadata = json.load( metadataFile )
0730             # When dest db = prep the hostname has to be set to dev.
0731             forceHost = False
0732             destDb = metadata['destinationDatabase']
0733             if destDb.startswith('oracle://cms_orcon_prod') or destDb.startswith('oracle://cms_orcoff_prep'):
0734                 hostName = defaultHostname
0735                 if destDb.startswith('oracle://cms_orcoff_prep'):
0736                      hostName = defaultDevHostname
0737                 dropBox.setHost( hostName )
0738                 authRet = dropBox.signIn( username, password )
0739                 if not authRet==0:
0740                     msg = "Error trying to connect to the server. Aborting."
0741                     if authRet==-2:
0742                         msg = "Error while signin in. Aborting."
0743                     logging.error(msg)
0744                     return { 'status' : authRet, 'error' : msg }
0745                 results[filename] = dropBox.uploadFile(filename, options.backend, options.temporaryFile)
0746             else:
0747                 results[filename] = False
0748                 logging.error("DestinationDatabase %s is not valid. Skipping the upload." %destDb)
0749             if not results[filename]:
0750                 if ret['status']<0:
0751                     ret['status'] = 0
0752                 ret['status'] += 1
0753         ret['files'] = results
0754         logging.debug("all files processed, logging out now.")
0755
0756         dropBox.signOut()
0757
0758     except HTTPError as e:
0759         logging.error('got HTTP error: %s', str(e))
0760         return { 'status' : -1, 'error' : str(e) }
0761
0762     return ret
0763
0764 def uploadTier0Files(filenames, username, password, cookieFileName = None):
0765     '''Uploads a bunch of files coming from Tier0.
0766     This has the following requirements:
0767         * Username/Password based authentication.
0768         * Uses the online backend.
0769         * Ignores errors related to the upload/content (e.g. duplicated file).
0770     '''
0771
0772     dropBox = ConditionsUploader()
0773
0774     dropBox.signIn(username, password)
0775
0776     for filename in filenames:
0777         try:
0778             result = dropBox.uploadFile(filename, backend = 'test')
0779         except HTTPError as e:
0780             if e.code == 400:
0781                 # 400 Bad Request: This is an exception related to the upload
0782                 # being wrong for some reason (e.g. duplicated file).
0783                 # Since for Tier0 this is not an issue, continue
0784                 logging.error('HTTP Exception 400 Bad Request: Upload-related, skipping. Message: %s', e)
0785                 continue
0786
0787             # In any other case, re-raise.
0788             raise
0789
0790         #-toDo: add a flag to say if we should retry or not. So far, all retries are done server-side (Tier-0),
0791         #       if we flag as failed any retry would not help and would result in the same error (e.g.
0792         #       when a file with an identical hash is uploaded again)
0793         #-review(2015-09-25): get feedback from tests at Tier-0 (action: AP)
0794
0795         if not result: # dropbox reported an error when uploading, do not retry.
0796             logging.error('Error from dropbox, upload-related, skipping.')
0797             continue
0798
0799     dropBox.signOut()
0800
0801 def re_upload( options ):
0802     netrcPath = None
0803     logDbSrv = prodLogDbSrv
0804     if options.hostname == defaultDevHostname:
0805         logDbSrv = devLogDbSrv
0806     if options.authPath is not None:
0807         netrcPath = os.path.join( options.authPath,'.netrc' )
0808     try:
0809         netrcKey = '%s/%s' %(logDbSrv,logDbSchema)
0810         # Try to find the netrc entry
0811         (username, account, password) = netrc.netrc( netrcPath ).authenticators( netrcKey )
0812     except IOError as e:
0813         logging.error('Cannot access netrc file.')
0814         return 1
0815     except Exception as e:
0816         logging.error('Netrc file is invalid: %s' %str(e))
0817         return 1
0818     conStr = '%s/%s@%s' %(username,password,logDbSrv)
0819     con = cx_Oracle.connect( conStr )
0820     cur = con.cursor()
0821     fh = options.reUpload
0822     cur.execute('SELECT FILECONTENT, STATE FROM FILES WHERE FILEHASH = :HASH',{'HASH':fh})
0823     res = cur.fetchall()
0824     found = False
0825     fdata = None
0826     for r in res:
0827         found = True
0828         logging.info("Found file %s in state '%s;" %(fh,r[1]))
0829         fdata = r[0].read().decode('bz2')
0830     con.close()
0831     if not found:
0832         logging.error("No file uploaded found with hash %s" %fh)
0833         return 1
0834     # writing as a tar file and open it ( is there a why to open it in memory?)
0835     fname = '%s.tar' %fh
0836     with open(fname, "wb" ) as f:
0837         f.write(fdata)
0838     rname = 'reupload_%s' %fh
0839     with tarfile.open(fname) as tar:
0840         tar.extractall()
0841     os.remove(fname)
0842     dfile = 'data.db'
0843     mdfile = 'metadata.txt'
0844     if os.path.exists(dfile):
0845         os.utime(dfile,None)
0846         os.chmod(dfile,0o755)
0847         os.rename(dfile,'%s.db' %rname)
0848     else:
0849         logging.error('Tar file does not contain the data file')
0850         return 1
0851     if os.path.exists(mdfile):
0852         os.utime(mdfile,None)
0853         os.chmod(mdfile,0o755)
0854         mdata = None
0855         with open(mdfile) as md:
0856             mdata = json.load(md)
0857         datelabel = datetime.now().strftime("%y-%m-%d %H:%M:%S")
0858         if mdata is None:
0859             logging.error('Metadata file is empty.')
0860             return 1
0861         logging.debug('Preparing new metadata file...')
0862         mdata['userText'] = 'reupload %s : %s' %(datelabel,mdata['userText'])
0863         with open( '%s.txt' %rname, 'wb') as jf:
0864             jf.write( json.dumps( mdata, sort_keys=True, indent = 2 ) )
0865             jf.write('\n')
0866         os.remove(mdfile)
0867     else:
0868         logging.error('Tar file does not contain the metadata file')
0869         return 1
0870     logging.info('Files %s prepared for the upload.' %rname)
0871     arguments = [rname]
0872     return upload(options, arguments)
0873
0874 def upload(options, arguments):
0875     results = uploadAllFiles(options, arguments)
0876
0877     if 'status' not in results:
0878         print('Unexpected error.')
0879         return -1
0880     ret = results['status']
0881     print(results)
0882     print("upload ended with code: %s" %ret)
0883     return ret
0884
0885 def main():
0886     '''Entry point.
0887     '''
0888
0889     parser = optparse.OptionParser(usage =
0890         'Usage: %prog [options] <file> [<file> ...]\n'
0891     )
0892
0893     parser.add_option('-d', '--debug',
0894         dest = 'debug',
0895         action="store_true",
0896         default = False,
0897         help = 'Switch on printing debug information. Default: %default',
0898     )
0899
0900     parser.add_option('-b', '--backend',
0901         dest = 'backend',
0902         default = defaultBackend,
0903         help = 'dropBox\'s backend to upload to. Default: %default',
0904     )
0905
0906     parser.add_option('-H', '--hostname',
0907         dest = 'hostname',
0908         default = defaultHostname,
0909         help = 'dropBox\'s hostname. Default: %default',
0910     )
0911
0912     parser.add_option('-u', '--urlTemplate',
0913         dest = 'urlTemplate',
0914         default = defaultUrlTemplate,
0915         help = 'dropBox\'s URL template. Default: %default',
0916     )
0917
0918     parser.add_option('-f', '--temporaryFile',
0919         dest = 'temporaryFile',
0920         default = defaultTemporaryFile,
0921         help = 'Temporary file that will be used to store the first tar file. Note that it then will be moved to a file with the hash of the file as its name, so there will be two temporary files created in fact. Default: %default',
0922     )
0923
0924     parser.add_option('-n', '--netrcHost',
0925         dest = 'netrcHost',
0926         default = defaultNetrcHost,
0927         help = 'The netrc host (machine) from where the username and password will be read. Default: %default',
0928     )
0929
0930     parser.add_option('-a', '--authPath',
0931         dest = 'authPath',
0932         default = None,
0933         help = 'The path of the .netrc file for the authentication. Default: $HOME',
0934     )
0935
0936     parser.add_option('-r', '--reUpload',
0937         dest = 'reUpload',
0938         default = None,
0939         help = 'The hash of the file to upload again.',
0940     )
0941
0942     (options, arguments) = parser.parse_args()
0943
0944     logLevel = logging.INFO
0945     if options.debug:
0946         logLevel = logging.DEBUG
0947     logging.basicConfig(
0948         format = '[%(asctime)s] %(levelname)s: %(message)s',
0949         level = logLevel,
0950     )
0951
0952     if len(arguments) < 1:
0953         if options.reUpload is None:
0954             parser.print_help()
0955             return -2
0956         else:
0957             return re_upload(options)
0958     if options.reUpload is not None:
0959         print("ERROR: options -r can't be specified on a new file upload.")
0960         return -2
0961
0962     return upload(options, arguments)
0963
0964 def testTier0Upload():
0965
0966     global defaultNetrcHost
0967
0968     (username, account, password) = netrc.netrc().authenticators(defaultNetrcHost)
0969
0970     filenames = ['testFiles/localSqlite-top2']
0971
0972     uploadTier0Files(filenames, username, password, cookieFileName = None)
0973
0974
0975 if __name__ == '__main__':
0976
0977     sys.exit(main())
0978     # testTier0Upload()