File indexing completed on 2024-11-25 02:29:20
0001
0002 '''Script that uploads to the new CMS conditions uploader.
0003 Adapted to the new infrastructure from v6 of the upload.py script for the DropBox from Miguel Ojeda.
0004 '''
0005
0006 __author__ = 'Andreas Pfeiffer'
0007 __copyright__ = 'Copyright 2015, CERN CMS'
0008 __credits__ = ['Giacomo Govi', 'Salvatore Di Guida', 'Miguel Ojeda', 'Andreas Pfeiffer']
0009 __license__ = 'Unknown'
0010 __maintainer__ = 'Giacomo Govi'
0011 __email__ = 'giacomo.govi@cern.ch'
0012 __version__ = 1
0013
0014
0015 import os
0016 import sys
0017 import optparse
0018 import hashlib
0019 import tarfile
0020 import netrc
0021 import getpass
0022 import errno
0023 import sqlite3
0024 import cx_Oracle
0025 import json
0026 import tempfile
0027 from datetime import datetime
0028
0029 defaultBackend = 'online'
0030 defaultHostname = 'cms-conddb-prod.cern.ch'
0031 defaultDevHostname = 'cms-conddb-dev.cern.ch'
0032 defaultUrlTemplate = 'https://%s/cmsDbUpload/'
0033 defaultTemporaryFile = 'upload.tar.bz2'
0034 defaultNetrcHost = 'ConditionUploader'
0035 defaultWorkflow = 'offline'
0036 prodLogDbSrv = 'cms_orcoff_prod'
0037 devLogDbSrv = 'cms_orcoff_prep'
0038 logDbSchema = 'CMS_COND_DROPBOX'
0039 authPathEnvVar = 'COND_AUTH_PATH'
0040 waitForRetry = 15
0041
0042
0043 import time
0044 import logging
0045 import io
0046
0047 import pycurl
0048 import socket
0049 import copy
0050
0051 def getInput(default, prompt = ''):
0052 '''Like input() but with a default and automatic strip().
0053 '''
0054
0055 answer = input(prompt)
0056 if answer:
0057 return answer.strip()
0058
0059 return default.strip()
0060
0061
0062 def getInputWorkflow(prompt = ''):
0063 '''Like getInput() but tailored to get target workflows (synchronization options).
0064 '''
0065
0066 while True:
0067 workflow = getInput(defaultWorkflow, prompt)
0068
0069 if workflow in frozenset(['offline', 'hlt', 'express', 'prompt', 'pcl']):
0070 return workflow
0071
0072 logging.error('Please specify one of the allowed workflows. See above for the explanation on each of them.')
0073
0074
0075 def getInputChoose(optionsList, default, prompt = ''):
0076 '''Makes the user choose from a list of options.
0077 '''
0078
0079 while True:
0080 index = getInput(default, prompt)
0081
0082 try:
0083 return optionsList[int(index)]
0084 except ValueError:
0085 logging.error('Please specify an index of the list (i.e. integer).')
0086 except IndexError:
0087 logging.error('The index you provided is not in the given list.')
0088
0089
0090 def getInputRepeat(prompt = ''):
0091 '''Like input() but repeats if nothing is provided and automatic strip().
0092 '''
0093
0094 while True:
0095 answer = input(prompt)
0096 if answer:
0097 return answer.strip()
0098
0099 logging.error('You need to provide a value.')
0100
0101
0102 def runWizard(basename, dataFilename, metadataFilename):
0103 while True:
0104 print('''\nWizard for metadata for %s
0105
0106 I will ask you some questions to fill the metadata file. For some of the questions there are defaults between square brackets (i.e. []), leave empty (i.e. hit Enter) to use them.''' % basename)
0107
0108
0109 dataConnection = sqlite3.connect(dataFilename)
0110 dataCursor = dataConnection.cursor()
0111
0112 dataCursor.execute('select NAME from TAG')
0113 records = dataCursor.fetchall()
0114 inputTags = []
0115 for rec in records:
0116 inputTags.append(rec[0])
0117
0118 if len(inputTags) == 0:
0119 raise Exception("Could not find any input tag in the data file.")
0120
0121 else:
0122 print('\nI found the following input tags in your SQLite data file:')
0123 for (index, inputTag) in enumerate(inputTags):
0124 print(' %s) %s' % (index, inputTag))
0125
0126 inputTag = getInputChoose(inputTags, '0',
0127 '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. 0 (you select the first in the list)\ninputTag [0]: ')
0128
0129 destinationDatabase = ''
0130 ntry = 0
0131 while ( destinationDatabase != 'oracle://cms_orcon_prod/CMS_CONDITIONS' and destinationDatabase != 'oracle://cms_orcoff_prep/CMS_CONDITIONS' ):
0132 if ntry==0:
0133 inputMessage = \
0134 '\nWhich is the destination database where the tags should be exported? \nPossible choices: oracle://cms_orcon_prod/CMS_CONDITIONS (or prod); oracle://cms_orcoff_prep/CMS_CONDITIONS (or prep) \ndestinationDatabase: '
0135 elif ntry==1:
0136 inputMessage = \
0137 '\nPlease choose one of the two valid destinations: \noracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \
0138 \ndestinationDatabase: '
0139 else:
0140 raise Exception('No valid destination chosen. Bailing out...')
0141 destinationDatabase = getInputRepeat(inputMessage)
0142 if destinationDatabase == 'prod':
0143 destinationDatabase = 'oracle://cms_orcon_prod/CMS_CONDITIONS'
0144 if destinationDatabase == 'prep':
0145 destinationDatabase = 'oracle://cms_orcoff_prep/CMS_CONDITIONS'
0146 ntry += 1
0147
0148 while True:
0149 since = getInput('',
0150 '\nWhich is the given since? (if not specified, the one from the SQLite data file will be taken -- note that even if specified, still this may not be the final since, depending on the synchronization options you select later: if the synchronization target is not offline, and the since you give is smaller than the next possible one (i.e. you give a run number earlier than the one which will be started/processed next in prompt/hlt/express), the DropBox will move the since ahead to go to the first safe run instead of the value you gave)\ne.g. 1234\nsince []: ')
0151 if not since:
0152 since = None
0153 break
0154 else:
0155 try:
0156 since = int(since)
0157 break
0158 except ValueError:
0159 logging.error('The since value has to be an integer or empty (null).')
0160
0161 userText = getInput('',
0162 '\nWrite any comments/text you may want to describe your request\ne.g. Muon alignment scenario for...\nuserText []: ')
0163
0164 destinationTags = {}
0165 while True:
0166 destinationTag = getInput('',
0167 '\nWhich is the next destination tag to be added (leave empty to stop)?\ne.g. BeamSpotObjects_PCL_byRun_v0_offline\ndestinationTag []: ')
0168 if not destinationTag:
0169 if len(destinationTags) == 0:
0170 logging.error('There must be at least one destination tag.')
0171 continue
0172 break
0173
0174 if destinationTag in destinationTags:
0175 logging.warning(
0176 'You already added this destination tag. Overwriting the previous one with this new one.')
0177
0178 destinationTags[destinationTag] = {
0179 }
0180
0181 metadata = {
0182 'destinationDatabase': destinationDatabase,
0183 'destinationTags': destinationTags,
0184 'inputTag': inputTag,
0185 'since': since,
0186 'userText': userText,
0187 }
0188
0189 metadata = json.dumps(metadata, sort_keys=True, indent=4)
0190 print('\nThis is the generated metadata:\n%s' % metadata)
0191
0192 if getInput('n',
0193 '\nIs it fine (i.e. save in %s and *upload* the conditions if this is the latest file)?\nAnswer [n]: ' % metadataFilename).lower() == 'y':
0194 break
0195 logging.info('Saving generated metadata in %s...', metadataFilename)
0196 with open(metadataFilename, 'w') as metadataFile:
0197 metadataFile.write(metadata)
0198
0199 class HTTPError(Exception):
0200 '''A common HTTP exception.
0201
0202 self.code is the response HTTP code as an integer.
0203 self.response is the response body (i.e. page).
0204 '''
0205
0206 def __init__(self, code, response):
0207 self.code = code
0208 self.response = response
0209
0210
0211 try:
0212 self.args = (response.split('<p>')[1].split('</p>')[0], )
0213 except Exception:
0214 self.args = (self.response, )
0215
0216
0217 CERN_SSO_CURL_CAPATH = '/etc/pki/tls/certs'
0218
0219 class HTTP(object):
0220 '''Class used for querying URLs using the HTTP protocol.
0221 '''
0222
0223 retryCodes = frozenset([502, 503])
0224
0225 def __init__(self):
0226 self.setBaseUrl()
0227 self.setRetries()
0228
0229 self.curl = pycurl.Curl()
0230 self.curl.setopt(self.curl.COOKIEFILE, '')
0231
0232
0233
0234
0235 self.curl.setopt(self.curl.SSL_VERIFYPEER, 0)
0236 self.curl.setopt(self.curl.SSL_VERIFYHOST, 2)
0237
0238 self.baseUrl = None
0239
0240 self.token = None
0241
0242 def getCookies(self):
0243 '''Returns the list of cookies.
0244 '''
0245 return self.curl.getinfo(self.curl.INFO_COOKIELIST)
0246
0247 def discardCookies(self):
0248 '''Discards cookies.
0249 '''
0250 self.curl.setopt(self.curl.COOKIELIST, 'ALL')
0251
0252
0253 def setBaseUrl(self, baseUrl = ''):
0254 '''Allows to set a base URL which will be prefixed to all the URLs
0255 that will be queried later.
0256 '''
0257 self.baseUrl = baseUrl
0258
0259
0260 def setProxy(self, proxy = ''):
0261 '''Allows to set a proxy.
0262 '''
0263 self.curl.setopt(self.curl.PROXY, proxy)
0264
0265
0266 def setTimeout(self, timeout = 0):
0267 '''Allows to set a timeout.
0268 '''
0269 self.curl.setopt(self.curl.TIMEOUT, timeout)
0270
0271
0272 def setRetries(self, retries = ()):
0273 '''Allows to set retries.
0274
0275 The retries are a sequence of the seconds to wait per retry.
0276
0277 The retries are done on:
0278 * PyCurl errors (includes network problems, e.g. not being able
0279 to connect to the host).
0280 * 502 Bad Gateway (for the moment, to avoid temporary
0281 Apache-CherryPy issues).
0282 * 503 Service Temporarily Unavailable (for when we update
0283 the frontends).
0284 '''
0285 self.retries = retries
0286
0287 def getToken(self, username, password):
0288
0289 url = self.baseUrl + 'token'
0290
0291 self.curl.setopt(pycurl.URL, url)
0292 self.curl.setopt(pycurl.VERBOSE, 0)
0293
0294
0295
0296
0297
0298
0299
0300
0301 self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
0302
0303 self.curl.setopt(self.curl.HTTPGET, 0)
0304
0305 response = io.BytesIO()
0306 self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
0307 self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
0308 logging.debug('going to connect to server at: %s' % url )
0309
0310 self.curl.perform()
0311 code = self.curl.getinfo(pycurl.RESPONSE_CODE)
0312 logging.debug('got: %s ', str(code))
0313 if code in ( 502,503,504 ):
0314 logging.debug('Trying again after %d seconds...', waitForRetry)
0315 time.sleep( waitForRetry )
0316 response = io.StringIO()
0317 self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
0318 self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
0319 self.curl.perform()
0320 code = self.curl.getinfo(pycurl.RESPONSE_CODE)
0321 resp = response.getvalue().decode('UTF-8')
0322 errorMsg = None
0323 if code==500 and not resp.find("INVALID_CREDENTIALS")==-1:
0324 logging.error("Invalid credentials provided.")
0325 return None
0326 if code==403 and not resp.find("Unauthorized access")==-1:
0327 logging.error("Unauthorized access. Please check the membership of group 'cms-cond-dropbox'")
0328 return None
0329 if code==200:
0330 try:
0331 self.token = json.loads( resp )['token']
0332 except Exception as e:
0333 errorMsg = 'Error while decoding returned json string'
0334 logging.debug('http::getToken> error while decoding json: %s ', str(resp) )
0335 logging.debug("error getting token: %s", str(e))
0336 resp = None
0337 else:
0338 errorMsg = 'HTTP Error code %s ' %code
0339 logging.debug('got: %s ', str(code))
0340 logging.debug('http::getToken> got error from server: %s ', str(resp) )
0341 resp = None
0342 if resp is None:
0343 raise Exception(errorMsg)
0344
0345 logging.debug('token: %s', self.token)
0346 logging.debug('returning: %s', response.getvalue().decode('UTF-8'))
0347
0348 return response.getvalue()
0349
0350 def query(self, url, data = None, files = None, keepCookies = True):
0351 '''Queries a URL, optionally with some data (dictionary).
0352
0353 If no data is specified, a GET request will be used.
0354 If some data is specified, a POST request will be used.
0355
0356 If files is specified, it must be a dictionary like data but
0357 the values are filenames.
0358
0359 By default, cookies are kept in-between requests.
0360
0361 A HTTPError exception is raised if the response's HTTP code is not 200.
0362 '''
0363
0364 if not keepCookies:
0365 self.discardCookies()
0366
0367 url = self.baseUrl + url
0368
0369
0370 data4log = copy.copy(data)
0371 if data4log:
0372 if 'password' in data4log.keys():
0373 data4log['password'] = '*'
0374
0375 retries = [0] + list(self.retries)
0376
0377 while True:
0378 logging.debug('Querying %s with data %s and files %s (retries left: %s, current sleep: %s)...', url, data4log, files, len(retries), retries[0])
0379
0380 time.sleep(retries.pop(0))
0381
0382 try:
0383 self.curl.setopt(self.curl.URL, url)
0384 self.curl.setopt(self.curl.HTTPGET, 1)
0385
0386
0387 self.curl.setopt(pycurl.USERPWD, '%s:""' % ( str(self.token), ) )
0388 self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
0389
0390 if data is not None or files is not None:
0391
0392
0393 finalData = {}
0394
0395 if data is not None:
0396 finalData.update(data)
0397
0398 if files is not None:
0399 for (key, fileName) in files.items():
0400 finalData[key] = (self.curl.FORM_FILE, fileName)
0401 self.curl.setopt( self.curl.HTTPPOST, list(finalData.items()) )
0402
0403 self.curl.setopt(pycurl.VERBOSE, 0)
0404
0405 response = io.BytesIO()
0406 self.curl.setopt(self.curl.WRITEFUNCTION, response.write)
0407 self.curl.perform()
0408
0409 code = self.curl.getinfo(self.curl.RESPONSE_CODE)
0410
0411 if code in self.retryCodes and len(retries) > 0:
0412 logging.debug('Retrying since we got the %s error code...', code)
0413 continue
0414
0415 if code != 200:
0416 raise HTTPError(code, response.getvalue())
0417
0418 return response.getvalue()
0419
0420 except pycurl.error as e:
0421 if len(retries) == 0:
0422 raise e
0423 logging.debug('Retrying since we got the %s pycurl exception...', str(e))
0424
0425
0426
0427 def addToTarFile(tarFile, fileobj, arcname):
0428 tarInfo = tarFile.gettarinfo(fileobj = fileobj, arcname = arcname)
0429 tarInfo.mode = 0o400
0430 tarInfo.uid = tarInfo.gid = tarInfo.mtime = 0
0431 tarInfo.uname = tarInfo.gname = 'root'
0432 tarFile.addfile(tarInfo, fileobj)
0433
0434 class ConditionsUploader(object):
0435 '''Upload conditions to the CMS conditions uploader service.
0436 '''
0437
0438 def __init__(self, hostname = defaultHostname, urlTemplate = defaultUrlTemplate):
0439 self.hostname = hostname
0440 self.urlTemplate = urlTemplate
0441 self.userName = None
0442 self.http = None
0443 self.password = None
0444 self.token = None
0445
0446 def setHost( self, hostname ):
0447 if not hostname==self.hostname:
0448 self.token = None
0449 self.hostname = hostname
0450
0451 def signIn(self, username, password ):
0452 if self.token is None:
0453 logging.debug("Initializing connection with server %s",self.hostname)
0454 ''' init the server.
0455 '''
0456 self.http = HTTP()
0457 if socket.getfqdn().strip().endswith('.cms'):
0458 self.http.setProxy('https://cmsproxy.cms:3128/')
0459 self.http.setBaseUrl(self.urlTemplate % self.hostname)
0460 '''Signs in the server.
0461 '''
0462 logging.info('%s: Signing in user %s ...', self.hostname, username)
0463 try:
0464 self.token = self.http.getToken(username, password)
0465 except Exception as e:
0466 ret = -1
0467
0468
0469
0470
0471 logging.error("Caught exception when trying to connect to %s: %s" % (self.hostname, str(e)) )
0472 return ret
0473
0474 if not self.token:
0475 logging.error("could not get token for user %s from %s" % (username, self.hostname) )
0476 return -2
0477 logging.debug( "got: '%s'", str(self.token) )
0478 self.userName = username
0479 self.password = password
0480 else:
0481 logging.debug("User %s has been already authenticated." %username)
0482 return 0
0483
0484 def signOut(self):
0485 '''Signs out the server.
0486 '''
0487
0488 logging.info('%s: Signing out...', self.hostname)
0489
0490 self.token = None
0491
0492
0493 def _checkForUpdates(self):
0494 '''Updates this script, if a new version is found.
0495 '''
0496
0497 logging.debug('%s: Checking if a newer version of this script is available ...', self.hostname)
0498 version = int(self.http.query('getUploadScriptVersion'))
0499
0500 if version <= __version__:
0501 logging.debug('%s: Script is up-to-date.', self.hostname)
0502 return
0503
0504 logging.info('%s: Updating to a newer version (%s) than the current one (%s): downloading ...', self.hostname, version, __version__)
0505
0506 uploadScript = self.http.query('getUploadScript')
0507
0508 self.signOut()
0509
0510 logging.info('%s: ... saving the new version ...', self.hostname)
0511 with open(sys.argv[0], 'wb') as f:
0512 f.write(uploadScript)
0513
0514 logging.info('%s: ... executing the new version...', self.hostname)
0515 os.execl(sys.executable, *([sys.executable] + sys.argv))
0516
0517
0518 def uploadFile(self, filename, backend = defaultBackend, temporaryFile = defaultTemporaryFile):
0519 '''Uploads a file to the dropBox.
0520
0521 The filename can be without extension, with .db or with .txt extension.
0522 It will be stripped and then both .db and .txt files are used.
0523 '''
0524
0525 basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
0526 basename = os.path.basename(basepath)
0527
0528 logging.debug('%s: %s: Creating tar file for upload ...', self.hostname, basename)
0529
0530 try:
0531 tarFile = tarfile.open(temporaryFile, 'w:bz2')
0532
0533 with open('%s.db' % basepath, 'rb') as data:
0534 addToTarFile(tarFile, data, 'data.db')
0535 except Exception as e:
0536 msg = 'Error when creating tar file. \n'
0537 msg += 'Please check that you have write access to the directory you are running,\n'
0538 msg += 'and that you have enough space on this disk (df -h .)\n'
0539 logging.error(msg)
0540 raise Exception(msg)
0541
0542 with tempfile.NamedTemporaryFile(mode='rb+') as metadata:
0543 with open('%s.txt' % basepath, 'r') as originalMetadata:
0544 metadata.write(json.dumps(json.load(originalMetadata), sort_keys = True, indent = 4).encode())
0545
0546 metadata.seek(0)
0547 addToTarFile(tarFile, metadata, 'metadata.txt')
0548
0549 tarFile.close()
0550
0551 logging.debug('%s: %s: Calculating hash...', self.hostname, basename)
0552
0553 fileHash = hashlib.sha1()
0554 with open(temporaryFile, 'rb') as f:
0555 while True:
0556 data = f.read(4 * 1024 * 1024)
0557 if not data:
0558 break
0559 fileHash.update(data)
0560
0561 fileHash = fileHash.hexdigest()
0562 fileInfo = os.stat(temporaryFile)
0563 fileSize = fileInfo.st_size
0564
0565 logging.debug('%s: %s: Hash: %s', self.hostname, basename, fileHash)
0566
0567 logging.info('%s: %s: Uploading file (%s, size %s) to the %s backend...', self.hostname, basename, fileHash, fileSize, backend)
0568 os.rename(temporaryFile, fileHash)
0569 try:
0570 ret = self.http.query('uploadFile',
0571 {
0572 'backend': backend,
0573 'fileName': basename,
0574 'userName': self.userName,
0575 },
0576 files = {
0577 'uploadedFile': fileHash,
0578 }
0579 )
0580 except Exception as e:
0581 logging.error('Error from uploading: %s' % str(e))
0582 ret = json.dumps( { "status": -1, "upload" : { 'itemStatus' : { basename : {'status':'failed', 'info':str(e)}}}, "error" : str(e)} )
0583
0584 os.unlink(fileHash)
0585
0586 statusInfo = json.loads(ret)['upload']
0587 logging.debug( 'upload returned: %s', statusInfo )
0588
0589 okTags = []
0590 skippedTags = []
0591 failedTags = []
0592 for tag, info in statusInfo['itemStatus'].items():
0593 logging.debug('checking tag %s, info %s', tag, str(json.dumps(info, indent=4,sort_keys=True)) )
0594 if 'ok' in info['status'].lower() :
0595 okTags.append( tag )
0596 logging.info('tag %s successfully uploaded', tag)
0597 if 'skip' in info['status'].lower() :
0598 skippedTags.append( tag )
0599 logging.warning('found tag %s to be skipped. reason: \n ... \t%s ', tag, info['info'])
0600 if 'fail' in info['status'].lower() :
0601 failedTags.append( tag )
0602 logging.error('found tag %s failed to upload. reason: \n ... \t%s ', tag, info['info'])
0603
0604 if len(okTags) > 0: logging.info ("tags sucessfully uploaded: %s ", str(okTags) )
0605 if len(skippedTags) > 0: logging.warning("tags SKIPped to upload : %s ", str(skippedTags) )
0606 if len(failedTags) > 0: logging.error ("tags FAILed to upload : %s ", str(failedTags) )
0607
0608 fileLogURL = 'https://cms-conddb.cern.ch/cmsDbBrowser/logs/show_cond_uploader_log/%s/%s'
0609 backend = 'Prod'
0610 if self.hostname=='cms-conddb-dev.cern.ch':
0611 backend = 'Prep'
0612 logging.info('file log at: %s', fileLogURL % (backend,fileHash))
0613
0614 return len(okTags)>0
0615
0616 def getCredentials( options ):
0617
0618 username = None
0619 password = None
0620 netrcPath = None
0621 if authPathEnvVar in os.environ:
0622 authPath = os.environ[authPathEnvVar]
0623 netrcPath = os.path.join(authPath,'.netrc')
0624 if options.authPath is not None:
0625 netrcPath = os.path.join( options.authPath,'.netrc' )
0626 try:
0627
0628 (username, account, password) = netrc.netrc( netrcPath ).authenticators(options.netrcHost)
0629 except Exception:
0630
0631 logging.info(
0632 'netrc entry "%s" not found: if you wish not to have to retype your password, you can add an entry in your .netrc file. However, beware of the risks of having your password stored as plaintext. Instead.',
0633 options.netrcHost)
0634
0635
0636 defaultUsername = getpass.getuser()
0637 if defaultUsername is None:
0638 defaultUsername = '(not found)'
0639
0640 username = getInput(defaultUsername, '\nUsername [%s]: ' % defaultUsername)
0641 password = getpass.getpass('Password: ')
0642
0643 return username, password
0644
0645
0646 def uploadAllFiles(options, arguments):
0647
0648 ret = {}
0649 ret['status'] = 0
0650
0651
0652
0653 for filename in arguments:
0654 basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
0655 basename = os.path.basename(basepath)
0656 dataFilename = '%s.db' % basepath
0657 metadataFilename = '%s.txt' % basepath
0658
0659 logging.info('Checking %s...', basename)
0660
0661
0662 try:
0663 with open(dataFilename, 'rb') as dataFile:
0664 pass
0665 except IOError as e:
0666 errMsg = 'Impossible to open SQLite data file %s' %dataFilename
0667 logging.error( errMsg )
0668 ret['status'] = -3
0669 ret['error'] = errMsg
0670 return ret
0671
0672
0673 empty = True
0674 try:
0675 dbcon = sqlite3.connect( dataFilename )
0676 dbcur = dbcon.cursor()
0677 dbcur.execute('SELECT * FROM IOV')
0678 rows = dbcur.fetchall()
0679 for r in rows:
0680 empty = False
0681 dbcon.close()
0682 if empty:
0683 errMsg = 'The input SQLite data file %s contains no data.' %dataFilename
0684 logging.error( errMsg )
0685 ret['status'] = -4
0686 ret['error'] = errMsg
0687 return ret
0688 except Exception as e:
0689 errMsg = 'Check on input SQLite data file %s failed: %s' %(dataFilename,str(e))
0690 logging.error( errMsg )
0691 ret['status'] = -5
0692 ret['error'] = errMsg
0693 return ret
0694
0695
0696 try:
0697 with open(metadataFilename, 'rb') as metadataFile:
0698 pass
0699 except IOError as e:
0700 if e.errno != errno.ENOENT:
0701 errMsg = 'Impossible to open file %s (for other reason than not existing)' %metadataFilename
0702 logging.error( errMsg )
0703 ret['status'] = -4
0704 ret['error'] = errMsg
0705 return ret
0706
0707 if getInput('y', '\nIt looks like the metadata file %s does not exist. Do you want me to create it and help you fill it?\nAnswer [y]: ' % metadataFilename).lower() != 'y':
0708 errMsg = 'Metadata file %s does not exist' %metadataFilename
0709 logging.error( errMsg )
0710 ret['status'] = -5
0711 ret['error'] = errMsg
0712 return ret
0713
0714 runWizard(basename, dataFilename, metadataFilename)
0715
0716
0717 try:
0718 dropBox = ConditionsUploader(options.hostname, options.urlTemplate)
0719
0720
0721 username, password = getCredentials(options)
0722
0723 results = {}
0724 for filename in arguments:
0725 backend = options.backend
0726 basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
0727 metadataFilename = '%s.txt' % basepath
0728 with open(metadataFilename, 'rb') as metadataFile:
0729 metadata = json.load( metadataFile )
0730
0731 forceHost = False
0732 destDb = metadata['destinationDatabase']
0733 if destDb.startswith('oracle://cms_orcon_prod') or destDb.startswith('oracle://cms_orcoff_prep'):
0734 hostName = defaultHostname
0735 if destDb.startswith('oracle://cms_orcoff_prep'):
0736 hostName = defaultDevHostname
0737 dropBox.setHost( hostName )
0738 authRet = dropBox.signIn( username, password )
0739 if not authRet==0:
0740 msg = "Error trying to connect to the server. Aborting."
0741 if authRet==-2:
0742 msg = "Error while signin in. Aborting."
0743 logging.error(msg)
0744 return { 'status' : authRet, 'error' : msg }
0745 results[filename] = dropBox.uploadFile(filename, options.backend, options.temporaryFile)
0746 else:
0747 results[filename] = False
0748 logging.error("DestinationDatabase %s is not valid. Skipping the upload." %destDb)
0749 if not results[filename]:
0750 if ret['status']<0:
0751 ret['status'] = 0
0752 ret['status'] += 1
0753 ret['files'] = results
0754 logging.debug("all files processed, logging out now.")
0755
0756 dropBox.signOut()
0757
0758 except HTTPError as e:
0759 logging.error('got HTTP error: %s', str(e))
0760 return { 'status' : -1, 'error' : str(e) }
0761
0762 return ret
0763
0764 def uploadTier0Files(filenames, username, password, cookieFileName = None):
0765 '''Uploads a bunch of files coming from Tier0.
0766 This has the following requirements:
0767 * Username/Password based authentication.
0768 * Uses the online backend.
0769 * Ignores errors related to the upload/content (e.g. duplicated file).
0770 '''
0771
0772 dropBox = ConditionsUploader()
0773
0774 dropBox.signIn(username, password)
0775
0776 for filename in filenames:
0777 try:
0778 result = dropBox.uploadFile(filename, backend = 'test')
0779 except HTTPError as e:
0780 if e.code == 400:
0781
0782
0783
0784 logging.error('HTTP Exception 400 Bad Request: Upload-related, skipping. Message: %s', e)
0785 continue
0786
0787
0788 raise
0789
0790
0791
0792
0793
0794
0795 if not result:
0796 logging.error('Error from dropbox, upload-related, skipping.')
0797 continue
0798
0799 dropBox.signOut()
0800
0801 def re_upload( options ):
0802 netrcPath = None
0803 logDbSrv = prodLogDbSrv
0804 if options.hostname == defaultDevHostname:
0805 logDbSrv = devLogDbSrv
0806 if options.authPath is not None:
0807 netrcPath = os.path.join( options.authPath,'.netrc' )
0808 try:
0809 netrcKey = '%s/%s' %(logDbSrv,logDbSchema)
0810
0811 (username, account, password) = netrc.netrc( netrcPath ).authenticators( netrcKey )
0812 except IOError as e:
0813 logging.error('Cannot access netrc file.')
0814 return 1
0815 except Exception as e:
0816 logging.error('Netrc file is invalid: %s' %str(e))
0817 return 1
0818 conStr = '%s/%s@%s' %(username,password,logDbSrv)
0819 con = cx_Oracle.connect( conStr )
0820 cur = con.cursor()
0821 fh = options.reUpload
0822 cur.execute('SELECT FILECONTENT, STATE FROM FILES WHERE FILEHASH = :HASH',{'HASH':fh})
0823 res = cur.fetchall()
0824 found = False
0825 fdata = None
0826 for r in res:
0827 found = True
0828 logging.info("Found file %s in state '%s;" %(fh,r[1]))
0829 fdata = r[0].read().decode('bz2')
0830 con.close()
0831 if not found:
0832 logging.error("No file uploaded found with hash %s" %fh)
0833 return 1
0834
0835 fname = '%s.tar' %fh
0836 with open(fname, "wb" ) as f:
0837 f.write(fdata)
0838 rname = 'reupload_%s' %fh
0839 with tarfile.open(fname) as tar:
0840 tar.extractall()
0841 os.remove(fname)
0842 dfile = 'data.db'
0843 mdfile = 'metadata.txt'
0844 if os.path.exists(dfile):
0845 os.utime(dfile,None)
0846 os.chmod(dfile,0o755)
0847 os.rename(dfile,'%s.db' %rname)
0848 else:
0849 logging.error('Tar file does not contain the data file')
0850 return 1
0851 if os.path.exists(mdfile):
0852 os.utime(mdfile,None)
0853 os.chmod(mdfile,0o755)
0854 mdata = None
0855 with open(mdfile) as md:
0856 mdata = json.load(md)
0857 datelabel = datetime.now().strftime("%y-%m-%d %H:%M:%S")
0858 if mdata is None:
0859 logging.error('Metadata file is empty.')
0860 return 1
0861 logging.debug('Preparing new metadata file...')
0862 mdata['userText'] = 'reupload %s : %s' %(datelabel,mdata['userText'])
0863 with open( '%s.txt' %rname, 'wb') as jf:
0864 jf.write( json.dumps( mdata, sort_keys=True, indent = 2 ) )
0865 jf.write('\n')
0866 os.remove(mdfile)
0867 else:
0868 logging.error('Tar file does not contain the metadata file')
0869 return 1
0870 logging.info('Files %s prepared for the upload.' %rname)
0871 arguments = [rname]
0872 return upload(options, arguments)
0873
0874 def upload(options, arguments):
0875 results = uploadAllFiles(options, arguments)
0876
0877 if 'status' not in results:
0878 print('Unexpected error.')
0879 return -1
0880 ret = results['status']
0881 print(results)
0882 print("upload ended with code: %s" %ret)
0883 return ret
0884
0885 def main():
0886 '''Entry point.
0887 '''
0888
0889 parser = optparse.OptionParser(usage =
0890 'Usage: %prog [options] <file> [<file> ...]\n'
0891 )
0892
0893 parser.add_option('-d', '--debug',
0894 dest = 'debug',
0895 action="store_true",
0896 default = False,
0897 help = 'Switch on printing debug information. Default: %default',
0898 )
0899
0900 parser.add_option('-b', '--backend',
0901 dest = 'backend',
0902 default = defaultBackend,
0903 help = 'dropBox\'s backend to upload to. Default: %default',
0904 )
0905
0906 parser.add_option('-H', '--hostname',
0907 dest = 'hostname',
0908 default = defaultHostname,
0909 help = 'dropBox\'s hostname. Default: %default',
0910 )
0911
0912 parser.add_option('-u', '--urlTemplate',
0913 dest = 'urlTemplate',
0914 default = defaultUrlTemplate,
0915 help = 'dropBox\'s URL template. Default: %default',
0916 )
0917
0918 parser.add_option('-f', '--temporaryFile',
0919 dest = 'temporaryFile',
0920 default = defaultTemporaryFile,
0921 help = 'Temporary file that will be used to store the first tar file. Note that it then will be moved to a file with the hash of the file as its name, so there will be two temporary files created in fact. Default: %default',
0922 )
0923
0924 parser.add_option('-n', '--netrcHost',
0925 dest = 'netrcHost',
0926 default = defaultNetrcHost,
0927 help = 'The netrc host (machine) from where the username and password will be read. Default: %default',
0928 )
0929
0930 parser.add_option('-a', '--authPath',
0931 dest = 'authPath',
0932 default = None,
0933 help = 'The path of the .netrc file for the authentication. Default: $HOME',
0934 )
0935
0936 parser.add_option('-r', '--reUpload',
0937 dest = 'reUpload',
0938 default = None,
0939 help = 'The hash of the file to upload again.',
0940 )
0941
0942 (options, arguments) = parser.parse_args()
0943
0944 logLevel = logging.INFO
0945 if options.debug:
0946 logLevel = logging.DEBUG
0947 logging.basicConfig(
0948 format = '[%(asctime)s] %(levelname)s: %(message)s',
0949 level = logLevel,
0950 )
0951
0952 if len(arguments) < 1:
0953 if options.reUpload is None:
0954 parser.print_help()
0955 return -2
0956 else:
0957 return re_upload(options)
0958 if options.reUpload is not None:
0959 print("ERROR: options -r can't be specified on a new file upload.")
0960 return -2
0961
0962 return upload(options, arguments)
0963
0964 def testTier0Upload():
0965
0966 global defaultNetrcHost
0967
0968 (username, account, password) = netrc.netrc().authenticators(defaultNetrcHost)
0969
0970 filenames = ['testFiles/localSqlite-top2']
0971
0972 uploadTier0Files(filenames, username, password, cookieFileName = None)
0973
0974
0975 if __name__ == '__main__':
0976
0977 sys.exit(main())
0978