python/Tools/confdbOfflineConverter.py

0001 #! /usr/bin/env python3
0002 import sys, os
0003 import re
0004 import hashlib
0005 import os.path
0006 import tempfile
0007 import requests
0008 import shutil
0009 import subprocess
0010 import atexit
0011 from collections import Counter
0012
0013 class OfflineConverter:
0014
0015     # the machine aliases and interfaces for the *online* database are
0016     #   cmsonr1-s.cms, cmsonr2-s.cms, cmsonr3-s.cms
0017     #   cmsonr1-v.cms, cmsonr2-v.cms, cmsonr3-v.cms
0018     # but the -s and -v interfaces resolve to the same hosts.
0019     # The actual machines and interfaces are
0020     #   CMSRAC11-S.cms, CMSRAC12-S.cms, CMSRAC21-S.cms
0021     #   CMSRAC11-V.cms, CMSRAC12-V.cms, CMSRAC21-V.cms
0022
0023     # the possible machines and interfaces for the *offline* database are
0024     #   cmsr1-s.cms, cmsr2-s.cms, cmsr3-s.cms
0025     #   cmsr1-v.cms, cmsr2-v.cms, cmsr3-v.cms
0026     # but the -s and -v interfaces resolve to the same hosts
0027     # The actual machines and interfaces are
0028     #   itrac50011-s.cern.ch, itrac50063-s.cern.ch, itrac50078-s.cern.ch
0029     #   itrac50011-v.cern.ch, itrac50063-v.cern.ch, itrac50078-v.cern.ch
0030
0031     databases = {}
0032     databases['v1'] = {}
0033     databases['v1']['offline'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch',        '-d', 'cms_cond.cern.ch',      '-u', 'cms_hltdev_reader', '-s', 'convertMe!' )
0034     databases['v1']['hltdev']  = databases['v1']['offline']     # for backwards compatibility
0035     databases['v1']['online']  = ( '-t', 'oracle', '-h', 'cmsonr1-s.cms',          '-d', 'cms_rcms.cern.ch',      '-u', 'cms_hlt_r',         '-s', 'convertMe!' )
0036     databases['v1']['adg']     = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch',        '-d', 'cms_cond.cern.ch',      '-u', 'cms_hlt_gui_r',     '-s', 'convertMe!' )
0037     databases['v1']['orcoff']  = databases['v1']['adg']         # for backwards compatibility
0038     databases['v3'] = {}
0039     databases['v3']['run2'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch',        '-d', 'cms_hlt.cern.ch',      '-u', 'cms_hlt_gdr_r',     '-s', 'convertMe!' )
0040     databases['v3']['run3'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch',        '-d', 'cms_hlt.cern.ch',      '-u', 'cms_hlt_v3_r',     '-s', 'convertMe!' )
0041     databases['v3']['dev'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch',        '-d', 'cms_hlt.cern.ch',      '-u', 'cms_hlt_gdrdev_r',     '-s', 'convertMe1!' )
0042     databases['v3']['online']  = ( '-t', 'oracle', '-h', 'cmsonr1-s.cms',          '-d', 'cms_rcms.cern.ch',      '-u', 'cms_hlt_gdr_r',     '-s', 'convertMe!' )
0043     databases['v3']['adg']     = ( '-t', 'oracle', '-h', 'cmsonr1-adg1-s.cern.ch', '-d', 'cms_orcon_adg.cern.ch', '-u', 'cms_hlt_gdr_r',     '-s', 'convertMe!' )
0044
0045     #ip addresses, there is a bug where we cant do dns over the socks server, sigh
0046     ips_for_proxy = {
0047         'cmsr1-s.cern.ch' : '10.116.96.89',
0048         'cmsr2-s.cern.ch' : '10.116.96.139',
0049         'cmsr3-s.cern.ch' : '10.116.96.105'
0050     }
0051
0052     databases['v3-beta'] = dict(databases['v3'])
0053     databases['v3-test'] = dict(databases['v3'])
0054     databases['v2'] = dict(databases['v3'])
0055     #old converter can only handle a single host so we modify the params accordingly
0056     for dbkey in databases['v2']:
0057         dbparams  = databases['v2'][dbkey]
0058         if dbparams[3]=='cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch':
0059             databases['v2'][dbkey] = dbparams[0:3]+('cmsr1-s.cern.ch',)+dbparams[4:]
0060
0061     @staticmethod
0062     def CheckTempDirectory(dir):
0063         dir = os.path.realpath(dir)
0064         if not os.path.isdir(dir):
0065             try:
0066                 os.makedirs(dir)
0067             except:
0068                 return None
0069         return dir
0070
0071
0072     def __init__(self, version = 'v3', database = 'run3', url = None, verbose = False,
0073                  proxy = False, proxyHost = 'localhost', proxyPort = '8080'):
0074         self.verbose = verbose
0075         self.version = version
0076         self.baseDir = '/afs/cern.ch/user/c/confdb/www/%s/lib' % version
0077         self.baseUrl = 'https://confdb.web.cern.ch/confdb/%s/lib' % version
0078         self.jars    = ( 'ojdbc8.jar', 'cmssw-evf-confdb-converter.jar' )
0079         if version=='v2':
0080             #legacy driver for run2 gui
0081             self.jars = ( 'ojdbc6.jar', 'cmssw-evf-confdb-converter.jar' )
0082         self.workDir = ''
0083         self.proxy = proxy
0084         self.proxyHost = proxyHost
0085         self.proxyPort = proxyPort
0086
0087         # check the schema version
0088         if version not in self.databases:
0089             # unsupported database version
0090             sys.stderr.write( "ERROR: unsupported database version \"%s\"\n" % version)
0091
0092         # check the database
0093         if database in self.databases[version]:
0094             # load the connection parameters for the given database
0095             self.connect = self.databases[version][database]
0096         else:
0097             # unsupported database
0098             sys.stderr.write( "ERROR: unknown database \"%s\" for version \"%s\"\n" % (database, version))
0099             sys.exit(1)
0100
0101         if self.proxy:
0102             self.proxy_connect_args = ('--dbproxy', '--dbproxyport', self.proxyPort, '--dbproxyhost', self.proxyHost)
0103             temp_connect = []
0104             for entry in self.connect:
0105                 for key,item in self.ips_for_proxy.items():
0106                     entry = entry.replace(key,item)
0107                 temp_connect.append(entry.replace(key,item))
0108             self.connect  = tuple(temp_connect)
0109         else:
0110             self.proxy_connect_args = ()
0111
0112         # check for a custom base URL
0113         if url is not None:
0114             self.baseUrl = url
0115
0116         # try to read the .jar files from AFS, or download them
0117         if os.path.isdir(self.baseDir) and all(os.path.isfile(self.baseDir + '/' + jar) for jar in self.jars):
0118             # read the .jar fles from AFS
0119             self.workDir = self.baseDir
0120         else:
0121             # try to use $CMSSW_BASE/tmp
0122             self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['CMSSW_BASE'],'tmp','confdb',self.version))
0123             if not self.workDir:
0124                 # try to use $TMP
0125                 self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['TMP'],'confdb',self.version))
0126             if not self.workDir:
0127                 # create a new temporary directory, and install a cleanup callback
0128                 self.workDir = tempfile.mkdtemp()
0129                 atexit.register(shutil.rmtree, self.workDir)
0130             # download the .jar files
0131             version_website = requests.get(self.baseUrl+"/../confdb.version").text
0132             jars_require_update = True
0133             if os.path.exists(os.path.join(self.workDir,"confdb.version")):
0134                 with open(os.path.join(self.workDir,"confdb.version")) as f:
0135                     version_existing = f.read()
0136                     if version_existing==version_website:
0137                         jars_require_update = False
0138
0139             if jars_require_update:
0140                 for jar in self.jars:
0141                     # download to a temporay name and use an atomic rename (in case an other istance is downloading the same file
0142                     handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = jar + '.')
0143                     os.close(handle)
0144                     request = requests.get(self.baseUrl + '/' + jar)
0145                     with open(temp,'wb') as f:
0146                         f.write(request.content)
0147                     os.rename(temp, self.workDir + '/' + jar)
0148                 #jars updated, write their version
0149                 handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = "confdb.version" + '.')
0150                 os.close(handle)
0151                 with open(temp,'w') as f:
0152                     f.write(version_website)
0153                 os.rename(temp,os.path.join(self.workDir,"confdb.version"))
0154
0155         # setup the java command line and CLASSPATH
0156         if self.verbose:
0157             sys.stderr.write("workDir = %s\n" % self.workDir)
0158         # use non-blocking random number source /dev/urandom (instead of /dev/random), see:
0159         #   http://blockdump.blogspot.fr/2012/07/connection-problems-inbound-connection.html
0160         # deal with timezone region not found
0161         #   http://stackoverflow.com/questions/9156379/ora-01882-timezone-region-not-found
0162         # increase the thread stack size from the default of 1 MB to work around java.lang.StackOverflowError errors, see
0163         #   man java
0164         self.javaCmd = ( 'java', '-cp', ':'.join(self.workDir + '/' + jar for jar in self.jars), '-Djava.security.egd=file:///dev/urandom', '-Doracle.jdbc.timezoneAsRegion=false', '-Xss32M', 'confdb.converter.BrowserConverter' )
0165
0166
0167     def query(self, *args):
0168         args = self.javaCmd + self.connect + self.proxy_connect_args + args
0169         if self.verbose:
0170             sys.stderr.write("\n" + ' '.join(args) + "\n\n" )
0171         sub = subprocess.Popen(
0172             args,
0173             stdin = None,
0174             stdout = subprocess.PIPE,
0175             stderr = subprocess.PIPE,
0176             shell = False,
0177             universal_newlines = True )
0178         return sub.communicate()
0179
0180 def help():
0181     sys.stdout.write("""Usage: %s OPTIONS
0182
0183         --v1|--v2|--v3|--v3-beta|--v3-test  (specify the ConfDB version [default: v3])
0184
0185         --run3|--run2|--dev|--online|--adg    (specify the target db [default: run3], online will only work inside p5 network)
0186
0187         Note that for v1
0188             --orcoff  is a synonim of --adg
0189             --offline is a synonim of --hltdev
0190
0191         --configId <id>             (specify the configuration by id)
0192         --configName <name>         (specify the configuration by name)
0193         --runNumber <run>           (specify the configuration by run number)
0194           [exactly one of --configId OR --configName OR --runNumber is required]
0195
0196         --cff                       (retrieve configuration *fragment*)
0197         --input <f1.root[,f2.root]> (insert PoolSource with specified fileNames)
0198         --input <files.list>        (read a text file which lists input ROOT files)
0199         --output <out.root>         (insert PoolOutputModule w/ specified fileName)
0200         --nopsets                   (exclude all globale psets)
0201         --noedsources               (exclude all edsources)
0202         --noes                      (exclude all essources *and* esmodules)
0203         --noessources               (exclude all essources)
0204         --noesmodules               (exclude all esmodules)
0205         --noservices                (exclude all services)
0206         --nooutput                  (exclude all output modules)
0207         --nopaths                   (exclude all paths [+=referenced seqs&mods])
0208         --nosequences               (don't define sequences [+=referenced s&m])
0209         --nomodules                 (don't define modules)
0210         --psets <pset1[,pset2]>     (include only specified global psets)
0211         --psets <-pset1[,-pset2]>   (include all global psets but the specified)
0212         --essources <ess1[,ess2]>   (include only specified essources)
0213         --essources <-ess1[,-ess2]> (include all essources but the specified)
0214         --esmodules <esm1[,esm2]>   (include only specified esmodules)
0215         --esmodules <-esm1[,-esm2]> (include all esmodules but the specified)
0216         --services <svc1[,svc2]>    (include only specified services)
0217         --services <-svc1[,-svc2]>  (include all services but the specified)
0218         --paths <p1[,p2]>           (include only specified paths)
0219         --paths <-p1[,-p2]>         (include all paths but the specified)
0220         --streams <s1[,s2]>         (include only specified streams)
0221         --datasets <d1[,d2]>        (include only specified datasets)
0222         --sequences <s1[,s2]>       (include sequences, referenced or not!)
0223         --modules <p1[,p2]>         (include modules, referenced or not!)
0224         --blocks <m1::p1[,p2][,m2]> (generate parameter blocks)
0225
0226         --verbose                   (print additional details)
0227 """)
0228
0229
0230 def main():
0231     args = sys.argv[1:]
0232     version = 'v3'
0233     db      = 'run3'
0234     verbose = False
0235
0236     if not args:
0237         help()
0238         sys.exit(1)
0239
0240     if '--help' in args or '-h' in args:
0241         help()
0242         sys.exit(0)
0243
0244     if '--verbose' in args:
0245         verbose = True
0246         args.remove('--verbose')
0247
0248     arg_count = Counter(args)
0249     db_count = arg_count['--v1'] + arg_count['--v2'] + arg_count['--v3'] + arg_count['--v3-beta'] + arg_count['--v3-test']
0250     if db_count>1:
0251         sys.stderr.write( 'ERROR: conflicting database version specifications: "--v1", "--v2", "--v3", "--v3-beta", and "--v3-test" are mutually exclusive options' )
0252         sys.exit(1)
0253
0254     if '--v1' in args:
0255         version = 'v1'
0256         db      = 'offline'
0257         args.remove('--v1')
0258
0259     if '--v2' in args:
0260         version = 'v2'
0261         db      = 'run2'
0262         args.remove('--v2')
0263
0264     if '--v3' in args:
0265         version = 'v3'
0266         db      = 'run3'
0267         args.remove('--v3')
0268
0269     if '--v3-beta' in args:
0270         version = 'v3-beta'
0271         db      = 'run3'
0272         args.remove('--v3-beta')
0273
0274     if '--v3-test' in args:
0275         version = 'v3-test'
0276         db      = 'dev'
0277         args.remove('--v3-test')
0278
0279     proxy=False
0280     proxy_host = "localhost"
0281     proxy_port = "8080"
0282     if '--dbproxy' in args:
0283         proxy = True
0284         args.remove('--dbproxy')
0285     if '--dbproxyhost' in args:
0286         proxy_host = args.pop(args.index('--dbproxyhost')+1)
0287         args.remove('--dbproxyhost')
0288     if '--dbproxyport' in args:
0289         proxy_port = args.pop(args.index('--dbproxyport')+1)
0290         args.remove('--dbproxyport')
0291
0292
0293     _dbs = {}
0294     _dbs['v1'] = [ '--%s' % _db for _db in OfflineConverter.databases['v1'] ] + [ '--runNumber' ]
0295     _dbs['v2'] = [ '--%s' % _db for _db in OfflineConverter.databases['v2'] ] + [ '--runNumber' ]
0296     _dbs['v3'] = [ '--%s' % _db for _db in OfflineConverter.databases['v3'] ] + [ '--runNumber']
0297     _dbs['v3-beta'] = [ '--%s' % _db for _db in OfflineConverter.databases['v3-beta'] ] + [ '--runNumber' ]
0298     _dbs['v3-test'] = [ '--%s' % _db for _db in OfflineConverter.databases['v3-test'] ] + [ '--runNumber' ]
0299     _dbargs = set(args) & set(sum(_dbs.values(), []))
0300
0301     if _dbargs:
0302         if len(_dbargs) > 1:
0303             sys.stderr.write( "ERROR: too many database specifications: \"" + "\", \"".join( _dbargs) + "\"\n" )
0304             sys.exit(1)
0305
0306         _arg = _dbargs.pop()
0307         db   = _arg[2:]
0308         if db == 'runNumber':
0309             db = 'adg'
0310         else:
0311             args.remove(_arg)
0312
0313         if not db in OfflineConverter.databases[version]:
0314             sys.stderr.write( "ERROR: database version \"%s\" incompatible with specification \"%s\"\n" % (version, db) )
0315             sys.exit(1)
0316
0317     converter = OfflineConverter(version = version, database = db, verbose = verbose,
0318                                  proxy = proxy, proxyHost = proxy_host, proxyPort=proxy_port)
0319     out, err = converter.query( * args )
0320     if 'ERROR' in err:
0321         sys.stderr.write( "%s: error while retriving the HLT menu\n\n%s\n\n" % (sys.argv[0], err) )
0322         sys.exit(1)
0323     else:
0324         sys.stdout.write( out )
0325
0326
0327 if __name__ == "__main__":
0328     main()