python/Tools/confdbOfflineConverter.py

0001 #! /usr/bin/env python3
0002 import sys, os
0003 import re
0004 import hashlib
0005 import os.path
0006 import tempfile
0007 import requests
0008 import shutil
0009 import subprocess
0010 import atexit
0011 from collections import Counter
0012
0013 class OfflineConverter:
0014
0015     # the machine aliases and interfaces for the *online* database are
0016     #   cmsonr1-s.cms, cmsonr2-s.cms, cmsonr3-s.cms
0017     #   cmsonr1-v.cms, cmsonr2-v.cms, cmsonr3-v.cms
0018     # but the -s and -v interfaces resolve to the same hosts.
0019     # The actual machines and interfaces are
0020     #   CMSRAC11-S.cms, CMSRAC12-S.cms, CMSRAC21-S.cms
0021     #   CMSRAC11-V.cms, CMSRAC12-V.cms, CMSRAC21-V.cms
0022
0023     # the possible machines and interfaces for the *offline* database are
0024     #   cmsr1-s.cms, cmsr2-s.cms, cmsr3-s.cms
0025     #   cmsr1-v.cms, cmsr2-v.cms, cmsr3-v.cms
0026     # but the -s and -v interfaces resolve to the same hosts
0027     # The actual machines and interfaces are
0028     #   itrac50011-s.cern.ch, itrac50063-s.cern.ch, itrac50078-s.cern.ch
0029     #   itrac50011-v.cern.ch, itrac50063-v.cern.ch, itrac50078-v.cern.ch
0030
0031     databases = {}
0032     databases['v1'] = {}
0033     databases['v1']['offline'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch',        '-d', 'cms_cond.cern.ch',      '-u', 'cms_hltdev_reader', '-s', 'convertMe!' )
0034     databases['v1']['hltdev']  = databases['v1']['offline']     # for backwards compatibility
0035     databases['v1']['online']  = ( '-t', 'oracle', '-h', 'cmsonr1-s.cms',          '-d', 'cms_rcms.cern.ch',      '-u', 'cms_hlt_r',         '-s', 'convertMe!' )
0036     databases['v1']['adg']     = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch',        '-d', 'cms_cond.cern.ch',      '-u', 'cms_hlt_gui_r',     '-s', 'convertMe!' )
0037     databases['v1']['orcoff']  = databases['v1']['adg']         # for backwards compatibility
0038     databases['v3'] = {}
0039     databases['v3']['run2'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch',        '-d', 'cms_hlt.cern.ch',      '-u', 'cms_hlt_gdr_r',     '-s', 'convertMe!' )
0040     databases['v3']['run3'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch',        '-d', 'cms_hlt.cern.ch',      '-u', 'cms_hlt_v3_r',     '-s', 'convertMe!' )
0041     databases['v3']['dev'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch',        '-d', 'cms_hlt.cern.ch',      '-u', 'cms_hlt_gdrdev_r',     '-s', 'convertMe1!' )
0042     databases['v3']['online']  = ( '-t', 'oracle', '-h', 'cmsonr1-s.cms',          '-d', 'cms_rcms.cern.ch',      '-u', 'cms_hlt_gdr_r',     '-s', 'convertMe!' )
0043     databases['v3']['adg']     = ( '-t', 'oracle', '-h', 'cmsonr1-adg1-s.cern.ch', '-d', 'cms_orcon_adg.cern.ch', '-u', 'cms_hlt_gdr_r',     '-s', 'convertMe!' )
0044
0045     #ip addresses, there is a bug where we cant do dns over the socks server, sigh
0046     ips_for_proxy = {
0047         'cmsr1-s.cern.ch' : '10.116.96.89',
0048         'cmsr2-s.cern.ch' : '10.116.96.139',
0049         'cmsr3-s.cern.ch' : '10.116.96.105',
0050         'cmsonr1-adg1-s.cern.ch' : '10.116.96.109',
0051         'cmsonr1-s.cms' : '10.176.84.78'
0052     }
0053
0054     databases['v3-beta'] = dict(databases['v3'])
0055     databases['v3-test'] = dict(databases['v3'])
0056     databases['v2'] = dict(databases['v3'])
0057     #old converter can only handle a single host so we modify the params accordingly
0058     for dbkey in databases['v2']:
0059         dbparams  = databases['v2'][dbkey]
0060         if dbparams[3]=='cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch':
0061             databases['v2'][dbkey] = dbparams[0:3]+('cmsr1-s.cern.ch',)+dbparams[4:]
0062
0063     @staticmethod
0064     def CheckTempDirectory(dir):
0065         dir = os.path.realpath(dir)
0066         if not os.path.isdir(dir):
0067             try:
0068                 os.makedirs(dir)
0069             except:
0070                 return None
0071         return dir
0072
0073
0074     def __init__(self, version = 'v3', database = 'run3', url = None, verbose = False,
0075                  proxy = False, proxyHost = 'localhost', proxyPort = '8080',
0076                  tunnel = False, tunnelPort = '10121'):
0077         self.verbose = verbose
0078         self.version = version
0079         self.baseDir = '/afs/cern.ch/user/c/confdb/www/%s/lib' % version
0080         self.baseUrl = 'https://confdb.web.cern.ch/confdb/%s/lib' % version
0081         self.jars    = ( 'ojdbc8.jar', 'cmssw-evf-confdb-converter.jar' )
0082         if version=='v2':
0083             #legacy driver for run2 gui
0084             self.jars = ( 'ojdbc6.jar', 'cmssw-evf-confdb-converter.jar' )
0085         self.workDir = ''
0086         self.proxy = proxy
0087         self.proxyHost = proxyHost
0088         self.proxyPort = proxyPort
0089         self.tunnel = tunnel
0090         self.tunnelPort = tunnelPort
0091
0092         if self.proxy and self.tunnel:
0093             sys.stderr.write( "ERROR: proxy and tunnel options can not both be true" )
0094             sys.exit(1)
0095
0096         # check the schema version
0097         if version not in self.databases:
0098             # unsupported database version
0099             sys.stderr.write( "ERROR: unsupported database version \"%s\"\n" % version)
0100
0101         # check the database
0102         if database in self.databases[version]:
0103             # load the connection parameters for the given database
0104             self.connect = self.databases[version][database]
0105         else:
0106             # unsupported database
0107             sys.stderr.write( "ERROR: unknown database \"%s\" for version \"%s\"\n" % (database, version))
0108             sys.exit(1)
0109
0110         if self.proxy:
0111             self.proxy_connect_args = ('--dbproxy', '--dbproxyport', self.proxyPort, '--dbproxyhost', self.proxyHost)
0112             temp_connect = []
0113             for entry in self.connect:
0114                 for key,item in self.ips_for_proxy.items():
0115                     entry = entry.replace(key,item)
0116                 temp_connect.append(entry.replace(key,item))
0117             self.connect  = tuple(temp_connect)
0118         else:
0119             self.proxy_connect_args = ()
0120
0121         # this sets the host to localhost
0122         if self.tunnel:
0123             temp_connect = list(self.connect)
0124             host_index = temp_connect.index('-h')
0125             temp_connect[host_index+1] = "localhost"
0126             self.connect = tuple(temp_connect)
0127             self.tunnel_connect_args = ('--dbport', self.tunnelPort)
0128         else:
0129             self.tunnel_connect_args = ()
0130
0131         # check for a custom base URL
0132         if url is not None:
0133             self.baseUrl = url
0134
0135         # try to read the .jar files from AFS, or download them
0136         if os.path.isdir(self.baseDir) and all(os.path.isfile(self.baseDir + '/' + jar) for jar in self.jars):
0137             # read the .jar fles from AFS
0138             self.workDir = self.baseDir
0139         else:
0140             # try to use $CMSSW_BASE/tmp
0141             self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['CMSSW_BASE'],'tmp','confdb',self.version))
0142             if not self.workDir:
0143                 # try to use $TMP
0144                 self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['TMP'],'confdb',self.version))
0145             if not self.workDir:
0146                 # create a new temporary directory, and install a cleanup callback
0147                 self.workDir = tempfile.mkdtemp()
0148                 atexit.register(shutil.rmtree, self.workDir)
0149             # download the .jar files
0150             version_website = requests.get(self.baseUrl+"/../confdb.version").text
0151             jars_require_update = True
0152             if os.path.exists(os.path.join(self.workDir,"confdb.version")):
0153                 with open(os.path.join(self.workDir,"confdb.version")) as f:
0154                     version_existing = f.read()
0155                     if version_existing==version_website:
0156                         jars_require_update = False
0157
0158             if jars_require_update:
0159                 for jar in self.jars:
0160                     # download to a temporay name and use an atomic rename (in case an other istance is downloading the same file
0161                     handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = jar + '.')
0162                     os.close(handle)
0163                     request = requests.get(self.baseUrl + '/' + jar)
0164                     with open(temp,'wb') as f:
0165                         f.write(request.content)
0166                     os.rename(temp, self.workDir + '/' + jar)
0167                 #jars updated, write their version
0168                 handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = "confdb.version" + '.')
0169                 os.close(handle)
0170                 with open(temp,'w') as f:
0171                     f.write(version_website)
0172                 os.rename(temp,os.path.join(self.workDir,"confdb.version"))
0173
0174         # setup the java command line and CLASSPATH
0175         if self.verbose:
0176             sys.stderr.write("workDir = %s\n" % self.workDir)
0177         # use non-blocking random number source /dev/urandom (instead of /dev/random), see:
0178         #   http://blockdump.blogspot.fr/2012/07/connection-problems-inbound-connection.html
0179         # deal with timezone region not found
0180         #   http://stackoverflow.com/questions/9156379/ora-01882-timezone-region-not-found
0181         # increase the thread stack size from the default of 1 MB to work around java.lang.StackOverflowError errors, see
0182         #   man java
0183         self.javaCmd = ( 'java', '-cp', ':'.join(self.workDir + '/' + jar for jar in self.jars), '-Djava.security.egd=file:///dev/urandom', '-Doracle.jdbc.timezoneAsRegion=false', '-Xss32M', 'confdb.converter.BrowserConverter' )
0184
0185
0186     def query(self, *args):
0187         args = self.javaCmd + self.connect + self.proxy_connect_args + self.tunnel_connect_args + args
0188         if self.verbose:
0189             sys.stderr.write("\n" + ' '.join(args) + "\n\n" )
0190         sub = subprocess.Popen(
0191             args,
0192             stdin = None,
0193             stdout = subprocess.PIPE,
0194             stderr = subprocess.PIPE,
0195             shell = False,
0196             universal_newlines = True )
0197         return sub.communicate()
0198
0199 def help():
0200     sys.stdout.write("""Usage: %s OPTIONS
0201
0202         --v1|--v2|--v3|--v3-beta|--v3-test  (specify the ConfDB version [default: v3])
0203
0204         --run3|--run2|--dev|--online|--adg    (specify the target db [default: run3], online will only work inside p5 network)
0205
0206         Note that for v1
0207             --orcoff  is a synonim of --adg
0208             --offline is a synonim of --hltdev
0209
0210         --configId <id>             (specify the configuration by id)
0211         --configName <name>         (specify the configuration by name)
0212         --runNumber <run>           (specify the configuration by run number)
0213           [exactly one of --configId OR --configName OR --runNumber is required]
0214
0215         --cff                       (retrieve configuration *fragment*)
0216         --input <f1.root[,f2.root]> (insert PoolSource with specified fileNames)
0217         --input <files.list>        (read a text file which lists input ROOT files)
0218         --output <out.root>         (insert PoolOutputModule w/ specified fileName)
0219         --nopsets                   (exclude all globale psets)
0220         --noedsources               (exclude all edsources)
0221         --noes                      (exclude all essources *and* esmodules)
0222         --noessources               (exclude all essources)
0223         --noesmodules               (exclude all esmodules)
0224         --noservices                (exclude all services)
0225         --nooutput                  (exclude all output modules)
0226         --nopaths                   (exclude all paths [+=referenced seqs&mods])
0227         --nosequences               (don't define sequences [+=referenced s&m])
0228         --nomodules                 (don't define modules)
0229         --psets <pset1[,pset2]>     (include only specified global psets)
0230         --psets <-pset1[,-pset2]>   (include all global psets but the specified)
0231         --essources <ess1[,ess2]>   (include only specified essources)
0232         --essources <-ess1[,-ess2]> (include all essources but the specified)
0233         --esmodules <esm1[,esm2]>   (include only specified esmodules)
0234         --esmodules <-esm1[,-esm2]> (include all esmodules but the specified)
0235         --services <svc1[,svc2]>    (include only specified services)
0236         --services <-svc1[,-svc2]>  (include all services but the specified)
0237         --paths <p1[,p2]>           (include only specified paths)
0238         --paths <-p1[,-p2]>         (include all paths but the specified)
0239         --streams <s1[,s2]>         (include only specified streams)
0240         --datasets <d1[,d2]>        (include only specified datasets)
0241         --sequences <s1[,s2]>       (include sequences, referenced or not!)
0242         --modules <p1[,p2]>         (include modules, referenced or not!)
0243         --blocks <m1::p1[,p2][,m2]> (generate parameter blocks)
0244
0245         Options to connect to target db via SOCKS proxy, or direct tunnel:
0246           [the options --dbproxy and --dbtunnel are mutually exclusive]
0247         --dbproxy                   (use a SOCKS proxy to connect outside CERN network [default: False])
0248         --dbproxyhost <hostname>    (host of the SOCKS proxy [default: "localhost"])
0249         --dbproxyport <port>        (port of the SOCKS proxy [default: 8080])
0250         --dbtunnel                  (use direct tunnel to connect outside CERN network [default: False])
0251         --dbtunnelport <port>       (port when using a direct tunnel on localhost [default: 10121])
0252
0253         --verbose                   (print additional details)
0254 """)
0255
0256
0257 def main():
0258     args = sys.argv[1:]
0259     version = 'v3'
0260     db      = 'run3'
0261     verbose = False
0262
0263     if not args:
0264         help()
0265         sys.exit(1)
0266
0267     if '--help' in args or '-h' in args:
0268         help()
0269         sys.exit(0)
0270
0271     if '--verbose' in args:
0272         verbose = True
0273         args.remove('--verbose')
0274
0275     arg_count = Counter(args)
0276     db_count = arg_count['--v1'] + arg_count['--v2'] + arg_count['--v3'] + arg_count['--v3-beta'] + arg_count['--v3-test']
0277     if db_count>1:
0278         sys.stderr.write( 'ERROR: conflicting database version specifications: "--v1", "--v2", "--v3", "--v3-beta", and "--v3-test" are mutually exclusive options' )
0279         sys.exit(1)
0280
0281     if '--v1' in args:
0282         version = 'v1'
0283         db      = 'offline'
0284         args.remove('--v1')
0285
0286     if '--v2' in args:
0287         version = 'v2'
0288         db      = 'run2'
0289         args.remove('--v2')
0290
0291     if '--v3' in args:
0292         version = 'v3'
0293         db      = 'run3'
0294         args.remove('--v3')
0295
0296     if '--v3-beta' in args:
0297         version = 'v3-beta'
0298         db      = 'run3'
0299         args.remove('--v3-beta')
0300
0301     if '--v3-test' in args:
0302         version = 'v3-test'
0303         db      = 'dev'
0304         args.remove('--v3-test')
0305
0306     proxy = False
0307     proxy_host = "localhost"
0308     proxy_port = "8080"
0309     if '--dbproxy' in args:
0310         proxy = True
0311         args.remove('--dbproxy')
0312     if '--dbproxyhost' in args:
0313         proxy_host = args.pop(args.index('--dbproxyhost')+1)
0314         args.remove('--dbproxyhost')
0315     if '--dbproxyport' in args:
0316         proxy_port = args.pop(args.index('--dbproxyport')+1)
0317         args.remove('--dbproxyport')
0318
0319     tunnel = False
0320     tunnel_port = "10121"
0321     if '--dbtunnel' in args:
0322         tunnel = True
0323         args.remove('--dbtunnel')
0324
0325     if '--dbtunnelport' in args:
0326         tunnel_port = args.pop(args.index('--dbtunnelport')+1)
0327         args.remove('--dbtunnelport')
0328
0329     if tunnel and proxy:
0330         sys.stderr.write( 'ERROR: conflicting connection specifications, "--dbtunnel" and "--dbproxy" are mutually exclusive options\n' )
0331         sys.exit(1)
0332
0333     _dbs = {}
0334     _dbs['v1'] = [ '--%s' % _db for _db in OfflineConverter.databases['v1'] ] + [ '--runNumber' ]
0335     _dbs['v2'] = [ '--%s' % _db for _db in OfflineConverter.databases['v2'] ] + [ '--runNumber' ]
0336     _dbs['v3'] = [ '--%s' % _db for _db in OfflineConverter.databases['v3'] ] + [ '--runNumber']
0337     _dbs['v3-beta'] = [ '--%s' % _db for _db in OfflineConverter.databases['v3-beta'] ] + [ '--runNumber' ]
0338     _dbs['v3-test'] = [ '--%s' % _db for _db in OfflineConverter.databases['v3-test'] ] + [ '--runNumber' ]
0339     _dbargs = set(args) & set(sum(_dbs.values(), []))
0340
0341     if _dbargs:
0342         if len(_dbargs) > 1:
0343             sys.stderr.write( "ERROR: too many database specifications: \"" + "\", \"".join( _dbargs) + "\"\n" )
0344             sys.exit(1)
0345
0346         _arg = _dbargs.pop()
0347         db   = _arg[2:]
0348         if db == 'runNumber':
0349             db = 'adg'
0350         else:
0351             args.remove(_arg)
0352
0353         if not db in OfflineConverter.databases[version]:
0354             sys.stderr.write( "ERROR: database version \"%s\" incompatible with specification \"%s\"\n" % (version, db) )
0355             sys.exit(1)
0356
0357     converter = OfflineConverter(version = version, database = db, verbose = verbose,
0358                                  proxy = proxy, proxyHost = proxy_host, proxyPort = proxy_port,
0359                                  tunnel = tunnel, tunnelPort = tunnel_port)
0360     out, err = converter.query( * args )
0361     if 'ERROR' in err:
0362         sys.stderr.write( "%s: error while retrieving the HLT menu\n\n%s\n\n" % (sys.argv[0], err) )
0363         sys.exit(1)
0364     else:
0365         sys.stdout.write( out )
0366
0367
0368 if __name__ == "__main__":
0369     main()