python/Tools/confdbOfflineConverter.py

0001 #! /usr/bin/env python3
0002 import sys, os
0003 import re
0004 import hashlib
0005 import os.path
0006 import tempfile
0007 import requests
0008 import shutil
0009 import subprocess
0010 import atexit
0011 from collections import Counter
0012
0013 class OfflineConverter:
0014
0015     # the machine aliases and interfaces for the *online* database are
0016     #   cmsonr1-s.cms, cmsonr2-s.cms, cmsonr3-s.cms
0017     #   cmsonr1-v.cms, cmsonr2-v.cms, cmsonr3-v.cms
0018     # but the -s and -v interfaces resolve to the same hosts.
0019     # The actual machines and interfaces are
0020     #   CMSRAC11-S.cms, CMSRAC12-S.cms, CMSRAC21-S.cms
0021     #   CMSRAC11-V.cms, CMSRAC12-V.cms, CMSRAC21-V.cms
0022
0023     # the possible machines and interfaces for the *offline* database are
0024     #   cmsr1-s.cern.ch, cmsr2-s.cern.ch, cmsr3-s.cern.ch
0025     #   cmsr1-v.cern.ch, cmsr2-v.cern.ch, cmsr3-v.cern.ch
0026     # but the -s and -v interfaces resolve to the same hosts
0027     # The actual machines and interfaces are
0028     #   itrac5404-s.cern.ch, itrac5413-s.cern.ch, itrac5433-s.cern.ch
0029     #   itrac5404-v.cern.ch, itrac5413-v.cern.ch, itrac5433-v.cern.ch
0030
0031     databases = {}
0032     databases['v1'] = {}
0033     databases['v1']['offline'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch',        '-d', 'cms_cond.cern.ch',      '-u', 'cms_hltdev_reader', '-s', 'convertMe!' )
0034     databases['v1']['hltdev']  = databases['v1']['offline']     # for backwards compatibility
0035     databases['v1']['online']  = ( '-t', 'oracle', '-h', 'cmsonr1-s.cms',          '-d', 'cms_rcms.cern.ch',      '-u', 'cms_hlt_r',         '-s', 'convertMe!' )
0036     databases['v1']['adg']     = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch',        '-d', 'cms_cond.cern.ch',      '-u', 'cms_hlt_gui_r',     '-s', 'convertMe!' )
0037     databases['v1']['orcoff']  = databases['v1']['adg']         # for backwards compatibility
0038     databases['v3'] = {}
0039     databases['v3']['run2'] = ( '-t', 'oracle', '-h', 'cmsr1-v.cern.ch,cmsr2-v.cern.ch,cmsr3-v.cern.ch',        '-d', 'cms_hlt.cern.ch',      '-u', 'cms_hlt_gdr_r',     '-s', 'convertMe!' )
0040     databases['v3']['run3'] = ( '-t', 'oracle', '-h', 'cmsr1-v.cern.ch,cmsr2-v.cern.ch,cmsr3-v.cern.ch',        '-d', 'cms_hlt.cern.ch',      '-u', 'cms_hlt_v3_r',     '-s', 'convertMe!' )
0041     databases['v3']['dev'] = ( '-t', 'oracle', '-h', 'cmsr1-v.cern.ch,cmsr2-v.cern.ch,cmsr3-v.cern.ch',        '-d', 'cms_hlt.cern.ch',      '-u', 'cms_hlt_gdrdev_r',     '-s', 'convertMe1!' )
0042     databases['v3']['online']  = ( '-t', 'oracle', '-h', 'cmsonr1-v.cms',          '-d', 'cms_rcms.cern.ch',      '-u', 'cms_hlt_gdr_r',     '-s', 'convertMe!' )
0043     databases['v3']['adg']     = ( '-t', 'oracle', '-h', 'cmsonr1-adg-v.cern.ch,cmsonr2-adg-v.cern.ch', '-d', 'cms_orcon_adg.cern.ch', '-u', 'cms_hlt_gdr_r',     '-s', 'convertMe!' )
0044
0045     databases['v3-beta'] = dict(databases['v3'])
0046     databases['v3-test'] = dict(databases['v3'])
0047     databases['v2'] = dict(databases['v3'])
0048     #old converter can only handle a single host so we modify the params accordingly
0049     for dbkey in databases['v2']:
0050         dbparams  = databases['v2'][dbkey]
0051         if dbparams[3]=='cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch':
0052             databases['v2'][dbkey] = dbparams[0:3]+('cmsr1-s.cern.ch',)+dbparams[4:]
0053
0054     @staticmethod
0055     def CheckTempDirectory(dir):
0056         dir = os.path.realpath(dir)
0057         if not os.path.isdir(dir):
0058             try:
0059                 os.makedirs(dir)
0060             except:
0061                 return None
0062         return dir
0063
0064
0065     def __init__(self, version = 'v3', database = 'run3', url = None, verbose = False,
0066                  proxy = False, proxyHost = 'localhost', proxyPort = '8080',
0067                  tunnel = False, tunnelPort = '10121'):
0068         self.verbose = verbose
0069         self.version = version
0070         self.baseDir = '/afs/cern.ch/user/c/confdb/www/%s/lib' % version
0071         self.baseUrl = 'https://confdb.web.cern.ch/confdb/%s/lib' % version
0072         self.jars    = ( 'ojdbc8.jar', 'cmssw-evf-confdb-converter.jar' )
0073         if version=='v2':
0074             #legacy driver for run2 gui
0075             self.jars = ( 'ojdbc6.jar', 'cmssw-evf-confdb-converter.jar' )
0076         self.workDir = ''
0077         self.proxy = proxy
0078         self.proxyHost = proxyHost
0079         self.proxyPort = proxyPort
0080         self.tunnel = tunnel
0081         self.tunnelPort = tunnelPort
0082
0083         if self.proxy and self.tunnel:
0084             sys.stderr.write( "ERROR: proxy and tunnel options can not both be true" )
0085             sys.exit(1)
0086
0087         # check the schema version
0088         if version not in self.databases:
0089             # unsupported database version
0090             sys.stderr.write( "ERROR: unsupported database version \"%s\"\n" % version)
0091
0092         # check the database
0093         if database in self.databases[version]:
0094             # load the connection parameters for the given database
0095             self.connect = self.databases[version][database]
0096         else:
0097             # unsupported database
0098             sys.stderr.write( "ERROR: unknown database \"%s\" for version \"%s\"\n" % (database, version))
0099             sys.exit(1)
0100
0101         if self.proxy:
0102             self.proxy_connect_args = ('--dbproxy', '--dbproxyport', self.proxyPort, '--dbproxyhost', self.proxyHost)
0103             temp_connect = []
0104             for entry in self.connect:
0105                 temp_connect.append(entry.replace(key,item))
0106             self.connect  = tuple(temp_connect)
0107         else:
0108             self.proxy_connect_args = ()
0109
0110         # this sets the host to localhost
0111         if self.tunnel:
0112             temp_connect = list(self.connect)
0113             host_index = temp_connect.index('-h')
0114             temp_connect[host_index+1] = "localhost"
0115             self.connect = tuple(temp_connect)
0116             self.tunnel_connect_args = ('--dbport', self.tunnelPort)
0117         else:
0118             self.tunnel_connect_args = ()
0119
0120         # check for a custom base URL
0121         if url is not None:
0122             self.baseUrl = url
0123
0124         # try to read the .jar files from AFS, or download them
0125         if os.path.isdir(self.baseDir) and all(os.path.isfile(self.baseDir + '/' + jar) for jar in self.jars):
0126             # read the .jar fles from AFS
0127             self.workDir = self.baseDir
0128         else:
0129             # try to use $CMSSW_BASE/tmp
0130             self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['CMSSW_BASE'],'tmp','confdb',self.version))
0131             if not self.workDir:
0132                 # try to use $TMP
0133                 self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['TMP'],'confdb',self.version))
0134             if not self.workDir:
0135                 # create a new temporary directory, and install a cleanup callback
0136                 self.workDir = tempfile.mkdtemp()
0137                 atexit.register(shutil.rmtree, self.workDir)
0138             # download the .jar files
0139             version_website = requests.get(self.baseUrl+"/../confdb.version").text
0140             jars_require_update = True
0141             if os.path.exists(os.path.join(self.workDir,"confdb.version")):
0142                 with open(os.path.join(self.workDir,"confdb.version")) as f:
0143                     version_existing = f.read()
0144                     if version_existing==version_website:
0145                         jars_require_update = False
0146
0147             if jars_require_update:
0148                 for jar in self.jars:
0149                     # download to a temporay name and use an atomic rename (in case an other istance is downloading the same file
0150                     handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = jar + '.')
0151                     os.close(handle)
0152                     request = requests.get(self.baseUrl + '/' + jar)
0153                     with open(temp,'wb') as f:
0154                         f.write(request.content)
0155                     os.rename(temp, self.workDir + '/' + jar)
0156                 #jars updated, write their version
0157                 handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = "confdb.version" + '.')
0158                 os.close(handle)
0159                 with open(temp,'w') as f:
0160                     f.write(version_website)
0161                 os.rename(temp,os.path.join(self.workDir,"confdb.version"))
0162
0163         # setup the java command line and CLASSPATH
0164         if self.verbose:
0165             sys.stderr.write("workDir = %s\n" % self.workDir)
0166         # use non-blocking random number source /dev/urandom (instead of /dev/random), see:
0167         #   http://blockdump.blogspot.fr/2012/07/connection-problems-inbound-connection.html
0168         # deal with timezone region not found
0169         #   http://stackoverflow.com/questions/9156379/ora-01882-timezone-region-not-found
0170         # increase the thread stack size from the default of 1 MB to work around java.lang.StackOverflowError errors, see
0171         #   man java
0172         self.javaCmd = ( 'java', '-cp', ':'.join(self.workDir + '/' + jar for jar in self.jars), '-Djava.security.egd=file:///dev/urandom', '-Doracle.jdbc.timezoneAsRegion=false', '-Xss32M', 'confdb.converter.BrowserConverter' )
0173
0174
0175     def query(self, *args):
0176         args = self.javaCmd + self.connect + self.proxy_connect_args + self.tunnel_connect_args + args
0177         if self.verbose:
0178             sys.stderr.write("\n" + ' '.join(args) + "\n\n" )
0179         sub = subprocess.Popen(
0180             args,
0181             stdin = None,
0182             stdout = subprocess.PIPE,
0183             stderr = subprocess.PIPE,
0184             shell = False,
0185             universal_newlines = True )
0186         return sub.communicate()
0187
0188 def help():
0189     sys.stdout.write("""Usage: %s OPTIONS
0190
0191         --v1|--v2|--v3|--v3-beta|--v3-test  (specify the ConfDB version [default: v3])
0192
0193         --run3|--run2|--dev|--online|--adg    (specify the target db [default: run3], online will only work inside p5 network)
0194
0195         Note that for v1
0196             --orcoff  is a synonim of --adg
0197             --offline is a synonim of --hltdev
0198
0199         --configId <id>             (specify the configuration by id)
0200         --configName <name>         (specify the configuration by name)
0201         --runNumber <run>           (specify the configuration by run number)
0202           [exactly one of --configId OR --configName OR --runNumber is required]
0203
0204         --cff                       (retrieve configuration *fragment*)
0205         --input <f1.root[,f2.root]> (insert PoolSource with specified fileNames)
0206         --input <files.list>        (read a text file which lists input ROOT files)
0207         --output <out.root>         (insert PoolOutputModule w/ specified fileName)
0208         --nopsets                   (exclude all globale psets)
0209         --noedsources               (exclude all edsources)
0210         --noes                      (exclude all essources *and* esmodules)
0211         --noessources               (exclude all essources)
0212         --noesmodules               (exclude all esmodules)
0213         --noservices                (exclude all services)
0214         --nooutput                  (exclude all output modules)
0215         --nopaths                   (exclude all paths [+=referenced seqs&mods])
0216         --nosequences               (don't define sequences [+=referenced s&m])
0217         --nomodules                 (don't define modules)
0218         --psets <pset1[,pset2]>     (include only specified global psets)
0219         --psets <-pset1[,-pset2]>   (include all global psets but the specified)
0220         --essources <ess1[,ess2]>   (include only specified essources)
0221         --essources <-ess1[,-ess2]> (include all essources but the specified)
0222         --esmodules <esm1[,esm2]>   (include only specified esmodules)
0223         --esmodules <-esm1[,-esm2]> (include all esmodules but the specified)
0224         --services <svc1[,svc2]>    (include only specified services)
0225         --services <-svc1[,-svc2]>  (include all services but the specified)
0226         --paths <p1[,p2]>           (include only specified paths)
0227         --paths <-p1[,-p2]>         (include all paths but the specified)
0228         --streams <s1[,s2]>         (include only specified streams)
0229         --datasets <d1[,d2]>        (include only specified datasets)
0230         --sequences <s1[,s2]>       (include sequences, referenced or not!)
0231         --modules <p1[,p2]>         (include modules, referenced or not!)
0232         --blocks <m1::p1[,p2][,m2]> (generate parameter blocks)
0233
0234         Options to connect to target db via SOCKS proxy, or direct tunnel:
0235           [the options --dbproxy and --dbtunnel are mutually exclusive]
0236         --dbproxy                   (use a SOCKS proxy to connect outside CERN network [default: False])
0237         --dbproxyhost <hostname>    (host of the SOCKS proxy [default: "localhost"])
0238         --dbproxyport <port>        (port of the SOCKS proxy [default: 8080])
0239         --dbtunnel                  (use direct tunnel to connect outside CERN network [default: False])
0240         --dbtunnelport <port>       (port when using a direct tunnel on localhost [default: 10121])
0241
0242         --verbose                   (print additional details)
0243 """)
0244
0245
0246 def main():
0247     args = sys.argv[1:]
0248     version = 'v3'
0249     db      = 'run3'
0250     verbose = False
0251
0252     if not args:
0253         help()
0254         sys.exit(1)
0255
0256     if '--help' in args or '-h' in args:
0257         help()
0258         sys.exit(0)
0259
0260     if '--verbose' in args:
0261         verbose = True
0262         args.remove('--verbose')
0263
0264     arg_count = Counter(args)
0265     db_count = arg_count['--v1'] + arg_count['--v2'] + arg_count['--v3'] + arg_count['--v3-beta'] + arg_count['--v3-test']
0266     if db_count>1:
0267         sys.stderr.write( 'ERROR: conflicting database version specifications: "--v1", "--v2", "--v3", "--v3-beta", and "--v3-test" are mutually exclusive options' )
0268         sys.exit(1)
0269
0270     if '--v1' in args:
0271         version = 'v1'
0272         db      = 'offline'
0273         args.remove('--v1')
0274
0275     if '--v2' in args:
0276         version = 'v2'
0277         db      = 'run2'
0278         args.remove('--v2')
0279
0280     if '--v3' in args:
0281         version = 'v3'
0282         db      = 'run3'
0283         args.remove('--v3')
0284
0285     if '--v3-beta' in args:
0286         version = 'v3-beta'
0287         db      = 'run3'
0288         args.remove('--v3-beta')
0289
0290     if '--v3-test' in args:
0291         version = 'v3-test'
0292         db      = 'dev'
0293         args.remove('--v3-test')
0294
0295     proxy = False
0296     proxy_host = "localhost"
0297     proxy_port = "8080"
0298     if '--dbproxy' in args:
0299         proxy = True
0300         args.remove('--dbproxy')
0301     if '--dbproxyhost' in args:
0302         proxy_host = args.pop(args.index('--dbproxyhost')+1)
0303         args.remove('--dbproxyhost')
0304     if '--dbproxyport' in args:
0305         proxy_port = args.pop(args.index('--dbproxyport')+1)
0306         args.remove('--dbproxyport')
0307
0308     tunnel = False
0309     tunnel_port = "10121"
0310     if '--dbtunnel' in args:
0311         tunnel = True
0312         args.remove('--dbtunnel')
0313
0314     if '--dbtunnelport' in args:
0315         tunnel_port = args.pop(args.index('--dbtunnelport')+1)
0316         args.remove('--dbtunnelport')
0317
0318     if tunnel and proxy:
0319         sys.stderr.write( 'ERROR: conflicting connection specifications, "--dbtunnel" and "--dbproxy" are mutually exclusive options\n' )
0320         sys.exit(1)
0321
0322     _dbs = {}
0323     _dbs['v1'] = [ '--%s' % _db for _db in OfflineConverter.databases['v1'] ] + [ '--runNumber' ]
0324     _dbs['v2'] = [ '--%s' % _db for _db in OfflineConverter.databases['v2'] ] + [ '--runNumber' ]
0325     _dbs['v3'] = [ '--%s' % _db for _db in OfflineConverter.databases['v3'] ] + [ '--runNumber']
0326     _dbs['v3-beta'] = [ '--%s' % _db for _db in OfflineConverter.databases['v3-beta'] ] + [ '--runNumber' ]
0327     _dbs['v3-test'] = [ '--%s' % _db for _db in OfflineConverter.databases['v3-test'] ] + [ '--runNumber' ]
0328     _dbargs = set(args) & set(sum(_dbs.values(), []))
0329
0330     if _dbargs:
0331         if len(_dbargs) > 1:
0332             sys.stderr.write( "ERROR: too many database specifications: \"" + "\", \"".join( _dbargs) + "\"\n" )
0333             sys.exit(1)
0334
0335         _arg = _dbargs.pop()
0336         db   = _arg[2:]
0337         if db == 'runNumber':
0338             db = 'adg'
0339         else:
0340             args.remove(_arg)
0341
0342         if not db in OfflineConverter.databases[version]:
0343             sys.stderr.write( "ERROR: database version \"%s\" incompatible with specification \"%s\"\n" % (version, db) )
0344             sys.exit(1)
0345
0346     converter = OfflineConverter(version = version, database = db, verbose = verbose,
0347                                  proxy = proxy, proxyHost = proxy_host, proxyPort = proxy_port,
0348                                  tunnel = tunnel, tunnelPort = tunnel_port)
0349     out, err = converter.query( * args )
0350     if 'ERROR' in err:
0351         sys.stderr.write( "%s: error while retrieving the HLT menu\n\n%s\n\n" % (sys.argv[0], err) )
0352         sys.exit(1)
0353     else:
0354         sys.stdout.write( out )
0355
0356
0357 if __name__ == "__main__":
0358     main()