File indexing completed on 2023-03-17 11:06:08
0001
0002 import sys, os
0003 import re
0004 import hashlib
0005 import os.path
0006 import tempfile
0007 import requests
0008 import shutil
0009 import subprocess
0010 import atexit
0011 from collections import Counter
0012
0013 class OfflineConverter:
0014
0015
0016
0017
0018
0019
0020
0021
0022
0023
0024
0025
0026
0027
0028
0029
0030
0031 databases = {}
0032 databases['v1'] = {}
0033 databases['v1']['offline'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch', '-d', 'cms_cond.cern.ch', '-u', 'cms_hltdev_reader', '-s', 'convertMe!' )
0034 databases['v1']['hltdev'] = databases['v1']['offline']
0035 databases['v1']['online'] = ( '-t', 'oracle', '-h', 'cmsonr1-s.cms', '-d', 'cms_rcms.cern.ch', '-u', 'cms_hlt_r', '-s', 'convertMe!' )
0036 databases['v1']['adg'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch', '-d', 'cms_cond.cern.ch', '-u', 'cms_hlt_gui_r', '-s', 'convertMe!' )
0037 databases['v1']['orcoff'] = databases['v1']['adg']
0038 databases['v3'] = {}
0039 databases['v3']['run2'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch', '-d', 'cms_hlt.cern.ch', '-u', 'cms_hlt_gdr_r', '-s', 'convertMe!' )
0040 databases['v3']['run3'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch', '-d', 'cms_hlt.cern.ch', '-u', 'cms_hlt_v3_r', '-s', 'convertMe!' )
0041 databases['v3']['dev'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch', '-d', 'cms_hlt.cern.ch', '-u', 'cms_hlt_gdrdev_r', '-s', 'convertMe1!' )
0042 databases['v3']['online'] = ( '-t', 'oracle', '-h', 'cmsonr1-s.cms', '-d', 'cms_rcms.cern.ch', '-u', 'cms_hlt_gdr_r', '-s', 'convertMe!' )
0043 databases['v3']['adg'] = ( '-t', 'oracle', '-h', 'cmsonr1-adg1-s.cern.ch', '-d', 'cms_orcon_adg.cern.ch', '-u', 'cms_hlt_gdr_r', '-s', 'convertMe!' )
0044
0045
0046 ips_for_proxy = {
0047 'cmsr1-s.cern.ch' : '10.116.96.89',
0048 'cmsr2-s.cern.ch' : '10.116.96.139',
0049 'cmsr3-s.cern.ch' : '10.116.96.105',
0050 'cmsonr1-adg1-s.cern.ch' : '10.116.96.109',
0051 'cmsonr1-s.cms' : '10.176.84.78'
0052 }
0053
0054 databases['v3-beta'] = dict(databases['v3'])
0055 databases['v3-test'] = dict(databases['v3'])
0056 databases['v2'] = dict(databases['v3'])
0057
0058 for dbkey in databases['v2']:
0059 dbparams = databases['v2'][dbkey]
0060 if dbparams[3]=='cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch':
0061 databases['v2'][dbkey] = dbparams[0:3]+('cmsr1-s.cern.ch',)+dbparams[4:]
0062
0063 @staticmethod
0064 def CheckTempDirectory(dir):
0065 dir = os.path.realpath(dir)
0066 if not os.path.isdir(dir):
0067 try:
0068 os.makedirs(dir)
0069 except:
0070 return None
0071 return dir
0072
0073
0074 def __init__(self, version = 'v3', database = 'run3', url = None, verbose = False,
0075 proxy = False, proxyHost = 'localhost', proxyPort = '8080',
0076 tunnel = False, tunnelPort = '10121'):
0077 self.verbose = verbose
0078 self.version = version
0079 self.baseDir = '/afs/cern.ch/user/c/confdb/www/%s/lib' % version
0080 self.baseUrl = 'https://confdb.web.cern.ch/confdb/%s/lib' % version
0081 self.jars = ( 'ojdbc8.jar', 'cmssw-evf-confdb-converter.jar' )
0082 if version=='v2':
0083
0084 self.jars = ( 'ojdbc6.jar', 'cmssw-evf-confdb-converter.jar' )
0085 self.workDir = ''
0086 self.proxy = proxy
0087 self.proxyHost = proxyHost
0088 self.proxyPort = proxyPort
0089 self.tunnel = tunnel
0090 self.tunnelPort = tunnelPort
0091
0092 if self.proxy and self.tunnel:
0093 sys.stderr.write( "ERROR: proxy and tunnel options can not both be true" )
0094 sys.exit(1)
0095
0096
0097 if version not in self.databases:
0098
0099 sys.stderr.write( "ERROR: unsupported database version \"%s\"\n" % version)
0100
0101
0102 if database in self.databases[version]:
0103
0104 self.connect = self.databases[version][database]
0105 else:
0106
0107 sys.stderr.write( "ERROR: unknown database \"%s\" for version \"%s\"\n" % (database, version))
0108 sys.exit(1)
0109
0110 if self.proxy:
0111 self.proxy_connect_args = ('--dbproxy', '--dbproxyport', self.proxyPort, '--dbproxyhost', self.proxyHost)
0112 temp_connect = []
0113 for entry in self.connect:
0114 for key,item in self.ips_for_proxy.items():
0115 entry = entry.replace(key,item)
0116 temp_connect.append(entry.replace(key,item))
0117 self.connect = tuple(temp_connect)
0118 else:
0119 self.proxy_connect_args = ()
0120
0121
0122 if self.tunnel:
0123 temp_connect = list(self.connect)
0124 host_index = temp_connect.index('-h')
0125 temp_connect[host_index+1] = "localhost"
0126 self.connect = tuple(temp_connect)
0127 self.tunnel_connect_args = ('--dbport', self.tunnelPort)
0128 else:
0129 self.tunnel_connect_args = ()
0130
0131
0132 if url is not None:
0133 self.baseUrl = url
0134
0135
0136 if os.path.isdir(self.baseDir) and all(os.path.isfile(self.baseDir + '/' + jar) for jar in self.jars):
0137
0138 self.workDir = self.baseDir
0139 else:
0140
0141 self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['CMSSW_BASE'],'tmp','confdb',self.version))
0142 if not self.workDir:
0143
0144 self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['TMP'],'confdb',self.version))
0145 if not self.workDir:
0146
0147 self.workDir = tempfile.mkdtemp()
0148 atexit.register(shutil.rmtree, self.workDir)
0149
0150 version_website = requests.get(self.baseUrl+"/../confdb.version").text
0151 jars_require_update = True
0152 if os.path.exists(os.path.join(self.workDir,"confdb.version")):
0153 with open(os.path.join(self.workDir,"confdb.version")) as f:
0154 version_existing = f.read()
0155 if version_existing==version_website:
0156 jars_require_update = False
0157
0158 if jars_require_update:
0159 for jar in self.jars:
0160
0161 handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = jar + '.')
0162 os.close(handle)
0163 request = requests.get(self.baseUrl + '/' + jar)
0164 with open(temp,'wb') as f:
0165 f.write(request.content)
0166 os.rename(temp, self.workDir + '/' + jar)
0167
0168 handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = "confdb.version" + '.')
0169 os.close(handle)
0170 with open(temp,'w') as f:
0171 f.write(version_website)
0172 os.rename(temp,os.path.join(self.workDir,"confdb.version"))
0173
0174
0175 if self.verbose:
0176 sys.stderr.write("workDir = %s\n" % self.workDir)
0177
0178
0179
0180
0181
0182
0183 self.javaCmd = ( 'java', '-cp', ':'.join(self.workDir + '/' + jar for jar in self.jars), '-Djava.security.egd=file:///dev/urandom', '-Doracle.jdbc.timezoneAsRegion=false', '-Xss32M', 'confdb.converter.BrowserConverter' )
0184
0185
0186 def query(self, *args):
0187 args = self.javaCmd + self.connect + self.proxy_connect_args + self.tunnel_connect_args + args
0188 if self.verbose:
0189 sys.stderr.write("\n" + ' '.join(args) + "\n\n" )
0190 sub = subprocess.Popen(
0191 args,
0192 stdin = None,
0193 stdout = subprocess.PIPE,
0194 stderr = subprocess.PIPE,
0195 shell = False,
0196 universal_newlines = True )
0197 return sub.communicate()
0198
0199 def help():
0200 sys.stdout.write("""Usage: %s OPTIONS
0201
0202 --v1|--v2|--v3|--v3-beta|--v3-test (specify the ConfDB version [default: v3])
0203
0204 --run3|--run2|--dev|--online|--adg (specify the target db [default: run3], online will only work inside p5 network)
0205
0206 Note that for v1
0207 --orcoff is a synonim of --adg
0208 --offline is a synonim of --hltdev
0209
0210 --configId <id> (specify the configuration by id)
0211 --configName <name> (specify the configuration by name)
0212 --runNumber <run> (specify the configuration by run number)
0213 [exactly one of --configId OR --configName OR --runNumber is required]
0214
0215 --cff (retrieve configuration *fragment*)
0216 --input <f1.root[,f2.root]> (insert PoolSource with specified fileNames)
0217 --input <files.list> (read a text file which lists input ROOT files)
0218 --output <out.root> (insert PoolOutputModule w/ specified fileName)
0219 --nopsets (exclude all globale psets)
0220 --noedsources (exclude all edsources)
0221 --noes (exclude all essources *and* esmodules)
0222 --noessources (exclude all essources)
0223 --noesmodules (exclude all esmodules)
0224 --noservices (exclude all services)
0225 --nooutput (exclude all output modules)
0226 --nopaths (exclude all paths [+=referenced seqs&mods])
0227 --nosequences (don't define sequences [+=referenced s&m])
0228 --nomodules (don't define modules)
0229 --psets <pset1[,pset2]> (include only specified global psets)
0230 --psets <-pset1[,-pset2]> (include all global psets but the specified)
0231 --essources <ess1[,ess2]> (include only specified essources)
0232 --essources <-ess1[,-ess2]> (include all essources but the specified)
0233 --esmodules <esm1[,esm2]> (include only specified esmodules)
0234 --esmodules <-esm1[,-esm2]> (include all esmodules but the specified)
0235 --services <svc1[,svc2]> (include only specified services)
0236 --services <-svc1[,-svc2]> (include all services but the specified)
0237 --paths <p1[,p2]> (include only specified paths)
0238 --paths <-p1[,-p2]> (include all paths but the specified)
0239 --streams <s1[,s2]> (include only specified streams)
0240 --datasets <d1[,d2]> (include only specified datasets)
0241 --sequences <s1[,s2]> (include sequences, referenced or not!)
0242 --modules <p1[,p2]> (include modules, referenced or not!)
0243 --blocks <m1::p1[,p2][,m2]> (generate parameter blocks)
0244
0245 Options to connect to target db via SOCKS proxy, or direct tunnel:
0246 [the options --dbproxy and --dbtunnel are mutually exclusive]
0247 --dbproxy (use a SOCKS proxy to connect outside CERN network [default: False])
0248 --dbproxyhost <hostname> (host of the SOCKS proxy [default: "localhost"])
0249 --dbproxyport <port> (port of the SOCKS proxy [default: 8080])
0250 --dbtunnel (use direct tunnel to connect outside CERN network [default: False])
0251 --dbtunnelport <port> (port when using a direct tunnel on localhost [default: 10121])
0252
0253 --verbose (print additional details)
0254 """)
0255
0256
0257 def main():
0258 args = sys.argv[1:]
0259 version = 'v3'
0260 db = 'run3'
0261 verbose = False
0262
0263 if not args:
0264 help()
0265 sys.exit(1)
0266
0267 if '--help' in args or '-h' in args:
0268 help()
0269 sys.exit(0)
0270
0271 if '--verbose' in args:
0272 verbose = True
0273 args.remove('--verbose')
0274
0275 arg_count = Counter(args)
0276 db_count = arg_count['--v1'] + arg_count['--v2'] + arg_count['--v3'] + arg_count['--v3-beta'] + arg_count['--v3-test']
0277 if db_count>1:
0278 sys.stderr.write( 'ERROR: conflicting database version specifications: "--v1", "--v2", "--v3", "--v3-beta", and "--v3-test" are mutually exclusive options' )
0279 sys.exit(1)
0280
0281 if '--v1' in args:
0282 version = 'v1'
0283 db = 'offline'
0284 args.remove('--v1')
0285
0286 if '--v2' in args:
0287 version = 'v2'
0288 db = 'run2'
0289 args.remove('--v2')
0290
0291 if '--v3' in args:
0292 version = 'v3'
0293 db = 'run3'
0294 args.remove('--v3')
0295
0296 if '--v3-beta' in args:
0297 version = 'v3-beta'
0298 db = 'run3'
0299 args.remove('--v3-beta')
0300
0301 if '--v3-test' in args:
0302 version = 'v3-test'
0303 db = 'dev'
0304 args.remove('--v3-test')
0305
0306 proxy = False
0307 proxy_host = "localhost"
0308 proxy_port = "8080"
0309 if '--dbproxy' in args:
0310 proxy = True
0311 args.remove('--dbproxy')
0312 if '--dbproxyhost' in args:
0313 proxy_host = args.pop(args.index('--dbproxyhost')+1)
0314 args.remove('--dbproxyhost')
0315 if '--dbproxyport' in args:
0316 proxy_port = args.pop(args.index('--dbproxyport')+1)
0317 args.remove('--dbproxyport')
0318
0319 tunnel = False
0320 tunnel_port = "10121"
0321 if '--dbtunnel' in args:
0322 tunnel = True
0323 args.remove('--dbtunnel')
0324
0325 if '--dbtunnelport' in args:
0326 tunnel_port = args.pop(args.index('--dbtunnelport')+1)
0327 args.remove('--dbtunnelport')
0328
0329 if tunnel and proxy:
0330 sys.stderr.write( 'ERROR: conflicting connection specifications, "--dbtunnel" and "--dbproxy" are mutually exclusive options\n' )
0331 sys.exit(1)
0332
0333 _dbs = {}
0334 _dbs['v1'] = [ '--%s' % _db for _db in OfflineConverter.databases['v1'] ] + [ '--runNumber' ]
0335 _dbs['v2'] = [ '--%s' % _db for _db in OfflineConverter.databases['v2'] ] + [ '--runNumber' ]
0336 _dbs['v3'] = [ '--%s' % _db for _db in OfflineConverter.databases['v3'] ] + [ '--runNumber']
0337 _dbs['v3-beta'] = [ '--%s' % _db for _db in OfflineConverter.databases['v3-beta'] ] + [ '--runNumber' ]
0338 _dbs['v3-test'] = [ '--%s' % _db for _db in OfflineConverter.databases['v3-test'] ] + [ '--runNumber' ]
0339 _dbargs = set(args) & set(sum(_dbs.values(), []))
0340
0341 if _dbargs:
0342 if len(_dbargs) > 1:
0343 sys.stderr.write( "ERROR: too many database specifications: \"" + "\", \"".join( _dbargs) + "\"\n" )
0344 sys.exit(1)
0345
0346 _arg = _dbargs.pop()
0347 db = _arg[2:]
0348 if db == 'runNumber':
0349 db = 'adg'
0350 else:
0351 args.remove(_arg)
0352
0353 if not db in OfflineConverter.databases[version]:
0354 sys.stderr.write( "ERROR: database version \"%s\" incompatible with specification \"%s\"\n" % (version, db) )
0355 sys.exit(1)
0356
0357 converter = OfflineConverter(version = version, database = db, verbose = verbose,
0358 proxy = proxy, proxyHost = proxy_host, proxyPort = proxy_port,
0359 tunnel = tunnel, tunnelPort = tunnel_port)
0360 out, err = converter.query( * args )
0361 if 'ERROR' in err:
0362 sys.stderr.write( "%s: error while retrieving the HLT menu\n\n%s\n\n" % (sys.argv[0], err) )
0363 sys.exit(1)
0364 else:
0365 sys.stdout.write( out )
0366
0367
0368 if __name__ == "__main__":
0369 main()