File indexing completed on 2021-11-09 23:32:03
0001
0002 import sys, os
0003 import re
0004 import hashlib
0005 import os.path
0006 import tempfile
0007 import requests
0008 import shutil
0009 import subprocess
0010 import atexit
0011 from collections import Counter
0012
0013 class OfflineConverter:
0014
0015
0016
0017
0018
0019
0020
0021
0022
0023
0024
0025
0026
0027
0028
0029
0030
0031 databases = {}
0032 databases['v1'] = {}
0033 databases['v1']['offline'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch', '-d', 'cms_cond.cern.ch', '-u', 'cms_hltdev_reader', '-s', 'convertMe!' )
0034 databases['v1']['hltdev'] = databases['v1']['offline']
0035 databases['v1']['online'] = ( '-t', 'oracle', '-h', 'cmsonr1-s.cms', '-d', 'cms_rcms.cern.ch', '-u', 'cms_hlt_r', '-s', 'convertMe!' )
0036 databases['v1']['adg'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch', '-d', 'cms_cond.cern.ch', '-u', 'cms_hlt_gui_r', '-s', 'convertMe!' )
0037 databases['v1']['orcoff'] = databases['v1']['adg']
0038 databases['v3'] = {}
0039 databases['v3']['run2'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch', '-d', 'cms_hlt.cern.ch', '-u', 'cms_hlt_gdr_r', '-s', 'convertMe!' )
0040 databases['v3']['run3'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch', '-d', 'cms_hlt.cern.ch', '-u', 'cms_hlt_v3_r', '-s', 'convertMe!' )
0041 databases['v3']['dev'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch', '-d', 'cms_hlt.cern.ch', '-u', 'cms_hlt_gdrdev_r', '-s', 'convertMe1!' )
0042 databases['v3']['online'] = ( '-t', 'oracle', '-h', 'cmsonr1-s.cms', '-d', 'cms_rcms.cern.ch', '-u', 'cms_hlt_gdr_r', '-s', 'convertMe!' )
0043 databases['v3']['adg'] = ( '-t', 'oracle', '-h', 'cmsonr1-adg1-s.cern.ch', '-d', 'cms_orcon_adg.cern.ch', '-u', 'cms_hlt_gdr_r', '-s', 'convertMe!' )
0044
0045
0046 ips_for_proxy = {
0047 'cmsr1-s.cern.ch' : '10.116.96.89',
0048 'cmsr2-s.cern.ch' : '10.116.96.139',
0049 'cmsr3-s.cern.ch' : '10.116.96.105'
0050 }
0051
0052 databases['v3-beta'] = dict(databases['v3'])
0053 databases['v3-test'] = dict(databases['v3'])
0054 databases['v2'] = dict(databases['v3'])
0055
0056 for dbkey in databases['v2']:
0057 dbparams = databases['v2'][dbkey]
0058 if dbparams[3]=='cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch':
0059 databases['v2'][dbkey] = dbparams[0:3]+('cmsr1-s.cern.ch',)+dbparams[4:]
0060
0061 @staticmethod
0062 def CheckTempDirectory(dir):
0063 dir = os.path.realpath(dir)
0064 if not os.path.isdir(dir):
0065 try:
0066 os.makedirs(dir)
0067 except:
0068 return None
0069 return dir
0070
0071
0072 def __init__(self, version = 'v3', database = 'run3', url = None, verbose = False,
0073 proxy = False, proxyHost = 'localhost', proxyPort = '8080'):
0074 self.verbose = verbose
0075 self.version = version
0076 self.baseDir = '/afs/cern.ch/user/c/confdb/www/%s/lib' % version
0077 self.baseUrl = 'https://confdb.web.cern.ch/confdb/%s/lib' % version
0078 self.jars = ( 'ojdbc8.jar', 'cmssw-evf-confdb-converter.jar' )
0079 if version=='v2':
0080
0081 self.jars = ( 'ojdbc6.jar', 'cmssw-evf-confdb-converter.jar' )
0082 self.workDir = ''
0083 self.proxy = proxy
0084 self.proxyHost = proxyHost
0085 self.proxyPort = proxyPort
0086
0087
0088 if version not in self.databases:
0089
0090 sys.stderr.write( "ERROR: unsupported database version \"%s\"\n" % version)
0091
0092
0093 if database in self.databases[version]:
0094
0095 self.connect = self.databases[version][database]
0096 else:
0097
0098 sys.stderr.write( "ERROR: unknown database \"%s\" for version \"%s\"\n" % (database, version))
0099 sys.exit(1)
0100
0101 if self.proxy:
0102 self.proxy_connect_args = ('--dbproxy', '--dbproxyport', self.proxyPort, '--dbproxyhost', self.proxyHost)
0103 temp_connect = []
0104 for entry in self.connect:
0105 for key,item in self.ips_for_proxy.items():
0106 entry = entry.replace(key,item)
0107 temp_connect.append(entry.replace(key,item))
0108 self.connect = tuple(temp_connect)
0109 else:
0110 self.proxy_connect_args = ()
0111
0112
0113 if url is not None:
0114 self.baseUrl = url
0115
0116
0117 if os.path.isdir(self.baseDir) and all(os.path.isfile(self.baseDir + '/' + jar) for jar in self.jars):
0118
0119 self.workDir = self.baseDir
0120 else:
0121
0122 self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['CMSSW_BASE'],'tmp','confdb',self.version))
0123 if not self.workDir:
0124
0125 self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['TMP'],'confdb',self.version))
0126 if not self.workDir:
0127
0128 self.workDir = tempfile.mkdtemp()
0129 atexit.register(shutil.rmtree, self.workDir)
0130
0131 version_website = requests.get(self.baseUrl+"/../confdb.version").text
0132 jars_require_update = True
0133 if os.path.exists(os.path.join(self.workDir,"confdb.version")):
0134 with open(os.path.join(self.workDir,"confdb.version")) as f:
0135 version_existing = f.read()
0136 if version_existing==version_website:
0137 jars_require_update = False
0138
0139 if jars_require_update:
0140 for jar in self.jars:
0141
0142 handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = jar + '.')
0143 os.close(handle)
0144 request = requests.get(self.baseUrl + '/' + jar)
0145 with open(temp,'wb') as f:
0146 f.write(request.content)
0147 os.rename(temp, self.workDir + '/' + jar)
0148
0149 handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = "confdb.version" + '.')
0150 os.close(handle)
0151 with open(temp,'w') as f:
0152 f.write(version_website)
0153 os.rename(temp,os.path.join(self.workDir,"confdb.version"))
0154
0155
0156 if self.verbose:
0157 sys.stderr.write("workDir = %s\n" % self.workDir)
0158
0159
0160
0161
0162
0163
0164 self.javaCmd = ( 'java', '-cp', ':'.join(self.workDir + '/' + jar for jar in self.jars), '-Djava.security.egd=file:///dev/urandom', '-Doracle.jdbc.timezoneAsRegion=false', '-Xss32M', 'confdb.converter.BrowserConverter' )
0165
0166
0167 def query(self, *args):
0168 args = self.javaCmd + self.connect + self.proxy_connect_args + args
0169 if self.verbose:
0170 sys.stderr.write("\n" + ' '.join(args) + "\n\n" )
0171 sub = subprocess.Popen(
0172 args,
0173 stdin = None,
0174 stdout = subprocess.PIPE,
0175 stderr = subprocess.PIPE,
0176 shell = False,
0177 universal_newlines = True )
0178 return sub.communicate()
0179
0180 def help():
0181 sys.stdout.write("""Usage: %s OPTIONS
0182
0183 --v1|--v2|--v3|--v3-beta|--v3-test (specify the ConfDB version [default: v3])
0184
0185 --run3|--run2|--dev|--online|--adg (specify the target db [default: run3], online will only work inside p5 network)
0186
0187 Note that for v1
0188 --orcoff is a synonim of --adg
0189 --offline is a synonim of --hltdev
0190
0191 --configId <id> (specify the configuration by id)
0192 --configName <name> (specify the configuration by name)
0193 --runNumber <run> (specify the configuration by run number)
0194 [exactly one of --configId OR --configName OR --runNumber is required]
0195
0196 --cff (retrieve configuration *fragment*)
0197 --input <f1.root[,f2.root]> (insert PoolSource with specified fileNames)
0198 --input <files.list> (read a text file which lists input ROOT files)
0199 --output <out.root> (insert PoolOutputModule w/ specified fileName)
0200 --nopsets (exclude all globale psets)
0201 --noedsources (exclude all edsources)
0202 --noes (exclude all essources *and* esmodules)
0203 --noessources (exclude all essources)
0204 --noesmodules (exclude all esmodules)
0205 --noservices (exclude all services)
0206 --nooutput (exclude all output modules)
0207 --nopaths (exclude all paths [+=referenced seqs&mods])
0208 --nosequences (don't define sequences [+=referenced s&m])
0209 --nomodules (don't define modules)
0210 --psets <pset1[,pset2]> (include only specified global psets)
0211 --psets <-pset1[,-pset2]> (include all global psets but the specified)
0212 --essources <ess1[,ess2]> (include only specified essources)
0213 --essources <-ess1[,-ess2]> (include all essources but the specified)
0214 --esmodules <esm1[,esm2]> (include only specified esmodules)
0215 --esmodules <-esm1[,-esm2]> (include all esmodules but the specified)
0216 --services <svc1[,svc2]> (include only specified services)
0217 --services <-svc1[,-svc2]> (include all services but the specified)
0218 --paths <p1[,p2]> (include only specified paths)
0219 --paths <-p1[,-p2]> (include all paths but the specified)
0220 --streams <s1[,s2]> (include only specified streams)
0221 --datasets <d1[,d2]> (include only specified datasets)
0222 --sequences <s1[,s2]> (include sequences, referenced or not!)
0223 --modules <p1[,p2]> (include modules, referenced or not!)
0224 --blocks <m1::p1[,p2][,m2]> (generate parameter blocks)
0225
0226 --verbose (print additional details)
0227 """)
0228
0229
0230 def main():
0231 args = sys.argv[1:]
0232 version = 'v3'
0233 db = 'run3'
0234 verbose = False
0235
0236 if not args:
0237 help()
0238 sys.exit(1)
0239
0240 if '--help' in args or '-h' in args:
0241 help()
0242 sys.exit(0)
0243
0244 if '--verbose' in args:
0245 verbose = True
0246 args.remove('--verbose')
0247
0248 arg_count = Counter(args)
0249 db_count = arg_count['--v1'] + arg_count['--v2'] + arg_count['--v3'] + arg_count['--v3-beta'] + arg_count['--v3-test']
0250 if db_count>1:
0251 sys.stderr.write( 'ERROR: conflicting database version specifications: "--v1", "--v2", "--v3", "--v3-beta", and "--v3-test" are mutually exclusive options' )
0252 sys.exit(1)
0253
0254 if '--v1' in args:
0255 version = 'v1'
0256 db = 'offline'
0257 args.remove('--v1')
0258
0259 if '--v2' in args:
0260 version = 'v2'
0261 db = 'run2'
0262 args.remove('--v2')
0263
0264 if '--v3' in args:
0265 version = 'v3'
0266 db = 'run3'
0267 args.remove('--v3')
0268
0269 if '--v3-beta' in args:
0270 version = 'v3-beta'
0271 db = 'run3'
0272 args.remove('--v3-beta')
0273
0274 if '--v3-test' in args:
0275 version = 'v3-test'
0276 db = 'dev'
0277 args.remove('--v3-test')
0278
0279 proxy=False
0280 proxy_host = "localhost"
0281 proxy_port = "8080"
0282 if '--dbproxy' in args:
0283 proxy = True
0284 args.remove('--dbproxy')
0285 if '--dbproxyhost' in args:
0286 proxy_host = args.pop(args.index('--dbproxyhost')+1)
0287 args.remove('--dbproxyhost')
0288 if '--dbproxyport' in args:
0289 proxy_port = args.pop(args.index('--dbproxyport')+1)
0290 args.remove('--dbproxyport')
0291
0292
0293 _dbs = {}
0294 _dbs['v1'] = [ '--%s' % _db for _db in OfflineConverter.databases['v1'] ] + [ '--runNumber' ]
0295 _dbs['v2'] = [ '--%s' % _db for _db in OfflineConverter.databases['v2'] ] + [ '--runNumber' ]
0296 _dbs['v3'] = [ '--%s' % _db for _db in OfflineConverter.databases['v3'] ] + [ '--runNumber']
0297 _dbs['v3-beta'] = [ '--%s' % _db for _db in OfflineConverter.databases['v3-beta'] ] + [ '--runNumber' ]
0298 _dbs['v3-test'] = [ '--%s' % _db for _db in OfflineConverter.databases['v3-test'] ] + [ '--runNumber' ]
0299 _dbargs = set(args) & set(sum(_dbs.values(), []))
0300
0301 if _dbargs:
0302 if len(_dbargs) > 1:
0303 sys.stderr.write( "ERROR: too many database specifications: \"" + "\", \"".join( _dbargs) + "\"\n" )
0304 sys.exit(1)
0305
0306 _arg = _dbargs.pop()
0307 db = _arg[2:]
0308 if db == 'runNumber':
0309 db = 'adg'
0310 else:
0311 args.remove(_arg)
0312
0313 if not db in OfflineConverter.databases[version]:
0314 sys.stderr.write( "ERROR: database version \"%s\" incompatible with specification \"%s\"\n" % (version, db) )
0315 sys.exit(1)
0316
0317 converter = OfflineConverter(version = version, database = db, verbose = verbose,
0318 proxy = proxy, proxyHost = proxy_host, proxyPort=proxy_port)
0319 out, err = converter.query( * args )
0320 if 'ERROR' in err:
0321 sys.stderr.write( "%s: error while retriving the HLT menu\n\n%s\n\n" % (sys.argv[0], err) )
0322 sys.exit(1)
0323 else:
0324 sys.stdout.write( out )
0325
0326
0327 if __name__ == "__main__":
0328 main()