File indexing completed on 2024-11-26 02:34:10
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013 import sys
0014 import os
0015 import string
0016 import urllib
0017 import time
0018 import datetime
0019 import getpass
0020
0021
0022
0023
0024 TD_shiftUTC = datetime.timedelta(hours = 2)
0025 INT_offset = 8
0026
0027 STR_p5 = 'cmsusr0.cern.ch'
0028 STR_wwwWBM = 'http://cmswbm/cmsdb/servlet'
0029 STR_SiStrip = 'SIST'
0030 STR_wwwDBSData = 'dbs_discovery/getData'
0031 LSTR_dbsInstances = ['cms_dbs_caf_analysis_01',
0032 'cms_dbs_prod_global' ]
0033 STR_headDatasets = 'datasets'
0034 STR_headFiles = 'available data files'
0035 DICT_tagsRunRegistry = {}
0036 DICT_tagsRunRegistry['GLOBAL_NAME'] = 'global name '
0037 DICT_tagsRunRegistry['STATUS'] = 'status '
0038 DICT_tagsRunRegistry['IN_DBS'] = 'in DBS '
0039 DICT_tagsRunRegistry['SUBSYSTEMS'] = 'subsystems '
0040 DICT_tagsRunRegistry['EVENTS'] = '# of triggers '
0041 DICT_tagsRunRegistry['START_TIME'] = 'start time (local) '
0042 DICT_tagsRunRegistry['END_TIME'] = 'end time (local) '
0043 DICT_tagsRunRegistry['L1KEY'] = 'L1 key '
0044 DICT_tagsRunRegistry['HLTKEY'] = 'HLT key '
0045 DICT_tagsRunRegistry['L1SOURCES'] = 'L1 sources '
0046 DICT_tagsRunRegistry['RUN_RATE'] = 'event rate (Hz) '
0047 DICT_tagsRunRegistry['STOP_REASON'] = 'stop reason '
0048 DICT_tagsRunRegistry['SHIFTER'] = 'DQM shifter '
0049 DICT_tagsRunRegistry['CREATE_USER'] = 'entry created by '
0050 DICT_tagsRunRegistry['CREATE_TIME'] = 'entry creation time '
0051 DICT_tagsRunRegistry['ONLINE_COMMENT'] = 'DQM online shifter\'s comment '
0052 DICT_tagsRunRegistry['OFFLINE_COMMENT'] = 'DQM offline shifter\'s comment '
0053 DICT_tagsRunRegistry['OFFLINE_COMMENT'] = 'DQM offline shifter\'s comment '
0054 DICT_tagsRunRegistry['BFIELD'] = 'magnetic field at run creation time'
0055 DICT_tagsRunRegistry['BFIELD_COMMENT'] = 'comment on magnetic field '
0056 STR_htlConfig = 'HLT Config ID'
0057 STR_runStart = 'START_TIME'
0058 STR_runEnd = 'STOP_TIME'
0059 DICT_keysRunSummary = {}
0060 DICT_keysRunSummary[STR_runStart] = 'start time '
0061 DICT_keysRunSummary[STR_runEnd] = 'end time '
0062 DICT_keysRunSummary['BField'] = 'magnetic field '
0063 DICT_keysRunSummary['HLT Version'] = 'HLT version '
0064 DICT_keysRunSummary['L1 Rate'] = 'L1 rate '
0065 DICT_keysRunSummary['HLT Rate'] = 'HLT rate '
0066 DICT_keysRunSummary['L1 Triggers'] = 'L1 triggers '
0067 DICT_keysRunSummary['HLT Triggers'] = 'HLT triggers '
0068 DICT_keysRunSummary['LHC Fill'] = 'LHC fill '
0069 DICT_keysRunSummary['LHC Energy'] = 'LHC energy '
0070 DICT_keysRunSummary['Initial Lumi'] = 'initial luminosity '
0071 DICT_keysRunSummary['Ending Lumi'] = 'ending luminosity '
0072 DICT_keysRunSummary['Run Lumi'] = 'run luminosity '
0073 DICT_keysRunSummary['Run Live Lumi'] = 'run live luminosity'
0074 DICT_keysRunSummaryTrigger = {}
0075 DICT_keysRunSummaryTrigger['L1 Key'] = 'L1 key '
0076 DICT_keysRunSummaryTrigger['HLT Key'] = 'HLT key '
0077 DICT_keysRunSummaryTrigger[STR_htlConfig] = 'HLT config ID '
0078
0079
0080
0081 global Str_passwd
0082 global Str_userID
0083 global Str_run
0084 global Dict_runRegistry
0085 global Float_magneticField
0086 global Dict_wbmRunSummary
0087 global Lstr_hltPaths
0088 global DictDict_dbsDatasets
0089 global DictDict_dbsEvents
0090 global Dict_dbsDatasets
0091 global Dict_maxLenDbsDatasets
0092
0093 Str_run = sys.argv[1]
0094 Dict_runRegistry = {}
0095 Float_magneticField = -999.0
0096 Dict_wbmRunSummary = {}
0097 Lstr_hltPaths = []
0098 DictDict_dbsDatasets = {}
0099 DictDict_dbsEvents = {}
0100 Dict_dbsDatasets = {}
0101 Dict_maxLenDbsDatasets = {}
0102
0103
0104
0105
0106
0107
0108 def Func_GetHtmlTags(str_text):
0109 """ Func_GetHtmlTags(str_text):
0110 Gets HTML tags from a string
0111 """
0112 dict_tags = {}
0113
0114 lstr_split = str_text.split('</')
0115 for str_split in lstr_split[1:]:
0116 str_key = str_split.split('>')[0]
0117 dict_tags[str_key] = str_key in dict_tags
0118
0119 lstr_split = str_text.split('/>')
0120 for str_split in lstr_split[:-1]:
0121 str_key = str_split.split('<')[-1].split()[0]
0122 dict_tags[str_key] = str_key in dict_tags
0123 return dict_tags
0124
0125
0126
0127
0128 def Func_GetHtmlTagValue(str_tag, str_text, int_index = 1):
0129 """ Func_GetHtmlTagValue(str_tag, str_text):
0130 Gets the value of the n-th oocurence a given HTML tag from a string
0131 """
0132 if int_index > str_text.count('<'+str_tag):
0133 return ''
0134 str_1 = str_text.split('<'+str_tag)[int_index]
0135 if str_1[0] != '>':
0136 if str_1.split('>')[0][-1] == '/':
0137 return ''
0138 return str_1.split('>',1)[1].split('</'+str_tag+'>')[0]
0139
0140
0141
0142
0143 def Func_GetHtmlTagValues(str_text):
0144 """ Func_GetHtmlTagValues(str_text):
0145 Gets HTML tag values from a string
0146 """
0147 lstr_split = str_text.split('</')
0148 lstr_values = []
0149 for str_split in lstr_split[:-1]:
0150 lstr_values.append(str_split.split('>')[-1])
0151 return lstr_values
0152
0153
0154
0155
0156 def Func_GetHtmlTagValueAttr(str_value, str_text):
0157 """ Func_GetHtmlTagValueAttr(str_value, str_text):
0158 Gets the (last) attributes of a given HTML tag value from a string
0159 """
0160 return str_text.split('\">'+str_value+'<')[0].split('=\"')[-1]
0161
0162
0163
0164
0165 def Func_MakeShellWord(str_python):
0166 """ Func_MakeShellWord(str_python)
0167 Adds shell escape charakters to Python strings
0168 """
0169 return str_python.replace('?','\\?').replace('=','\\=').replace(' ','\\ ').replace('&','\\&').replace(':','\\:')
0170
0171
0172
0173
0174 def Func_GetWBMInfo(str_name, str_path):
0175 """ Func_GetWBMInfo(str_name, str_path)
0176 Logs in on cmsusr0, retrieves WBM information and stores it locally
0177 """
0178 pid, fd = os.forkpty()
0179 if pid == 0:
0180 os.execv('/usr/bin/ssh', ['/usr/bin/ssh', '-l', Str_userID, STR_p5] + ['rm', '-f', '\"'+str_name + '\" && ' + 'wget', '\"'+str_path+'/'+str_name+'\"'])
0181 else:
0182 time.sleep(1)
0183 os.read(fd, 1000)
0184 time.sleep(1)
0185 os.write(fd, Str_passwd)
0186 time.sleep(1)
0187 c = 0
0188 s = os.read(fd, 1)
0189 while s:
0190 c += 1
0191 s = os.read(fd, 1)
0192 if c >= 2:
0193 break
0194
0195
0196
0197
0198 def Func_CopyWBMInfo(str_name):
0199 """ Func_CopyWBMInfo(str_name)
0200 Logs in on cmsusr0 and copies file from there
0201 """
0202 pid, fd = os.forkpty()
0203 if pid == 0:
0204 os.execv('/usr/bin/scp', ['/usr/bin/scp', Str_userID+'@'+STR_p5+':~/'+str_name, '.'])
0205 else:
0206 time.sleep(1)
0207 os.read(fd, 1000)
0208 time.sleep(1)
0209 os.write(fd, Str_passwd)
0210 time.sleep(1)
0211 c = 0
0212 s = os.read(fd, 1)
0213 while s:
0214 c += 1
0215 s = os.read(fd, 1)
0216 if c >= 163:
0217 break
0218
0219
0220
0221
0222 def Func_FillInfoRunRegistry():
0223 """ Func_FillInfoRunRegistry():
0224 Retrieves run info from RunRegistry and fills it into containers
0225 """
0226 str_runRegistry = urllib.urlencode({'format':'xml', 'intpl':'xml', 'qtype':'RUN_NUMBER', 'sortname':'RUN_NUMBER'})
0227 file_runRegistry = urllib.urlopen("http://pccmsdqm04.cern.ch/runregistry/runregisterdata", str_runRegistry)
0228 str_runRegistryLong = ''
0229 for str_runRegistry in file_runRegistry.readlines():
0230 str_runRegistryLong += str_runRegistry.splitlines()[0]
0231 bool_foundRun = False
0232 str_runRunRegistry = ''
0233 for int_runIndex in range(1,int(str_runRegistryLong.split('<RUNS')[1].split('>')[0].split('total=\"')[1].split('\"')[0])):
0234 str_runRunRegistry = Func_GetHtmlTagValue('RUN', str_runRegistryLong, int_runIndex)
0235 if Func_GetHtmlTagValue('NUMBER', str_runRunRegistry) == Str_run:
0236 bool_foundRun = True
0237 break
0238 if not bool_foundRun:
0239 print('> getRunInfo.py > run %s not found in run registry' %(Str_run))
0240 return False
0241 dict_tagsRunRegistry = Func_GetHtmlTags(str_runRunRegistry)
0242 for str_tagRunRegistry in dict_tagsRunRegistry.keys():
0243 if dict_tagsRunRegistry[str_tagRunRegistry] == False:
0244 Dict_runRegistry[str_tagRunRegistry] = Func_GetHtmlTagValue(str_tagRunRegistry, str_runRunRegistry)
0245 if Dict_runRegistry['SUBSYSTEMS'].find(STR_SiStrip) < 0:
0246 print('> getRunInfo.py > SiStrip was not in this run')
0247 return False
0248 return True
0249
0250
0251
0252
0253 def Func_FillInfoRunSummary():
0254 """ Func_FillInfoRunSummary():
0255 Retrieves run info from RunSummary and fills it into containers
0256 """
0257 str_nameRunSummary = 'RunSummary?RUN=' + Str_run
0258 Func_GetWBMInfo(str_nameRunSummary, STR_wwwWBM)
0259 Func_CopyWBMInfo(Func_MakeShellWord(str_nameRunSummary))
0260 file_wbmRunSummary = file(str_nameRunSummary, 'r')
0261 bool_table = False
0262 int_tableHeader = 0
0263 int_tableItem = 0
0264 int_startItem = 0
0265 int_endItem = 0
0266 for str_wbmRunSummary in file_wbmRunSummary.readlines():
0267 if str_wbmRunSummary.find('<TABLE CLASS="params"><THEAD><TR>') >= 0:
0268 bool_table = True
0269 if str_wbmRunSummary.find('</TBODY></TABLE>') >= 0:
0270 bool_table = False
0271 if bool_table:
0272 if str_wbmRunSummary.startswith('<TH>'):
0273 int_tableHeader += 1
0274 if str_wbmRunSummary.find(STR_runStart) >= 0:
0275 int_startItem = int_tableHeader
0276 if str_wbmRunSummary.find(STR_runEnd) >= 0:
0277 int_endItem = int_tableHeader
0278 if str_wbmRunSummary.startswith('<TD'):
0279 int_tableItem += 1
0280 if int_tableItem == int_startItem:
0281 Dict_wbmRunSummary[STR_runStart] = str_wbmRunSummary.split(' </TD>')[0].split('<TD>')[-1]
0282 if int_tableItem == int_endItem:
0283 Dict_wbmRunSummary[STR_runEnd] = str_wbmRunSummary.split(' </TD>')[0].split('<TD>')[-1]
0284 continue
0285 for str_keyRunSummary in DICT_keysRunSummary.keys():
0286 if str_wbmRunSummary.find(str_keyRunSummary) >= 0:
0287 Dict_wbmRunSummary[str_keyRunSummary] = str_wbmRunSummary.split('</TD></TR>')[0].split('>')[-1]
0288 break
0289 for str_summaryKeysTrigger in DICT_keysRunSummaryTrigger.keys():
0290 if str_wbmRunSummary.find(str_summaryKeysTrigger) >= 0:
0291 Dict_wbmRunSummary[str_summaryKeysTrigger] = str_wbmRunSummary.split('</A></TD></TR>')[0].split('>')[-1]
0292 if str_summaryKeysTrigger == 'HLT Key':
0293 Dict_wbmRunSummary[STR_htlConfig] = str_wbmRunSummary.split('HLTConfiguration?KEY=')[1].split('>')[0]
0294 file_wbmRunSummary.close()
0295 os.remove(str_nameRunSummary)
0296
0297
0298
0299
0300 def Func_FillInfoMagnetHistory(str_timeStart, str_timeEnd):
0301 """ Func_FillInfoMagnetHistory():
0302 Retrieves run info from MagnetHistory and fills it into containers
0303 """
0304 str_nameMagnetHistory = 'MagnetHistory?TIME_BEGIN=' + str_timeStart + '&TIME_END=' + str_timeEnd
0305 Func_GetWBMInfo(str_nameMagnetHistory, STR_wwwWBM)
0306 Func_CopyWBMInfo(Func_MakeShellWord(str_nameMagnetHistory))
0307 file_wbmMagnetHistory = file(str_nameMagnetHistory, 'r')
0308 float_avMagMeasure = Float_magneticField
0309 for str_wbmMagnetHistory in file_wbmMagnetHistory.readlines():
0310 if str_wbmMagnetHistory.find('BFIELD, Tesla') >= 0:
0311 float_avMagMeasure = float(str_wbmMagnetHistory.split('</A>')[0].split('>')[-1])
0312 file_wbmMagnetHistory.close()
0313 os.remove(str_nameMagnetHistory)
0314 return float_avMagMeasure
0315
0316
0317
0318
0319 def Func_FillInfoHlt():
0320 """ Func_FillInfoHlt():
0321 Retrieves run info from Hlt and fills it into containers
0322 """
0323 str_nameHlt = 'HLTConfiguration?KEY=' + Dict_wbmRunSummary[STR_htlConfig]
0324 Func_GetWBMInfo(str_nameHlt, STR_wwwWBM)
0325 Func_CopyWBMInfo(Func_MakeShellWord(str_nameHlt))
0326 file_wbmHlt = file(str_nameHlt, 'r')
0327 bool_foundPaths = False
0328 bool_foundPath = False
0329 for str_wbmHlt in file_wbmHlt.readlines():
0330 if str_wbmHlt.find('<H3>Paths</H3>') >= 0:
0331 bool_foundPaths = True
0332 if bool_foundPaths and str_wbmHlt.find('<HR><H3>') >= 0:
0333 bool_foundPaths = False
0334 if bool_foundPaths and str_wbmHlt.startswith('<TR><TD ALIGN=RIGHT>'):
0335 Lstr_hltPaths.append(str_wbmHlt.split('</TD>')[1].split('<TD>')[-1])
0336 file_wbmHlt.close()
0337 os.remove(str_nameHlt)
0338 return (len(Lstr_hltPaths)>0)
0339
0340
0341
0342
0343 def Func_FillInfoDBS(str_dbsInstance):
0344 """ Func_FillInfoDBS(str_dbsInstance)
0345 Retrieves run info from DBS and fills it into containers
0346 """
0347 str_dbsRuns = urllib.urlencode({'ajax':'0', '_idx':'0', 'pagerStep':'0', 'userMode':'user', 'release':'Any', 'tier':'Any', 'dbsInst':str_dbsInstance, 'primType':'Any', 'primD':'Any', 'minRun':Str_run, 'maxRun':Str_run})
0348 file_dbsRuns = urllib.urlopen("https://cmsweb.cern.ch/dbs_discovery/getRunsFromRange", str_dbsRuns)
0349 lstr_dbsRuns = []
0350 lstr_dbsDatasets = []
0351 dict_dbsDatasets = {}
0352 dict_dbsEvents = {}
0353 for str_dbsRuns in file_dbsRuns.readlines():
0354 lstr_dbsRuns.append(str_dbsRuns)
0355 if str_dbsRuns.find(STR_wwwDBSData) >= 0:
0356 if str_dbsRuns.split('&proc=')[1].find('&') >= 0:
0357 lstr_dbsDatasets.append(str_dbsRuns.split('&proc=')[1].split('&')[0])
0358 else:
0359 lstr_dbsDatasets.append(str_dbsRuns.split('&proc=')[1])
0360 int_maxLenDbsDatasets = 0
0361 for str_dbsDataset in lstr_dbsDatasets:
0362 str_dbsLFN = urllib.urlencode({'dbsInst':str_dbsInstance, 'blockName':'*', 'dataset':str_dbsDataset, 'userMode':'user', 'run':Str_run})
0363 file_dbsLFN = urllib.urlopen("https://cmsweb.cern.ch/dbs_discovery/getLFNlist", str_dbsLFN)
0364 lstr_dbsLFN = []
0365 int_events = 0
0366 for str_dbsLFN in file_dbsLFN.readlines():
0367 lstr_dbsLFN.append(str_dbsLFN)
0368 if str_dbsLFN.find('contians') >= 0 and str_dbsLFN.find('file(s)'):
0369 dict_dbsDatasets[str_dbsDataset] = str_dbsLFN.split()[1]
0370 if str_dbsLFN.startswith('/store/data/'):
0371 int_events += int(Func_GetHtmlTagValue('td' ,lstr_dbsLFN[len(lstr_dbsLFN)-4]))
0372 dict_dbsEvents[str_dbsDataset] = str(int_events)
0373 if len(str_dbsDataset) > int_maxLenDbsDatasets:
0374 int_maxLenDbsDatasets = len(str_dbsDataset)
0375 DictDict_dbsDatasets[str_dbsInstance] = dict_dbsDatasets
0376 DictDict_dbsEvents[str_dbsInstance] = dict_dbsEvents
0377 Dict_dbsDatasets[str_dbsInstance] = lstr_dbsDatasets
0378 Dict_maxLenDbsDatasets[str_dbsInstance] = int_maxLenDbsDatasets
0379
0380
0381
0382 print()
0383 print('> getRunInfo.py > information on run \t*** %s ***' %(Str_run))
0384 print()
0385
0386
0387
0388 Str_userID = getpass.getuser()
0389 Str_passwd = getpass.getpass('> getRunInfo.py > '+Str_userID+'@'+STR_p5+'\'s password: ') + '\n'
0390
0391
0392
0393 bool_runRegistry = Func_FillInfoRunRegistry()
0394
0395
0396
0397 if bool_runRegistry:
0398 print()
0399 print('> getRunInfo.py > * information from run registry *')
0400 print()
0401 for str_htmlTag in DICT_tagsRunRegistry.keys():
0402 if str_htmlTag in Dict_runRegistry:
0403 print('> getRunInfo.py > %s: %s' %(DICT_tagsRunRegistry[str_htmlTag],Dict_runRegistry[str_htmlTag]))
0404
0405
0406
0407 Func_FillInfoRunSummary()
0408
0409
0410
0411 print()
0412 print('> getRunInfo.py > * information from run summary *')
0413 print()
0414 for str_key in DICT_keysRunSummary.keys():
0415 if str_key in Dict_wbmRunSummary:
0416 print('> getRunInfo.py > %s: %s' %(DICT_keysRunSummary[str_key],Dict_wbmRunSummary[str_key]))
0417 for str_key in DICT_keysRunSummaryTrigger.keys():
0418 if str_key in Dict_wbmRunSummary:
0419 print('> getRunInfo.py > %s: %s' %(DICT_keysRunSummaryTrigger[str_key],Dict_wbmRunSummary[str_key]))
0420
0421
0422
0423 if STR_runStart in Dict_wbmRunSummary and STR_runEnd in Dict_wbmRunSummary:
0424 Float_magneticField = Func_FillInfoMagnetHistory(Dict_wbmRunSummary[STR_runStart],Dict_wbmRunSummary[STR_runEnd])
0425
0426
0427
0428 if Float_magneticField >= 0.0:
0429 print()
0430 print('> getRunInfo.py > * information from magnet history *')
0431 print()
0432 print('> getRunInfo.py > (average) magnetic field: %s T' %(str(Float_magneticField)))
0433
0434
0435
0436 bool_hlt = False
0437 if STR_htlConfig in Dict_wbmRunSummary:
0438 bool_hlt = Func_FillInfoHlt()
0439
0440
0441
0442 if bool_hlt:
0443 print()
0444 print('> getRunInfo.py > * information from HLT configuration %s *' %(Dict_wbmRunSummary[STR_htlConfig]))
0445 print()
0446 print('> getRunInfo.py > HLT paths included:')
0447 print('> -----------------------------------')
0448 for str_hltPaths in Lstr_hltPaths:
0449 if str_hltPaths.find('CandHLTTrackerCosmics') >= 0 or str_hltPaths.find('HLT_TrackerCosmics') >= 0 or str_hltPaths.find('HLTTrackerCosmics') >= 0:
0450 print(' %s \t<====== FOR SURE!' %(str_hltPaths))
0451 elif str_hltPaths.find('Tracker') >= 0:
0452 print(' %s \t<====== maybe?' %(str_hltPaths))
0453 else:
0454 print(' %s' %(str_hltPaths))
0455
0456
0457
0458 for str_dbsInstance in LSTR_dbsInstances:
0459 Func_FillInfoDBS(str_dbsInstance)
0460
0461
0462
0463 print()
0464 print('> getRunInfo.py > * information from DBS *')
0465 for str_dbsInstance in LSTR_dbsInstances:
0466 print()
0467 print('> getRunInfo.py > DBS instance: %s' %(str_dbsInstance))
0468 if str_dbsInstance == LSTR_dbsInstances[0]:
0469 print(' (This is the instance used at CAF!)')
0470 str_print = '> getRunInfo.py > ' + STR_headDatasets
0471 for int_i in range(Dict_maxLenDbsDatasets[str_dbsInstance]-len(STR_headDatasets)):
0472 str_print += ' '
0473 str_print += ' '
0474 int_length = len(str_print)
0475 print('%s%s' %(str_print,STR_headFiles))
0476 str_print = ' '
0477 for int_i in range(int_length-16+len(STR_headFiles)/2+INT_offset+8):
0478 str_print += '-'
0479 print(str_print)
0480 for str_dbsDataset in Dict_dbsDatasets[str_dbsInstance]:
0481 str_print = ' ' + str_dbsDataset
0482 for int_i in range(Dict_maxLenDbsDatasets[str_dbsInstance]-len(str_dbsDataset)):
0483 str_print += ' '
0484 str_print += ' '
0485 for int_i in range(len(STR_headFiles)/2-len(DictDict_dbsDatasets[str_dbsInstance][str_dbsDataset])):
0486 str_print += ' '
0487 str_print += DictDict_dbsDatasets[str_dbsInstance][str_dbsDataset] + ' ('
0488 for int_i in range(INT_offset-len(DictDict_dbsEvents[str_dbsInstance][str_dbsDataset])):
0489 str_print += ' '
0490 print('%s%s events)' %(str_print,DictDict_dbsEvents[str_dbsInstance][str_dbsDataset]))
0491
0492 print()
