File indexing completed on 2023-03-17 10:56:40
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013 from __future__ import print_function
0014 import sys
0015 import os
0016 import string
0017 import urllib
0018 import time
0019 import datetime
0020 import getpass
0021
0022
0023
0024
0025 TD_shiftUTC = datetime.timedelta(hours = 2)
0026 INT_offset = 8
0027
0028 STR_p5 = 'cmsusr0.cern.ch'
0029 STR_wwwWBM = 'http://cmswbm/cmsdb/servlet'
0030 STR_SiStrip = 'SIST'
0031 STR_wwwDBSData = 'dbs_discovery/getData'
0032 LSTR_dbsInstances = ['cms_dbs_caf_analysis_01',
0033 'cms_dbs_prod_global' ]
0034 STR_headDatasets = 'datasets'
0035 STR_headFiles = 'available data files'
0036 DICT_tagsRunRegistry = {}
0037 DICT_tagsRunRegistry['GLOBAL_NAME'] = 'global name '
0038 DICT_tagsRunRegistry['STATUS'] = 'status '
0039 DICT_tagsRunRegistry['IN_DBS'] = 'in DBS '
0040 DICT_tagsRunRegistry['SUBSYSTEMS'] = 'subsystems '
0041 DICT_tagsRunRegistry['EVENTS'] = '# of triggers '
0042 DICT_tagsRunRegistry['START_TIME'] = 'start time (local) '
0043 DICT_tagsRunRegistry['END_TIME'] = 'end time (local) '
0044 DICT_tagsRunRegistry['L1KEY'] = 'L1 key '
0045 DICT_tagsRunRegistry['HLTKEY'] = 'HLT key '
0046 DICT_tagsRunRegistry['L1SOURCES'] = 'L1 sources '
0047 DICT_tagsRunRegistry['RUN_RATE'] = 'event rate (Hz) '
0048 DICT_tagsRunRegistry['STOP_REASON'] = 'stop reason '
0049 DICT_tagsRunRegistry['SHIFTER'] = 'DQM shifter '
0050 DICT_tagsRunRegistry['CREATE_USER'] = 'entry created by '
0051 DICT_tagsRunRegistry['CREATE_TIME'] = 'entry creation time '
0052 DICT_tagsRunRegistry['ONLINE_COMMENT'] = 'DQM online shifter\'s comment '
0053 DICT_tagsRunRegistry['OFFLINE_COMMENT'] = 'DQM offline shifter\'s comment '
0054 DICT_tagsRunRegistry['OFFLINE_COMMENT'] = 'DQM offline shifter\'s comment '
0055 DICT_tagsRunRegistry['BFIELD'] = 'magnetic field at run creation time'
0056 DICT_tagsRunRegistry['BFIELD_COMMENT'] = 'comment on magnetic field '
0057 STR_htlConfig = 'HLT Config ID'
0058 STR_runStart = 'START_TIME'
0059 STR_runEnd = 'STOP_TIME'
0060 DICT_keysRunSummary = {}
0061 DICT_keysRunSummary[STR_runStart] = 'start time '
0062 DICT_keysRunSummary[STR_runEnd] = 'end time '
0063 DICT_keysRunSummary['BField'] = 'magnetic field '
0064 DICT_keysRunSummary['HLT Version'] = 'HLT version '
0065 DICT_keysRunSummary['L1 Rate'] = 'L1 rate '
0066 DICT_keysRunSummary['HLT Rate'] = 'HLT rate '
0067 DICT_keysRunSummary['L1 Triggers'] = 'L1 triggers '
0068 DICT_keysRunSummary['HLT Triggers'] = 'HLT triggers '
0069 DICT_keysRunSummary['LHC Fill'] = 'LHC fill '
0070 DICT_keysRunSummary['LHC Energy'] = 'LHC energy '
0071 DICT_keysRunSummary['Initial Lumi'] = 'initial luminosity '
0072 DICT_keysRunSummary['Ending Lumi'] = 'ending luminosity '
0073 DICT_keysRunSummary['Run Lumi'] = 'run luminosity '
0074 DICT_keysRunSummary['Run Live Lumi'] = 'run live luminosity'
0075 DICT_keysRunSummaryTrigger = {}
0076 DICT_keysRunSummaryTrigger['L1 Key'] = 'L1 key '
0077 DICT_keysRunSummaryTrigger['HLT Key'] = 'HLT key '
0078 DICT_keysRunSummaryTrigger[STR_htlConfig] = 'HLT config ID '
0079
0080
0081
0082 global Str_passwd
0083 global Str_userID
0084 global Str_run
0085 global Dict_runRegistry
0086 global Float_magneticField
0087 global Dict_wbmRunSummary
0088 global Lstr_hltPaths
0089 global DictDict_dbsDatasets
0090 global DictDict_dbsEvents
0091 global Dict_dbsDatasets
0092 global Dict_maxLenDbsDatasets
0093
0094 Str_run = sys.argv[1]
0095 Dict_runRegistry = {}
0096 Float_magneticField = -999.0
0097 Dict_wbmRunSummary = {}
0098 Lstr_hltPaths = []
0099 DictDict_dbsDatasets = {}
0100 DictDict_dbsEvents = {}
0101 Dict_dbsDatasets = {}
0102 Dict_maxLenDbsDatasets = {}
0103
0104
0105
0106
0107
0108
0109 def Func_GetHtmlTags(str_text):
0110 """ Func_GetHtmlTags(str_text):
0111 Gets HTML tags from a string
0112 """
0113 dict_tags = {}
0114
0115 lstr_split = str_text.split('</')
0116 for str_split in lstr_split[1:]:
0117 str_key = str_split.split('>')[0]
0118 dict_tags[str_key] = str_key in dict_tags
0119
0120 lstr_split = str_text.split('/>')
0121 for str_split in lstr_split[:-1]:
0122 str_key = str_split.split('<')[-1].split()[0]
0123 dict_tags[str_key] = str_key in dict_tags
0124 return dict_tags
0125
0126
0127
0128
0129 def Func_GetHtmlTagValue(str_tag, str_text, int_index = 1):
0130 """ Func_GetHtmlTagValue(str_tag, str_text):
0131 Gets the value of the n-th oocurence a given HTML tag from a string
0132 """
0133 if int_index > str_text.count('<'+str_tag):
0134 return ''
0135 str_1 = str_text.split('<'+str_tag)[int_index]
0136 if str_1[0] != '>':
0137 if str_1.split('>')[0][-1] == '/':
0138 return ''
0139 return str_1.split('>',1)[1].split('</'+str_tag+'>')[0]
0140
0141
0142
0143
0144 def Func_GetHtmlTagValues(str_text):
0145 """ Func_GetHtmlTagValues(str_text):
0146 Gets HTML tag values from a string
0147 """
0148 lstr_split = str_text.split('</')
0149 lstr_values = []
0150 for str_split in lstr_split[:-1]:
0151 lstr_values.append(str_split.split('>')[-1])
0152 return lstr_values
0153
0154
0155
0156
0157 def Func_GetHtmlTagValueAttr(str_value, str_text):
0158 """ Func_GetHtmlTagValueAttr(str_value, str_text):
0159 Gets the (last) attributes of a given HTML tag value from a string
0160 """
0161 return str_text.split('\">'+str_value+'<')[0].split('=\"')[-1]
0162
0163
0164
0165
0166 def Func_MakeShellWord(str_python):
0167 """ Func_MakeShellWord(str_python)
0168 Adds shell escape charakters to Python strings
0169 """
0170 return str_python.replace('?','\\?').replace('=','\\=').replace(' ','\\ ').replace('&','\\&').replace(':','\\:')
0171
0172
0173
0174
0175 def Func_GetWBMInfo(str_name, str_path):
0176 """ Func_GetWBMInfo(str_name, str_path)
0177 Logs in on cmsusr0, retrieves WBM information and stores it locally
0178 """
0179 pid, fd = os.forkpty()
0180 if pid == 0:
0181 os.execv('/usr/bin/ssh', ['/usr/bin/ssh', '-l', Str_userID, STR_p5] + ['rm', '-f', '\"'+str_name + '\" && ' + 'wget', '\"'+str_path+'/'+str_name+'\"'])
0182 else:
0183 time.sleep(1)
0184 os.read(fd, 1000)
0185 time.sleep(1)
0186 os.write(fd, Str_passwd)
0187 time.sleep(1)
0188 c = 0
0189 s = os.read(fd, 1)
0190 while s:
0191 c += 1
0192 s = os.read(fd, 1)
0193 if c >= 2:
0194 break
0195
0196
0197
0198
0199 def Func_CopyWBMInfo(str_name):
0200 """ Func_CopyWBMInfo(str_name)
0201 Logs in on cmsusr0 and copies file from there
0202 """
0203 pid, fd = os.forkpty()
0204 if pid == 0:
0205 os.execv('/usr/bin/scp', ['/usr/bin/scp', Str_userID+'@'+STR_p5+':~/'+str_name, '.'])
0206 else:
0207 time.sleep(1)
0208 os.read(fd, 1000)
0209 time.sleep(1)
0210 os.write(fd, Str_passwd)
0211 time.sleep(1)
0212 c = 0
0213 s = os.read(fd, 1)
0214 while s:
0215 c += 1
0216 s = os.read(fd, 1)
0217 if c >= 163:
0218 break
0219
0220
0221
0222
0223 def Func_FillInfoRunRegistry():
0224 """ Func_FillInfoRunRegistry():
0225 Retrieves run info from RunRegistry and fills it into containers
0226 """
0227 str_runRegistry = urllib.urlencode({'format':'xml', 'intpl':'xml', 'qtype':'RUN_NUMBER', 'sortname':'RUN_NUMBER'})
0228 file_runRegistry = urllib.urlopen("http://pccmsdqm04.cern.ch/runregistry/runregisterdata", str_runRegistry)
0229 str_runRegistryLong = ''
0230 for str_runRegistry in file_runRegistry.readlines():
0231 str_runRegistryLong += str_runRegistry.splitlines()[0]
0232 bool_foundRun = False
0233 str_runRunRegistry = ''
0234 for int_runIndex in range(1,int(str_runRegistryLong.split('<RUNS')[1].split('>')[0].split('total=\"')[1].split('\"')[0])):
0235 str_runRunRegistry = Func_GetHtmlTagValue('RUN', str_runRegistryLong, int_runIndex)
0236 if Func_GetHtmlTagValue('NUMBER', str_runRunRegistry) == Str_run:
0237 bool_foundRun = True
0238 break
0239 if not bool_foundRun:
0240 print('> getRunInfo.py > run %s not found in run registry' %(Str_run))
0241 return False
0242 dict_tagsRunRegistry = Func_GetHtmlTags(str_runRunRegistry)
0243 for str_tagRunRegistry in dict_tagsRunRegistry.keys():
0244 if dict_tagsRunRegistry[str_tagRunRegistry] == False:
0245 Dict_runRegistry[str_tagRunRegistry] = Func_GetHtmlTagValue(str_tagRunRegistry, str_runRunRegistry)
0246 if Dict_runRegistry['SUBSYSTEMS'].find(STR_SiStrip) < 0:
0247 print('> getRunInfo.py > SiStrip was not in this run')
0248 return False
0249 return True
0250
0251
0252
0253
0254 def Func_FillInfoRunSummary():
0255 """ Func_FillInfoRunSummary():
0256 Retrieves run info from RunSummary and fills it into containers
0257 """
0258 str_nameRunSummary = 'RunSummary?RUN=' + Str_run
0259 Func_GetWBMInfo(str_nameRunSummary, STR_wwwWBM)
0260 Func_CopyWBMInfo(Func_MakeShellWord(str_nameRunSummary))
0261 file_wbmRunSummary = file(str_nameRunSummary, 'r')
0262 bool_table = False
0263 int_tableHeader = 0
0264 int_tableItem = 0
0265 int_startItem = 0
0266 int_endItem = 0
0267 for str_wbmRunSummary in file_wbmRunSummary.readlines():
0268 if str_wbmRunSummary.find('<TABLE CLASS="params"><THEAD><TR>') >= 0:
0269 bool_table = True
0270 if str_wbmRunSummary.find('</TBODY></TABLE>') >= 0:
0271 bool_table = False
0272 if bool_table:
0273 if str_wbmRunSummary.startswith('<TH>'):
0274 int_tableHeader += 1
0275 if str_wbmRunSummary.find(STR_runStart) >= 0:
0276 int_startItem = int_tableHeader
0277 if str_wbmRunSummary.find(STR_runEnd) >= 0:
0278 int_endItem = int_tableHeader
0279 if str_wbmRunSummary.startswith('<TD'):
0280 int_tableItem += 1
0281 if int_tableItem == int_startItem:
0282 Dict_wbmRunSummary[STR_runStart] = str_wbmRunSummary.split(' </TD>')[0].split('<TD>')[-1]
0283 if int_tableItem == int_endItem:
0284 Dict_wbmRunSummary[STR_runEnd] = str_wbmRunSummary.split(' </TD>')[0].split('<TD>')[-1]
0285 continue
0286 for str_keyRunSummary in DICT_keysRunSummary.keys():
0287 if str_wbmRunSummary.find(str_keyRunSummary) >= 0:
0288 Dict_wbmRunSummary[str_keyRunSummary] = str_wbmRunSummary.split('</TD></TR>')[0].split('>')[-1]
0289 break
0290 for str_summaryKeysTrigger in DICT_keysRunSummaryTrigger.keys():
0291 if str_wbmRunSummary.find(str_summaryKeysTrigger) >= 0:
0292 Dict_wbmRunSummary[str_summaryKeysTrigger] = str_wbmRunSummary.split('</A></TD></TR>')[0].split('>')[-1]
0293 if str_summaryKeysTrigger == 'HLT Key':
0294 Dict_wbmRunSummary[STR_htlConfig] = str_wbmRunSummary.split('HLTConfiguration?KEY=')[1].split('>')[0]
0295 file_wbmRunSummary.close()
0296 os.remove(str_nameRunSummary)
0297
0298
0299
0300
0301 def Func_FillInfoMagnetHistory(str_timeStart, str_timeEnd):
0302 """ Func_FillInfoMagnetHistory():
0303 Retrieves run info from MagnetHistory and fills it into containers
0304 """
0305 str_nameMagnetHistory = 'MagnetHistory?TIME_BEGIN=' + str_timeStart + '&TIME_END=' + str_timeEnd
0306 Func_GetWBMInfo(str_nameMagnetHistory, STR_wwwWBM)
0307 Func_CopyWBMInfo(Func_MakeShellWord(str_nameMagnetHistory))
0308 file_wbmMagnetHistory = file(str_nameMagnetHistory, 'r')
0309 float_avMagMeasure = Float_magneticField
0310 for str_wbmMagnetHistory in file_wbmMagnetHistory.readlines():
0311 if str_wbmMagnetHistory.find('BFIELD, Tesla') >= 0:
0312 float_avMagMeasure = float(str_wbmMagnetHistory.split('</A>')[0].split('>')[-1])
0313 file_wbmMagnetHistory.close()
0314 os.remove(str_nameMagnetHistory)
0315 return float_avMagMeasure
0316
0317
0318
0319
0320 def Func_FillInfoHlt():
0321 """ Func_FillInfoHlt():
0322 Retrieves run info from Hlt and fills it into containers
0323 """
0324 str_nameHlt = 'HLTConfiguration?KEY=' + Dict_wbmRunSummary[STR_htlConfig]
0325 Func_GetWBMInfo(str_nameHlt, STR_wwwWBM)
0326 Func_CopyWBMInfo(Func_MakeShellWord(str_nameHlt))
0327 file_wbmHlt = file(str_nameHlt, 'r')
0328 bool_foundPaths = False
0329 bool_foundPath = False
0330 for str_wbmHlt in file_wbmHlt.readlines():
0331 if str_wbmHlt.find('<H3>Paths</H3>') >= 0:
0332 bool_foundPaths = True
0333 if bool_foundPaths and str_wbmHlt.find('<HR><H3>') >= 0:
0334 bool_foundPaths = False
0335 if bool_foundPaths and str_wbmHlt.startswith('<TR><TD ALIGN=RIGHT>'):
0336 Lstr_hltPaths.append(str_wbmHlt.split('</TD>')[1].split('<TD>')[-1])
0337 file_wbmHlt.close()
0338 os.remove(str_nameHlt)
0339 return (len(Lstr_hltPaths)>0)
0340
0341
0342
0343
0344 def Func_FillInfoDBS(str_dbsInstance):
0345 """ Func_FillInfoDBS(str_dbsInstance)
0346 Retrieves run info from DBS and fills it into containers
0347 """
0348 str_dbsRuns = urllib.urlencode({'ajax':'0', '_idx':'0', 'pagerStep':'0', 'userMode':'user', 'release':'Any', 'tier':'Any', 'dbsInst':str_dbsInstance, 'primType':'Any', 'primD':'Any', 'minRun':Str_run, 'maxRun':Str_run})
0349 file_dbsRuns = urllib.urlopen("https://cmsweb.cern.ch/dbs_discovery/getRunsFromRange", str_dbsRuns)
0350 lstr_dbsRuns = []
0351 lstr_dbsDatasets = []
0352 dict_dbsDatasets = {}
0353 dict_dbsEvents = {}
0354 for str_dbsRuns in file_dbsRuns.readlines():
0355 lstr_dbsRuns.append(str_dbsRuns)
0356 if str_dbsRuns.find(STR_wwwDBSData) >= 0:
0357 if str_dbsRuns.split('&proc=')[1].find('&') >= 0:
0358 lstr_dbsDatasets.append(str_dbsRuns.split('&proc=')[1].split('&')[0])
0359 else:
0360 lstr_dbsDatasets.append(str_dbsRuns.split('&proc=')[1])
0361 int_maxLenDbsDatasets = 0
0362 for str_dbsDataset in lstr_dbsDatasets:
0363 str_dbsLFN = urllib.urlencode({'dbsInst':str_dbsInstance, 'blockName':'*', 'dataset':str_dbsDataset, 'userMode':'user', 'run':Str_run})
0364 file_dbsLFN = urllib.urlopen("https://cmsweb.cern.ch/dbs_discovery/getLFNlist", str_dbsLFN)
0365 lstr_dbsLFN = []
0366 int_events = 0
0367 for str_dbsLFN in file_dbsLFN.readlines():
0368 lstr_dbsLFN.append(str_dbsLFN)
0369 if str_dbsLFN.find('contians') >= 0 and str_dbsLFN.find('file(s)'):
0370 dict_dbsDatasets[str_dbsDataset] = str_dbsLFN.split()[1]
0371 if str_dbsLFN.startswith('/store/data/'):
0372 int_events += int(Func_GetHtmlTagValue('td' ,lstr_dbsLFN[len(lstr_dbsLFN)-4]))
0373 dict_dbsEvents[str_dbsDataset] = str(int_events)
0374 if len(str_dbsDataset) > int_maxLenDbsDatasets:
0375 int_maxLenDbsDatasets = len(str_dbsDataset)
0376 DictDict_dbsDatasets[str_dbsInstance] = dict_dbsDatasets
0377 DictDict_dbsEvents[str_dbsInstance] = dict_dbsEvents
0378 Dict_dbsDatasets[str_dbsInstance] = lstr_dbsDatasets
0379 Dict_maxLenDbsDatasets[str_dbsInstance] = int_maxLenDbsDatasets
0380
0381
0382
0383 print()
0384 print('> getRunInfo.py > information on run \t*** %s ***' %(Str_run))
0385 print()
0386
0387
0388
0389 Str_userID = getpass.getuser()
0390 Str_passwd = getpass.getpass('> getRunInfo.py > '+Str_userID+'@'+STR_p5+'\'s password: ') + '\n'
0391
0392
0393
0394 bool_runRegistry = Func_FillInfoRunRegistry()
0395
0396
0397
0398 if bool_runRegistry:
0399 print()
0400 print('> getRunInfo.py > * information from run registry *')
0401 print()
0402 for str_htmlTag in DICT_tagsRunRegistry.keys():
0403 if str_htmlTag in Dict_runRegistry:
0404 print('> getRunInfo.py > %s: %s' %(DICT_tagsRunRegistry[str_htmlTag],Dict_runRegistry[str_htmlTag]))
0405
0406
0407
0408 Func_FillInfoRunSummary()
0409
0410
0411
0412 print()
0413 print('> getRunInfo.py > * information from run summary *')
0414 print()
0415 for str_key in DICT_keysRunSummary.keys():
0416 if str_key in Dict_wbmRunSummary:
0417 print('> getRunInfo.py > %s: %s' %(DICT_keysRunSummary[str_key],Dict_wbmRunSummary[str_key]))
0418 for str_key in DICT_keysRunSummaryTrigger.keys():
0419 if str_key in Dict_wbmRunSummary:
0420 print('> getRunInfo.py > %s: %s' %(DICT_keysRunSummaryTrigger[str_key],Dict_wbmRunSummary[str_key]))
0421
0422
0423
0424 if STR_runStart in Dict_wbmRunSummary and STR_runEnd in Dict_wbmRunSummary:
0425 Float_magneticField = Func_FillInfoMagnetHistory(Dict_wbmRunSummary[STR_runStart],Dict_wbmRunSummary[STR_runEnd])
0426
0427
0428
0429 if Float_magneticField >= 0.0:
0430 print()
0431 print('> getRunInfo.py > * information from magnet history *')
0432 print()
0433 print('> getRunInfo.py > (average) magnetic field: %s T' %(str(Float_magneticField)))
0434
0435
0436
0437 bool_hlt = False
0438 if STR_htlConfig in Dict_wbmRunSummary:
0439 bool_hlt = Func_FillInfoHlt()
0440
0441
0442
0443 if bool_hlt:
0444 print()
0445 print('> getRunInfo.py > * information from HLT configuration %s *' %(Dict_wbmRunSummary[STR_htlConfig]))
0446 print()
0447 print('> getRunInfo.py > HLT paths included:')
0448 print('> -----------------------------------')
0449 for str_hltPaths in Lstr_hltPaths:
0450 if str_hltPaths.find('CandHLTTrackerCosmics') >= 0 or str_hltPaths.find('HLT_TrackerCosmics') >= 0 or str_hltPaths.find('HLTTrackerCosmics') >= 0:
0451 print(' %s \t<====== FOR SURE!' %(str_hltPaths))
0452 elif str_hltPaths.find('Tracker') >= 0:
0453 print(' %s \t<====== maybe?' %(str_hltPaths))
0454 else:
0455 print(' %s' %(str_hltPaths))
0456
0457
0458
0459 for str_dbsInstance in LSTR_dbsInstances:
0460 Func_FillInfoDBS(str_dbsInstance)
0461
0462
0463
0464 print()
0465 print('> getRunInfo.py > * information from DBS *')
0466 for str_dbsInstance in LSTR_dbsInstances:
0467 print()
0468 print('> getRunInfo.py > DBS instance: %s' %(str_dbsInstance))
0469 if str_dbsInstance == LSTR_dbsInstances[0]:
0470 print(' (This is the instance used at CAF!)')
0471 str_print = '> getRunInfo.py > ' + STR_headDatasets
0472 for int_i in range(Dict_maxLenDbsDatasets[str_dbsInstance]-len(STR_headDatasets)):
0473 str_print += ' '
0474 str_print += ' '
0475 int_length = len(str_print)
0476 print('%s%s' %(str_print,STR_headFiles))
0477 str_print = ' '
0478 for int_i in range(int_length-16+len(STR_headFiles)/2+INT_offset+8):
0479 str_print += '-'
0480 print(str_print)
0481 for str_dbsDataset in Dict_dbsDatasets[str_dbsInstance]:
0482 str_print = ' ' + str_dbsDataset
0483 for int_i in range(Dict_maxLenDbsDatasets[str_dbsInstance]-len(str_dbsDataset)):
0484 str_print += ' '
0485 str_print += ' '
0486 for int_i in range(len(STR_headFiles)/2-len(DictDict_dbsDatasets[str_dbsInstance][str_dbsDataset])):
0487 str_print += ' '
0488 str_print += DictDict_dbsDatasets[str_dbsInstance][str_dbsDataset] + ' ('
0489 for int_i in range(INT_offset-len(DictDict_dbsEvents[str_dbsInstance][str_dbsDataset])):
0490 str_print += ' '
0491 print('%s%s events)' %(str_print,DictDict_dbsEvents[str_dbsInstance][str_dbsDataset]))
0492
0493 print()
