File indexing completed on 2021-07-07 22:33:05
0001
0002
0003 from __future__ import print_function
0004 import sys
0005 from ROOT import *
0006 import os
0007 from subprocess import call
0008 import os.path
0009 import shutil
0010 import subprocess
0011 import codecs
0012 import re
0013 import errno
0014 from getGTfromDQMFile_V2 import getGTfromDQMFile
0015
0016 def setRunDirectory(runNumber):
0017
0018 dirDict = { 325799:['Data2018', 'HIRun2018'],\
0019 315252:['Data2018', 'Run2018'],\
0020 308336:['Data2018', 'Commissioning2018'],\
0021 294644:['Data2017', 'Run2017'],\
0022 290123:['Data2017', 'Commissioning2017'],\
0023 284500:['Data2016', 'PARun2016'],\
0024 271024:['Data2016', 'Run2016'],\
0025 264200:['Data2016', 'Commissioning2016'],\
0026 246907:['Data2015', 'Run2015'],\
0027 232881:['Data2015', 'Commissioning2015'],\
0028 211658:['Data2013', 'Run2013'],\
0029 209634:['Data2013', 'HIRun2013'],\
0030 190450:['Data2012', 'Run2012']}
0031 runKey=0
0032 for key in sorted(dirDict):
0033 if runNumber > key:
0034 runKey=key
0035 return dirDict[runKey]
0036
0037 def downloadOfflineDQMhisto(run, Run_type,rereco):
0038 runDirval=setRunDirectory(run)
0039 DataLocalDir=runDirval[0]
0040 DataOfflineDir=runDirval[1]
0041 nnn=run/100
0042 print('Processing '+ Run_type + ' in '+DataOfflineDir+"...")
0043 File_Name = ''
0044 print('Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/')
0045 url = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
0046 os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url+" > index.html")
0047 f=codecs.open("index.html", 'r')
0048 index = f.readlines()
0049 if any(str(Run_Number[i]) in s for s in index):
0050 for s in index:
0051 if rereco:
0052 if (str(Run_Number[i]) in s) and ("__DQMIO.root" in s) and ("17Sep2018" in s):
0053 File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
0054 else:
0055 if (str(Run_Number[i]) in s) and ("__DQMIO.root" in s):
0056 File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
0057
0058 else:
0059 print('No DQM file available. Please check the Offline server')
0060 sys.exit(0)
0061
0062 print('Downloading DQM file:'+File_Name)
0063 os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'+File_Name+' > /tmp/'+File_Name)
0064
0065 return File_Name
0066
0067 def downloadOfflinePCLhisto(run, Run_type):
0068 runDirval=setRunDirectory(run)
0069 DataLocalDir=runDirval[0]
0070 DataOfflineDir=runDirval[1]
0071 nnn=run/100
0072 print('Processing '+ Run_type + ' in '+DataOfflineDir+"...")
0073 File_Name = 'Temp'
0074 print('Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/')
0075 url = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
0076 os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url+" > index.html")
0077 f=codecs.open("index.html", 'r')
0078 index = f.readlines()
0079 if any(str(Run_Number[i]) in s for s in index):
0080 for s in index:
0081 if (str(Run_Number[i]) in s) and ("PromptCalibProdSiPixel-Express" in s) and ("__ALCAPROMPT.root" in s):
0082 File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
0083 else:
0084 print('No DQM file available. Please check the Offline server')
0085 sys.exit(0)
0086 if File_Name!='Temp':
0087 print('Downloading DQM file:'+File_Name)
0088 os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'+File_Name+' > /tmp/'+File_Name)
0089
0090 return File_Name
0091
0092
0093 def downloadnlineDQMhisto(run, Run_type):
0094 runDirval=setRunDirectory(run)
0095 DataLocalDir=runDirval[0]
0096 DataOfflineDir=runDirval[1]
0097 nnn=run/100
0098 nnnOnline = run/10000
0099 File_Name_online=''
0100 deadRocMap = False
0101
0102 url1 = 'https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
0103 os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url1+" > index_online.html")
0104
0105 url2 = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OnlineData/original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
0106 os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url2+" > index_online_backup.html")
0107 f_online_backup=codecs.open("index_online_backup.html", 'r')
0108 index_online_backup = f_online_backup.readlines()
0109
0110 f_online=codecs.open("index_online.html", 'r')
0111 index_online = f_online.readlines()
0112 if any(str(run) in x for x in index_online):
0113 for x in index_online:
0114 if (str(run) in x) and ("_PixelPhase1_" in x):
0115 File_Name_online=str(str(x).split(".root'>")[1].split("</a></td><td>")[0])
0116 deadRocMap = True
0117 else:
0118 print("Can't find any file in offline server, trying the online server")
0119 if any(str(run) in y for y in index_online_backup):
0120 for y in index_online:
0121 if (str(run) in y) and ("_PixelPhase1_" in y):
0122 File_Name_online=str(str(y).split(".root'>")[1].split("</a></td><td>")[0])
0123 deadRocMap = True
0124 else:
0125 print('No Online DQM file available. Skip dead roc map')
0126 deadRocMap = False
0127
0128 print('Downloading DQM file:'+File_Name_online)
0129
0130
0131 os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'+File_Name_online+' > /tmp/'+File_Name_online)
0132
0133 os.remove('index_online.html')
0134 os.remove('index_online_backup.html')
0135 return deadRocMap, File_Name_online;
0136
0137
0138 def getGT(DQMfile, RunNumber, globalTagVar):
0139 globalTag_v0 = getGTfromDQMFile(DQMfile, RunNumber, globalTagVar)
0140 print("Global Tag: " + globalTag_v0)
0141 globalTag = globalTag_v0
0142
0143 for z in range(len(globalTag_v0)-2):
0144 if (globalTag_v0[z].isdigit()) and (globalTag_v0[z+1].isdigit()) and (globalTag_v0[z+2].isdigit()) and(globalTag_v0[z+3].isupper()):
0145 globalTag = globalTag_v0[z:]
0146 break
0147 if globalTag == "":
0148 print(" No GlobalTag found: trying from DAS.... ")
0149 globalTag = str(os.popen('getGTscript.sh '+filepath+ File_Name+' ' +str(Run_Number[i])));
0150 if globalTag == "":
0151 print(" No GlobalTag found for run: "+str(Run_Number[i]))
0152
0153 return globalTag
0154
0155
0156 Run_type = sys.argv[1]
0157 Run_Number = [int(x) for x in sys.argv[2:]]
0158 CMSSW_BASE = str(os.popen('echo ${CMSSW_BASE}').read().strip())
0159 rereco=False
0160
0161
0162 if Run_type == 'Cosmics' or Run_type == 'StreamExpress' or Run_type == 'StreamExpressCosmics' or Run_type == 'ZeroBias' or Run_type == 'StreamHIExpress' or Run_type == 'HIMinimumBias1' or re.match('ZeroBias([0-9]+?)',Run_type) or re.match('HIMinimumBias([0-9]+?)',Run_type):
0163 print(Run_type)
0164 elif Run_type == 'ReReco':
0165 rereco=True
0166 Run_type='ZeroBias'
0167 else:
0168 print("please enter a valid run type: Cosmics | ZeroBias | StreamExpress | StreamExpressCosmics ")
0169 sys.exit(0)
0170
0171 for i in range(len(Run_Number)):
0172
0173 print("Downloading File!!")
0174 nnnOut = Run_Number[i]/1000
0175
0176 filepath = '/tmp/'
0177 File_Name = downloadOfflineDQMhisto(Run_Number[i], Run_type, rereco)
0178 if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
0179 File_Name_PCL = downloadOfflinePCLhisto(Run_Number[i], Run_type)
0180 deadRocMap, File_Name_online = downloadnlineDQMhisto(Run_Number[i], Run_type)
0181
0182 runDirval=setRunDirectory(Run_Number[i])
0183 DataLocalDir=runDirval[0]
0184
0185
0186
0187
0188 print("get the run status from DQMFile")
0189
0190
0191 check_command = 'check_runcomplete '+filepath+File_Name
0192 Check_output = subprocess.call(check_command, shell=True)
0193
0194
0195 if Check_output == 0:
0196 print('Using DQM file: '+File_Name)
0197 else:
0198 print('*****************Warning: DQM file is not ready************************')
0199 input_var = input("DQM file is incompleted, do you want to continue? (y/n): ")
0200 if (input_var == 'y') or (input_var == 'Y'):
0201 print('Using DQM file: '+File_Name)
0202 else:
0203 sys.exit(0)
0204 if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
0205 if File_Name_PCL=='Temp':
0206 print(' ')
0207 print(' ')
0208 print('*****************Warning: PCL file is not ready************************')
0209 input_var = input("PCL file is not ready, you will need to re-run the script later for PCL plots, do you want to continue? (y/n): ")
0210 if (input_var == 'y') or (input_var == 'Y'):
0211 print('--------> Remember to re-run the script later!!!!!')
0212 else:
0213 sys.exit(0)
0214
0215
0216
0217 checkfolder = os.path.exists(str(Run_Number[i]))
0218 if checkfolder == True:
0219 shutil.rmtree(str(Run_Number[i]))
0220 os.makedirs(str(Run_Number[i])+'/'+Run_type)
0221 else:
0222 os.makedirs(str(Run_Number[i])+'/'+Run_type)
0223
0224
0225
0226 globalTag = getGT(filepath+File_Name, str(Run_Number[i]), 'globalTag_Step1')
0227
0228
0229
0230
0231 print(" Creating the TrackerMap.... ")
0232
0233 detIdInfoFileName = 'TkDetIdInfo_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
0234 workPath = os.popen('pwd').readline().strip()
0235
0236 os.chdir(str(Run_Number[i])+'/'+Run_type)
0237
0238
0239 os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/SiStripDQM_OfflineTkMap_Template_cfg_DB.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name+' detIdInfoFile='+detIdInfoFileName)
0240 os.system('rm -f *svg')
0241
0242 sefile = 'QualityTest_run'+str(Run_Number[i])+'.txt'
0243 shutil.move('QTBadModules.log',sefile)
0244
0245
0246
0247
0248
0249
0250
0251
0252
0253 if Run_type == "Cosmics" or Run_type == "StreamExpressCosmics":
0254 os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap_cosmics.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
0255 elif Run_type == "StreamExpress":
0256 os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_Express_TKMap.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
0257 else:
0258 os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
0259
0260 shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/fedmap.html','fedmap.html')
0261 shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/psumap.html','psumap.html')
0262
0263 print(" Check TrackerMap on "+str(Run_Number[i])+'/'+Run_type+" folder")
0264
0265 output =[]
0266 output.append(os.popen("/bin/ls ").readline().strip())
0267 print(output)
0268
0269
0270 print(" Creating the list of bad modules ")
0271
0272 os.system('listbadmodule '+filepath+'/'+File_Name+' PCLBadComponents.log')
0273
0274
0275
0276
0277
0278
0279
0280
0281
0282
0283
0284
0285
0286
0287
0288
0289
0290
0291
0292 os.system('bs_bad_ls_harvester . '+str(Run_Number[i]))
0293
0294
0295
0296
0297 dest='Beam'
0298
0299 if (Run_type == "Cosmics") or (Run_type == "StreamExpressCosmics"):
0300 dest="Cosmics"
0301
0302
0303
0304 os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/mergeBadChannel_Template_cfg.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name)
0305 shutil.move('MergedBadComponents.log','MergedBadComponents_run'+str(Run_Number[i])+'.txt')
0306
0307 os.system("mkdir -p /data/users/event_display/TkCommissioner_runs/"+DataLocalDir+"/"+dest+" 2> /dev/null")
0308
0309
0310 shutil.copyfile(detIdInfoFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+detIdInfoFileName)
0311
0312 os.remove(detIdInfoFileName)
0313 os.remove('MergedBadComponentsTkMap_Canvas.root')
0314 os.remove('MergedBadComponentsTkMap.root')
0315
0316 print("countig dead pixel ROCs" )
0317 if (Run_Number[i] < 290124) :
0318
0319 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter.py '+filepath+'/'+File_Name)
0320 else:
0321 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter_Phase1.py '+filepath+'/'+File_Name)
0322
0323 if rereco:
0324 os.system('mkdir -p /data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco 2> /dev/null')
0325 else:
0326 os.system('mkdir -p /data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type+' 2> /dev/null')
0327
0328 shutil.move('PixZeroOccROCs_run'+str(Run_Number[i])+'.txt',workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
0329
0330
0331
0332
0333
0334
0335 if deadRocMap == True:
0336
0337 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROC_duringRun.py '+filepath+File_Name_online+' '+filepath+File_Name)
0338 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/change_name.py')
0339 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PixelMapPlotter.py MaskedROC_sum.txt -c')
0340 os.remove('MaskedROC_sum.txt')
0341 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/InefficientDoubleROC.py '+filepath+File_Name_online)
0342 else:
0343 print('No Online DQM file available, Dead ROC maps will not be produced')
0344 print('No Online DQM file available, inefficient DC list will also not be produced')
0345
0346
0347
0348 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergeOccDeadROC.py '+filepath+File_Name)
0349
0350
0351 if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
0352 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergePCLDeadROC.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
0353 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergePCLFedErr.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
0354 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PCLOthers.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
0355
0356
0357 strip_files = os.listdir('.')
0358 for file_name in strip_files:
0359 full_stripfile_name = os.path.join('.', file_name)
0360 if (os.path.isfile(full_stripfile_name)):
0361 if rereco:
0362 shutil.copy(full_stripfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco')
0363 else:
0364 shutil.copy(full_stripfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
0365
0366
0367
0368
0369
0370 os.chdir(workPath)
0371 shutil.rmtree(str(Run_Number[i]))
0372 os.remove('index.html')
0373
0374
0375
0376 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/TH2PolyOfflineMaps.py ' + filepath+'/'+File_Name+' 3000 2000')
0377 shutil.move(workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt', 'OUT/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
0378
0379
0380
0381 pixel_files = os.listdir('./OUT')
0382 for file_name in pixel_files:
0383 full_pixelfile_name = os.path.join('./OUT/', file_name)
0384 if (os.path.isfile(full_pixelfile_name)):
0385 if rereco:
0386 shutil.copy(full_pixelfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco')
0387 else:
0388 shutil.copy(full_pixelfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
0389
0390
0391 shutil.rmtree('OUT')
0392
0393
0394
0395 pixelTreeFileName = 'PixelPhase1Tree_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
0396 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PhaseITreeProducer.py ' + filepath+'/'+File_Name + ' ' + pixelTreeFileName)
0397
0398 shutil.copyfile(pixelTreeFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+pixelTreeFileName)
0399 os.remove(pixelTreeFileName)
0400 if File_Name:
0401 os.remove(filepath+File_Name)
0402 if File_Name_online:
0403 os.remove(filepath+File_Name_online)
0404 os.chdir(workPath)