File indexing completed on 2024-11-26 02:34:11
0001
0002
0003 import sys
0004 import os
0005 from subprocess import call
0006 import os.path
0007 import shutil
0008 import subprocess
0009 import codecs
0010 import re
0011 import errno
0012 from getGTfromDQMFile_V2 import getGTfromDQMFile
0013
0014 def setRunDirectory(runNumber):
0015
0016 dirDict = { 325799:['Data2018', 'HIRun2018'],\
0017 315252:['Data2018', 'Run2018'],\
0018 308336:['Data2018', 'Commissioning2018'],\
0019 294644:['Data2017', 'Run2017'],\
0020 290123:['Data2017', 'Commissioning2017'],\
0021 284500:['Data2016', 'PARun2016'],\
0022 271024:['Data2016', 'Run2016'],\
0023 264200:['Data2016', 'Commissioning2016'],\
0024 246907:['Data2015', 'Run2015'],\
0025 232881:['Data2015', 'Commissioning2015'],\
0026 211658:['Data2013', 'Run2013'],\
0027 209634:['Data2013', 'HIRun2013'],\
0028 190450:['Data2012', 'Run2012']}
0029 runKey=0
0030 for key in sorted(dirDict):
0031 if runNumber > key:
0032 runKey=key
0033 return dirDict[runKey]
0034
0035 def downloadOfflineDQMhisto(run, Run_type,rereco):
0036 runDirval=setRunDirectory(run)
0037 DataLocalDir=runDirval[0]
0038 DataOfflineDir=runDirval[1]
0039 nnn=run/100
0040 print('Processing '+ Run_type + ' in '+DataOfflineDir+"...")
0041 File_Name = ''
0042 print('Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/')
0043 url = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
0044 os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url+" > index.html")
0045 f=codecs.open("index.html", 'r')
0046 index = f.readlines()
0047 if any(str(Run_Number[i]) in s for s in index):
0048 for s in index:
0049 if rereco:
0050 if (str(Run_Number[i]) in s) and ("__DQMIO.root" in s) and ("17Sep2018" in s):
0051 File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
0052 else:
0053 if (str(Run_Number[i]) in s) and ("__DQMIO.root" in s):
0054 File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
0055
0056 else:
0057 print('No DQM file available. Please check the Offline server')
0058 sys.exit(0)
0059
0060 print('Downloading DQM file:'+File_Name)
0061 os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'+File_Name+' > /tmp/'+File_Name)
0062
0063 return File_Name
0064
0065 def downloadOfflinePCLhisto(run, Run_type):
0066 runDirval=setRunDirectory(run)
0067 DataLocalDir=runDirval[0]
0068 DataOfflineDir=runDirval[1]
0069 nnn=run/100
0070 print('Processing '+ Run_type + ' in '+DataOfflineDir+"...")
0071 File_Name = 'Temp'
0072 print('Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/')
0073 url = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
0074 os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url+" > index.html")
0075 f=codecs.open("index.html", 'r')
0076 index = f.readlines()
0077 if any(str(Run_Number[i]) in s for s in index):
0078 for s in index:
0079 if (str(Run_Number[i]) in s) and ("PromptCalibProdSiPixel-Express" in s) and ("__ALCAPROMPT.root" in s):
0080 File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
0081 else:
0082 print('No DQM file available. Please check the Offline server')
0083 sys.exit(0)
0084 if File_Name!='Temp':
0085 print('Downloading DQM file:'+File_Name)
0086 os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'+File_Name+' > /tmp/'+File_Name)
0087
0088 return File_Name
0089
0090
0091 def downloadnlineDQMhisto(run, Run_type):
0092 runDirval=setRunDirectory(run)
0093 DataLocalDir=runDirval[0]
0094 DataOfflineDir=runDirval[1]
0095 nnn=run/100
0096 nnnOnline = run/10000
0097 File_Name_online=''
0098 deadRocMap = False
0099
0100 url1 = 'https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
0101 os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url1+" > index_online.html")
0102
0103 url2 = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OnlineData/original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
0104 os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url2+" > index_online_backup.html")
0105 f_online_backup=codecs.open("index_online_backup.html", 'r')
0106 index_online_backup = f_online_backup.readlines()
0107
0108 f_online=codecs.open("index_online.html", 'r')
0109 index_online = f_online.readlines()
0110 if any(str(run) in x for x in index_online):
0111 for x in index_online:
0112 if (str(run) in x) and ("_PixelPhase1_" in x):
0113 File_Name_online=str(str(x).split(".root'>")[1].split("</a></td><td>")[0])
0114 deadRocMap = True
0115 else:
0116 print("Can't find any file in offline server, trying the online server")
0117 if any(str(run) in y for y in index_online_backup):
0118 for y in index_online:
0119 if (str(run) in y) and ("_PixelPhase1_" in y):
0120 File_Name_online=str(str(y).split(".root'>")[1].split("</a></td><td>")[0])
0121 deadRocMap = True
0122 else:
0123 print('No Online DQM file available. Skip dead roc map')
0124 deadRocMap = False
0125
0126 print('Downloading DQM file:'+File_Name_online)
0127
0128
0129 os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'+File_Name_online+' > /tmp/'+File_Name_online)
0130
0131 os.remove('index_online.html')
0132 os.remove('index_online_backup.html')
0133 return deadRocMap, File_Name_online;
0134
0135
0136 def getGT(DQMfile, RunNumber, globalTagVar):
0137 globalTag_v0 = getGTfromDQMFile(DQMfile, RunNumber, globalTagVar)
0138 print("Global Tag: " + globalTag_v0)
0139 globalTag = globalTag_v0
0140
0141 for z in range(len(globalTag_v0)-2):
0142 if (globalTag_v0[z].isdigit()) and (globalTag_v0[z+1].isdigit()) and (globalTag_v0[z+2].isdigit()) and(globalTag_v0[z+3].isupper()):
0143 globalTag = globalTag_v0[z:]
0144 break
0145 if globalTag == "":
0146 print(" No GlobalTag found: trying from DAS.... ")
0147 globalTag = str(os.popen('getGTscript.sh '+filepath+ File_Name+' ' +str(Run_Number[i])));
0148 if globalTag == "":
0149 print(" No GlobalTag found for run: "+str(Run_Number[i]))
0150
0151 return globalTag
0152
0153
0154 Run_type = sys.argv[1]
0155 Run_Number = [int(x) for x in sys.argv[2:]]
0156 CMSSW_BASE = str(os.popen('echo ${CMSSW_BASE}').read().strip())
0157 rereco=False
0158
0159
0160 if Run_type == 'Cosmics' or Run_type == 'StreamExpress' or Run_type == 'StreamExpressCosmics' or Run_type == 'ZeroBias' or Run_type == 'StreamHIExpress' or Run_type == 'HIMinimumBias1' or re.match('ZeroBias([0-9]+?)',Run_type) or re.match('HIMinimumBias([0-9]+?)',Run_type):
0161 print(Run_type)
0162 elif Run_type == 'ReReco':
0163 rereco=True
0164 Run_type='ZeroBias'
0165 else:
0166 print("please enter a valid run type: Cosmics | ZeroBias | StreamExpress | StreamExpressCosmics ")
0167 sys.exit(0)
0168
0169 for i in range(len(Run_Number)):
0170
0171 print("Downloading File!!")
0172 nnnOut = Run_Number[i]/1000
0173
0174 filepath = '/tmp/'
0175 File_Name = downloadOfflineDQMhisto(Run_Number[i], Run_type, rereco)
0176 if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
0177 File_Name_PCL = downloadOfflinePCLhisto(Run_Number[i], Run_type)
0178 deadRocMap, File_Name_online = downloadnlineDQMhisto(Run_Number[i], Run_type)
0179
0180 runDirval=setRunDirectory(Run_Number[i])
0181 DataLocalDir=runDirval[0]
0182
0183
0184
0185
0186 print("get the run status from DQMFile")
0187
0188
0189 check_command = 'check_runcomplete '+filepath+File_Name
0190 Check_output = subprocess.call(check_command, shell=True)
0191
0192
0193 if Check_output == 0:
0194 print('Using DQM file: '+File_Name)
0195 else:
0196 print('*****************Warning: DQM file is not ready************************')
0197 input_var = input("DQM file is incompleted, do you want to continue? (y/n): ")
0198 if (input_var == 'y') or (input_var == 'Y'):
0199 print('Using DQM file: '+File_Name)
0200 else:
0201 sys.exit(0)
0202 if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
0203 if File_Name_PCL=='Temp':
0204 print(' ')
0205 print(' ')
0206 print('*****************Warning: PCL file is not ready************************')
0207 input_var = input("PCL file is not ready, you will need to re-run the script later for PCL plots, do you want to continue? (y/n): ")
0208 if (input_var == 'y') or (input_var == 'Y'):
0209 print('--------> Remember to re-run the script later!!!!!')
0210 else:
0211 sys.exit(0)
0212
0213
0214
0215 checkfolder = os.path.exists(str(Run_Number[i]))
0216 if checkfolder == True:
0217 shutil.rmtree(str(Run_Number[i]))
0218 os.makedirs(str(Run_Number[i])+'/'+Run_type)
0219 else:
0220 os.makedirs(str(Run_Number[i])+'/'+Run_type)
0221
0222
0223
0224 globalTag = getGT(filepath+File_Name, str(Run_Number[i]), 'globalTag_Step1')
0225
0226
0227
0228
0229 print(" Creating the TrackerMap.... ")
0230
0231 detIdInfoFileName = 'TkDetIdInfo_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
0232 workPath = os.popen('pwd').readline().strip()
0233
0234 os.chdir(str(Run_Number[i])+'/'+Run_type)
0235
0236
0237 os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/SiStripDQM_OfflineTkMap_Template_cfg_DB.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name+' detIdInfoFile='+detIdInfoFileName)
0238 os.system('rm -f *svg')
0239
0240 sefile = 'QualityTest_run'+str(Run_Number[i])+'.txt'
0241 shutil.move('QTBadModules.log',sefile)
0242
0243
0244
0245
0246
0247
0248
0249
0250
0251 if Run_type == "Cosmics" or Run_type == "StreamExpressCosmics":
0252 os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap_cosmics.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
0253 elif Run_type == "StreamExpress":
0254 os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_Express_TKMap.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
0255 else:
0256 os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
0257
0258 shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/fedmap.html','fedmap.html')
0259 shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/psumap.html','psumap.html')
0260
0261 print(" Check TrackerMap on "+str(Run_Number[i])+'/'+Run_type+" folder")
0262
0263 output =[]
0264 output.append(os.popen("/bin/ls ").readline().strip())
0265 print(output)
0266
0267
0268 print(" Creating the list of bad modules ")
0269
0270 os.system('listbadmodule '+filepath+'/'+File_Name+' PCLBadComponents.log')
0271
0272
0273
0274
0275
0276
0277
0278
0279
0280
0281
0282
0283
0284
0285
0286
0287
0288
0289
0290 os.system('bs_bad_ls_harvester . '+str(Run_Number[i]))
0291
0292
0293
0294
0295 dest='Beam'
0296
0297 if (Run_type == "Cosmics") or (Run_type == "StreamExpressCosmics"):
0298 dest="Cosmics"
0299
0300
0301
0302 os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/mergeBadChannel_Template_cfg.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name)
0303 shutil.move('MergedBadComponents.log','MergedBadComponents_run'+str(Run_Number[i])+'.txt')
0304
0305 os.system("mkdir -p /data/users/event_display/TkCommissioner_runs/"+DataLocalDir+"/"+dest+" 2> /dev/null")
0306
0307
0308 shutil.copyfile(detIdInfoFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+detIdInfoFileName)
0309
0310 os.remove(detIdInfoFileName)
0311 os.remove('MergedBadComponentsTkMap_Canvas.root')
0312 os.remove('MergedBadComponentsTkMap.root')
0313
0314 print("countig dead pixel ROCs" )
0315 if (Run_Number[i] < 290124) :
0316
0317 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter.py '+filepath+'/'+File_Name)
0318 else:
0319 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter_Phase1.py '+filepath+'/'+File_Name)
0320
0321 if rereco:
0322 os.system('mkdir -p /data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco 2> /dev/null')
0323 else:
0324 os.system('mkdir -p /data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type+' 2> /dev/null')
0325
0326 shutil.move('PixZeroOccROCs_run'+str(Run_Number[i])+'.txt',workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
0327
0328
0329
0330
0331
0332
0333 if deadRocMap == True:
0334
0335 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROC_duringRun.py '+filepath+File_Name_online+' '+filepath+File_Name)
0336 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/change_name.py')
0337 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PixelMapPlotter.py MaskedROC_sum.txt -c')
0338 os.remove('MaskedROC_sum.txt')
0339 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/InefficientDoubleROC.py '+filepath+File_Name_online)
0340 else:
0341 print('No Online DQM file available, Dead ROC maps will not be produced')
0342 print('No Online DQM file available, inefficient DC list will also not be produced')
0343
0344
0345
0346 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergeOccDeadROC.py '+filepath+File_Name)
0347
0348
0349 if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
0350 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergePCLDeadROC.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
0351 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergePCLFedErr.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
0352 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PCLOthers.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
0353
0354
0355 strip_files = os.listdir('.')
0356 for file_name in strip_files:
0357 full_stripfile_name = os.path.join('.', file_name)
0358 if (os.path.isfile(full_stripfile_name)):
0359 if rereco:
0360 shutil.copy(full_stripfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco')
0361 else:
0362 shutil.copy(full_stripfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
0363
0364
0365
0366
0367
0368 os.chdir(workPath)
0369 shutil.rmtree(str(Run_Number[i]))
0370 os.remove('index.html')
0371
0372
0373
0374 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/TH2PolyOfflineMaps.py ' + filepath+'/'+File_Name+' 3000 2000')
0375 shutil.move(workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt', 'OUT/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
0376
0377
0378
0379 pixel_files = os.listdir('./OUT')
0380 for file_name in pixel_files:
0381 full_pixelfile_name = os.path.join('./OUT/', file_name)
0382 if (os.path.isfile(full_pixelfile_name)):
0383 if rereco:
0384 shutil.copy(full_pixelfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco')
0385 else:
0386 shutil.copy(full_pixelfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
0387
0388
0389 shutil.rmtree('OUT')
0390
0391
0392
0393 pixelTreeFileName = 'PixelPhase1Tree_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
0394 os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PhaseITreeProducer.py ' + filepath+'/'+File_Name + ' ' + pixelTreeFileName)
0395
0396 shutil.copyfile(pixelTreeFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+pixelTreeFileName)
0397 os.remove(pixelTreeFileName)
0398 if File_Name:
0399 os.remove(filepath+File_Name)
0400 if File_Name_online:
0401 os.remove(filepath+File_Name_online)
0402 os.chdir(workPath)