SiStripMonitorClient/scripts/TkMap_script_phase1.py

0001 #!/usr/bin/env python3
0002
0003 from __future__ import print_function
0004 import sys
0005 from ROOT import *
0006 import os
0007 from subprocess import call
0008 import os.path
0009 import shutil
0010 import subprocess
0011 import codecs
0012 import re
0013 import errno
0014 from getGTfromDQMFile_V2 import getGTfromDQMFile
0015
0016 def setRunDirectory(runNumber):
0017     # Don't forget to add an entry when there is a new era
0018     dirDict = { 325799:['Data2018', 'HIRun2018'],\
0019                 315252:['Data2018', 'Run2018'],\
0020                 308336:['Data2018', 'Commissioning2018'],\
0021                 294644:['Data2017', 'Run2017'],\
0022                 290123:['Data2017', 'Commissioning2017'],\
0023                 284500:['Data2016', 'PARun2016'],\
0024                 271024:['Data2016', 'Run2016'],\
0025                 264200:['Data2016', 'Commissioning2016'],\
0026                 246907:['Data2015', 'Run2015'],\
0027                 232881:['Data2015', 'Commissioning2015'],\
0028                 211658:['Data2013', 'Run2013'],\
0029                 209634:['Data2013', 'HIRun2013'],\
0030                 190450:['Data2012', 'Run2012']}
0031     runKey=0
0032     for key in sorted(dirDict):
0033         if runNumber > key:
0034             runKey=key
0035     return dirDict[runKey]
0036
0037 def downloadOfflineDQMhisto(run, Run_type,rereco):
0038     runDirval=setRunDirectory(run)
0039     DataLocalDir=runDirval[0]
0040     DataOfflineDir=runDirval[1]
0041     nnn=run/100
0042     print('Processing '+ Run_type + ' in '+DataOfflineDir+"...")
0043     File_Name = ''
0044     print('Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/')
0045     url = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
0046     os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url+" > index.html")
0047     f=codecs.open("index.html", 'r')
0048     index = f.readlines()
0049     if any(str(Run_Number[i]) in s for s in index):
0050         for s in index:
0051             if rereco:
0052                 if (str(Run_Number[i]) in s) and ("__DQMIO.root" in s) and ("17Sep2018" in s):
0053                     File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
0054             else:
0055                 if (str(Run_Number[i]) in s) and ("__DQMIO.root" in s):
0056                     File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
0057
0058     else:
0059         print('No DQM file available. Please check the Offline server')
0060         sys.exit(0)
0061
0062     print('Downloading DQM file:'+File_Name)
0063     os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'+File_Name+' > /tmp/'+File_Name)
0064
0065     return File_Name
0066
0067 def downloadOfflinePCLhisto(run, Run_type):
0068     runDirval=setRunDirectory(run)
0069     DataLocalDir=runDirval[0]
0070     DataOfflineDir=runDirval[1]
0071     nnn=run/100
0072     print('Processing '+ Run_type + ' in '+DataOfflineDir+"...")
0073     File_Name = 'Temp'
0074     print('Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/')
0075     url = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
0076     os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url+" > index.html")
0077     f=codecs.open("index.html", 'r')
0078     index = f.readlines()
0079     if any(str(Run_Number[i]) in s for s in index):
0080         for s in index:
0081             if (str(Run_Number[i]) in s) and ("PromptCalibProdSiPixel-Express" in s) and ("__ALCAPROMPT.root" in s):
0082                 File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
0083     else:
0084         print('No DQM file available. Please check the Offline server')
0085         sys.exit(0)
0086     if File_Name!='Temp':
0087         print('Downloading DQM file:'+File_Name)
0088         os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'+File_Name+' > /tmp/'+File_Name)
0089
0090     return File_Name
0091
0092
0093 def downloadnlineDQMhisto(run, Run_type):
0094     runDirval=setRunDirectory(run)
0095     DataLocalDir=runDirval[0]
0096     DataOfflineDir=runDirval[1]
0097     nnn=run/100
0098     nnnOnline = run/10000
0099     File_Name_online=''
0100     deadRocMap = False
0101     ##################online file########
0102     url1 = 'https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
0103     os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url1+" > index_online.html")
0104
0105     url2 = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OnlineData/original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
0106     os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url2+" > index_online_backup.html")
0107     f_online_backup=codecs.open("index_online_backup.html", 'r')
0108     index_online_backup = f_online_backup.readlines()
0109
0110     f_online=codecs.open("index_online.html", 'r')
0111     index_online = f_online.readlines()
0112     if any(str(run) in x for x in index_online):
0113         for x in index_online:
0114             if (str(run) in x) and ("_PixelPhase1_" in x):
0115                 File_Name_online=str(str(x).split(".root'>")[1].split("</a></td><td>")[0])
0116                 deadRocMap = True
0117     else:
0118         print("Can't find any file in offline server, trying the online server")
0119         if any(str(run) in y for y in index_online_backup):
0120             for y in index_online:
0121                 if (str(run) in y) and ("_PixelPhase1_" in y):
0122                     File_Name_online=str(str(y).split(".root'>")[1].split("</a></td><td>")[0])
0123                     deadRocMap = True
0124         else:
0125             print('No Online DQM file available. Skip dead roc map')
0126             deadRocMap = False
0127
0128     print('Downloading DQM file:'+File_Name_online)
0129
0130
0131     os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'+File_Name_online+' > /tmp/'+File_Name_online)
0132
0133     os.remove('index_online.html')
0134     os.remove('index_online_backup.html')
0135     return deadRocMap, File_Name_online;
0136
0137
0138 def getGT(DQMfile, RunNumber, globalTagVar):
0139     globalTag_v0 = getGTfromDQMFile(DQMfile, RunNumber, globalTagVar)
0140     print("Global Tag: " + globalTag_v0)
0141     globalTag = globalTag_v0
0142
0143     for z in range(len(globalTag_v0)-2):#clean up the garbage string in the GT
0144         if (globalTag_v0[z].isdigit()) and  (globalTag_v0[z+1].isdigit()) and (globalTag_v0[z+2].isdigit()) and(globalTag_v0[z+3].isupper()):
0145             globalTag = globalTag_v0[z:]
0146             break
0147     if globalTag == "":
0148         print(" No GlobalTag found: trying from DAS.... ")
0149         globalTag = str(os.popen('getGTscript.sh '+filepath+ File_Name+' ' +str(Run_Number[i])));
0150         if globalTag == "":
0151             print(" No GlobalTag found for run: "+str(Run_Number[i]))
0152
0153     return globalTag
0154
0155
0156 Run_type = sys.argv[1]
0157 Run_Number = [int(x) for x in sys.argv[2:]]
0158 CMSSW_BASE = str(os.popen('echo ${CMSSW_BASE}').read().strip())
0159 rereco=False
0160
0161 ###########Check if user enter the right run type######################
0162 if Run_type == 'Cosmics' or Run_type == 'StreamExpress' or Run_type == 'StreamExpressCosmics' or Run_type == 'ZeroBias' or Run_type == 'StreamHIExpress' or Run_type == 'HIMinimumBias1' or re.match('ZeroBias([0-9]+?)',Run_type) or re.match('HIMinimumBias([0-9]+?)',Run_type):
0163     print(Run_type)
0164 elif Run_type == 'ReReco':
0165     rereco=True
0166     Run_type='ZeroBias'
0167 else:
0168     print("please enter a valid run type: Cosmics | ZeroBias | StreamExpress | StreamExpressCosmics ")
0169     sys.exit(0)
0170
0171 for i in range(len(Run_Number)):
0172 #################Downloading DQM file############################
0173     print("Downloading File!!")
0174     nnnOut = Run_Number[i]/1000
0175
0176     filepath = '/tmp/'
0177     File_Name = downloadOfflineDQMhisto(Run_Number[i], Run_type, rereco)
0178     if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
0179         File_Name_PCL = downloadOfflinePCLhisto(Run_Number[i], Run_type)
0180     deadRocMap, File_Name_online = downloadnlineDQMhisto(Run_Number[i], Run_type)
0181
0182     runDirval=setRunDirectory(Run_Number[i])
0183     DataLocalDir=runDirval[0]
0184     #DataOfflineDir=runDirval[1]
0185
0186 ################Check if run is complete##############
0187
0188     print("get the run status from DQMFile")
0189
0190
0191     check_command = 'check_runcomplete '+filepath+File_Name
0192     Check_output = subprocess.call(check_command, shell=True)
0193
0194
0195     if Check_output == 0:
0196         print('Using DQM file: '+File_Name)
0197     else:
0198         print('*****************Warning: DQM file is not ready************************')
0199         input_var = input("DQM file is incompleted, do you want to continue? (y/n): ")
0200         if (input_var == 'y') or (input_var == 'Y'):
0201             print('Using DQM file: '+File_Name)
0202         else:
0203             sys.exit(0)
0204     if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
0205         if File_Name_PCL=='Temp':
0206             print(' ')
0207             print(' ')
0208             print('*****************Warning: PCL file is not ready************************')
0209             input_var = input("PCL file is not ready, you will need to re-run the script later for PCL plots, do you want to continue? (y/n): ")
0210             if (input_var == 'y') or (input_var == 'Y'):
0211                 print('-------->   Remember to re-run the script later!!!!!')
0212             else:
0213                 sys.exit(0)
0214
0215 ###################Start making TkMaps################
0216
0217     checkfolder = os.path.exists(str(Run_Number[i]))
0218     if checkfolder == True:
0219         shutil.rmtree(str(Run_Number[i]))
0220         os.makedirs(str(Run_Number[i])+'/'+Run_type)
0221     else:
0222         os.makedirs(str(Run_Number[i])+'/'+Run_type)
0223
0224 #######Getting GT##############
0225     ####After switch production to 10_X_X release, the clean up section need to be reviewed and modified  ##########
0226     globalTag = getGT(filepath+File_Name, str(Run_Number[i]), 'globalTag_Step1')
0227     ####################################################
0228
0229
0230
0231     print(" Creating the TrackerMap.... ")
0232
0233     detIdInfoFileName = 'TkDetIdInfo_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
0234     workPath = os.popen('pwd').readline().strip()
0235
0236     os.chdir(str(Run_Number[i])+'/'+Run_type)
0237
0238
0239     os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/SiStripDQM_OfflineTkMap_Template_cfg_DB.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name+' detIdInfoFile='+detIdInfoFileName)
0240     os.system('rm -f *svg')
0241 ####################### rename bad module list file######################
0242     sefile = 'QualityTest_run'+str(Run_Number[i])+'.txt'
0243     shutil.move('QTBadModules.log',sefile)
0244
0245 ################### put color legend in the TrackerMap###################
0246
0247 # PLEASE UNCOMMENT THE LINES BELOW TO GET THE LEGEND ON THE QT TkMAP (it will work only on vocms061)
0248 #    os.system('/usr/bin/python ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/LegendToQT.py QTestAlarm.png /data/users/cctrack/FinalLegendTrans.png')
0249 #    shutil.move('result.png', 'QTestAlarm.png')
0250
0251 ####################Copying the template html file to index.html########################
0252
0253     if Run_type == "Cosmics" or Run_type == "StreamExpressCosmics":
0254         os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap_cosmics.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
0255     elif Run_type == "StreamExpress":
0256          os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_Express_TKMap.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
0257     else:
0258         os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
0259
0260     shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/fedmap.html','fedmap.html')
0261     shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/psumap.html','psumap.html')
0262
0263     print(" Check TrackerMap on "+str(Run_Number[i])+'/'+Run_type+" folder")
0264
0265     output =[]
0266     output.append(os.popen("/bin/ls ").readline().strip())
0267     print(output)
0268
0269 ## Producing the list of bad modules
0270     print(" Creating the list of bad modules ")
0271
0272     os.system('listbadmodule '+filepath+'/'+File_Name+' PCLBadComponents.log')
0273
0274  ##   if Run_type != "StreamExpress":
0275  ##       shutil.copyfile(sefile, checkdir+'/'+sefile)
0276  ##       os.system('/usr/bin/python ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/findBadModT9.py -p '+sefile+' -s /'+checkdir+'/'+sefile);
0277
0278
0279 ## Producing the run certification by lumisection
0280
0281 #    print(" Creating the lumisection certification:")
0282
0283 #    if (Run_type.startswith("ZeroBias")) or (Run_type == "StreamExpress"):
0284 #        os.system('ls_cert 0.95 0.95 '+filepath+'/'+File_Name)
0285
0286 ## Producing the PrimaryVertex/BeamSpot quality test by LS..
0287 #    if (Run_type != "Cosmics") and ( Run_type != "StreamExpress") and (Run_type != "StreamExpressCosmics"):
0288 #        print(" Creating the BeamSpot Calibration certification summary:")
0289 #        os.system('lsbs_cert '+filepath+'/'+File_Name)
0290
0291 ## .. and harvest the bad beamspot LS with automatic emailing (if in period and if bad LS found)
0292     os.system('bs_bad_ls_harvester . '+str(Run_Number[i]))
0293
0294
0295 ## Producing the Module difference for ExpressStream
0296
0297     dest='Beam'
0298
0299     if (Run_type == "Cosmics") or (Run_type == "StreamExpressCosmics"):
0300         dest="Cosmics"
0301
0302
0303 ## create merged list of BadComponent from (PCL, RunInfo and FED Errors) ignore for now
0304     os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/mergeBadChannel_Template_cfg.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name)
0305     shutil.move('MergedBadComponents.log','MergedBadComponents_run'+str(Run_Number[i])+'.txt')
0306
0307     os.system("mkdir -p /data/users/event_display/TkCommissioner_runs/"+DataLocalDir+"/"+dest+" 2> /dev/null")
0308
0309
0310     shutil.copyfile(detIdInfoFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+detIdInfoFileName)
0311
0312     os.remove(detIdInfoFileName)
0313     os.remove('MergedBadComponentsTkMap_Canvas.root')
0314     os.remove('MergedBadComponentsTkMap.root')
0315 ##############counting dead pixel#######################
0316     print("countig dead pixel ROCs" )
0317     if (Run_Number[i] < 290124) :
0318
0319         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter.py '+filepath+'/'+File_Name)
0320     else:
0321         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter_Phase1.py '+filepath+'/'+File_Name)
0322
0323     if rereco:
0324         os.system('mkdir -p /data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco 2> /dev/null')
0325     else:
0326         os.system('mkdir -p /data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type+' 2> /dev/null')
0327
0328     shutil.move('PixZeroOccROCs_run'+str(Run_Number[i])+'.txt',workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
0329
0330
0331
0332
0333 ######Counting Dead ROCs and Inefficient DC during the run#########################
0334
0335     if deadRocMap == True:
0336
0337         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROC_duringRun.py '+filepath+File_Name_online+' '+filepath+File_Name)
0338         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/change_name.py')
0339         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PixelMapPlotter.py MaskedROC_sum.txt -c')
0340         os.remove('MaskedROC_sum.txt')
0341         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/InefficientDoubleROC.py '+filepath+File_Name_online)
0342     else:
0343         print('No Online DQM file available, Dead ROC maps will not be produced')
0344         print('No Online DQM file available, inefficient DC list  will also not be produced')
0345
0346 #######Merge Dead ROCs and Occupoancy Plot ###################
0347
0348     os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergeOccDeadROC.py '+filepath+File_Name)
0349
0350 #######Merge PCL and DQM Plot (only StreamExpress) ###################
0351     if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
0352         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergePCLDeadROC.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
0353         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergePCLFedErr.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
0354         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PCLOthers.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
0355
0356 ###################copy ouput files###################
0357     strip_files = os.listdir('.')
0358     for file_name in strip_files:
0359         full_stripfile_name = os.path.join('.', file_name)
0360         if (os.path.isfile(full_stripfile_name)):
0361             if rereco:
0362                 shutil.copy(full_stripfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco')
0363             else:
0364                 shutil.copy(full_stripfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
0365
0366
0367
0368
0369 #########################Start making pixel maps#####################
0370     os.chdir(workPath)
0371     shutil.rmtree(str(Run_Number[i]))
0372     os.remove('index.html')
0373
0374     # produce pixel phase1 TH2Poly maps
0375 #    os.chdir(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/scripts/PhaseIMaps/')
0376     os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/TH2PolyOfflineMaps.py ' + filepath+'/'+File_Name+' 3000 2000')
0377     shutil.move(workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt', 'OUT/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
0378
0379
0380 ###################copy ouput files##########
0381     pixel_files = os.listdir('./OUT')
0382     for file_name in pixel_files:
0383         full_pixelfile_name = os.path.join('./OUT/', file_name)
0384         if (os.path.isfile(full_pixelfile_name)):
0385             if rereco:
0386                 shutil.copy(full_pixelfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco')
0387             else:
0388                 shutil.copy(full_pixelfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
0389
0390
0391     shutil.rmtree('OUT')
0392
0393
0394     # produce pixel phase1 tree for Offline TkCommissioner
0395     pixelTreeFileName = 'PixelPhase1Tree_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
0396     os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PhaseITreeProducer.py ' + filepath+'/'+File_Name + ' ' + pixelTreeFileName)
0397
0398     shutil.copyfile(pixelTreeFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+pixelTreeFileName)
0399     os.remove(pixelTreeFileName)
0400     if File_Name:
0401         os.remove(filepath+File_Name)
0402     if File_Name_online:
0403         os.remove(filepath+File_Name_online)
0404     os.chdir(workPath)