SiStripMonitorClient/scripts/TkMap_script_phase1.py

0001 #!/usr/bin/env python3
0002
0003 from __future__ import print_function
0004 import sys
0005 import os
0006 from subprocess import call
0007 import os.path
0008 import shutil
0009 import subprocess
0010 import codecs
0011 import re
0012 import errno
0013 from getGTfromDQMFile_V2 import getGTfromDQMFile
0014
0015 def setRunDirectory(runNumber):
0016     # Don't forget to add an entry when there is a new era
0017     dirDict = { 325799:['Data2018', 'HIRun2018'],\
0018                 315252:['Data2018', 'Run2018'],\
0019                 308336:['Data2018', 'Commissioning2018'],\
0020                 294644:['Data2017', 'Run2017'],\
0021                 290123:['Data2017', 'Commissioning2017'],\
0022                 284500:['Data2016', 'PARun2016'],\
0023                 271024:['Data2016', 'Run2016'],\
0024                 264200:['Data2016', 'Commissioning2016'],\
0025                 246907:['Data2015', 'Run2015'],\
0026                 232881:['Data2015', 'Commissioning2015'],\
0027                 211658:['Data2013', 'Run2013'],\
0028                 209634:['Data2013', 'HIRun2013'],\
0029                 190450:['Data2012', 'Run2012']}
0030     runKey=0
0031     for key in sorted(dirDict):
0032         if runNumber > key:
0033             runKey=key
0034     return dirDict[runKey]
0035
0036 def downloadOfflineDQMhisto(run, Run_type,rereco):
0037     runDirval=setRunDirectory(run)
0038     DataLocalDir=runDirval[0]
0039     DataOfflineDir=runDirval[1]
0040     nnn=run/100
0041     print('Processing '+ Run_type + ' in '+DataOfflineDir+"...")
0042     File_Name = ''
0043     print('Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/')
0044     url = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
0045     os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url+" > index.html")
0046     f=codecs.open("index.html", 'r')
0047     index = f.readlines()
0048     if any(str(Run_Number[i]) in s for s in index):
0049         for s in index:
0050             if rereco:
0051                 if (str(Run_Number[i]) in s) and ("__DQMIO.root" in s) and ("17Sep2018" in s):
0052                     File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
0053             else:
0054                 if (str(Run_Number[i]) in s) and ("__DQMIO.root" in s):
0055                     File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
0056
0057     else:
0058         print('No DQM file available. Please check the Offline server')
0059         sys.exit(0)
0060
0061     print('Downloading DQM file:'+File_Name)
0062     os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'+File_Name+' > /tmp/'+File_Name)
0063
0064     return File_Name
0065
0066 def downloadOfflinePCLhisto(run, Run_type):
0067     runDirval=setRunDirectory(run)
0068     DataLocalDir=runDirval[0]
0069     DataOfflineDir=runDirval[1]
0070     nnn=run/100
0071     print('Processing '+ Run_type + ' in '+DataOfflineDir+"...")
0072     File_Name = 'Temp'
0073     print('Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/')
0074     url = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
0075     os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url+" > index.html")
0076     f=codecs.open("index.html", 'r')
0077     index = f.readlines()
0078     if any(str(Run_Number[i]) in s for s in index):
0079         for s in index:
0080             if (str(Run_Number[i]) in s) and ("PromptCalibProdSiPixel-Express" in s) and ("__ALCAPROMPT.root" in s):
0081                 File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
0082     else:
0083         print('No DQM file available. Please check the Offline server')
0084         sys.exit(0)
0085     if File_Name!='Temp':
0086         print('Downloading DQM file:'+File_Name)
0087         os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'+File_Name+' > /tmp/'+File_Name)
0088
0089     return File_Name
0090
0091
0092 def downloadnlineDQMhisto(run, Run_type):
0093     runDirval=setRunDirectory(run)
0094     DataLocalDir=runDirval[0]
0095     DataOfflineDir=runDirval[1]
0096     nnn=run/100
0097     nnnOnline = run/10000
0098     File_Name_online=''
0099     deadRocMap = False
0100     ##################online file########
0101     url1 = 'https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
0102     os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url1+" > index_online.html")
0103
0104     url2 = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OnlineData/original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
0105     os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url2+" > index_online_backup.html")
0106     f_online_backup=codecs.open("index_online_backup.html", 'r')
0107     index_online_backup = f_online_backup.readlines()
0108
0109     f_online=codecs.open("index_online.html", 'r')
0110     index_online = f_online.readlines()
0111     if any(str(run) in x for x in index_online):
0112         for x in index_online:
0113             if (str(run) in x) and ("_PixelPhase1_" in x):
0114                 File_Name_online=str(str(x).split(".root'>")[1].split("</a></td><td>")[0])
0115                 deadRocMap = True
0116     else:
0117         print("Can't find any file in offline server, trying the online server")
0118         if any(str(run) in y for y in index_online_backup):
0119             for y in index_online:
0120                 if (str(run) in y) and ("_PixelPhase1_" in y):
0121                     File_Name_online=str(str(y).split(".root'>")[1].split("</a></td><td>")[0])
0122                     deadRocMap = True
0123         else:
0124             print('No Online DQM file available. Skip dead roc map')
0125             deadRocMap = False
0126
0127     print('Downloading DQM file:'+File_Name_online)
0128
0129
0130     os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'+File_Name_online+' > /tmp/'+File_Name_online)
0131
0132     os.remove('index_online.html')
0133     os.remove('index_online_backup.html')
0134     return deadRocMap, File_Name_online;
0135
0136
0137 def getGT(DQMfile, RunNumber, globalTagVar):
0138     globalTag_v0 = getGTfromDQMFile(DQMfile, RunNumber, globalTagVar)
0139     print("Global Tag: " + globalTag_v0)
0140     globalTag = globalTag_v0
0141
0142     for z in range(len(globalTag_v0)-2):#clean up the garbage string in the GT
0143         if (globalTag_v0[z].isdigit()) and  (globalTag_v0[z+1].isdigit()) and (globalTag_v0[z+2].isdigit()) and(globalTag_v0[z+3].isupper()):
0144             globalTag = globalTag_v0[z:]
0145             break
0146     if globalTag == "":
0147         print(" No GlobalTag found: trying from DAS.... ")
0148         globalTag = str(os.popen('getGTscript.sh '+filepath+ File_Name+' ' +str(Run_Number[i])));
0149         if globalTag == "":
0150             print(" No GlobalTag found for run: "+str(Run_Number[i]))
0151
0152     return globalTag
0153
0154
0155 Run_type = sys.argv[1]
0156 Run_Number = [int(x) for x in sys.argv[2:]]
0157 CMSSW_BASE = str(os.popen('echo ${CMSSW_BASE}').read().strip())
0158 rereco=False
0159
0160 ###########Check if user enter the right run type######################
0161 if Run_type == 'Cosmics' or Run_type == 'StreamExpress' or Run_type == 'StreamExpressCosmics' or Run_type == 'ZeroBias' or Run_type == 'StreamHIExpress' or Run_type == 'HIMinimumBias1' or re.match('ZeroBias([0-9]+?)',Run_type) or re.match('HIMinimumBias([0-9]+?)',Run_type):
0162     print(Run_type)
0163 elif Run_type == 'ReReco':
0164     rereco=True
0165     Run_type='ZeroBias'
0166 else:
0167     print("please enter a valid run type: Cosmics | ZeroBias | StreamExpress | StreamExpressCosmics ")
0168     sys.exit(0)
0169
0170 for i in range(len(Run_Number)):
0171 #################Downloading DQM file############################
0172     print("Downloading File!!")
0173     nnnOut = Run_Number[i]/1000
0174
0175     filepath = '/tmp/'
0176     File_Name = downloadOfflineDQMhisto(Run_Number[i], Run_type, rereco)
0177     if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
0178         File_Name_PCL = downloadOfflinePCLhisto(Run_Number[i], Run_type)
0179     deadRocMap, File_Name_online = downloadnlineDQMhisto(Run_Number[i], Run_type)
0180
0181     runDirval=setRunDirectory(Run_Number[i])
0182     DataLocalDir=runDirval[0]
0183     #DataOfflineDir=runDirval[1]
0184
0185 ################Check if run is complete##############
0186
0187     print("get the run status from DQMFile")
0188
0189
0190     check_command = 'check_runcomplete '+filepath+File_Name
0191     Check_output = subprocess.call(check_command, shell=True)
0192
0193
0194     if Check_output == 0:
0195         print('Using DQM file: '+File_Name)
0196     else:
0197         print('*****************Warning: DQM file is not ready************************')
0198         input_var = input("DQM file is incompleted, do you want to continue? (y/n): ")
0199         if (input_var == 'y') or (input_var == 'Y'):
0200             print('Using DQM file: '+File_Name)
0201         else:
0202             sys.exit(0)
0203     if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
0204         if File_Name_PCL=='Temp':
0205             print(' ')
0206             print(' ')
0207             print('*****************Warning: PCL file is not ready************************')
0208             input_var = input("PCL file is not ready, you will need to re-run the script later for PCL plots, do you want to continue? (y/n): ")
0209             if (input_var == 'y') or (input_var == 'Y'):
0210                 print('-------->   Remember to re-run the script later!!!!!')
0211             else:
0212                 sys.exit(0)
0213
0214 ###################Start making TkMaps################
0215
0216     checkfolder = os.path.exists(str(Run_Number[i]))
0217     if checkfolder == True:
0218         shutil.rmtree(str(Run_Number[i]))
0219         os.makedirs(str(Run_Number[i])+'/'+Run_type)
0220     else:
0221         os.makedirs(str(Run_Number[i])+'/'+Run_type)
0222
0223 #######Getting GT##############
0224     ####After switch production to 10_X_X release, the clean up section need to be reviewed and modified  ##########
0225     globalTag = getGT(filepath+File_Name, str(Run_Number[i]), 'globalTag_Step1')
0226     ####################################################
0227
0228
0229
0230     print(" Creating the TrackerMap.... ")
0231
0232     detIdInfoFileName = 'TkDetIdInfo_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
0233     workPath = os.popen('pwd').readline().strip()
0234
0235     os.chdir(str(Run_Number[i])+'/'+Run_type)
0236
0237
0238     os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/SiStripDQM_OfflineTkMap_Template_cfg_DB.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name+' detIdInfoFile='+detIdInfoFileName)
0239     os.system('rm -f *svg')
0240 ####################### rename bad module list file######################
0241     sefile = 'QualityTest_run'+str(Run_Number[i])+'.txt'
0242     shutil.move('QTBadModules.log',sefile)
0243
0244 ################### put color legend in the TrackerMap###################
0245
0246 # PLEASE UNCOMMENT THE LINES BELOW TO GET THE LEGEND ON THE QT TkMAP (it will work only on vocms061)
0247 #    os.system('/usr/bin/python ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/LegendToQT.py QTestAlarm.png /data/users/cctrack/FinalLegendTrans.png')
0248 #    shutil.move('result.png', 'QTestAlarm.png')
0249
0250 ####################Copying the template html file to index.html########################
0251
0252     if Run_type == "Cosmics" or Run_type == "StreamExpressCosmics":
0253         os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap_cosmics.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
0254     elif Run_type == "StreamExpress":
0255          os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_Express_TKMap.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
0256     else:
0257         os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
0258
0259     shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/fedmap.html','fedmap.html')
0260     shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/psumap.html','psumap.html')
0261
0262     print(" Check TrackerMap on "+str(Run_Number[i])+'/'+Run_type+" folder")
0263
0264     output =[]
0265     output.append(os.popen("/bin/ls ").readline().strip())
0266     print(output)
0267
0268 ## Producing the list of bad modules
0269     print(" Creating the list of bad modules ")
0270
0271     os.system('listbadmodule '+filepath+'/'+File_Name+' PCLBadComponents.log')
0272
0273  ##   if Run_type != "StreamExpress":
0274  ##       shutil.copyfile(sefile, checkdir+'/'+sefile)
0275  ##       os.system('/usr/bin/python ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/findBadModT9.py -p '+sefile+' -s /'+checkdir+'/'+sefile);
0276
0277
0278 ## Producing the run certification by lumisection
0279
0280 #    print(" Creating the lumisection certification:")
0281
0282 #    if (Run_type.startswith("ZeroBias")) or (Run_type == "StreamExpress"):
0283 #        os.system('ls_cert 0.95 0.95 '+filepath+'/'+File_Name)
0284
0285 ## Producing the PrimaryVertex/BeamSpot quality test by LS..
0286 #    if (Run_type != "Cosmics") and ( Run_type != "StreamExpress") and (Run_type != "StreamExpressCosmics"):
0287 #        print(" Creating the BeamSpot Calibration certification summary:")
0288 #        os.system('lsbs_cert '+filepath+'/'+File_Name)
0289
0290 ## .. and harvest the bad beamspot LS with automatic emailing (if in period and if bad LS found)
0291     os.system('bs_bad_ls_harvester . '+str(Run_Number[i]))
0292
0293
0294 ## Producing the Module difference for ExpressStream
0295
0296     dest='Beam'
0297
0298     if (Run_type == "Cosmics") or (Run_type == "StreamExpressCosmics"):
0299         dest="Cosmics"
0300
0301
0302 ## create merged list of BadComponent from (PCL, RunInfo and FED Errors) ignore for now
0303     os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/mergeBadChannel_Template_cfg.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name)
0304     shutil.move('MergedBadComponents.log','MergedBadComponents_run'+str(Run_Number[i])+'.txt')
0305
0306     os.system("mkdir -p /data/users/event_display/TkCommissioner_runs/"+DataLocalDir+"/"+dest+" 2> /dev/null")
0307
0308
0309     shutil.copyfile(detIdInfoFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+detIdInfoFileName)
0310
0311     os.remove(detIdInfoFileName)
0312     os.remove('MergedBadComponentsTkMap_Canvas.root')
0313     os.remove('MergedBadComponentsTkMap.root')
0314 ##############counting dead pixel#######################
0315     print("countig dead pixel ROCs" )
0316     if (Run_Number[i] < 290124) :
0317
0318         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter.py '+filepath+'/'+File_Name)
0319     else:
0320         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter_Phase1.py '+filepath+'/'+File_Name)
0321
0322     if rereco:
0323         os.system('mkdir -p /data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco 2> /dev/null')
0324     else:
0325         os.system('mkdir -p /data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type+' 2> /dev/null')
0326
0327     shutil.move('PixZeroOccROCs_run'+str(Run_Number[i])+'.txt',workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
0328
0329
0330
0331
0332 ######Counting Dead ROCs and Inefficient DC during the run#########################
0333
0334     if deadRocMap == True:
0335
0336         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROC_duringRun.py '+filepath+File_Name_online+' '+filepath+File_Name)
0337         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/change_name.py')
0338         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PixelMapPlotter.py MaskedROC_sum.txt -c')
0339         os.remove('MaskedROC_sum.txt')
0340         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/InefficientDoubleROC.py '+filepath+File_Name_online)
0341     else:
0342         print('No Online DQM file available, Dead ROC maps will not be produced')
0343         print('No Online DQM file available, inefficient DC list  will also not be produced')
0344
0345 #######Merge Dead ROCs and Occupoancy Plot ###################
0346
0347     os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergeOccDeadROC.py '+filepath+File_Name)
0348
0349 #######Merge PCL and DQM Plot (only StreamExpress) ###################
0350     if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
0351         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergePCLDeadROC.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
0352         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergePCLFedErr.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
0353         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PCLOthers.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
0354
0355 ###################copy ouput files###################
0356     strip_files = os.listdir('.')
0357     for file_name in strip_files:
0358         full_stripfile_name = os.path.join('.', file_name)
0359         if (os.path.isfile(full_stripfile_name)):
0360             if rereco:
0361                 shutil.copy(full_stripfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco')
0362             else:
0363                 shutil.copy(full_stripfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
0364
0365
0366
0367
0368 #########################Start making pixel maps#####################
0369     os.chdir(workPath)
0370     shutil.rmtree(str(Run_Number[i]))
0371     os.remove('index.html')
0372
0373     # produce pixel phase1 TH2Poly maps
0374 #    os.chdir(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/scripts/PhaseIMaps/')
0375     os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/TH2PolyOfflineMaps.py ' + filepath+'/'+File_Name+' 3000 2000')
0376     shutil.move(workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt', 'OUT/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
0377
0378
0379 ###################copy ouput files##########
0380     pixel_files = os.listdir('./OUT')
0381     for file_name in pixel_files:
0382         full_pixelfile_name = os.path.join('./OUT/', file_name)
0383         if (os.path.isfile(full_pixelfile_name)):
0384             if rereco:
0385                 shutil.copy(full_pixelfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco')
0386             else:
0387                 shutil.copy(full_pixelfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
0388
0389
0390     shutil.rmtree('OUT')
0391
0392
0393     # produce pixel phase1 tree for Offline TkCommissioner
0394     pixelTreeFileName = 'PixelPhase1Tree_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
0395     os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PhaseITreeProducer.py ' + filepath+'/'+File_Name + ' ' + pixelTreeFileName)
0396
0397     shutil.copyfile(pixelTreeFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+pixelTreeFileName)
0398     os.remove(pixelTreeFileName)
0399     if File_Name:
0400         os.remove(filepath+File_Name)
0401     if File_Name_online:
0402         os.remove(filepath+File_Name_online)
0403     os.chdir(workPath)