Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-11-26 02:34:11

0001 #!/usr/bin/env python3
0002 
0003 import sys
0004 import os
0005 from subprocess import call
0006 import os.path
0007 import shutil
0008 import subprocess
0009 import codecs
0010 import re
0011 import errno
0012 from getGTfromDQMFile_V2 import getGTfromDQMFile
0013 
0014 def setRunDirectory(runNumber):
0015     # Don't forget to add an entry when there is a new era
0016     dirDict = { 325799:['Data2018', 'HIRun2018'],\
0017                 315252:['Data2018', 'Run2018'],\
0018                 308336:['Data2018', 'Commissioning2018'],\
0019                 294644:['Data2017', 'Run2017'],\
0020                 290123:['Data2017', 'Commissioning2017'],\
0021                 284500:['Data2016', 'PARun2016'],\
0022                 271024:['Data2016', 'Run2016'],\
0023                 264200:['Data2016', 'Commissioning2016'],\
0024                 246907:['Data2015', 'Run2015'],\
0025                 232881:['Data2015', 'Commissioning2015'],\
0026                 211658:['Data2013', 'Run2013'],\
0027                 209634:['Data2013', 'HIRun2013'],\
0028                 190450:['Data2012', 'Run2012']}
0029     runKey=0
0030     for key in sorted(dirDict):
0031         if runNumber > key:
0032             runKey=key
0033     return dirDict[runKey]  
0034 
0035 def downloadOfflineDQMhisto(run, Run_type,rereco):
0036     runDirval=setRunDirectory(run)
0037     DataLocalDir=runDirval[0]
0038     DataOfflineDir=runDirval[1]
0039     nnn=run/100
0040     print('Processing '+ Run_type + ' in '+DataOfflineDir+"...")
0041     File_Name = ''
0042     print('Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/')
0043     url = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
0044     os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url+" > index.html") 
0045     f=codecs.open("index.html", 'r')
0046     index = f.readlines()
0047     if any(str(Run_Number[i]) in s for s in index):
0048         for s in index:
0049             if rereco:
0050                 if (str(Run_Number[i]) in s) and ("__DQMIO.root" in s) and ("17Sep2018" in s):
0051                     File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
0052             else:
0053                 if (str(Run_Number[i]) in s) and ("__DQMIO.root" in s):
0054                     File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
0055 
0056     else:
0057         print('No DQM file available. Please check the Offline server')
0058         sys.exit(0)
0059 
0060     print('Downloading DQM file:'+File_Name)
0061     os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'+File_Name+' > /tmp/'+File_Name)
0062     
0063     return File_Name
0064 
0065 def downloadOfflinePCLhisto(run, Run_type):
0066     runDirval=setRunDirectory(run)
0067     DataLocalDir=runDirval[0]
0068     DataOfflineDir=runDirval[1]
0069     nnn=run/100
0070     print('Processing '+ Run_type + ' in '+DataOfflineDir+"...")
0071     File_Name = 'Temp'
0072     print('Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/')
0073     url = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
0074     os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url+" > index.html") 
0075     f=codecs.open("index.html", 'r')
0076     index = f.readlines()
0077     if any(str(Run_Number[i]) in s for s in index):
0078         for s in index:
0079             if (str(Run_Number[i]) in s) and ("PromptCalibProdSiPixel-Express" in s) and ("__ALCAPROMPT.root" in s):
0080                 File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
0081     else:
0082         print('No DQM file available. Please check the Offline server')
0083         sys.exit(0)
0084     if File_Name!='Temp':
0085         print('Downloading DQM file:'+File_Name)
0086         os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'+File_Name+' > /tmp/'+File_Name)
0087     
0088     return File_Name
0089 
0090 
0091 def downloadnlineDQMhisto(run, Run_type):
0092     runDirval=setRunDirectory(run)
0093     DataLocalDir=runDirval[0]
0094     DataOfflineDir=runDirval[1]
0095     nnn=run/100
0096     nnnOnline = run/10000
0097     File_Name_online=''
0098     deadRocMap = False
0099     ##################online file########    
0100     url1 = 'https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
0101     os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url1+" > index_online.html")
0102     
0103     url2 = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OnlineData/original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
0104     os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url2+" > index_online_backup.html")
0105     f_online_backup=codecs.open("index_online_backup.html", 'r')
0106     index_online_backup = f_online_backup.readlines()
0107 
0108     f_online=codecs.open("index_online.html", 'r')
0109     index_online = f_online.readlines()
0110     if any(str(run) in x for x in index_online):
0111         for x in index_online:
0112             if (str(run) in x) and ("_PixelPhase1_" in x):
0113                 File_Name_online=str(str(x).split(".root'>")[1].split("</a></td><td>")[0])
0114                 deadRocMap = True
0115     else:
0116         print("Can't find any file in offline server, trying the online server")
0117         if any(str(run) in y for y in index_online_backup):
0118             for y in index_online:
0119                 if (str(run) in y) and ("_PixelPhase1_" in y):
0120                     File_Name_online=str(str(y).split(".root'>")[1].split("</a></td><td>")[0])
0121                     deadRocMap = True
0122         else:
0123             print('No Online DQM file available. Skip dead roc map')
0124             deadRocMap = False
0125 
0126     print('Downloading DQM file:'+File_Name_online)
0127 
0128 
0129     os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'+File_Name_online+' > /tmp/'+File_Name_online)
0130 
0131     os.remove('index_online.html')
0132     os.remove('index_online_backup.html')
0133     return deadRocMap, File_Name_online;
0134 
0135 
0136 def getGT(DQMfile, RunNumber, globalTagVar):
0137     globalTag_v0 = getGTfromDQMFile(DQMfile, RunNumber, globalTagVar)
0138     print("Global Tag: " + globalTag_v0)
0139     globalTag = globalTag_v0
0140 
0141     for z in range(len(globalTag_v0)-2):#clean up the garbage string in the GT
0142         if (globalTag_v0[z].isdigit()) and  (globalTag_v0[z+1].isdigit()) and (globalTag_v0[z+2].isdigit()) and(globalTag_v0[z+3].isupper()):
0143             globalTag = globalTag_v0[z:]
0144             break
0145     if globalTag == "":
0146         print(" No GlobalTag found: trying from DAS.... ")
0147         globalTag = str(os.popen('getGTscript.sh '+filepath+ File_Name+' ' +str(Run_Number[i])));
0148         if globalTag == "":
0149             print(" No GlobalTag found for run: "+str(Run_Number[i]))
0150 
0151     return globalTag
0152 
0153 
0154 Run_type = sys.argv[1]
0155 Run_Number = [int(x) for x in sys.argv[2:]]
0156 CMSSW_BASE = str(os.popen('echo ${CMSSW_BASE}').read().strip())
0157 rereco=False
0158 
0159 ###########Check if user enter the right run type######################
0160 if Run_type == 'Cosmics' or Run_type == 'StreamExpress' or Run_type == 'StreamExpressCosmics' or Run_type == 'ZeroBias' or Run_type == 'StreamHIExpress' or Run_type == 'HIMinimumBias1' or re.match('ZeroBias([0-9]+?)',Run_type) or re.match('HIMinimumBias([0-9]+?)',Run_type):
0161     print(Run_type)
0162 elif Run_type == 'ReReco':
0163     rereco=True    
0164     Run_type='ZeroBias'
0165 else: 
0166     print("please enter a valid run type: Cosmics | ZeroBias | StreamExpress | StreamExpressCosmics ")
0167     sys.exit(0)
0168 
0169 for i in range(len(Run_Number)):
0170 #################Downloading DQM file############################
0171     print("Downloading File!!")
0172     nnnOut = Run_Number[i]/1000
0173 
0174     filepath = '/tmp/'
0175     File_Name = downloadOfflineDQMhisto(Run_Number[i], Run_type, rereco)
0176     if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
0177         File_Name_PCL = downloadOfflinePCLhisto(Run_Number[i], Run_type)
0178     deadRocMap, File_Name_online = downloadnlineDQMhisto(Run_Number[i], Run_type)
0179 
0180     runDirval=setRunDirectory(Run_Number[i])
0181     DataLocalDir=runDirval[0]
0182     #DataOfflineDir=runDirval[1]
0183 
0184 ################Check if run is complete##############
0185 
0186     print("get the run status from DQMFile")
0187 
0188 
0189     check_command = 'check_runcomplete '+filepath+File_Name
0190     Check_output = subprocess.call(check_command, shell=True)
0191 
0192 
0193     if Check_output == 0:
0194         print('Using DQM file: '+File_Name)
0195     else:
0196         print('*****************Warning: DQM file is not ready************************')
0197         input_var = input("DQM file is incompleted, do you want to continue? (y/n): ")
0198         if (input_var == 'y') or (input_var == 'Y'):
0199             print('Using DQM file: '+File_Name)
0200         else:
0201             sys.exit(0)
0202     if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
0203         if File_Name_PCL=='Temp':
0204             print(' ')
0205             print(' ')
0206             print('*****************Warning: PCL file is not ready************************')
0207             input_var = input("PCL file is not ready, you will need to re-run the script later for PCL plots, do you want to continue? (y/n): ")
0208             if (input_var == 'y') or (input_var == 'Y'):
0209                 print('-------->   Remember to re-run the script later!!!!!')
0210             else:
0211                 sys.exit(0)
0212     
0213 ###################Start making TkMaps################
0214 
0215     checkfolder = os.path.exists(str(Run_Number[i]))
0216     if checkfolder == True:
0217         shutil.rmtree(str(Run_Number[i]))
0218         os.makedirs(str(Run_Number[i])+'/'+Run_type)
0219     else:
0220         os.makedirs(str(Run_Number[i])+'/'+Run_type)
0221         
0222 #######Getting GT##############
0223     ####After switch production to 10_X_X release, the clean up section need to be reviewed and modified  ##########
0224     globalTag = getGT(filepath+File_Name, str(Run_Number[i]), 'globalTag_Step1')
0225     ####################################################
0226 
0227 
0228     
0229     print(" Creating the TrackerMap.... ")
0230 
0231     detIdInfoFileName = 'TkDetIdInfo_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
0232     workPath = os.popen('pwd').readline().strip()
0233 
0234     os.chdir(str(Run_Number[i])+'/'+Run_type)
0235    
0236 
0237     os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/SiStripDQM_OfflineTkMap_Template_cfg_DB.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name+' detIdInfoFile='+detIdInfoFileName)
0238     os.system('rm -f *svg')
0239 ####################### rename bad module list file######################
0240     sefile = 'QualityTest_run'+str(Run_Number[i])+'.txt'
0241     shutil.move('QTBadModules.log',sefile)
0242 
0243 ################### put color legend in the TrackerMap###################
0244 
0245 # PLEASE UNCOMMENT THE LINES BELOW TO GET THE LEGEND ON THE QT TkMAP (it will work only on vocms061)    
0246 #    os.system('/usr/bin/python ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/LegendToQT.py QTestAlarm.png /data/users/cctrack/FinalLegendTrans.png')
0247 #    shutil.move('result.png', 'QTestAlarm.png')
0248 
0249 ####################Copying the template html file to index.html########################
0250 
0251     if Run_type == "Cosmics" or Run_type == "StreamExpressCosmics":
0252         os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap_cosmics.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
0253     elif Run_type == "StreamExpress":
0254          os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_Express_TKMap.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
0255     else:
0256         os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
0257 
0258     shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/fedmap.html','fedmap.html')
0259     shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/psumap.html','psumap.html')
0260 
0261     print(" Check TrackerMap on "+str(Run_Number[i])+'/'+Run_type+" folder")
0262     
0263     output =[]
0264     output.append(os.popen("/bin/ls ").readline().strip())
0265     print(output)
0266 
0267 ## Producing the list of bad modules
0268     print(" Creating the list of bad modules ")
0269     
0270     os.system('listbadmodule '+filepath+'/'+File_Name+' PCLBadComponents.log')
0271 
0272  ##   if Run_type != "StreamExpress":
0273  ##       shutil.copyfile(sefile, checkdir+'/'+sefile)
0274  ##       os.system('/usr/bin/python ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/findBadModT9.py -p '+sefile+' -s /'+checkdir+'/'+sefile);
0275       
0276       
0277 ## Producing the run certification by lumisection
0278     
0279 #    print(" Creating the lumisection certification:")
0280 
0281 #    if (Run_type.startswith("ZeroBias")) or (Run_type == "StreamExpress"):
0282 #        os.system('ls_cert 0.95 0.95 '+filepath+'/'+File_Name)
0283 
0284 ## Producing the PrimaryVertex/BeamSpot quality test by LS..
0285 #    if (Run_type != "Cosmics") and ( Run_type != "StreamExpress") and (Run_type != "StreamExpressCosmics"):
0286 #        print(" Creating the BeamSpot Calibration certification summary:")
0287 #        os.system('lsbs_cert '+filepath+'/'+File_Name)
0288 
0289 ## .. and harvest the bad beamspot LS with automatic emailing (if in period and if bad LS found)
0290     os.system('bs_bad_ls_harvester . '+str(Run_Number[i]))
0291     
0292 
0293 ## Producing the Module difference for ExpressStream
0294 
0295     dest='Beam'
0296 
0297     if (Run_type == "Cosmics") or (Run_type == "StreamExpressCosmics"):
0298         dest="Cosmics"
0299 
0300 
0301 ## create merged list of BadComponent from (PCL, RunInfo and FED Errors) ignore for now
0302     os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/mergeBadChannel_Template_cfg.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name)
0303     shutil.move('MergedBadComponents.log','MergedBadComponents_run'+str(Run_Number[i])+'.txt')
0304 
0305     os.system("mkdir -p /data/users/event_display/TkCommissioner_runs/"+DataLocalDir+"/"+dest+" 2> /dev/null")
0306 
0307 
0308     shutil.copyfile(detIdInfoFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+detIdInfoFileName)
0309 
0310     os.remove(detIdInfoFileName)
0311     os.remove('MergedBadComponentsTkMap_Canvas.root')
0312     os.remove('MergedBadComponentsTkMap.root')
0313 ##############counting dead pixel#######################
0314     print("countig dead pixel ROCs" )
0315     if (Run_Number[i] < 290124) :
0316 
0317         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter.py '+filepath+'/'+File_Name)
0318     else: 
0319         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter_Phase1.py '+filepath+'/'+File_Name)
0320 
0321     if rereco:
0322         os.system('mkdir -p /data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco 2> /dev/null')
0323     else:
0324         os.system('mkdir -p /data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type+' 2> /dev/null')
0325     
0326     shutil.move('PixZeroOccROCs_run'+str(Run_Number[i])+'.txt',workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
0327 
0328 
0329 
0330 
0331 ######Counting Dead ROCs and Inefficient DC during the run#########################
0332     
0333     if deadRocMap == True:
0334 
0335         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROC_duringRun.py '+filepath+File_Name_online+' '+filepath+File_Name)
0336         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/change_name.py')
0337         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PixelMapPlotter.py MaskedROC_sum.txt -c')
0338         os.remove('MaskedROC_sum.txt')
0339         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/InefficientDoubleROC.py '+filepath+File_Name_online)
0340     else:
0341         print('No Online DQM file available, Dead ROC maps will not be produced')
0342         print('No Online DQM file available, inefficient DC list  will also not be produced')
0343 
0344 #######Merge Dead ROCs and Occupoancy Plot ###################
0345 
0346     os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergeOccDeadROC.py '+filepath+File_Name)
0347 
0348 #######Merge PCL and DQM Plot (only StreamExpress) ###################
0349     if Run_type=="StreamExpress" or Run_type=="StreamHIExpress":
0350         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergePCLDeadROC.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
0351         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergePCLFedErr.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
0352         os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PCLOthers.py '+filepath+File_Name+' '+filepath+File_Name_PCL)
0353 
0354 ###################copy ouput files###################
0355     strip_files = os.listdir('.')
0356     for file_name in strip_files:
0357         full_stripfile_name = os.path.join('.', file_name)
0358         if (os.path.isfile(full_stripfile_name)):
0359             if rereco:
0360                 shutil.copy(full_stripfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco')
0361             else:
0362                 shutil.copy(full_stripfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
0363 
0364 
0365 
0366 
0367 #########################Start making pixel maps#####################
0368     os.chdir(workPath)
0369     shutil.rmtree(str(Run_Number[i]))
0370     os.remove('index.html')
0371 
0372     # produce pixel phase1 TH2Poly maps
0373 #    os.chdir(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/scripts/PhaseIMaps/')
0374     os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/TH2PolyOfflineMaps.py ' + filepath+'/'+File_Name+' 3000 2000')
0375     shutil.move(workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt', 'OUT/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
0376 
0377 
0378 ###################copy ouput files##########
0379     pixel_files = os.listdir('./OUT')
0380     for file_name in pixel_files:
0381         full_pixelfile_name = os.path.join('./OUT/', file_name)
0382         if (os.path.isfile(full_pixelfile_name)):
0383             if rereco:
0384                 shutil.copy(full_pixelfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/ReReco')
0385             else:
0386                 shutil.copy(full_pixelfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
0387 
0388 
0389     shutil.rmtree('OUT')
0390 
0391 
0392     # produce pixel phase1 tree for Offline TkCommissioner
0393     pixelTreeFileName = 'PixelPhase1Tree_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
0394     os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PhaseITreeProducer.py ' + filepath+'/'+File_Name + ' ' + pixelTreeFileName)
0395 
0396     shutil.copyfile(pixelTreeFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+pixelTreeFileName)
0397     os.remove(pixelTreeFileName)
0398     if File_Name:
0399         os.remove(filepath+File_Name)
0400     if File_Name_online:
0401         os.remove(filepath+File_Name_online)
0402     os.chdir(workPath)