SiStripDCS/test/MakeTkMaps.py

0001 #! /usr/bin/env python
0002 #G.Benelli Aug 26 2010
0003 #Process the CheckAllIOVs.py DEBUG output log files to prepare them as inputs to tkmapcreator...
0004 #Incorporating also the next step: cloning the cfg.py files (1 per log file, i.e. 1 per IOV) and execute cmsRun
0005 #producing the wanted tkMaps.
0006
0007 #TODO:
0008 #Could use this same script to do other processing as input to the ValidateO2O.py for other type of plots etc...
0009
0010 #Script assumes that it is run in the same dir as the output of CheckAllIOVs.py
0011 #TODO:
0012 #In the future can put it in scripts/ and take dir to run from and write to as options.
0013
0014 from __future__ import print_function
0015 import os, subprocess
0016
0017 def ProduceTkMapVoltageInputFiles(workdir=os.getcwd()): #Setting the dir by default to the current working directory...
0018     """
0019     Function that processes the indicated workdir directory (defaults to current working directory) looking for CheckAllIOVs.py
0020     DetVOffReaderDebug*.log files.
0021     For each log file (that correspond to 1 IOV) 2 input files (one for LV one for HV status) are created for the TkVoltageMapCreator plugin.
0022     The function returns the 2 lists of HV and LV files.
0023     """
0024     #Get all the files in the directory workdir (1 file per IOV):
0025     print("Analysing %s directory"%workdir)
0026     logfilenames=[x for x in os.listdir(workdir) if x.startswith("DetVOffReaderDebug")]
0027     if logfilenames:
0028         print("Processing %s logfiles..."%len(logfilenames))
0029     else:
0030         print("No DetVIfReaderDebug files found!\nPlease run this script in the same dir you have run CheckAllIOVS.py, or in the dir where you store the output logfiles.")
0031
0032     #Let's dump for each IOV a TkVoltageMapCreator ready input file, i.e. with 1 entry per detID...
0033     #Need to read in our usual pkl to get all the strip detids
0034     #Get it from the release
0035     import pickle
0036     #FIX:
0037     #Until the tag is in the release, CMSSW_RELEASE_BASE should be replaced by CMSSW_BASE...
0038     StripDetIDAliasPickle=os.path.join(os.getenv("CMSSW_RELEASE_BASE"),"src/CalibTracker/SiStripDCS/data","StripDetIDAlias.pkl")
0039     #TODO:
0040     #Could use try/except to make this robust... but from next CVS commit should be fine...
0041     #Otherwise can use the file in ~gbenelli/O2O/StripDetIDAlias.pkl, use full AFS path!
0042     DetIDdict=open(StripDetIDAliasPickle,"r")
0043     StripDetIDAlias=pickle.load(DetIDdict)
0044
0045     #Process the logfiles!
0046     #Lists to keep the LV/HV input files, that will be returned by the function:
0047     LVFilenames=[]
0048     HVFilenames=[]
0049     for logfilename in logfilenames:
0050         print(logfilename)
0051         #Create LV/HV filenames for the input files we will write
0052         #TODO:
0053         #Could add here the possibility of writing in a different dir, by adding an extra outdir argument to the function
0054         #and use that instead of workdir...
0055         LVfilename=os.path.join(workdir,logfilename.replace("DetVOffReaderDebug_","LV"))
0056         HVfilename=os.path.join(workdir,logfilename.replace("DetVOffReaderDebug_","HV"))
0057
0058         #Adding the filenames to the respective lists that will be returned by the function:
0059         LVFilenames.append(LVfilename)
0060         HVFilenames.append(HVfilename)
0061
0062         #Open input/output files:
0063         logfile=open(logfilename,"r")
0064         LVfile=open(LVfilename,"w")
0065         HVfile=open(HVfilename,"w")
0066
0067         #File parsing and creation of 2 separate LV and HV status files:
0068         #Need dicts to handle the filling of files with all detIDs:
0069         LVDict={}
0070         HVDict={}
0071         for line in logfile:
0072             if "OFF" in line: #All the interesting lines contain "OFF" by definition
0073                 (detid,HV,LV)=line.split()
0074             LVDict.update({detid:LV})
0075             HVDict.update({detid:HV})
0076
0077         #Handle the LV/HV files:
0078         for detid in StripDetIDAlias.keys():
0079             detid=str(detid)
0080             if detid in LVDict.keys(): #Set the detids to whatever status they were reported (can be ON or OFF, since the detid would be reported in the case of HV OFF and LV ON of course...)
0081                 LVfile.write(detid+"\t"+LVDict[detid]+"\n")
0082             else: #Set the remaining detids as ON (they would be reported otherwise in the LVDict)
0083                 LVfile.write(detid+"\tON\n")
0084             if detid in HVDict.keys(): #Set the detids to whatever status they were reported (should only reported when OFF..., HV ON while LV OFF should be impossible)
0085                 HVfile.write(detid+"\t"+HVDict[detid]+"\n")
0086             else: #Set the remaining detids as ON (they would be reported otherwise in the HVDict)
0087                 HVfile.write(detid+"\tON\n")
0088
0089
0090         #Close files:
0091         logfile.close()
0092         LVfile.close()
0093         HVfile.close()
0094
0095     #Now finally return the 2 lists of LV and HV TkVoltageMapCreator input files:
0096     return LVFilenames,HVFilenames
0097
0098 #Function to run a (for us cmsRun) command:
0099 def runcmd(command):
0100     """
0101     Function that uses subprocess.Popen to run commands, it returns the exit code of the command.
0102     """
0103     try:
0104         process  = subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
0105         pid=process.pid
0106         exitstat= process.wait()
0107         cmdout   = process.stdout.read()
0108         exitstat = process.returncode
0109     except OSError as detail:
0110         print("Race condition in subprocess.Popen has robbed us of the exit code of the %s process (PID %s).Assume it failed!\n %s\n"%(command,pid,detail))
0111         exitstat=999
0112     if exitstat == None:
0113         print("Something strange is going on! Exit code was None for command %s: check if it really ran!"%command)
0114         exitstat=0
0115     return exitstat
0116
0117 def CreateTkVoltageMapsCfgs(workdir=os.getcwd()): #Default to current working directory (could pass HV/LV list)
0118     """
0119     Function that looks for TkVoltageMapCreator input files, creates 1 cfg.py for each IOV.
0120     It returns the list of cfgs ready to be cmsRun to produce the maps
0121     """
0122     #Use HV log files to loop... could use also LV logs...
0123     HVLogs=[x for x in os.listdir(workdir) if x.startswith("HV") and "FROM" in x and x.endswith(".log")]
0124
0125     #Open the file to use as template
0126     TkMapCreatorTemplateFile=open(os.path.join(os.getenv("CMSSW_BASE"),"src/CalibTracker/SiStripDCS/test","TkVoltageMapCreator_cfg.py"),"r")
0127     TkMapCreatorTemplateContent=TkMapCreatorTemplateFile.readlines()
0128     #Let's do it!
0129     TkMapCfgFilenames=[]
0130     for HVlog in HVLogs:
0131         #Use full path since workdir could be different than current working dir:
0132         HVlog=os.path.join(workdir,HVlog)
0133             #Check if the corresponding LV log is there!
0134         LVlog=os.path.join(workdir,HVlog.replace("HV","LV"))
0135         if not os.path.exists(LVlog):
0136             print("ARGH! Missing LV file for file %s"%HVlog)
0137             print("Will not process the HV file either!")
0138         TkMapCfgFilename=os.path.join(workdir,HVlog.replace("HV","TkVoltageMap").replace(".log","_cfg.py"))
0139         TkMapCfgFilenames.append(TkMapCfgFilename)
0140         TkMapCfg=open(TkMapCfgFilename,"w")
0141         for line in TkMapCreatorTemplateContent:
0142             if "LVStatusFile" in line and "#" not in line:
0143                 line='\tLVStatusFile = cms.string("%s"),\n'%LVlog
0144             if "LVTkMapName" in line and "#" not in line:
0145                 line='\tLVTkMapName = cms.string("%s"),\n'%LVlog.replace(".log",".png")
0146             if "HVStatusFile" in line and "#" not in line:
0147                 line='\tHVStatusFile = cms.string("%s"),\n'%HVlog
0148             if "HVTkMapName" in line and "#" not in line:
0149                 line='\tHVTkMapName = cms.string("%s")\n'%HVlog.replace(".log",".png")
0150             TkMapCfg.write(line)
0151         TkMapCfg.close()
0152
0153     TkMapCreatorTemplateFile.close()
0154     return TkMapCfgFilenames
0155
0156 def CreateTkVoltageMaps(workdir=os.getcwd()): #Default to current working directory for now...
0157     """
0158     Function that looks for TkVoltageMap*cfg.py in the workdir directory and launches each of them
0159     creating 2 TkVoltageMaps per IOV, one for LV and one of HV status (each as a png file).
0160     """
0161     TkMapCfgs=[x for x in os.listdir(workdir) if x.startswith("TkVoltageMap") and "FROM" in x and x.endswith("cfg.py")]
0162     for TkMapCfg in TkMapCfgs:
0163         #Make sure we run the cfg in the workdir and also the logfile is saved there...
0164         TkMapCfg=os.path.join(workdir,TkMapCfg)
0165         cmsRunCmd="cmsRun %s >& %s"%(TkMapCfg,TkMapCfg.replace(".py",".log"))
0166         print(cmsRunCmd)
0167         exitstat=runcmd(cmsRunCmd)
0168         if exitstat != 0:
0169             print("Uh-Oh!")
0170             print("Command %s FAILED!"%cmsRunCmd)
0171
0172 #Could put in a def main...
0173
0174 #Create the TkVoltageMapCreator input files in the test dir below:
0175 (LVfiles,HVfiles)=ProduceTkMapVoltageInputFiles()
0176 #print LVfiles
0177 #print HVfiles
0178
0179 #Create the actual TkVoltageMaps!
0180 TkMapCfgFilenames=CreateTkVoltageMapsCfgs()
0181 print(TkMapCfgFilenames)
0182 CreateTkVoltageMaps()
0183
0184 #Finish this up, so that it can be committed, but above all use it!
0185 #Check how to use CheckAllIOVs.py to access the Offline DB directly! Maybe can fold this in?
0186
0187
0188