SiStripDCS/test/MakeTkMaps.py

0001 #! /usr/bin/env python
0002 #G.Benelli Aug 26 2010
0003 #Process the CheckAllIOVs.py DEBUG output log files to prepare them as inputs to tkmapcreator...
0004 #Incorporating also the next step: cloning the cfg.py files (1 per log file, i.e. 1 per IOV) and execute cmsRun
0005 #producing the wanted tkMaps.
0006
0007 #TODO:
0008 #Could use this same script to do other processing as input to the ValidateO2O.py for other type of plots etc...
0009
0010 #Script assumes that it is run in the same dir as the output of CheckAllIOVs.py
0011 #TODO:
0012 #In the future can put it in scripts/ and take dir to run from and write to as options.
0013
0014 import os, subprocess
0015
0016 def ProduceTkMapVoltageInputFiles(workdir=os.getcwd()): #Setting the dir by default to the current working directory...
0017     """
0018     Function that processes the indicated workdir directory (defaults to current working directory) looking for CheckAllIOVs.py
0019     DetVOffReaderDebug*.log files.
0020     For each log file (that correspond to 1 IOV) 2 input files (one for LV one for HV status) are created for the TkVoltageMapCreator plugin.
0021     The function returns the 2 lists of HV and LV files.
0022     """
0023     #Get all the files in the directory workdir (1 file per IOV):
0024     print("Analysing %s directory"%workdir)
0025     logfilenames=[x for x in os.listdir(workdir) if x.startswith("DetVOffReaderDebug")]
0026     if logfilenames:
0027         print("Processing %s logfiles..."%len(logfilenames))
0028     else:
0029         print("No DetVIfReaderDebug files found!\nPlease run this script in the same dir you have run CheckAllIOVS.py, or in the dir where you store the output logfiles.")
0030
0031     #Let's dump for each IOV a TkVoltageMapCreator ready input file, i.e. with 1 entry per detID...
0032     #Need to read in our usual pkl to get all the strip detids
0033     #Get it from the release
0034     import pickle
0035     #FIX:
0036     #Until the tag is in the release, CMSSW_RELEASE_BASE should be replaced by CMSSW_BASE...
0037     StripDetIDAliasPickle=os.path.join(os.getenv("CMSSW_RELEASE_BASE"),"src/CalibTracker/SiStripDCS/data","StripDetIDAlias.pkl")
0038     #TODO:
0039     #Could use try/except to make this robust... but from next CVS commit should be fine...
0040     #Otherwise can use the file in ~gbenelli/O2O/StripDetIDAlias.pkl, use full AFS path!
0041     DetIDdict=open(StripDetIDAliasPickle,"r")
0042     StripDetIDAlias=pickle.load(DetIDdict)
0043
0044     #Process the logfiles!
0045     #Lists to keep the LV/HV input files, that will be returned by the function:
0046     LVFilenames=[]
0047     HVFilenames=[]
0048     for logfilename in logfilenames:
0049         print(logfilename)
0050         #Create LV/HV filenames for the input files we will write
0051         #TODO:
0052         #Could add here the possibility of writing in a different dir, by adding an extra outdir argument to the function
0053         #and use that instead of workdir...
0054         LVfilename=os.path.join(workdir,logfilename.replace("DetVOffReaderDebug_","LV"))
0055         HVfilename=os.path.join(workdir,logfilename.replace("DetVOffReaderDebug_","HV"))
0056
0057         #Adding the filenames to the respective lists that will be returned by the function:
0058         LVFilenames.append(LVfilename)
0059         HVFilenames.append(HVfilename)
0060
0061         #Open input/output files:
0062         logfile=open(logfilename,"r")
0063         LVfile=open(LVfilename,"w")
0064         HVfile=open(HVfilename,"w")
0065
0066         #File parsing and creation of 2 separate LV and HV status files:
0067         #Need dicts to handle the filling of files with all detIDs:
0068         LVDict={}
0069         HVDict={}
0070         for line in logfile:
0071             if "OFF" in line: #All the interesting lines contain "OFF" by definition
0072                 (detid,HV,LV)=line.split()
0073                 LVDict.update({detid:LV})
0074                 HVDict.update({detid:HV})
0075
0076         #Handle the LV/HV files:
0077         for detid in StripDetIDAlias.keys():
0078             detid=str(detid)
0079             if detid in LVDict.keys(): #Set the detids to whatever status they were reported (can be ON or OFF, since the detid would be reported in the case of HV OFF and LV ON of course...)
0080                 LVfile.write(detid+"\t"+LVDict[detid]+"\n")
0081             else: #Set the remaining detids as ON (they would be reported otherwise in the LVDict)
0082                 LVfile.write(detid+"\tON\n")
0083             if detid in HVDict.keys(): #Set the detids to whatever status they were reported (should only reported when OFF..., HV ON while LV OFF should be impossible)
0084                 HVfile.write(detid+"\t"+HVDict[detid]+"\n")
0085             else: #Set the remaining detids as ON (they would be reported otherwise in the HVDict)
0086                 HVfile.write(detid+"\tON\n")
0087
0088
0089         #Close files:
0090         logfile.close()
0091         LVfile.close()
0092         HVfile.close()
0093
0094     #Now finally return the 2 lists of LV and HV TkVoltageMapCreator input files:
0095     return LVFilenames,HVFilenames
0096
0097 #Function to run a (for us cmsRun) command:
0098 def runcmd(command):
0099     """
0100     Function that uses subprocess.Popen to run commands, it returns the exit code of the command.
0101     """
0102     try:
0103         process  = subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
0104         pid=process.pid
0105         exitstat= process.wait()
0106         cmdout   = process.stdout.read()
0107         exitstat = process.returncode
0108     except OSError as detail:
0109         print("Race condition in subprocess.Popen has robbed us of the exit code of the %s process (PID %s).Assume it failed!\n %s\n"%(command,pid,detail))
0110         exitstat=999
0111     if exitstat == None:
0112         print("Something strange is going on! Exit code was None for command %s: check if it really ran!"%command)
0113         exitstat=0
0114     return exitstat
0115
0116 def CreateTkVoltageMapsCfgs(workdir=os.getcwd()): #Default to current working directory (could pass HV/LV list)
0117     """
0118     Function that looks for TkVoltageMapCreator input files, creates 1 cfg.py for each IOV.
0119     It returns the list of cfgs ready to be cmsRun to produce the maps
0120     """
0121     #Use HV log files to loop... could use also LV logs...
0122     HVLogs=[x for x in os.listdir(workdir) if x.startswith("HV") and "FROM" in x and x.endswith(".log")]
0123
0124     #Open the file to use as template
0125     TkMapCreatorTemplateFile=open(os.path.join(os.getenv("CMSSW_BASE"),"src/CalibTracker/SiStripDCS/test","TkVoltageMapCreator_cfg.py"),"r")
0126     TkMapCreatorTemplateContent=TkMapCreatorTemplateFile.readlines()
0127     #Let's do it!
0128     TkMapCfgFilenames=[]
0129     for HVlog in HVLogs:
0130         #Use full path since workdir could be different than current working dir:
0131         HVlog=os.path.join(workdir,HVlog)
0132         #Check if the corresponding LV log is there!
0133         LVlog=os.path.join(workdir,HVlog.replace("HV","LV"))
0134         if not os.path.exists(LVlog):
0135             print("ARGH! Missing LV file for file %s"%HVlog)
0136             print("Will not process the HV file either!")
0137         TkMapCfgFilename=os.path.join(workdir,HVlog.replace("HV","TkVoltageMap").replace(".log","_cfg.py"))
0138         TkMapCfgFilenames.append(TkMapCfgFilename)
0139         TkMapCfg=open(TkMapCfgFilename,"w")
0140         for line in TkMapCreatorTemplateContent:
0141             if "LVStatusFile" in line and "#" not in line:
0142                 line='\tLVStatusFile = cms.string("%s"),\n'%LVlog
0143             if "LVTkMapName" in line and "#" not in line:
0144                 line='\tLVTkMapName = cms.string("%s"),\n'%LVlog.replace(".log",".png")
0145             if "HVStatusFile" in line and "#" not in line:
0146                 line='\tHVStatusFile = cms.string("%s"),\n'%HVlog
0147             if "HVTkMapName" in line and "#" not in line:
0148                 line='\tHVTkMapName = cms.string("%s")\n'%HVlog.replace(".log",".png")
0149             TkMapCfg.write(line)
0150         TkMapCfg.close()
0151
0152     TkMapCreatorTemplateFile.close()
0153     return TkMapCfgFilenames
0154
0155 def CreateTkVoltageMaps(workdir=os.getcwd()): #Default to current working directory for now...
0156     """
0157     Function that looks for TkVoltageMap*cfg.py in the workdir directory and launches each of them
0158     creating 2 TkVoltageMaps per IOV, one for LV and one of HV status (each as a png file).
0159     """
0160     TkMapCfgs=[x for x in os.listdir(workdir) if x.startswith("TkVoltageMap") and "FROM" in x and x.endswith("cfg.py")]
0161     for TkMapCfg in TkMapCfgs:
0162         #Make sure we run the cfg in the workdir and also the logfile is saved there...
0163         TkMapCfg=os.path.join(workdir,TkMapCfg)
0164         cmsRunCmd="cmsRun %s >& %s"%(TkMapCfg,TkMapCfg.replace(".py",".log"))
0165         print(cmsRunCmd)
0166         exitstat=runcmd(cmsRunCmd)
0167         if exitstat != 0:
0168             print("Uh-Oh!")
0169             print("Command %s FAILED!"%cmsRunCmd)
0170
0171 #Could put in a def main...
0172
0173 #Create the TkVoltageMapCreator input files in the test dir below:
0174 (LVfiles,HVfiles)=ProduceTkMapVoltageInputFiles()
0175 #print LVfiles
0176 #print HVfiles
0177
0178 #Create the actual TkVoltageMaps!
0179 TkMapCfgFilenames=CreateTkVoltageMapsCfgs()
0180 print(TkMapCfgFilenames)
0181 CreateTkVoltageMaps()
0182
0183 #Finish this up, so that it can be committed, but above all use it!
0184 #Check how to use CheckAllIOVs.py to access the Offline DB directly! Maybe can fold this in?
0185
0186
0187