Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2021-02-14 14:14:50

0001 #! /usr/bin/env python
0002 #Script to parse the output of ManualO2O.py for various debugging tasks
0003 #First use case is to debug issue with HV1/HV2 channels handling
0004 
0005 from __future__ import print_function
0006 import os,datetime, pickle
0007 def GetLogTimestamps(ManualO2Ologfilename):
0008     """
0009     This function opens the ManualO2O log chosen, and it prints out the Tmin/Tmax intervals
0010     for the individual O2O jobs that were run.
0011     This information can in turn be used with the function GetLogInfoByTimeStamp,
0012     to extract from the log the (DEBUG) information only from the wanted time interval.
0013     """
0014     ManualO2Olog=open(ManualO2Ologfilename,'r')
0015     LinesCounter=0
0016     IOVCounter=1
0017     TmaxDict={}#This dictionary will have Tmax as keys and the lines corresponding to that Tmax as value
0018     for line in ManualO2Olog:
0019         LinesCounter+=1
0020         if 'Tmin:' in line:
0021             Tmin=line.split(':')[1].strip()
0022         if 'Tmax:' in line:
0023             Tmax=line.split(':')[1].strip()
0024             #Assume that when Tmax is read Tmin was read too since we are parsing lines like this:
0025             #     Tmin: 2010 8 27 10 0 0 0
0026             #     Tmax: 2010 8 29 9 0 0 0
0027             print("%s- Tmin: %s and Tmax: %s (number log lines %s)"%(IOVCounter,Tmin,Tmax,LinesCounter))
0028             #Actually Tmin is not necessary... since they all will be set to the first one in the dcs_o2O_template.py used by the ManualO2O.py that created the log we're parsing.
0029             if Tmax not in ListOfTmax.keys():
0030                 TmaxDict.update({Tmax:LinesCounter})
0031             else:
0032                 print("This Tmax (%s) seems to be duplicated!!!"%Tmax)
0033             LinesCounter=0
0034             IOVCounter+=1
0035     ManualO2Olog.close()
0036     return TmaxDict
0037             
0038             
0039 def GetLogInfoByTimeStamp(ManualO2Ologfilename,StartTime,StopTime):
0040     """
0041     This function takes a ManualO2Ologfilename, a start and a stop timestamps (that should be
0042     in the "YYYY M D H M S"  format, e.g. Tmin: "2010 8 27 10 0 0 0",Tmax: "2010 8 27 11 0 0 0",
0043     and it parses the given logfile to just print out the lines relevant to the given timestamp. 
0044     """
0045     #Open the log:
0046     ManualO2Olog=open(ManualO2Ologfilename,'r')
0047     #Loop to extract only the wanted lines
0048     PrintLines=False
0049     WantedLines=[]
0050     for line in ManualO2Olog:
0051         if 'Tmax:' in line and StopTime in line:
0052             PrintLines=True
0053         if PrintLines:
0054             if 'Tmin:' in line:
0055                 break
0056             else:
0057                 print(line.strip())
0058                 WantedLines.append(line)
0059     ManualO2Olog.close()
0060     return WantedLines
0061 #TO BE DONE!
0062 #implement tzinfo subclasses to get GMT1 behaviour to translate always the timestamp right...
0063 #from datetime import timedelta, datetime, tzinfo
0064 #   import datetime
0065 #   class GMT1(datetime.tzinfo):
0066 #       def __init__(self,dt):         # DST starts last Sunday in March
0067 #           d = datetime.datetime(dt.year, 4, 1)   # ends last Sunday in October
0068 #           self.dston = d - datetime.timedelta(days=d.weekday() + 1)
0069 #           d = datetime.datetime(dt.year, 11, 1)
0070 #           self.dstoff = d - datetime.timedelta(days=d.weekday() + 1)
0071 #       def utcoffset(self, dt):
0072 #           return datetime.timedelta(hours=1) + self.dst(dt)
0073 #       def dst(self, dt):
0074 #           if self.dston <=  dt.replace(tzinfo=None) < self.dstoff:
0075 #               return datetime.timedelta(hours=1)
0076 #           else:
0077 #               return datetime.timedelta(0)
0078 #       def tzname(self,dt):
0079 #           return "GMT +1"
0080 #   
0081 #   import time as _time
0082 #   
0083 #   STDOFFSET = datetime.timedelta(seconds = -_time.timezone)
0084 #   if _time.daylight:
0085 #       DSTOFFSET = datetime.timedelta(seconds = -_time.altzone)
0086 #   else:
0087 #       DSTOFFSET = STDOFFSET
0088 #   
0089 #   DSTDIFF = DSTOFFSET - STDOFFSET
0090 #   
0091 #   class LocalTimezone(datetime.tzinfo):
0092 #   
0093 #       def utcoffset(self, dt):
0094 #           if self._isdst(dt):
0095 #               return DSTOFFSET
0096 #           else:
0097 #               return STDOFFSET
0098 #   
0099 #       def dst(self, dt):
0100 #           if self._isdst(dt):
0101 #               return DSTDIFF
0102 #           else:
0103 #               return ZERO
0104 #   
0105 #       def tzname(self, dt):
0106 #           return _time.tzname[self._isdst(dt)]
0107 #   
0108 #       def _isdst(self, dt):
0109 #           tt = (dt.year, dt.month, dt.day,
0110 #                 dt.hour, dt.minute, dt.second,
0111 #                 dt.weekday(), 0, -1)
0112 #           stamp = _time.mktime(tt)
0113 #           tt = _time.localtime(stamp)
0114 #           return tt.tm_isdst > 0
0115 #   
0116 #   Local = LocalTimezone()
0117 #   
0118 def GetQueryResults(ManualO2Ologfilename):
0119     """
0120     This function takes a ManualO2Ologfilename,
0121     and if it was run with the debug option True,
0122     it extracts all the query results.
0123     It uses StartTime/StopTime to identify the interval of the query (i.e. the ManualO2O.py running)
0124     and uses the tuple (StartTime,StopTime) as a key, as stores the results rows as a list.
0125     It returns the QueryResults dictionary that has for keys the (Tmin,Tmax) tuple, and as values lists of rows,
0126     where the rows are dictionaries with 3 keys:
0127     {
0128     'change_date':datetime.datetime,
0129     'actual_status':int,
0130     'dpname':str
0131     }
0132     The idea is that any massaging of this information is done later, while this function returns the query results in bulk.
0133     """
0134     print("Parsing file %s to extract query results"%ManualO2Ologfilename)
0135     #Open the log:
0136     ManualO2Olog=open(ManualO2Ologfilename,'r')
0137     #Loop to extract the start and stop times
0138     Tmin=""
0139     Tmax=""
0140     ReadRows=False
0141     QueryResultsDict={}
0142     for line in ManualO2Olog:
0143         if 'Tmin:' in line:
0144             if Tmin=="":#This is valid only for the first time... after the  first query use the previous Tmax as Tmin.
0145                 Tmin=datetime.datetime(*[int(x) for x in (line.split(':')[1].strip()).split()])
0146             else:
0147                 Tmin=Tmax
0148         if 'Tmax:' in line:
0149             Tmax=datetime.datetime(*[int(x) for x in (line.split(':')[1].strip()).split()])
0150             #Assume that when Tmax is read Tmin was read too since we are parsing lines like this:
0151             #     Tmin: 2010 8 27 10 0 0 0
0152             #     Tmax: 2010 8 29 9 0 0 0
0153             #print "%s- Tmin: %s and Tmax: %s (number log lines %s)"%(IOVCounter,Tmin,Tmax,LinesCounter)
0154             if (Tmin,Tmax) not in QueryResultsDict.keys():
0155                 QueryResultsDict.update({(Tmin,Tmax):[]})
0156             else:
0157                 print("This Tmax (%s) seems to be duplicated!!!"%Tmax)
0158         if "Dumping all query results" in line:
0159             ReadRows=True
0160         if "Finished dumping query results" in line:
0161             NumOfRows=int(line.split(",")[1].split()[0])
0162             ReadRows=False
0163             #Debug output:
0164             #print "Log reports %s rows, we counted %s rows"%(NumOfRows,len(QueryResultsDict[(Tmin,Tmax)]))
0165         if ReadRows:
0166             Row={}
0167             RowTokens=line.split(",")
0168             for token in RowTokens:
0169                 if 'CHANGE_DATE' in token:
0170                     try:
0171                         #Approximate time to the second, millisecond seems just to be too much information anyway
0172                         #(one can uncomment following line to do detailed tests if needed for a short time interval)
0173                         #Row.update({'change_date':datetime.datetime.strptime(token[token.index(":")+2:-5],"%Y/%m/%d %H:%M:%S")})
0174                         #Complete timestamp including millisecond is parsed with the following commented for now line:
0175                         changeDate=datetime.datetime.strptime(token[token.index(":")+2:-1],"%Y/%m/%d %H:%M:%S.%f")
0176                         #Now introduce the check on the timezone:
0177                         #Based on the fact that we have (in Tmin/Tmax) the 1 hr interval of the query time, we can univocally determine the timezone!
0178                         #FIXME:
0179                         #Should not recalculate this at every query result... just store it!
0180                         #deltaTimezone=datetime.timedelta(seconds=((changeDate-Tmin).seconds/3600)*3600) #Using int / as modulo...
0181                         #The above is wrong! I did not realize that Tmin is also in UTC!
0182                         #Need to implement a subclass of tzinfo to handle it properly all the time... to be done later!
0183                         #For now:
0184                         deltaTimezone=datetime.timedelta(seconds=7200) #DST+GMT+1 (not valid in non DST times but ok for Run144086...
0185                         Row.update({'change_date':changeDate+deltaTimezone})
0186                     except:
0187                         print("WEIRD!\n Timestamp had a format YYYY/MM/DD HH:MM:0SS.millisecond (extra 0 tabbing for seconds!)")
0188                         print(line)
0189                         #Approximate time to the second, millisecond seems just to be too much information anyway
0190                         #(one can uncomment following line to do detailed tests if needed for a short time interval)
0191                         #Row.update({'change_date':datetime.datetime.strptime(token[token.index(":")+2:-5],"%Y/%m/%d %H:%M:0%S")})
0192                         #Complete timestamp including millisecond is parsed with the following commented for now line:
0193                         changeDate=datetime.datetime.strptime(token[token.index(":")+2:-1],"%Y/%m/%d %H:%M:0%S.%f")
0194                         #Now introduce the check on the timezone:
0195                         #Based on the fact that we have (in Tmin/Tmax) the 1 hr interval of the query time, we can univocally determine the timezone!
0196                         #FIXME:
0197                         #Should not recalculate this at every query result... just store it!
0198                         #deltaTimezone=datetime.timedelta(seconds=((changeDate-Tmin).seconds/3600)*3600) #Using int / as modulo...
0199                         #The above is wrong! I did not realize that Tmin is also in UTC!
0200                         #Need to implement a subclass of tzinfo to handle it properly all the time... to be done later!
0201                         #For now:
0202                         deltaTimezone=datetime.timedelta(seconds=7200) #DST+GMT+1 (not valid in non DST times but ok for Run144086...
0203                         Row.update({'change_date':changeDate+deltaTimezone})
0204                 if 'ACTUAL_STATUS' in token:
0205                     Row.update({'actual_status':int(token[token.index(":")+2:-1])})
0206                 if 'DPNAME' in token:
0207                     Row.update({'dpname':token[token.index(":")+2:-2]})
0208             if len(Row)>0: #To avoid the case of an empty Row
0209                 QueryResultsDict[(Tmin,Tmax)].append(Row)
0210     ManualO2Olog.close()
0211     return QueryResultsDict 
0212 
0213 class TrkVoltageStatus:    
0214     """
0215     #How about we make a class and instantiate an object TrackerVoltageStatus that we update at each row:
0216     #1-initialize it with a list of detID (from the most complete maps) OFF and ON (the masked ones):
0217     #2-Has a member history, a dictionary, with keeps, by timestamp (key) a list of detIDs with their channel 0,1,2,[3] status (to decide later if we need to already handle here the checks that
0218     #   a-channel 0==1, for example by having a list of detIDs that do not have matching LV channels statuses (easy to check later, for sure it will be not empty during little transition IOVs due to different sub-second timing... we could introduce deadbands later)
0219     #   b-a LV ON/OFF bit
0220     #   c-a HV ON/OFF bit
0221     #   d-add extra (VMON, V0_SETTINGS) info in the query?
0222     #   e-add the power group turning off info in the query to check the effect
0223     #   f-add a RAMPING UP/DOWN flag on top of the conventional O2O ON/OFF?
0224     #3-define an update(row) method that for each row takes the timestamp:
0225     #   a-if timestamp already exist, updates the channel status of the relevant detIDs contained
0226     #   b-if timestamp does not exist, creates an entry for it carrying over the information for the relevant detIDs from previous timestamp and update it with the information in the row
0227     #4-define a getVoltageHistory(start,end) method to dump the list of IOVs, and for each IOV dump a list with the detIDs and their LV/HV status, the number of LV OFF modules and HV OFF modules, (check that there is no module with LV OFF and HV ON... basic check can be done at  the update level)
0228     """
0229     import datetime,sys,copy
0230     def __init__(self,detIDPSUmapfilename="map.txt",mask_mapfilename="mask_map.txt",DetIDAliasFile="StripDetIDAlias.pkl",startTime=datetime.datetime.now(),debug=False):
0231         """Initialize the object with a timestamp for the start time of the query (defaults to datetime.datetime.now()) and a list of modules OFF and ON depending on masking map (for now all OFF)"""
0232         self.debug=debug
0233         try:
0234             mapfile=open(detIDPSUmapfilename,'r')
0235             maplines=mapfile.readlines()
0236             mapfile.close()
0237             #Extract the detIDs from the map:
0238             self.detIDs=[int(line.split()[0]) for line in maplines]
0239             #Check number of detIDs
0240             #FIXME PIXEL:
0241             #Add a flag for pixels (name of the map?) to implement the check for either detector
0242             if len(list(set(self.detIDs)))!=15148:
0243                 print("There is an issue with the map provided: not reporting 15148 unique detIDs!")
0244             #Parse the PSU channel part of the map
0245             #This one does not list all channels, but follows this convention:
0246             #-detids that are HV mapped list channel002 or channel003
0247             #-detids that are NOT HV mapped list channel000
0248             #-detids that are connected to HV1/HV2 crosstalking PSUs list channel999
0249             self.PSUChannelListed=[line.split()[1] for line in maplines]
0250             #In any case all PSU are listed so we can extract them this way:
0251             self.PSUs=list(set([PSUChannel[:-10] for PSUChannel in self.PSUChannelListed]))
0252             #Check number of PSUs:
0253             if len(self.PSUs)!=1944:
0254                 print("There is an issue with the map provided: not reporting 1944 unique Power Supplies!")
0255             #Building now a list of all PSUChannels
0256             #(since queries to the DB will report results in DPNAMEs, i.e. PSU channels
0257             self.PSUChannels=[]
0258             for PSU in self.PSUs:
0259                 self.PSUChannels.append(PSU+"channel000")
0260                 self.PSUChannels.append(PSU+"channel001")
0261                 self.PSUChannels.append(PSU+"channel002")
0262                 self.PSUChannels.append(PSU+"channel003")
0263             #This part should be unnecessary, since we do not mask any detid anymore...
0264             #But we leave it since the possibility is in the O2O code itself still.
0265             try:
0266                 maskfile=open(mask_mapfilename,'r')
0267                 self.masked_detIDs=[int(detID.rstrip()) for detID in maskfile]
0268                 maskfile.close()
0269             except:
0270                 self.masked_detIDs=[] #empty list of detIDs to be "masked"
0271                 print("No mask map was provided, assuming to start from a complete Tracker OFF state")
0272             try:
0273                 DetIDAlias=open(DetIDAliasFile,"rb")
0274                 DetIDAliasDict=pickle.load(DetIDAlias)
0275                 DetIDAlias.close()
0276                 self.DetIDAliasDict=DetIDAliasDict
0277             except:
0278                 self.DetIDAliasDict={}
0279                 print("No valid detID-Alias map pickle file was provided!")
0280                 print("The TkrVoltageStatus object could not be initialized properly!")
0281                 print("Please use an existing detIDPSUChannel mapfilename as argument!")
0282                 sys.exit()
0283         except:
0284             print("Could not find detID-PSUchannel map to initialize the detIDs")
0285             print("The TkrVoltageStatus object could not be initialized properly! Please use an existing detIDPSUChannel mapfilename as argument!")
0286             sys.exit()
0287         #With the information from the map build the 2 map dictionaries of interest:
0288         #DetID->PSUChannelListed (same convention as the map)
0289         self.DetIDPSUmap={}
0290         for detID in self.detIDs:
0291             self.DetIDPSUmap.update({detID:self.PSUChannelListed[self.detIDs.index(detID)]})
0292         #PSUChannel->DetID map:
0293         self.PSUDetIDmap={}
0294         #This map is a bit more complicated than the DetIDPSUmap (that was just a dictionary form of the given detIDPSUChannel map passed as argument):
0295         #Here we make a 1 to 1 map for all the PSUChannels to all detIDs connected, so for a given PSU:
0296         #1-all detIDs listed as channel000 and as channel999 will be listed for all PSUchannels (000/001/002/003)
0297         #2-all detIDs listed as channel002 will be listed for channel002 and for channel000/001
0298         #3-all detIDs listed as channel003 will be listed for channel003 and for channel000/001
0299         #NOTE:
0300         #For Unmapped and Crosstalking channel even though we list all the detids as above (for all channels of the given PSU), we will handle them separately
0301         #when determining whether the detids are going ON or OFF... see below.
0302         for PSUChannel in self.PSUChannels:
0303             if PSUChannel.endswith("0") or PSUChannel.endswith("1"): #Report all channels that match at the PSU level!
0304                 self.PSUDetIDmap.update({PSUChannel:[detid for detid in self.DetIDPSUmap.keys() if PSUChannel[:-3] in self.DetIDPSUmap[detid]]}) #PSU matching (picks all detids by PSU) 
0305             else: #For channel002 and channel003 list in this map ONLY the actual mapped ones, the unmapped or crosstalking are listed in the corresponding PSUDetIDUnmappedMap and PSUDetIDCrosstalkingMap (see below).
0306                 self.PSUDetIDmap.update({PSUChannel:[detid for detid in self.DetIDPSUmap.keys() if (PSUChannel in self.DetIDPSUmap[detid])]})
0307 
0308         #Separate PSU-based maps for unmapped and crosstalking detids:
0309         self.PSUDetIDUnmappedMap={}
0310         UnmappedPSUs=list(set([psuchannel[:-10] for psuchannel in self.DetIDPSUmap.values() if psuchannel.endswith("000")]))
0311         self.PSUDetIDCrosstalkingMap={}
0312         CrosstalkingPSUs=list(set([psuchannel[:-10] for psuchannel in self.DetIDPSUmap.values() if psuchannel.endswith("999")]))
0313         for PSU in self.PSUs:
0314             if PSU in UnmappedPSUs:
0315                 self.PSUDetIDUnmappedMap.update({PSU:[detid for detid in self.DetIDPSUmap.keys() if (self.DetIDPSUmap[detid].endswith("000") and PSU in self.DetIDPSUmap[detid])]})
0316             if PSU in CrosstalkingPSUs:
0317                 self.PSUDetIDCrosstalkingMap.update({PSU:[detid for detid in self.DetIDPSUmap.keys() if (self.DetIDPSUmap[detid].endswith("999") and PSU in self.DetIDPSUmap[detid])]})
0318         #Need also the list of PSU channels that are unmapped and crosstalking with their status!
0319         #Since the state for those detIDs is determined by the knowledge of the other PSU channel, we need a dictionary for both that keeps their "last value" at all times.
0320         self.UnmappedPSUChannelStatus={}
0321         #Initialize the dictionary with all the unmapped (only HV is relevant, channel002/003) PSU channels set to 0 (off).
0322         for PSU in self.PSUDetIDUnmappedMap.keys():
0323             self.UnmappedPSUChannelStatus.update({PSU+"channel002":0})
0324             self.UnmappedPSUChannelStatus.update({PSU+"channel003":0})
0325         self.CrosstalkingPSUChannelStatus={}
0326         #Initialize the dictionary with all the crosstalking (only HV is relevant, channel002/003) PSU channels set to 0 (off).
0327         for PSU in self.PSUDetIDCrosstalkingMap.keys():
0328             self.CrosstalkingPSUChannelStatus.update({PSU+"channel002":0})
0329             self.CrosstalkingPSUChannelStatus.update({PSU+"channel003":0})
0330         
0331         #Now let's initialize the object (data member) we will use to keep track of the tracker status:
0332         
0333         #Make the startTime of the initialization an attribute of the object:
0334         self.startTime=startTime
0335         #Basic structure of the internal history dictionary, based on query results
0336         self.PSUChannelHistory={startTime:range(len(self.PSUChannels))}
0337         #NOTE:
0338         #Using the indeces to the self.PSUChannels list of PSUChannels to spare memory...
0339         #Strategy:
0340         #keep a timestamped list of indeces to PSUChannels
0341         #Dynamically fetch the PSUChannels (or corresponding detIDs via PSUDetIDmap dict) on the fly whey querying the status.
0342         
0343         #For Masked DetIDs set them always ON (since they would be detIDs that we don't know to with PSU to associate them to
0344         #This should NOT BE THE CASE any longer, since all detIDs are now mapped at least to a PSU
0345         #TOBEIMPLEMENTED:
0346         #(some missing HV channel, but we treat those differently: turn them off in case either HV1/HV2 is OFF)
0347         #(some having cross-talking PSUs, we treat those differently: turn them OFF only if both HV1/HV2 are OFF)
0348         for detID in self.masked_detIDs:
0349             #For each detID get the PSUChannelListed, then remove last digit (the channel number that can be 0,2 or 3)
0350             #and then make sure to remove all channels 000,001,002,003 
0351             self.PSUChannelHistory[startTime].remove(index(detIDPSUmap[detid][-1]+'0')) #Channel000
0352             self.PSUChannelHistory[startTime].remove(index(detIDPSUmap[detid][-1]+'1')) #Channel001
0353             self.PSUChannelHistory[startTime].remove(index(detIDPSUmap[detid][-1]+'2')) #Channel002
0354             self.PSUChannelHistory[startTime].remove(index(detIDPSUmap[detid][-1]+'3')) #Channel003
0355         #Let's try to keep the one following dict that reports the number of detIDs with HV OFF and LV OFF at a given timestamp...
0356         #Not sure even just this dict makes sense... but it is the most used information, quite minimal compare to the status of each detID
0357         #and it provides a global IOV for the whole tracker
0358         #NOTE:
0359         #it won't track the case of a channel going down at the exact time another goes up!
0360         #self.TkNumberOfDetIDsOff={startTime:(len([PSUChannel[i] for i in self.PSUChannelHistory]),len(self.detIDs))} 
0361 
0362     def updateO2OQuery(self,row):
0363         """Function to automatically handle the updating the of history dictionary
0364         with the results rows from the O2O query (PSUchannel name based).
0365         Note that row can be an actual row from SQLAlchemy from direct online DB querying
0366         but also a dictionary extracted parsing ManualO2O.log files.
0367         """
0368         #NOTE:May need to introduce IOV reduction at this level,
0369         #but I'd prefer to handle it separately so to conserve the original data for validation purposes.
0370         #For new IOV create a new entry in the dict and initialize it to the last IOV status
0371         #since only change is reported with STATUS_CHANGE queries.
0372 
0373         #First check if we care about this channel (could be a Pixel channel while we want only Strip channels or viceversa):
0374         if row['dpname'] not in self.PSUChannels:
0375             if self.debug:
0376                 print("Discarding results row since it is not in the currently considered map:\n%s"%row['dpname'])
0377             return        
0378         
0379         #NOTE:
0380         #Actually use the getIOV to allow for "asynchronous" updating:
0381         lastTimeStamp,nextTimeStamp=self.getIOV(row['change_date'],self.PSUChannelHistory)
0382         #Print a warning when the timeStamp is not the last one in the history!
0383         if self.debug:
0384             if row['change_date'] not in self.PSUChannelHistory.keys():
0385                 if nextTimeStamp!="Infinity":
0386                     print("WARNING! Asynchronous updating of the Tracker Voltage Status")
0387                     print("WARNING! Inserting an IOV between %s, and %s existing timestamps!"%(lastTimeStamp,nextTimeStamp))
0388                     print("The state will be correctly updated (if necessary) starting from the state at %s"%lastTimeStamp)
0389                 else:
0390                     print("Appending one more IOV to the PSUChannelHistory dictionary")
0391             else:
0392                 print("Updating already present IOV with timestamp %s"%row['change_date'])
0393         #The fact that we edit the lasttimestamp takes care of updating existing IOVs the same way as for new ones...
0394         #Update the internal dictionary modifying the last time stamp state...
0395         #Case of channel being ON (it's only ON if CAEN reports 1)
0396         PSUChannelOFFList=self.PSUChannelHistory[lastTimeStamp][:]
0397         if row['actual_status']==1: #Switching from string to int here to make sure it will be compatible with SQLAlchemy query for testing with direct DB access.
0398             try:
0399                 PSUChannelOFFList.remove(self.PSUChannels.index(row['dpname']))
0400                 self.PSUChannelHistory.update({row['change_date']:PSUChannelOFFList})
0401             except:
0402                 if self.debug:
0403                     print("Found a channel that turned ON but as already ON apparently!")
0404                     print(row)
0405         else: #Case of channel being OFF (it's considered OFF in any non 1 state, ramping, error, off)
0406             PSUChannelOFFList.append(self.PSUChannels.index(row['dpname']))
0407             self.PSUChannelHistory.update({row['change_date']:list(set(PSUChannelOFFList))})
0408         
0409         #First handle the detID based dict:
0410         #Use the map to translate the DPNAME (PSUChannel) into a detid via internal map:
0411         #CAREFUL... need to protect against pixel PSU channels being reported by the query!
0412         #try:
0413         #    detIDList=self.PSUDetIDmap[row['dpname']]
0414         #except:
0415         #    #FIXME: develop the pixel testing part here later...
0416         #    print "PSU channel (DPNAME) reported in not part of the wanted subdetector (only Pixel or Strips at one time)"
0417         #    print "DPNAME=%s"%row['dpname']
0418         #    detIDList=[] #Set it to empty list...
0419         ##print detID, type(detID)
0420         #for detID in detIDList:
0421         #    #Get the previous list of channel statuses for the relevant detID:
0422         #    ModuleStatus=self.history[row['change_date']][detID]
0423         #    #Update it with the current query result row:
0424         #    #DISTINGUISH LV and HV case:
0425         #    if row['dpname'].endswith('channel000') or row['dpname'].endswith('channel001'):
0426         #        ModuleStatus[0]=row['actual_status'] #{detid:[LV,HV]} convention is ModuleStatus[0]=0 is LV OFF
0427         #    else:
0428         #        ModuleStatus[1]=row['actual_status']
0429         #    #Update the history dictionary:
0430         #    self.history[row['change_date']].update({detID:ModuleStatus})
0431         ##Add a check for the possibility of having a second IOV when only channel000/channel001 changed,
0432         ##i.e. the status did not change as a consequence for the detID based history dictionary
0433         ##(this does not affect the historyPSUChannel dictionary by definition)
0434         #if self.history[row['change_date']]==self.history[TimeStamps['detID'][0]] and row['change_date']!=TimeStamps['detID'][0]: #This last condition is just in case something funky happens, to avoid removing good IOVs
0435         #    if self.debug:
0436         #        print "Found that the update for timestamp %s, does not change the detid status w.r.t timestamp %s"%(row['change_date'],TimeStamps['detID'][0])
0437         #        print "Eliminating the temporarily created entry in the history dictionary (it will still be logged in the historyPSUChannel dictionary"
0438         #    self.history.pop(row['change_date'])
0439         ##Then handle the PSUChannel based one:
0440         #self.historyPSUChannel[row['change_date']].update({row['dpname']:row['actual_status']})
0441     
0442                     
0443     def updateDetIDQuery(self,row):
0444         """Function to automatically handle the updating of the history dictionary
0445         with the result rows from detID based query
0446         """
0447         #FIXME:
0448         #this will have to be developed once the detID query is ready and being used in the Tracker DCS O2O
0449         if row['change_date'] not in self.history.keys():#New timestamp (i.e. new IOV)
0450             #For new IOV create a new entry in the dict and initialize it to the last IOV status
0451             #since only change is reported with STATUS_CHANGE queries.
0452             lastTimeStamp=sorted(self.history.keys()).pop(0)
0453             self.history.update({row['change_date']:self.history[lastTimeStamp]})
0454         #Get the previous list of channel statuses for the relevant detID:            
0455         channelStatus=self.history[row['change_date']][row['detid']]
0456         #Update it with the current query result row:
0457         channelStatus[row['channel']]=row['voltage_status']
0458         #Update the history dictionary:
0459         self.history[row['change_date']].update({row['detid']:channelStatus})
0460         
0461         
0462     #Define the getters:
0463 
0464     #First the TIMESTAMP based getters:
0465     
0466     def getPSUChannelsOff(self,timestamp):
0467         """
0468         Function that given a timestamp, returns the list of PSUChannels OFF, the list of PSUChannels ON and the IOV (start/stop timestamps) of the reported status.
0469         NOTE: No distinction between HV and LV is made since all PSUChannels (000/001 and 002/003) are reported.
0470         """
0471         StartTime,StopTime=self.getIOV(timestamp,self.PSUChannelHistory)
0472         PSUChannelsOFF=map(lambda x: self.PSUChannels[x],self.PSUChannelHistory[StartTime])
0473         PSUChannelsON=list(set(self.PSUChannels).difference(set(PSUChannelsOFF)))
0474         return PSUChannelsOFF,PSUChannelsON,StartTime,StopTime
0475     
0476     def getDetIDsHVOff(self,timestamp):
0477         """
0478         Function that given a timestamp, returns the list of detIDs with HV OFF, the list of detIDs with HV ON and the IOV (start/stop times) of the reported status.
0479         """
0480         StartTime,StopTime=self.getIOV(timestamp,self.PSUChannelHistory)
0481         DetIDsHVOFF=[]
0482         #A little too nasty python oneliner:
0483         #[DetIDsHVOFF.extend(i) for i in map(lambda x: self.PSUDetIDmap[self.PSUChannels[x]],[index for index in self.PSUChannelHistory[StartTime] if (self.PSUChannels[index].endswith("2") or self.PSUChannels[index].endswith("3"))])]
0484         #It actually does not work, since we need to consider unmapped and crosstalking channels and handle them differently, let's see:
0485         for index in self.PSUChannelHistory[StartTime]:
0486             #First check only for HV channels!
0487             if self.PSUChannels[index].endswith("2") or self.PSUChannels[index].endswith("3") :
0488                 #Then check HV MAPPED channels:
0489                 if self.PSUChannels[index] in self.PSUDetIDmap.keys():
0490                     #print "Extending the list of DetIdsHVOff with the positively matched detids:",self.PSUDetIDmap[self.PSUChannels[index]]
0491                     DetIDsHVOFF.extend(self.PSUDetIDmap[self.PSUChannels[index]])
0492                 #Furthermore check the unmapped channels:
0493                 if self.PSUChannels[index][:-10] in self.PSUDetIDUnmappedMap.keys():
0494                     #To turn OFF unmapped channels there is no need to check the "other" channel:
0495                     #print "Extending the list of DetIdsHVOff with the HV unmapped (PSU-)matched detids:",self.PSUDetIDUnmappedMap[self.PSUChannels[index][:-10]]
0496                     DetIDsHVOFF.extend(self.PSUDetIDUnmappedMap[self.PSUChannels[index][:-10]])
0497                 #Further check the crosstalking channels:
0498                 if self.PSUChannels[index][:-10] in self.PSUDetIDCrosstalkingMap.keys():
0499                     #To turn OFF crosstalking channels we need to check that the other channel is OFF too!
0500                     if (self.PSUChannels.index(self.PSUChannels[index][:-1]+"2") in self.PSUChannelHistory[StartTime]) and (self.PSUChannels.index(self.PSUChannels[index][:-1]+"3") in self.PSUChannelHistory[StartTime]):
0501                         #print "Extending the list of DetIdsHVOff with the HV-CROSSTALKING (PSU-)matched detids:",self.PSUDetIDCrosstalkingMap[self.PSUChannels[index][:-10]]
0502                         DetIDsHVOFF.extend(self.PSUDetIDCrosstalkingMap[self.PSUChannels[index][:-10]])
0503         DetIDsHVON=list(set(self.detIDs).difference(set(DetIDsHVOFF)))
0504         return list(set(DetIDsHVOFF)),DetIDsHVON,StartTime,StopTime
0505     
0506     def getDetIDsLVOff(self,timestamp):
0507         """
0508         Function that given a timestamp, returns the list of detIDs with LV OFF, the list of detIDs with LV ON and the IOV (start/stop times) of the reported status.
0509         """
0510         #Note that channels with LV OFF naturally should have HV OFF too!
0511         StartTime,StopTime=self.getIOV(timestamp,self.PSUChannelHistory)
0512         DetIDsLVOFF=[]
0513         for detids in map(lambda x: self.PSUDetIDmap[self.PSUChannels[x]],[index for index in self.PSUChannelHistory[StartTime] if (self.PSUChannels[index].endswith("0") or self.PSUChannels[index].endswith("1"))]):
0514             DetIDsLVOFF.extend(detids)
0515         DetIDsLVON=list(set(self.detIDs).difference(set(DetIDsLVOFF)))
0516         return list(set(DetIDsLVOFF)),DetIDsLVON,StartTime,StopTime
0517 
0518     def getAliasesHVOff(self,timestamp):
0519         """
0520         Function that given a timestamp, returns the list of (PSU) Aliases with HV OFF, the list of (PSU) Aliases with HV ON and the IOV (start/stop times) of the reported status. 
0521         """
0522         DetIDsHVOFF,DetIDsHVON,StartTime,StopTime=self.getDetIDsHVOff(timestamp)
0523         AliasesHVOFF=list(set([list(self.DetIDAliasDict[detid])[0] for detid in DetIDsHVOFF])) #FIXME: check on fixing the StripDetIDAlias.pkl dictionary... no need of a set as a result! actually need to check if there is more than one element, that would be an ERROR!
0524         AliasesHVON=list(set([list(self.DetIDAliasDict[detid])[0] for detid in DetIDsHVON]))
0525         return AliasesHVOFF,AliasesHVON,StartTime,StopTime
0526 
0527     def getAliasesLVOff(self,timestamp):
0528         """
0529         Function that given a timestamp, returns the list of (PSU) Aliases with HV OFF, the list of (PSU) Aliases with HV ON and the IOV (start/stop times) of the reported status.
0530         """
0531         DetIDsLVOFF,DetIDsLVON,StartTime,StopTime=self.getDetIDsLVOff(timestamp)
0532         AliasesLVOFF=list(set([list(self.DetIDAliasDict[detid])[0] for detid in DetIDsLVOFF])) #FIXME: check on fixing the StripDetIDAlias.pkl dictionary... no need of a set as a result! actually need to check if there is more than one element, that would be an ERROR!
0533         AliasesLVON=list(set([list(self.DetIDAliasDict[detid])[0] for detid in DetIDsLVON]))
0534         return AliasesLVOFF,AliasesLVON,StartTime,StopTime
0535 
0536     #Then the Value vs. time getters:
0537 
0538     def getDetIDsHVOffVsTime(self):
0539         """
0540         Function that returns the number of DetIDs with HV OFF vs time for all IOVs available in the PSUChannelHistory.
0541         The results is one list of tuples [(datetime.datetime,#DetIDsHVOFF),...] and the timestamp of the last entry processed in the PSUChannelHistory.
0542         This is important, so that the user can check until when in time the last tuple in the list is valid until (closing the last IOV).
0543         This can be easily used to
0544         -Do spot checks by IOV: picking a timestamp inside one of the IOVs one can use the getDetIDsHVOff function to get a list of the detIDs ON/OFF.
0545         -Plot the graph of number of HV OFF (or ON by complementing to 15148) vs time using pyRoot
0546         """
0547         #Loop over the timestamps:
0548         DetIDsHVOffVsTime=[]
0549         PreviousTimeStamp=None
0550         PreviousDetIDsHVOFF=[]
0551         for timestamp in sorted(self.PSUChannelHistory.keys()):
0552             DetIDsHVOFF=self.getDetIDsHVOff(timestamp)[0] #Use the first returned value (DetIDsHVOFF) from getDetIDsHVOff(timestamp)!
0553             #Check with respect to previous IOV, using set functionality, so that only relevant IOVs where there was an actual HV change are reported!
0554             if PreviousTimeStamp:
0555                 if DetIDsHVOFF!=PreviousDetIDsHVOFF: #If there is a previous timestamp to compare to, look for differences before reporting!
0556                     DetIDsHVOffVsTime.append((timestamp,len(DetIDsHVOFF)))
0557             else: #First IOV start... nothing to compare to:
0558                 DetIDsHVOffVsTime.append((timestamp,len(DetIDsHVOFF)))
0559             PreviousTimeStamp=timestamp
0560             PreviousDetIDsHVOFF=DetIDsHVOFF
0561         LastTimeStamp=PreviousTimeStamp
0562         return DetIDsHVOffVsTime,PreviousTimeStamp 
0563 
0564     def getDetIDsLVOffVsTime(self):
0565         """
0566         Function that returns the number of DetIDs with LV OFF vs time for all IOVs available in the PSUChannelHistory.
0567         The results is one list of tuples [(datetime.datetime,#DetIDsLVOFF),...], basically 1 entry per IOV (change in the LV channels only),
0568         and the timestamp of the last entry processed in the PSUChannelHistory. This timestamp is important, so that the user can check until when in time the last tuple in the list is valid until (closing the last IOV).
0569         This can be easily used to
0570         -Do spot checks by IOV: picking a timestamp inside one of the IOVs one can use the getDetIDsHVOff function to get a list of the detIDs ON/OFF.
0571         -Plot the graph of number of HV OFF (or ON by complementing to 15148) vs time using pyRoot
0572         """
0573         #Loop over the timestamps:
0574         DetIDsLVOffVsTime=[]
0575         PreviousTimeStamp=None
0576         for timestamp in sorted(self.PSUChannelHistory.keys()):
0577             DetIDsLVOFF=set(self.getDetIDsLVOff(timestamp)[0]) #Use the first returned value (DetIDsLVOFF) from getDetIDsLVOff(timestamp)!Use set() to be able to compare the sets with != otherwise for lists, the order also makes them different!
0578             #Check with respect to previous IOV, using set functionality, so that only relevant IOVs where there was an actual LV change are reported!
0579             if PreviousTimeStamp:
0580                 if DetIDsLVOFF!=PreviousDetIDsLVOFF: #If there is a previous timestamp to compare to, look for differences before reporting! 
0581                     DetIDsLVOffVsTime.append((timestamp,len(DetIDsLVOFF)))
0582             else: #First IOV start... nothing to compare to:
0583                 DetIDsLVOffVsTime.append((timestamp,len(DetIDsLVOFF)))
0584             PreviousTimeStamp=timestamp
0585             PreviousDetIDsLVOFF=DetIDsLVOFF
0586         LastTimeStamp=PreviousTimeStamp
0587         return DetIDsLVOffVsTime,LastTimeStamp
0588     
0589     #Data massagers:
0590     def getArraysForTimePlots(self,TimeStampsValuesTuple):
0591         """
0592         Function that given a tuple with (datetime.datetime,values) returns the arrays ready to the plotting vs time function (basically renders data histogram-like adding extra data points).
0593         """
0594         import array, time
0595 
0596         #Initialize the lists of timestamps (to be doubled to get the histogram/IOV look)
0597         Timestamps=[]
0598         Values=[]
0599         #Loop over the list of tuples with (timestamp,numberofdetidswithLVOFF):
0600         for item in sorted(TimeStampsValuesTuple):
0601            #FIXME:
0602            #NEED to check the effect/use of tzinfo to avoid issues with UTC/localtime DST etc.
0603            #Will have to massage the datetime.datetime object in a proper way not necessary now.
0604            #Also need to decide if we want add the milliseconds into the "timestamps for plotting in root... not straightforward so will implement only if needed. For now know that the approximation is by truncation for now!
0605            timestamp=int(time.mktime(item[0].timetuple())) #Need to get first a time tuple, then translate it into a unix timestamp (seconds since epoc). This means no resolution under 1 second by default
0606            if item==TimeStampsValuesTuple[0]: #First item does not need duplication
0607                Timestamps.append(timestamp)
0608                Value=item[1]
0609                Values.append(Value)
0610            else: #Add twice each timestamp except the first one (to make a histogram looking graph)
0611                Timestamps.append(timestamp)
0612                Values.append(Value) #Input previous value with new timestamp
0613                Value=item[1]
0614                Timestamps.append(timestamp)
0615                Values.append(Value) #Input new value with new timestamp
0616         #Need to render the two lists as arrays for pyROOT
0617         TimestampsArray=array.array('i',Timestamps)
0618         ValuesArray=array.array('i',Values)
0619 
0620         return TimestampsArray,ValuesArray
0621 
0622     def getReducedArraysForTimePlots(self,TimeStamps,Values):
0623         """
0624         Implement IOV reduction based on timestamp delta, following O2O algorithm.
0625         """
0626         
0627         return
0628     
0629     #The plotters:
0630     def plotGraphSeconds(self,TimeArray,ValuesArray,GraphTitle="Graph",YTitle="Number of channels",GraphFilename="test.gif"):
0631         """
0632         Function that given an array with timestamps (massaged to introduce a second timestamp for each value to produce an histogram-looking plot) and a corresponding array with values to be plotted, a title, a Y axis title and a plot filename, produces with pyROOT a time plot and saves it locally.
0633         The function can be used for cumulative plots (number of channels with HV/LV OFF/ON vs time) or for individual (single detID HV/LV status vs time) plots.
0634         """
0635         import ROOT
0636         canvas=ROOT.TCanvas()
0637         graph=ROOT.TGraph(len(TimeArray),TimeArray,ValuesArray)
0638         graph.GetXaxis().SetTimeDisplay(1)
0639         graph.GetXaxis().SetLabelOffset(0.02)
0640         #Set the time format for the X Axis labels depending on the total time interval of the plot!
0641         TotalTimeIntervalSecs=TimeArray[-1]-TimeArray[0]
0642         if TotalTimeIntervalSecs <= 120: #When zooming into less than 2 mins total interval report seconds too
0643             graph.GetXaxis().SetTimeFormat("#splitline{   %d/%m}{%H:%M:%S}")
0644         elif 120 < TotalTimeIntervalSecs < 6400: #When zooming into less than 2 hrs total interval report minutes too
0645             graph.GetXaxis().SetTimeFormat("#splitline{%d/%m}{%H:%M}")
0646         else: #When plotting more than 2 hrs only report the date and hour of day
0647             graph.GetXaxis().SetTimeFormat("#splitline{%d/%m}{  %H}")
0648         graph.GetYaxis().SetTitle(YTitle)
0649         graph.GetYaxis().SetTitleOffset(1.4)
0650         graph.SetTitle(GraphTitle)
0651         graph.Draw("APL")
0652         canvas.SaveAs(GraphFilename)
0653         print("Saved graph as %s"%GraphFilename)
0654         return
0655     def plotPSUChannelvsTime(self,TimeArray,ValuesArray,GraphTitle="PSUChannelGraph",YTitle="Channel HV Status",GraphFilename="PSUChannel.gif"):
0656         """
0657         Function that given an array with timestamps (massaged to introduce a second timestamp for each value to produce an histogram-looking plot) and a corresponding array with values to be plotted, a title, a Y axis title and a plot filename, produces with pyROOT a time plot and saves it locally.
0658         The function can be used for cumulative plots (number of channels with HV/LV OFF/ON vs time) or for individual (single detID HV/LV status vs time) plots.
0659         """
0660         import ROOT
0661         canvas=ROOT.TCanvas()
0662         graph=ROOT.TGraph(len(TimeArray),TimeArray,ValuesArray)
0663         graph.GetXaxis().SetTimeDisplay(1)
0664         graph.GetXaxis().SetLabelOffset(0.02)
0665         #Set the time format for the X Axis labels depending on the total time interval of the plot!
0666         TotalTimeIntervalSecs=TimeArray[-1]-TimeArray[0]
0667         if TotalTimeIntervalSecs <= 120: #When zooming into less than 2 mins total interval report seconds too
0668             graph.GetXaxis().SetTimeFormat("#splitline{   %d/%m}{%H:%M:%S}")
0669         elif 120 < TotalTimeIntervalSecs < 6400: #When zooming into less than 2 hrs total interval report minutes too
0670             graph.GetXaxis().SetTimeFormat("#splitline{%d/%m}{%H:%M}")
0671         else: #When plotting more than 2 hrs only report the date and hour of day
0672             graph.GetXaxis().SetTimeFormat("#splitline{%d/%m}{  %H}")
0673         graph.GetYaxis().SetTitle(YTitle)
0674         graph.GetYaxis().SetTitleOffset(1.4)
0675         graph.SetTitle(GraphTitle)
0676         graph.Draw("APL")
0677         canvas.SaveAs(GraphFilename)
0678         print("Saved graph as %s"%GraphFilename)
0679         return
0680     
0681     def plotHVOFFvsTime(self):
0682         """
0683         """
0684         return
0685     
0686     def plotLVOFFvsTime(self):
0687         """
0688         """
0689         return
0690 
0691     def plotDetIDHVOFFVsTime(self,detID):
0692         """
0693         """
0694         return
0695     def plotDetIDLVOFFVsTime(self,detID):
0696         """
0697         """
0698         return
0699 
0700     def plotDetIDHistory(self):
0701         return
0702 
0703     def getIOV(self,timeStamp,HistoryDict):
0704         """
0705         Function that given a timeStamp return the TimeStampStart and TimeStampStop of the IOV in the wanted HistoryDict (assumed to have timestamps as keys) that contains the timestamp.
0706         This can be used by all functions that need to access HistoryDict by timeStamp.
0707         """        
0708         
0709         TimeStamps=HistoryDict.keys()[:]
0710         #Add the wanted timeStamp to the list of TimeStamps (if it is not there already!)
0711         if timeStamp not in TimeStamps:
0712             TimeStamps.append(timeStamp)
0713             #Sort the list with the timestamp we added, so it will fall in between the wanted StartTime and EndTime.
0714             TimeStamps.sort()            
0715             TimeStampStart=TimeStamps[TimeStamps.index(timeStamp)-1]
0716             #Remember to remove it now, so that TimeStamps is still the list of available timestamps!
0717             TimeStamps.remove(timeStamp)
0718         else:#If the timeStamp matches one in the file, then the StartTime is that timestamp ;)
0719             TimeStamps.sort() #this is still needed since the keys of a dictionary are not automatically sorted, but we assume TimeStamps to be sorted to spot TimeStampStop...
0720             TimeStampStart=timeStamp
0721         #For the TimeStampStop check the case of hitting the last IOV that is valid until infinity...
0722         if len(TimeStamps)>TimeStamps.index(TimeStampStart)+1:
0723             TimeStampStop=TimeStamps[TimeStamps.index(TimeStampStart)+1]
0724         else:
0725             TimeStampStop="Infinity"
0726         if self.debug:
0727             print("TimeStamp %s falls into IOV starting at %s and ending at %s"%(timeStamp,TimeStampStart,TimeStampStop))
0728         return (TimeStampStart,TimeStampStop)
0729 
0730     def getIOVsInTimeInterval(self,StartTime,StopTime,HistoryDict):
0731         """
0732         Function that returns the IOV timestamps (ordered) contained in a given interval.
0733         """
0734         #Copy and sort the timestamps in a list
0735         TimeStamps=sorted(HistoryDict.keys()[:])
0736         IOVsInTimeInterval=[]
0737         #loop over them:
0738         for timestamp in TimeStamps:
0739             if timestamp>=StartTime and timestamp<=StopTime: #Pick only the timestamps inside the time interval specified
0740                 if self.debug:
0741                     print("Found timestamp %s in the wanted interval [%s,%s]"%(timestamp,StartTime,StopTime))
0742                 IOVsInTimeInterval.append(timestamp)
0743         return IOVsInTimeInterval
0744 
0745     def getReducedIOVs (self,StartTime,StopTime,HistoryDict,deltaT,maxIOVLength):
0746         """
0747         Apply the reduction algorithm to the timeintervals and return them so that one can test the reduction (and do plots more easily).
0748         """
0749         deltaTime=datetime.timedelta(seconds=deltaT)
0750         maxSequenceLength=datetime.timedelta(seconds=maxIOVLength)
0751         #Copy and sort the timestamps in a list:
0752         TimeStamps=sorted(HistoryDict.keys()[:])
0753         ReducedIOVs=TimeStamps[:]
0754         PreviousTimestamp=TimeStamps[0] 
0755         SequenceStart=TimeStamps[0]  #Initialization irrelevant see loop
0756         SequenceOn=False
0757         #Loop over timestamps:
0758         #Note:Leave the ramp-up/down logic for later (an other function), for now we just do IOV reduction: don't care if we are ramping up or down!
0759         for timestamp in TimeStamps[1:]: #Skip the first timestamp!(since we initialize PreviousTimeStamp to the first timestamp!)
0760             #Check only within the interval specified:
0761             if timestamp>=StartTime and timestamp<=StopTime:
0762                 #Check if the timestamp is within the wanted deltaT from previous timestamp
0763                 if timestamp-PreviousTimestamp<=deltaTime:
0764                     #Check if there is already an ongoing sequence of IOVs:
0765                     if SequenceOn:
0766                         #Check that this timestamp is not farther away than the maximum IOV sequence
0767                         if (timestamp-SequenceStart)<=maxSequenceLength and timestamp!=TimeStamps[-1]:#need to handle the last time stamp differently!
0768                             if self.debug:
0769                                 print("Eliminating timestamp %s since it is %s (<=%s)seconds from previous timestamp %s and %s (<=%s) seconds from the IOV sequence start %s!"%(timestamp,timestamp-PreviousTimestamp,deltaTime,PreviousTimestamp,timestamp-SequenceStart,maxSequenceLength,SequenceStart))
0770                             ReducedIOVs.remove(timestamp)
0771                         elif timestamp==TimeStamps[-1]: #special case of last timestamp in the list of input timestamps!
0772                             if self.debug:
0773                                 print("###Terminating the IOV sequence started with %s, since current timestamp %s is the last one in the input list of timestamps to REDUCE!"%(SequenceStart,timestamp))
0774                         else:
0775                             #Terminate the sequence (keep the timestamp):
0776                             SequenceOn=False
0777                             #Reappend the last item of the sequence (that was automatically removed not knowing it would be the last of the sequence):
0778                             ReducedIOVs.append(PreviousTimestamp)
0779                             #Re-order the list via sort():
0780                             ReducedIOVs.sort()
0781                             if self.debug:
0782                                 print("###Terminating the IOV sequence started with %s, since current timestamp %s is %s seconds (>%s) away from the IOV sequence starting timestamp (%s). Re-appending the last timestamp in the sequence %s."%(SequenceStart,timestamp,timestamp-SequenceStart,maxIOVLength,PreviousTimestamp,PreviousTimestamp))
0783                     else:
0784                         #Start the sequence
0785                         SequenceOn=True
0786                         #Save the first timestamp of the sequence (it will be used to check sequence length)
0787                         if self.debug:
0788                             print("@@@Starting a new IOV sequence with previous timestamp %s (current being %s)"%(PreviousTimestamp,timestamp))
0789                         #Still get rid of the current (second in the sequence) IOV:
0790                         ReducedIOVs.remove(timestamp)
0791                         SequenceStart=PreviousTimestamp
0792                 else:
0793                     #Check if there is already an ongoing sequence of IOVs:
0794                     if SequenceOn:
0795                         #Terminate the sequence since timestamp is further than deltaT from previous timestamp:
0796                         SequenceOn=False
0797                         #Reappend the last item of the sequence (that was automatically removed not knowing it would be the last of the sequence):
0798                         ReducedIOVs.append(PreviousTimestamp)
0799                         #Re-order the list via sort():
0800                         ReducedIOVs.sort()
0801                         if self.debug:
0802                             print("$$$Terminating the IOV sequence started with %s, since current timestamp %s is %s seconds (>%s) away from the previous timestamp (%s) in the sequence. Re-appending the last timestamp in the sequence %s."%(SequenceStart,timestamp,timestamp-PreviousTimestamp,deltaT,PreviousTimestamp,PreviousTimestamp))
0803                                                             
0804             else:
0805                 #The following is conventional of course:
0806                 #Since we are outside the wanted time interval we should still kick out the timestamp,
0807                 #Basically we will only report a reduce sequence of timestamps within a certain interval
0808                 ReducedIOVs.remove(timestamp)
0809                 #If there is an ongoing sequence that stretches outside the StopTime:
0810                 if SequenceOn:
0811                     #Terminate sequence (should check the results if they finish with a timestamp>StopTime):
0812                     SequenceOn=False
0813                     #Reappend the last item of the sequence (that was automatically removed not knowing it would be the last of the sequence):
0814                     ReducedIOVs.append(PreviousTimestamp)
0815                     #Re-order the list via sort():
0816                     ReducedIOVs.sort()
0817                     if self.debug:
0818                         print("^^^ Terminating the IOV sequence started with %s, since current timestamp %s is no more in the wanted time interval under investigation ([%s,%s]). Sequence might have been continuing beyond the StopTime %s limit! Re-appending the last timestamp in the sequence %s."%(SequenceStart,timestamp,StartTime,StopTime,StopTime,PreviousTimestamp))
0819             PreviousTimestamp=timestamp
0820         return ReducedIOVs
0821 
0822                     
0823 #Messing around...
0824 TkStatus=TrkVoltageStatus(detIDPSUmapfilename=os.path.join(os.getenv('CMSSW_BASE'),'src/CalibTracker/SiStripDCS/data/StripPSUDetIDMap_FromJan132010_Crosstalk.dat'),DetIDAliasFile=os.path.join(os.getenv('CMSSW_BASE'),'src/CalibTracker/SiStripDCS/data/StripDetIDAlias.pkl'),startTime=datetime.datetime(2010,8,27,12,00,00),debug=True)
0825 
0826 #row1={'change_date':datetime.datetime(2010,8,27,10,37,54,732),'actual_status':1,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel001"}
0827 #
0828 #row2={'change_date':datetime.datetime(2010,8,27,11,37,54,732),'actual_status':0,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel001"}
0829 #
0830 #row3={'change_date':datetime.datetime(2010,8,27,12,37,54,732),'actual_status':1,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel001"}
0831 #
0832 #row4={'change_date':datetime.datetime(2010,8,27,10,37,54,732),'actual_status':1,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel002"}
0833 #
0834 #row5={'change_date':datetime.datetime(2010,8,27,11,37,54,732),'actual_status':0,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel002"}
0835 #
0836 #row6={'change_date':datetime.datetime(2010,8,27,12,37,54,732),'actual_status':1,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel002"}
0837 #
0838 #row7={'change_date':datetime.datetime(2010,8,27,11,37,54,732),'actual_status':0,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel002"}
0839 #
0840 #row8={'change_date':datetime.datetime(2010,8,27,10,37,54,735),'actual_status':1,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel002"}
0841 #
0842 ##Initialization IOV:
0843 #print "detID IOVs",TkStatus.history.keys()
0844 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0845 #
0846 #TkStatus.updateO2OQuery(row1)
0847 #print "detID IOVs",TkStatus.history.keys()
0848 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0849 #
0850 #TkStatus.updateO2OQuery(row2)
0851 #print "detID IOVs",TkStatus.history.keys()
0852 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0853 #
0854 #TkStatus.updateO2OQuery(row3)
0855 #print "detID IOVs",TkStatus.history.keys()
0856 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0857 #
0858 #TkStatus.updateO2OQuery(row4)
0859 #print "detID IOVs",TkStatus.history.keys()
0860 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0861 #
0862 #TkStatus.updateO2OQuery(row5)
0863 #print "detID IOVs",TkStatus.history.keys()
0864 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0865 #
0866 #TkStatus.updateO2OQuery(row6)
0867 #print "detID IOVs",TkStatus.history.keys()
0868 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0869 #
0870 #TkStatus.updateO2OQuery(row7)
0871 #print "detID IOVs",TkStatus.history.keys()
0872 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0873 #
0874 #TkStatus.updateO2OQuery(row8)
0875 #print "detID IOVs",TkStatus.history.keys()
0876 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0877 #
0878 #TkStatus.getHVOnModules(datetime.datetime(2010, 8, 27, 10, 00, 54, 732))
0879 
0880 #Test Crosstalking channels handling:
0881 #rowCross1={'change_date':datetime.datetime(2010,8,27,10,37,54,732),'actual_status':1,'dpname':"cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel002"}
0882 #rowCross2={'change_date':datetime.datetime(2010,8,27,10,37,57,732),'actual_status':1,'dpname':"cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel003"}
0883 #rowCross3={'change_date':datetime.datetime(2010,8,27,10,38,00,732),'actual_status':0,'dpname':"cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel002"}
0884 #rowCross4={'change_date':datetime.datetime(2010,8,27,10,38,05,732),'actual_status':0,'dpname':"cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel003"}
0885 #print "Starting number of PSUChannels OFF is %s and of DetIDs ON is %s"%(len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,10,30,00),TkStatus.PSUChannelHistory)[0]]),len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010,8,27,10,30,00),TkStatus.PSUChannelHistory)[0])[1]))                                                                         
0886 #TkStatus.updateO2OQuery(rowCross1)
0887 #print "Turning ON HV crosstalking channel cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel002 number of channels OFF is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,10,37,55),TkStatus.PSUChannelHistory)[0]])
0888 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010,8,27,10,37,55),TkStatus.PSUChannelHistory)[0])[1])
0889 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010,8,27,10,37,55),TkStatus.PSUChannelHistory)[0])[1]
0890 #TkStatus.updateO2OQuery(rowCross2)
0891 #print "Turning ON HV crosstalking channel cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel003 number of channels OFF is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,10,37,58),TkStatus.PSUChannelHistory)[0]])
0892 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 10, 37, 58),TkStatus.PSUChannelHistory)[0])[1])
0893 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 10, 37, 58),TkStatus.PSUChannelHistory)[0])[1]
0894 #TkStatus.updateO2OQuery(rowCross3)
0895 #print "Turning OFF HV crosstalking channel cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel002 number of channels off is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,10,38,02),TkStatus.PSUChannelHistory)[0]])
0896 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 10, 38, 02),TkStatus.PSUChannelHistory)[0])[1])
0897 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 10, 38, 02),TkStatus.PSUChannelHistory)[0])[1]
0898 #TkStatus.updateO2OQuery(rowCross4)
0899 #print "Turning OFF HV crosstalking channel cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel003 number of channels off is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,10,38,07),TkStatus.PSUChannelHistory)[0]])
0900 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 10, 38, 07),TkStatus.PSUChannelHistory)[0])[1])
0901 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 10, 38, 07),TkStatus.PSUChannelHistory)[0])[1]
0902 #
0903 ##Test Unmapped channels handling:
0904 #rowUnmap1={'change_date':datetime.datetime(2010,8,27,11,37,54,732),'actual_status':1,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_2/branchController05/easyCrate1/easyBoard07/channel002"}
0905 #rowUnmap2={'change_date':datetime.datetime(2010,8,27,11,37,57,732),'actual_status':1,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_2/branchController05/easyCrate1/easyBoard07/channel003"}
0906 #rowUnmap3={'change_date':datetime.datetime(2010,8,27,11,38,00,732),'actual_status':0,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_2/branchController05/easyCrate1/easyBoard07/channel002"}
0907 #rowUnmap4={'change_date':datetime.datetime(2010,8,27,11,38,05,732),'actual_status':0,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_2/branchController05/easyCrate1/easyBoard07/channel003"}
0908 #print "Starting number of PSUChannels OFF is %s and of DetIDs ON is %s"%(len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,11,30,00),TkStatus.PSUChannelHistory)[0]]),len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010,8,27,11,30,00),TkStatus.PSUChannelHistory)[0])[1]))                                                                         
0909 #TkStatus.updateO2OQuery(rowUnmap1)
0910 #print "Turning ON HV crosstalking channel cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_2/branchController05/easyCrate1/easyBoard07/channel002  number of channels OFF is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,11,37,55),TkStatus.PSUChannelHistory)[0]])
0911 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010,8,27,11,37,55),TkStatus.PSUChannelHistory)[0])[1])
0912 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010,8,27,11,37,55),TkStatus.PSUChannelHistory)[0])[1]
0913 #TkStatus.updateO2OQuery(rowUnmap2)
0914 #print "Turning ON HV crosstalking channel cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_2/branchController05/easyCrate1/easyBoard07/channel003 number of channels OFF is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,11,37,58),TkStatus.PSUChannelHistory)[0]])
0915 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 11, 37, 58),TkStatus.PSUChannelHistory)[0])[1])
0916 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 11, 37, 58),TkStatus.PSUChannelHistory)[0])[1]
0917 #TkStatus.updateO2OQuery(rowUnmap3)
0918 #print "Turning OFF HV crosstalking channel cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_2/branchController05/easyCrate1/easyBoard07/channel002 number of channels off is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,11,38,02),TkStatus.PSUChannelHistory)[0]])
0919 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 11, 38, 02),TkStatus.PSUChannelHistory)[0])[1])
0920 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 11, 38, 02),TkStatus.PSUChannelHistory)[0])[1]
0921 #TkStatus.updateO2OQuery(rowUnmap4)
0922 #print "Turning OFF HV crosstalking channel cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel003 number of channels off is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,11,38,07),TkStatus.PSUChannelHistory)[0]])
0923 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 11, 38, 07),TkStatus.PSUChannelHistory)[0])[1])
0924 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 11, 38, 07),TkStatus.PSUChannelHistory)[0])[1]
0925 #
0926 #Test function that parses ManualO2O.log to get by timeinterval the results of each O2O query
0927 QueryResults=GetQueryResults("ManualO2O.log")
0928 #QueryResultsPickle=open("QueryResultsDict.pkl","wb")
0929 #pickle.dump(QueryResults,QueryResultsPickle)
0930 
0931 #Now that we have this we can do:
0932 counter=0
0933 hours=72 #introduced to test only a few hours, setting it to 1000 to process ALL IOVS. 
0934 for interval in sorted(QueryResults.keys()):
0935     counter+=1
0936     if counter<hours: #Hours
0937         print("Updating TkStatus with query results for time interval %s to %s"%interval)
0938         for row in QueryResults[interval]:
0939             TkStatus.updateO2OQuery(row)
0940             print(len(TkStatus.PSUChannelHistory))
0941 if counter-hours>0:
0942     print("Number of intervals skipped %s"%(counter-hours))
0943 #Dump the PSUChannelHistory dictionary!
0944 #TkHistoryPickle=open("TkPSUChannelHistory.pkl","wb")
0945 #pickle.dump(TkStatus.PSUChannelHistory,TkHistoryPickle)
0946 #TkHistoryPickle.close()
0947 #
0948 #for timestamp in sorted(TkStatus.PSUChannelHistory.keys()):
0949 #    if len(TkStatus.PSUChannelHistory[timestamp])<4000:
0950 #        print len(TkStatus.PSUChannelHistory[timestamp]),timestamp                     
0951 
0952 #Test getters as I implement them:
0953 #ChannelsOFF,ChannelsON,Start,Stop=TkStatus.getPSUChannelsOff(datetime.datetime(2010,8,27,10,38,26,700000))
0954 #print "IOV: %s -> %s"%(Start,Stop)
0955 #print "Number of PSU channels reported OFF: %s"%len(ChannelsOFF)
0956 #print "Number of PSU channels reported ON: %s"%len(ChannelsON)
0957 #DetIDsHVOFF,DetIDsHVON,Start,Stop=TkStatus.getDetIDsHVOff(datetime.datetime(2010,8,27,10,38,26,700000))
0958 #print "IOV: %s -> %s"%(Start,Stop)
0959 #print "Number of detID reported with HV OFF: %s"%len(DetIDsHVOFF)
0960 #print "Number of detID reported with HV ON: %s"%len(DetIDsHVON)
0961 #DetIDsLVOFF,DetIDsLVON,Start,Stop=TkStatus.getDetIDsLVOff(datetime.datetime(2010,8,27,10,38,26,700000))
0962 #print "IOV: %s -> %s"%(Start,Stop)
0963 #print "Number of detID reported with LV OFF: %s"%len(DetIDsLVOFF)
0964 #print "Number of detID reported LV ON: %s"%len(DetIDsLVON)
0965 #AliasesHVOFF,AliasesHVON,Start,Stop=TkStatus.getAliasesHVOff(datetime.datetime(2010,8,27,10,38,26,700000))
0966 #print "IOV: %s -> %s"%(Start,Stop)
0967 #print "Number of aliases reported with HV OFF: %s"%len(AliasesHVOFF)
0968 #print "Number of aliases reported HV ON: %s"%len(AliasesHVON)
0969 #AliasesLVOFF,AliasesLVON,Start,Stop=TkStatus.getAliasesLVOff(datetime.datetime(2010,8,27,10,38,26,700000))
0970 #print "IOV: %s -> %s"%(Start,Stop)
0971 #print "Number of aliases reported with LV OFF: %s"%len(AliasesLVOFF)
0972 #print "Number of aliases reported LV ON: %s"%len(AliasesLVON)
0973 
0974 #Check the number of channels off in the Run144086 IOV:
0975 #DetIDsHVOFF,DetIDsHVON,Start,Stop=TkStatus.getDetIDsHVOff(datetime.datetime(2010,8,29,8,0,0,0))
0976 #print "IOV: %s -> %s"%(Start,Stop)
0977 #print "Number of detID reported with HV OFF: %s"%len(DetIDsHVOFF)
0978 #print "Number of detID reported with HV ON: %s"%len(DetIDsHVON)
0979 #DetIDsLVOFF,DetIDsLVON,Start,Stop=TkStatus.getDetIDsLVOff(datetime.datetime(2010,8,29,8,0,0,0))
0980 #print "IOV: %s -> %s"%(Start,Stop)
0981 #print "Number of detID reported with LV OFF: %s"%len(DetIDsLVOFF)
0982 #print "Number of detID reported LV ON: %s"%len(DetIDsLVON)
0983 
0984 #Now implement the following test:
0985 #-Read in all the logs (either the DetVOffReaderDebug or the ones produced by the MakeTkMaps
0986 #-Extract the IOV from the filename
0987 #-Get the corresponding IOVs list from the PSUChannelHistory
0988 #-Print out the number of DetIDs with LV ON/OFF (and of crosstalking and unmapped detids separately)
0989 #-DetID comparison for each IOV (picking the stable one in PSUChannelHistory)
0990 #-Implement the reduction as a method of TrkVoltageStatus that takes as argument deltaTmin and  MaxIOVlength
0991 #and returns the lists of HV and LV channels off (IOVs are for any change of either HV or LV PSU channels)
0992 #-Check the reduction result (number of IOVs, suppression during ramp up/down, ...)
0993 #-DetID comparison for each IOV
0994 #-Implement the direct query check of the query results and report anomalies (old results at the interface of the 1-hour queries:
0995 #   -Get the query dumped by Coral by turning on verbosity and off
0996 #   -Pick the query from the code too
0997 #-Mapping part, integrate results of dictionaries from HV mapping code from Ben, dump all relevant dictionaries
0998 #Cross-check stuff with cabling info?
0999 #Check time changes and effect DST etc timezone issues and plotting limitations...
1000 #Add number of ramp up/downs plots minima/maxima of HV/LV OFF/ON
1001 #Look out for individual channels going down (trips) instead of full ramping
1002 #Talk to Online guys about cross-talking and HV mapping
1003 #Look into the query with the turn off commands and into giving the map for the new table to Robert
1004 #Talk with Frank about extra info needed
1005 #Look into Pixel integration extras
1006 
1007 
1008 ## #This next step is REALLY SLOW if there are thousands of IOVs to handle!
1009 ## 
1010 ## #Turn debug off to avoid a lot of printouts from getIOV:
1011 ## TkStatus.debug=False
1012 ## DetIDsHVOffVsTime,LastHVTimeStamp=TkStatus.getDetIDsHVOffVsTime()
1013 ## TkStatus.debug=True
1014 ## print "There are %s timestamps for DetIDs HV Off. The last timestamp (end of last IOV) is %s"%(len(DetIDsHVOffVsTime),LastHVTimeStamp)
1015 ## 
1016 ## #Turn debug off to avoid a lot of printouts from getIOV:
1017 ## TkStatus.debug=False
1018 ## DetIDsLVOffVsTime,LastLVTimeStamp=TkStatus.getDetIDsLVOffVsTime()
1019 ## TkStatus.debug=True
1020 ## print "There are %s timestamps for DetIDs LV Off. The last timestamp (end of last IOV) is %s"%(len(DetIDsLVOffVsTime),LastLVTimeStamp)
1021 ## 
1022 ## #Prepare the data for LV/HV OFF Graph plotting with PyROOT:
1023 ## import array, time
1024 ## 
1025 ## #Initialize the lists of timestamps (to be doubled to get the histogram/IOV look)
1026 ## LVOFFTimestamps=[]
1027 ## NumberOfLVOFFChannels=[]
1028 ## #Loop over the list of tuples with (timestamp,numberofdetidswithLVOFF):
1029 ## for item in sorted(DetIDsLVOffVsTime):
1030 ##     #FIXME:
1031 ##     #NEED to check the effect/use of tzinfo to avoid issues with UTC/localtime DST etc.
1032 ##     #Will have to massage the datetime.datetime object in a proper way not necessary now.
1033 ##     #Also need to decide if we want add the milliseconds into the "timestamps for plotting in root... not straightforward so will implement only if needed. For now know that the approximation is by truncation for now!
1034 ##     timestamp=int(time.mktime(item[0].timetuple())) #Need to get first a time tuple, then translate it into a unix timestamp (seconds since epoc). This means no resolution under 1 second by default
1035 ##     if item==DetIDsLVOffVsTime[0]: #First item does not need duplication
1036 ##         LVOFFTimestamps.append(timestamp)
1037 ##         LVOff=item[1]
1038 ##         NumberOfLVOFFChannels.append(LVOff)
1039 ##     else: #Add twice each timestamp except the first one (to make a histogram looking graph)
1040 ##         LVOFFTimestamps.append(timestamp)
1041 ##         NumberOfLVOFFChannels.append(LVOff) #Input previous value with new timestamp
1042 ##         LVOff=item[1]
1043 ##         LVOFFTimestamps.append(timestamp)
1044 ##         NumberOfLVOFFChannels.append(LVOff) #Input new value with new timestamp
1045 ##         
1046 ## LVOFFTimestampsArray=array.array('i',LVOFFTimestamps)
1047 ## NumberOfLVOFFChannelsArray=array.array('i',NumberOfLVOFFChannels)
1048 ## 
1049 ## #Initialize the lists of timestamps (to be doubled to get the histogram/IOV look)
1050 ## HVOFFTimestamps=[]
1051 ## NumberOfHVOFFChannels=[]
1052 ## #Loop over the list of tuples with (timestamp,numberofdetidswithHVOFF):
1053 ## for item in sorted(DetIDsHVOffVsTime):
1054 ##     timestamp=int(time.mktime(item[0].timetuple()))
1055 ##     if item==DetIDsHVOffVsTime[0]: #First item does not need duplication
1056 ##         HVOFFTimestamps.append(timestamp)
1057 ##         HVOff=item[1]
1058 ##         NumberOfHVOFFChannels.append(HVOff)
1059 ##     else: #Add twice each timestamp except the first one (to make a histogram looking graph)
1060 ##         HVOFFTimestamps.append(timestamp)
1061 ##         NumberOfHVOFFChannels.append(HVOff) #Input previous value with new timestamp
1062 ##         HVOff=item[1]
1063 ##         HVOFFTimestamps.append(timestamp)
1064 ##         NumberOfHVOFFChannels.append(HVOff) #Input new value with new timestamp
1065 ##         
1066 ## HVOFFTimestampsArray=array.array('i',HVOFFTimestamps) #NEED TO USE DOUBLES if we want the microseconds!!!! or it screws up approximating the timestamp
1067 ## NumberOfHVOFFChannelsArray=array.array('i',NumberOfHVOFFChannels)
1068 ## 
1069 ## #Testing the plotting function
1070 ## TkStatus.plotGraphSeconds(LVOFFTimestampsArray,NumberOfLVOFFChannelsArray,GraphTitle="Modules with LV OFF",YTitle="Number of modules with LV OFF",GraphFilename="Test2.gif")
1071 ## #TkStatus.plotGraphSeconds(LVOFFTimestampsArray,NumberOfLVOFFChannelsArray,"Modules with LV OFF","Number of modules with LV OFF","Test2.gif")
1072 
1073 #Develop the Validation based on the TkStatus object on one side and the actual output of the O2O reader (CheckAllIOVs) in the current dir:
1074 
1075 #FIXME:
1076 #Issues to look into:
1077 #1-first IOV not starting from beginning of the query: that should always be the case!
1078 #2-Reduction seems to not be doing the right thing...
1079 #3-timestamp timezone translation...
1080 
1081 #Get the ReducedIOVsTimestamps (using the TkVoltage.getReducedIOVs() using the same deltaT=15s, and maxIOVSequenceLength=120s):
1082 ReducedIOVsTimestamps=TkStatus.getReducedIOVs(datetime.datetime(2010, 8, 27, 12, 0),datetime.datetime(2010, 8, 29, 17, 45),TkStatus.PSUChannelHistory,15,120)
1083 #Print out for debugging the reduced timestamps with their index (to see when a sequence is present) and the number of LV/HV channels OFF for the given timestamp in the TkStatus object!
1084 print("Dumping ReducedIOVsTimestamps and the corresponding timestamp index and number of HV/LV channels off:")
1085 for timestamp in ReducedIOVsTimestamps:
1086     TkStatus.debug=False
1087     print(timestamp,sorted(TkStatus.PSUChannelHistory.keys()).index(timestamp),len(TkStatus.getDetIDsHVOff(timestamp)[0]), len(TkStatus.getDetIDsLVOff(timestamp)[0]))
1088 
1089 #Following function will be moved inside the TkVoltageStatus class once it's perfected:
1090 def ReducedIOVsHistory(ReducedIOVsTimestamps):
1091     """
1092     Function that given a list of reduced IOVs timestamps (output of getReducedIOVs()), analyses the timestamps to identify the IOV sequences and treats them differently when ramping-up or ramping-down, to return a dictionary that has timestamp as a key and the list of LV and HV channels OFF, that can be compared with the content of the CheckAllIOVs output DetVOffReaderDebug logs for validation.
1093     """
1094     
1095     AllTimestamps=sorted(TkStatus.PSUChannelHistory.keys())
1096     PreviousTimestampIndex=-1
1097     ReducedIOVsDict={}
1098     for timestamp in ReducedIOVsTimestamps:
1099         if AllTimestamps.index(timestamp)!=(PreviousTimestampIndex+1): #Sequence end!
1100             #Get the current timestamp LVOff channels:
1101             LVOffEnd=TkStatus.getDetIDsLVOff(timestamp)[0]
1102             #and the LV Off ones for the previous timestamp (beginning of the sequence):
1103             LVOffStart=TkStatus.getDetIDsLVOff(AllTimestamps[PreviousTimestampIndex])[0]
1104             #Get the current timestamp HVOff channels:
1105             HVOffEnd=TkStatus.getDetIDsHVOff(timestamp)[0]
1106             #and the HV Off ones for the previous timestamp (beginning of the sequence):
1107             HVOffStart=TkStatus.getDetIDsHVOff(AllTimestamps[PreviousTimestampIndex])[0]
1108             #This can be complicated... let's go for the same approach as the official O2O reduction
1109             #We can just test if the other conditions not captured by the ifs ever happen!
1110             #Turning OFF case:
1111             if len(LVOffEnd)>len(LVOffStart) or len(HVOffEnd)>len(HVOffStart): 
1112                 ReducedIOVsDict.update({AllTimestamps[PreviousTimestampIndex]:(TkStatus.getDetIDsHVOff(timestamp)[0],TkStatus.getDetIDsLVOff(timestamp)[0])}) #use the LVOff/HVOff form the last element in the sequence and set the first element of the sequence to it!
1113             #Turning (Staying) ON case:
1114             #Nothing special to do (same as not a sequence)! We're happy with having thrown away all the intermediate timestamps, and keep the validity of the first timestamp of the sequence throughout the sequence:
1115         #For all timestamps reported (if they are a start of a sequence they will be "overwritten" once we process the end of the sequence timestamp) in particular also the end of a ramp-up sequence does not need no special treatement:
1116         #Actually check if the LV Off or HVOff are the same as the previous timestamp: if they are do nothing, if they are not then add an IOV...
1117         if set(TkStatus.getDetIDsHVOff(timestamp)[0])!=set(TkStatus.getDetIDsHVOff(AllTimestamps[PreviousTimestampIndex])[0]) or set(TkStatus.getDetIDsLVOff(timestamp)[0])!=set(TkStatus.getDetIDsLVOff(AllTimestamps[PreviousTimestampIndex])[0]):
1118             ReducedIOVsDict.update({timestamp:(TkStatus.getDetIDsHVOff(timestamp)[0],TkStatus.getDetIDsLVOff(timestamp)[0])})
1119         PreviousTimestampIndex=AllTimestamps.index(timestamp)
1120     return ReducedIOVsDict
1121 
1122 #Now using the ReducedIOVs timestamps we can get the actual ReducedIOVs using the same algorithm as the O2O (implemented in the function ReducedIOVsHistory):
1123 ValidationReducedIOVsHistory=ReducedIOVsHistory(ReducedIOVsTimestamps)
1124 #Print out for debugging the timestamp, the number of HV/LV channels off from the TkStatus object for each of the REDUCED timestamps (AGAIN)
1125 #for timestamp in ReducedIOVsTimestamps:
1126 #    print timestamp,len(TkStatus.getDetIDsHVOff(timestamp)[0]),len(TkStatus.getDetIDsLVOff(timestamp)[0])
1127 
1128 #Print out for debugging the timestamp, the number of HV/LV channels OFF from the ValidationReducedIOVsHistory object directly!
1129 #i=0
1130 print("Dumping ValidationReducedIOVsHistory contents:")
1131 for timestamp in sorted(ValidationReducedIOVsHistory.keys()):
1132     print(timestamp, len(ValidationReducedIOVsHistory[timestamp][0]),len(ValidationReducedIOVsHistory[timestamp][1]))#,sorted(O2OReducedIOVs.keys())[i],len(O2OReducedIOVs[sorted(O2OReducedIOVs.keys())[i]][0]),len(O2OReducedIOVs[sorted(O2OReducedIOVs.keys())[i]][1])
1133 #    i=i+1
1134     
1135 #for i in range(42):
1136 #    print sorted(ValidationReducedIOVsHistory.keys())[i],sorted(O2OReducedIOVs.keys())[i]
1137 
1138 #Now extract the DetVOffInfo from the logfiles directly and then we can compare!
1139 
1140 #Cut&Paste of a quick and dirty script that reads the CheckAllIOVS.py output (ReaderDebug) logs and produces an O2OData dictionary
1141 #that has timestamps (from filename) as keys and a tuple (HVOff,LVOff) as value, where HVOff and LVOff are lists of detids that have HV/LV off respectively.
1142 #The union of the two would be the total number of detids listed as OFF (in either way, OFF-OFF or OFF-ON)
1143 #Can implement a direct IOV comparison using the reduction above (need to be careful on the matching of IOVs)
1144 #import datetime
1145 def ExtractDetVOffInfo(directory=os.getcwd()):
1146     """
1147     Function that given a directory (defaults to the local one in case no dir indicated), parses all local DetVOffReaderDebug*.log files and returna a dictionary with timestamps for keys and a tuple with the list of LV and HV channels OFF (LVOff,HVOff). 
1148     """
1149     ls=os.listdir(directory)
1150     TimesLogs=[]
1151     O2OData={}
1152     for log in ls:
1153         if "DetVOffReaderDebug__FROM" in log:
1154             (start,end)=log[:-4].split("FROM_")[1].split("_TO_")
1155             TimeStamp=datetime.datetime.strptime(start.replace("__","_0"),"%a_%b_%d_%H_%M_%S_%Y")
1156             #print start,TimeStamp
1157             file=open(log,'r')
1158             filelines=file.readlines()
1159             file.close()
1160             LVOff=[]
1161             HVOff=[]
1162             for line in filelines:
1163                 #print line
1164                 if "OFF" in line:
1165                     detid,hv,lv=line.split()
1166                     #print line,detid,hv,lv
1167                     if hv=="OFF":
1168                         HVOff.append(int(detid))
1169                     if lv=="OFF":
1170                         LVOff.append(int(detid))
1171                         
1172                     O2OData.update({TimeStamp:(HVOff,LVOff)})
1173     return O2OData
1174 
1175 #Extract the O2O Reduced IOVs data from the logfiles in the current directory 
1176 O2OReducedIOVs=ExtractDetVOffInfo()
1177 #Print out for debugging the timestamp, the number of HV/LV  channels OFF reported by the O2O
1178 print("Dumping the O2OReducedIOVs contents:")
1179 for timestamp in sorted(O2OReducedIOVs.keys()):
1180     print(timestamp, len(O2OReducedIOVs[timestamp][0]),len(O2OReducedIOVs[timestamp][1]))#,len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(timestamp,TkStatus.PSUChannelHistory)[0])[0]), len(TkStatus.getDetIDsLVOff(TkStatus.getIOV(timestamp,TkStatus.PSUChannelHistory)[0])[0])
1181     #Compare the actual detids after doing the reduction the way we want to do it!
1182     
1183 # len(TkStatus.getDetIDsHVOff(sorted(TkStatus.PSUChannelHistory.keys())[-1])[0])
1184 
1185 
1186 #Now compare the reduced histories:
1187 def CompareReducedDetIDs(FirstDict,SecondDict):
1188     """
1189     Function that given 2 Dictionaries (key=timestamp, value=(LVOff,HVOff)) loops through the first one and compared the content of its IOVs with the IOV in the other dict that overlaps with it.
1190     """
1191     DifferenceDict={}
1192     for timestamp in sorted(FirstDict.keys()):
1193         if timestamp.replace(microsecond=0) in SecondDict.keys():
1194             secondtimestamp=timestamp.replace(microsecond=0)
1195             print("Timestamp %s is present in both Dictionaries!"%timestamp)
1196         else:
1197             secondtimestamps=sorted(SecondDict.keys())
1198             secondtimestamps.append(timestamp)
1199             secondtimestamps.sort()
1200             if secondtimestamps.index(timestamp)!=0:#To avoid wrapping up to the end of the list of timestamps!!!
1201                 secondtimestamp=secondtimestamps[secondtimestamps.index(timestamp)-1]
1202             else:#Default to the earliest timestamp in the second dictionary...
1203                 secondtimestamp=secondtimestamps[secondtimestamps.index(timestamp)+1]
1204             print("Comparing the IOV with timestamp %s (1st dict) with IOV with timestamp %s (2nd dict)"%(timestamp,secondtimestamp)) 
1205         if set(map(lambda x:int(x),FirstDict[timestamp][0]))!=set(map(lambda x:int(x),SecondDict[secondtimestamp][0])) or set(map(lambda x:int(x),FirstDict[timestamp][1]))!=set(map(lambda x:int(x),SecondDict[secondtimestamp][1])): #Change!
1206             if len(set(FirstDict[timestamp][0]))<=len(set(SecondDict[secondtimestamp][0])):
1207                 differenceHV=set(map(lambda x:int(x),SecondDict[secondtimestamp][0]))-set(map(lambda x:int(x),FirstDict[timestamp][0]))
1208             else:
1209             #elif len(set(SecondDict[secondtimestamp][0]))<len(set(FirstDict[timestamp][0])):
1210                 differenceHV=set(map(lambda x:int(x),FirstDict[timestamp][0]))-set(map(lambda x:int(x),SecondDict[secondtimestamp][0]))
1211             #else:
1212             #    print "SCREAM! Something weird going on one of the two should be a subset of the other!"
1213             #    differenceLV=set([])
1214             #    differenceHV=set([])
1215             if len(set(FirstDict[timestamp][1]))<=len(set(SecondDict[secondtimestamp][1])):
1216                 differenceLV=set(map(lambda x:int(x),SecondDict[secondtimestamp][1]))-set(map(lambda x:int(x),FirstDict[timestamp][1]))
1217             else:
1218             #elif set(SecondDict[secondtimestamp][1]).issubset(set(FirstDict[timestamp][1])):
1219                 differenceLV=set(map(lambda x:int(x),FirstDict[timestamp][1]))-set(map(lambda x:int(x),SecondDict[secondtimestamp][1]))
1220             #else:
1221             #    print "SCREAM! Something weird going on one of the two should be a subset of the other!"
1222             #    differenceLV=set([])
1223             #    differenceHV=set([])
1224             DifferenceDict.update({(timestamp,secondtimestamp):(differenceHV,differenceLV)})
1225             print("Difference in timestamp %s (corresponding to %s):"%(timestamp,secondtimestamp))
1226             #print "LV OFF:"
1227             #for LVChannel in differenceLV:
1228             #    print LVChannel
1229             #print "HV OFF:"
1230             #for HVChannel in differenceHV:
1231             #    print HVChannel
1232         else:
1233             print("Timestamp %s is identical in both dictionaries"%timestamp)
1234     return DifferenceDict
1235 Comparison=CompareReducedDetIDs(ValidationReducedIOVsHistory,O2OReducedIOVs)
1236 print("Dumping the results of the comparisons of the two dictionaries:")
1237 for timestamps in sorted(Comparison.keys()):
1238     print(timestamps, Comparison[timestamps])
1239     if Comparison[timestamps][0]:
1240         print("HV:")
1241         if Comparison[timestamps][0].issubset(set(O2OReducedIOVs[timestamps[1]][0])):
1242             print("Only in O2O Dict!")
1243         else:
1244             print("Only in Validation Dict!")
1245         for detid in Comparison[timestamps][0]:
1246             print(detid,TkStatus.DetIDAliasDict[detid])
1247     if Comparison[timestamps][1]:
1248         print("LV:")
1249         if Comparison[timestamps][1].issubset(set(O2OReducedIOVs[timestamps[1]][1])):
1250             print("Only in O2O Dict!")
1251         else:
1252             print("Only in Validation Dict!")
1253         for detid in Comparison[timestamps][1]:
1254             print(detid,TkStatus.DetIDAliasDict[detid])
1255 
1256 
1257 #Add a check with the query using sqlalchemy
1258 #Add a check for channels that never show updates in the queries (even when several ramp cycles happened)
1259 #Add a check that inside the sequence change is consistently up or down
1260 
1261 #Check of the direct DB query:
1262 #Need to first run another python script (using the local python of cmstrko2ovm02 that has sqlalchemy, cx-oracle etc) that dumps a pkl with a list of the result rows of the O2O query....
1263 
1264 TkStatusFromQuery=TrkVoltageStatus(detIDPSUmapfilename=os.path.join(os.getenv('CMSSW_BASE'),'src/CalibTracker/SiStripDCS/data/StripPSUDetIDMap_FromJan132010_Crosstalk.dat'),DetIDAliasFile=os.path.join(os.getenv('CMSSW_BASE'),'src/CalibTracker/SiStripDCS/data/StripDetIDAlias.pkl'),startTime=datetime.datetime(2010,8,27,10,00,00),debug=True)
1265 DBQueryPickle=open('/afs/cern.ch/user/g/gbenelli/scratch0/O2OValidation/QueryResults.pkl','rb')
1266 DBQueryResults=[]
1267 DBQueryResults=pickle.load(DBQueryPickle)
1268 DBQueryPickle.close()
1269 for row in DBQueryResults:
1270     TkStatusFromQuery.updateO2OQuery(row)
1271     print(len(TkStatusFromQuery.PSUChannelHistory))
1272 
1273 delta=datetime.timedelta(seconds=7200)
1274 counter=0
1275 DifferingTimestamps=[]
1276 for timestamp in sorted(TkStatus.PSUChannelHistory.keys()):
1277     if timestamp-delta!=sorted(TkStatusFromQuery.PSUChannelHistory.keys())[counter]:
1278         print(timestamp, sorted(TkStatusFromQuery.PSUChannelHistory.keys())[counter])
1279         DifferingTimestamps.append(timestamp,sorted(TkStatusFromQuery.PSUChannelHistory.keys())[counter])
1280     counter=counter+1
1281 if DifferingTimestamps:
1282     print("There are %s timestamps that are different in the 2 TkVoltageStatus objects!"%len(DifferingTimestamps))
1283 
1284 #Test issue with channel000 and channel001 lagging:
1285 #LVChannelsHistory={}
1286 #for timestamp in sorted(TkStatus.PSUChannelHistory.keys()):
1287 #    LVOFF=[psuchannel for psuchannel in TkStatus.getPSUChannelsOff(timestamp) if (psuchannel.endswith('0') or psuchannel.endswith('1'))]
1288 #    for channel in LVOFF:
1289 LVChannelHistory={}
1290 HVChannelHistory={}
1291 for interval in QueryResults.keys():
1292     for row in QueryResults[interval]:
1293         if row['dpname'].endswith('0') or row['dpname'].endswith('1'):
1294             if row['dpname'][:-1] not in LVChannelHistory.keys():
1295                 LVChannelHistory.update({row['dpname'][:-1]:[(row['change_date'],row['actual_status'])]})
1296             else:
1297                 LVChannelHistory[row['dpname'][:-1]].append((row['change_date'],row['actual_status']))
1298         else:
1299             if row['dpname'] not in HVChannelHistory.keys():
1300                 HVChannelHistory.update({row['dpname']:[(row['change_date'],row['actual_status'])]})
1301             else:
1302                 HVChannelHistory[row['dpname']].append((row['change_date'],row['actual_status']))
1303         
1304         #if row['change_date']==datetime.datetime(2010, 8, 28, 22, 52, 8, 994000):
1305         #    print row
1306         #if row['change_date']==datetime.datetime(2010, 8, 28, 22, 51, 8, 994000):
1307         #    print row
1308         if row['change_date']>datetime.datetime(2010, 8, 28, 22, 44) and row['change_date']<datetime.datetime(2010, 8, 28, 23, 5,37):
1309             print(row)    
1310             
1311         
1312 
1313 for row in sorted(HVChannelHistory['cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_6/branchController02/easyCrate3/easyBoard15/channel002']):
1314     print(row[0],row[1])
1315 for row in sorted(HVChannelHistory['cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_6/branchController02/easyCrate3/easyBoard15/channel003']):
1316     print(row[0],row[1])
1317 print(TkStatus.PSUChannelHistory[sorted(TkStatus.PSUChannelHistory.keys())[sorted(TkStatus.PSUChannelHistory.keys()).index(datetime.datetime(2010, 8, 28, 22, 52, 8, 994000))-1]])
1318 a=TkStatus.getIOV(datetime.datetime(2010, 8, 28, 22, 51, 16),TkStatus.PSUChannelHistory)
1319 
1320 #PLOTTTING!
1321 import array, time
1322 TOBminus_1_3_1_4_HV1_Status=HVChannelHistory['cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_6/branchController02/easyCrate3/easyBoard15/channel002']
1323 #Select time range via:
1324 TOBminus_1_3_1_4_HV1_Status=[item for item in HVChannelHistory['cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_6/branchController02/easyCrate3/easyBoard15/channel002'] if item[0]>datetime.datetime(2010,8,28,22) and item[0]<datetime.datetime(2010,8,29,2)]
1325 TOBTimestamps=[]
1326 TOBHVStatus=[]
1327 for item in TOBminus_1_3_1_4_HV1_Status:
1328     timestamp=int(time.mktime(item[0].timetuple()))
1329     if item==TOBminus_1_3_1_4_HV1_Status[0]: #First item does not need duplication
1330         TOBTimestamps.append(timestamp)
1331         HVStatus=int(item[1]==1)
1332         TOBHVStatus.append(HVStatus)
1333     else: #Add twice each timestamp except the first one (to make a histogram looking graph)
1334         TOBTimestamps.append(timestamp)
1335         TOBHVStatus.append(HVStatus) #Input previous value with new timestamp
1336         HVStatus=int(item[1]==1)
1337         TOBTimestamps.append(timestamp)
1338         TOBHVStatus.append(HVStatus) #Input new value with new timestamp
1339 
1340 
1341 #TOBTimestamps=map(lambda x: int(time.mktime(x[0].timetuple())),TOBminus_1_3_1_4_HV1_Status)
1342 #TOBHVStatus=map(lambda y: int(y[1]==1),TOBminus_1_3_1_4_HV1_Status)
1343 ##print "FIRST the original arrays:"
1344 ###Duplication of timestamps...
1345 ##for timestamp in [item[0] for item in TOBminus_1_3_1_4_HV1_Status]:
1346 ##    print timestamp,TOBHVStatus[TOBTimestamps.index(timestamp)]
1347 ##    TOBTimestamps.insert(TOBTimestamps.index(timestamp)+1,timestamp)
1348 ##    TOBHVStatus.insert(TOBTimestamps.index(timestamp)+1,TOBHVStatus[TOBTimestamps.index(timestamp)-1])
1349 ##print 'NOW "duplicated" arrays'
1350 ##for timestamp in TOBTimestamps:
1351 ##    print timestamp, TOBHVStatus[TOBTimestamps.index(timestamp)]
1352 ##        
1353 TOBTimestampsArray=array.array('i',TOBTimestamps)
1354 TOBHVStatusArray=array.array('i',TOBHVStatus)
1355 
1356 TkStatus.plotGraphSeconds(TOBTimestampsArray,TOBHVStatusArray,GraphTitle="PSUChannelGraph",YTitle="Channel HV Status",GraphFilename="PSUChannel.gif")
1357 TOBminus_1_3_1_4_HV1_Status=HVChannelHistory['cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_6/branchController02/easyCrate3/easyBoard15/channel003']
1358 #Select time range via:
1359 TOBminus_1_3_1_4_HV1_Status=[item for item in HVChannelHistory['cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_6/branchController02/easyCrate3/easyBoard15/channel003'] if item[0]>datetime.datetime(2010,8,28,22) and item[0]<datetime.datetime(2010,8,29,2)]
1360 TOBTimestamps=[]
1361 TOBHVStatus=[]
1362 for item in TOBminus_1_3_1_4_HV1_Status:
1363     timestamp=int(time.mktime(item[0].timetuple()))
1364     if item==TOBminus_1_3_1_4_HV1_Status[0]: #First item does not need duplication
1365         TOBTimestamps.append(timestamp)
1366         HVStatus=int(item[1]==1)
1367         TOBHVStatus.append(HVStatus)
1368     else: #Add twice each timestamp except the first one (to make a histogram looking graph)
1369         TOBTimestamps.append(timestamp)
1370         TOBHVStatus.append(HVStatus) #Input previous value with new timestamp
1371         HVStatus=int(item[1]==1)
1372         TOBTimestamps.append(timestamp)
1373         TOBHVStatus.append(HVStatus) #Input new value with new timestamp
1374 
1375 
1376 #TOBTimestamps=map(lambda x: int(time.mktime(x[0].timetuple())),TOBminus_1_3_1_4_HV1_Status)
1377 #TOBHVStatus=map(lambda y: int(y[1]==1),TOBminus_1_3_1_4_HV1_Status)
1378 ##print "FIRST the original arrays:"
1379 ###Duplication of timestamps...
1380 ##for timestamp in [item[0] for item in TOBminus_1_3_1_4_HV1_Status]:
1381 ##    print timestamp,TOBHVStatus[TOBTimestamps.index(timestamp)]
1382 ##    TOBTimestamps.insert(TOBTimestamps.index(timestamp)+1,timestamp)
1383 ##    TOBHVStatus.insert(TOBTimestamps.index(timestamp)+1,TOBHVStatus[TOBTimestamps.index(timestamp)-1])
1384 ##print 'NOW "duplicated" arrays'
1385 ##for timestamp in TOBTimestamps:
1386 ##    print timestamp, TOBHVStatus[TOBTimestamps.index(timestamp)]
1387 ##        
1388 TOBTimestampsArray=array.array('i',TOBTimestamps)
1389 TOBHVStatusArray=array.array('i',TOBHVStatus)
1390 
1391 TkStatus.plotGraphSeconds(TOBTimestampsArray,TOBHVStatusArray,GraphTitle="PSUChannelGraph",YTitle="Channel HV Status",GraphFilename="PSUChannelHV2.gif")
1392 def plotPSUChannelvsTime(self,TimeArray,ValuesArray,GraphTitle="PSUChannelGraph",YTitle="Channel HV Status",GraphFilename="PSUChannel.gif"):
1393     """
1394     Function that given an array with timestamps (massaged to introduce a second timestamp for each value to produce an histogram-looking plot) and a corresponding array with values to be plotted, a title, a Y axis title and a plot filename, produces with pyROOT a time plot and saves it locally.
1395     The function can be used for cumulative plots (number of channels with HV/LV OFF/ON vs time) or for individual (single detID HV/LV status vs time) plots.
1396     """
1397     import ROOT
1398     canvas=ROOT.TCanvas()
1399     graph=ROOT.TGraph(len(TimeArray),TimeArray,ValuesArray)
1400     graph.GetXaxis().SetTimeDisplay(1)
1401     graph.GetXaxis().SetLabelOffset(0.02)
1402     #Set the time format for the X Axis labels depending on the total time interval of the plot!
1403     TotalTimeIntervalSecs=TimeArray[-1]-TimeArray[0]
1404     if TotalTimeIntervalSecs <= 120: #When zooming into less than 2 mins total interval report seconds too
1405         graph.GetXaxis().SetTimeFormat("#splitline{   %d/%m}{%H:%M:%S}")
1406     elif 120 < TotalTimeIntervalSecs < 6400: #When zooming into less than 2 hrs total interval report minutes too
1407         graph.GetXaxis().SetTimeFormat("#splitline{%d/%m}{%H:%M}")
1408     else: #When plotting more than 2 hrs only report the date and hour of day
1409         graph.GetXaxis().SetTimeFormat("#splitline{%d/%m}{  %H}")
1410     graph.GetYaxis().SetTitle(YTitle)
1411     graph.GetYaxis().SetTitleOffset(1.4)
1412     graph.SetTitle(GraphTitle)
1413     graph.Draw("APL")
1414     canvas.SaveAs(GraphFilename)
1415     print("Saved graph as %s"%GraphFilename)
1416     return
1417 
1418 
1419 ReducedIOVsTimestampsTEST=TkStatus.getReducedIOVs(datetime.datetime(2010, 8, 27, 12, 0),datetime.datetime(2010, 8, 29, 17, 45),TkStatus.PSUChannelHistory,2,90)
1420 #Print out for debugging the reduced timestamps with their index (to see when a sequence is present) and the number of LV/HV channels OFF for the given timestamp in the TkStatus object!
1421 print("Dumping ReducedIOVsTimestamps and the corresponding timestamp index and number of HV/LV channels off:")
1422 for timestamp in ReducedIOVsTimestampsTEST:
1423     TkStatus.debug=False
1424     print(timestamp,sorted(TkStatus.PSUChannelHistory.keys()).index(timestamp),len(TkStatus.getDetIDsHVOff(timestamp)[0]), len(TkStatus.getDetIDsLVOff(timestamp)[0]))
1425 ValidationReducedIOVsHistoryTEST=ReducedIOVsHistory(ReducedIOVsTimestampsTEST)
1426 print("Dumping ValidationReducedIOVsHistory contents:")
1427 for timestamp in sorted(ValidationReducedIOVsHistoryTEST.keys()):
1428     print(timestamp, len(ValidationReducedIOVsHistoryTEST[timestamp][0]),len(ValidationReducedIOVsHistoryTEST[timestamp][1]))#,sorted(O2OReducedIOVs.keys())[i],len(O2OReducedIOVs[sorted(O2OReducedIOVs.keys())[i]][0]),len(O2OR