Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-11-25 02:29:11

0001 #! /usr/bin/env python
0002 #Script to parse the output of ManualO2O.py for various debugging tasks
0003 #First use case is to debug issue with HV1/HV2 channels handling
0004 
0005 import os,datetime, pickle
0006 def GetLogTimestamps(ManualO2Ologfilename):
0007     """
0008     This function opens the ManualO2O log chosen, and it prints out the Tmin/Tmax intervals
0009     for the individual O2O jobs that were run.
0010     This information can in turn be used with the function GetLogInfoByTimeStamp,
0011     to extract from the log the (DEBUG) information only from the wanted time interval.
0012     """
0013     ManualO2Olog=open(ManualO2Ologfilename,'r')
0014     LinesCounter=0
0015     IOVCounter=1
0016     TmaxDict={}#This dictionary will have Tmax as keys and the lines corresponding to that Tmax as value
0017     for line in ManualO2Olog:
0018         LinesCounter+=1
0019         if 'Tmin:' in line:
0020             Tmin=line.split(':')[1].strip()
0021         if 'Tmax:' in line:
0022             Tmax=line.split(':')[1].strip()
0023             #Assume that when Tmax is read Tmin was read too since we are parsing lines like this:
0024             #     Tmin: 2010 8 27 10 0 0 0
0025             #     Tmax: 2010 8 29 9 0 0 0
0026             print("%s- Tmin: %s and Tmax: %s (number log lines %s)"%(IOVCounter,Tmin,Tmax,LinesCounter))
0027             #Actually Tmin is not necessary... since they all will be set to the first one in the dcs_o2O_template.py used by the ManualO2O.py that created the log we're parsing.
0028             if Tmax not in ListOfTmax.keys():
0029                 TmaxDict.update({Tmax:LinesCounter})
0030             else:
0031                 print("This Tmax (%s) seems to be duplicated!!!"%Tmax)
0032             LinesCounter=0
0033             IOVCounter+=1
0034     ManualO2Olog.close()
0035     return TmaxDict
0036             
0037             
0038 def GetLogInfoByTimeStamp(ManualO2Ologfilename,StartTime,StopTime):
0039     """
0040     This function takes a ManualO2Ologfilename, a start and a stop timestamps (that should be
0041     in the "YYYY M D H M S"  format, e.g. Tmin: "2010 8 27 10 0 0 0",Tmax: "2010 8 27 11 0 0 0",
0042     and it parses the given logfile to just print out the lines relevant to the given timestamp. 
0043     """
0044     #Open the log:
0045     ManualO2Olog=open(ManualO2Ologfilename,'r')
0046     #Loop to extract only the wanted lines
0047     PrintLines=False
0048     WantedLines=[]
0049     for line in ManualO2Olog:
0050         if 'Tmax:' in line and StopTime in line:
0051             PrintLines=True
0052         if PrintLines:
0053             if 'Tmin:' in line:
0054                 break
0055             else:
0056                 print(line.strip())
0057                 WantedLines.append(line)
0058     ManualO2Olog.close()
0059     return WantedLines
0060 #TO BE DONE!
0061 #implement tzinfo subclasses to get GMT1 behaviour to translate always the timestamp right...
0062 #from datetime import timedelta, datetime, tzinfo
0063 #   import datetime
0064 #   class GMT1(datetime.tzinfo):
0065 #       def __init__(self,dt):         # DST starts last Sunday in March
0066 #           d = datetime.datetime(dt.year, 4, 1)   # ends last Sunday in October
0067 #           self.dston = d - datetime.timedelta(days=d.weekday() + 1)
0068 #           d = datetime.datetime(dt.year, 11, 1)
0069 #           self.dstoff = d - datetime.timedelta(days=d.weekday() + 1)
0070 #       def utcoffset(self, dt):
0071 #           return datetime.timedelta(hours=1) + self.dst(dt)
0072 #       def dst(self, dt):
0073 #           if self.dston <=  dt.replace(tzinfo=None) < self.dstoff:
0074 #               return datetime.timedelta(hours=1)
0075 #           else:
0076 #               return datetime.timedelta(0)
0077 #       def tzname(self,dt):
0078 #           return "GMT +1"
0079 #   
0080 #   import time as _time
0081 #   
0082 #   STDOFFSET = datetime.timedelta(seconds = -_time.timezone)
0083 #   if _time.daylight:
0084 #       DSTOFFSET = datetime.timedelta(seconds = -_time.altzone)
0085 #   else:
0086 #       DSTOFFSET = STDOFFSET
0087 #   
0088 #   DSTDIFF = DSTOFFSET - STDOFFSET
0089 #   
0090 #   class LocalTimezone(datetime.tzinfo):
0091 #   
0092 #       def utcoffset(self, dt):
0093 #           if self._isdst(dt):
0094 #               return DSTOFFSET
0095 #           else:
0096 #               return STDOFFSET
0097 #   
0098 #       def dst(self, dt):
0099 #           if self._isdst(dt):
0100 #               return DSTDIFF
0101 #           else:
0102 #               return ZERO
0103 #   
0104 #       def tzname(self, dt):
0105 #           return _time.tzname[self._isdst(dt)]
0106 #   
0107 #       def _isdst(self, dt):
0108 #           tt = (dt.year, dt.month, dt.day,
0109 #                 dt.hour, dt.minute, dt.second,
0110 #                 dt.weekday(), 0, -1)
0111 #           stamp = _time.mktime(tt)
0112 #           tt = _time.localtime(stamp)
0113 #           return tt.tm_isdst > 0
0114 #   
0115 #   Local = LocalTimezone()
0116 #   
0117 def GetQueryResults(ManualO2Ologfilename):
0118     """
0119     This function takes a ManualO2Ologfilename,
0120     and if it was run with the debug option True,
0121     it extracts all the query results.
0122     It uses StartTime/StopTime to identify the interval of the query (i.e. the ManualO2O.py running)
0123     and uses the tuple (StartTime,StopTime) as a key, as stores the results rows as a list.
0124     It returns the QueryResults dictionary that has for keys the (Tmin,Tmax) tuple, and as values lists of rows,
0125     where the rows are dictionaries with 3 keys:
0126     {
0127     'change_date':datetime.datetime,
0128     'actual_status':int,
0129     'dpname':str
0130     }
0131     The idea is that any massaging of this information is done later, while this function returns the query results in bulk.
0132     """
0133     print("Parsing file %s to extract query results"%ManualO2Ologfilename)
0134     #Open the log:
0135     ManualO2Olog=open(ManualO2Ologfilename,'r')
0136     #Loop to extract the start and stop times
0137     Tmin=""
0138     Tmax=""
0139     ReadRows=False
0140     QueryResultsDict={}
0141     for line in ManualO2Olog:
0142         if 'Tmin:' in line:
0143             if Tmin=="":#This is valid only for the first time... after the  first query use the previous Tmax as Tmin.
0144                 Tmin=datetime.datetime(*[int(x) for x in (line.split(':')[1].strip()).split()])
0145             else:
0146                 Tmin=Tmax
0147         if 'Tmax:' in line:
0148             Tmax=datetime.datetime(*[int(x) for x in (line.split(':')[1].strip()).split()])
0149             #Assume that when Tmax is read Tmin was read too since we are parsing lines like this:
0150             #     Tmin: 2010 8 27 10 0 0 0
0151             #     Tmax: 2010 8 29 9 0 0 0
0152             #print "%s- Tmin: %s and Tmax: %s (number log lines %s)"%(IOVCounter,Tmin,Tmax,LinesCounter)
0153             if (Tmin,Tmax) not in QueryResultsDict.keys():
0154                 QueryResultsDict.update({(Tmin,Tmax):[]})
0155             else:
0156                 print("This Tmax (%s) seems to be duplicated!!!"%Tmax)
0157         if "Dumping all query results" in line:
0158             ReadRows=True
0159         if "Finished dumping query results" in line:
0160             NumOfRows=int(line.split(",")[1].split()[0])
0161             ReadRows=False
0162             #Debug output:
0163             #print "Log reports %s rows, we counted %s rows"%(NumOfRows,len(QueryResultsDict[(Tmin,Tmax)]))
0164         if ReadRows:
0165             Row={}
0166             RowTokens=line.split(",")
0167             for token in RowTokens:
0168                 if 'CHANGE_DATE' in token:
0169                     try:
0170                         #Approximate time to the second, millisecond seems just to be too much information anyway
0171                         #(one can uncomment following line to do detailed tests if needed for a short time interval)
0172                         #Row.update({'change_date':datetime.datetime.strptime(token[token.index(":")+2:-5],"%Y/%m/%d %H:%M:%S")})
0173                         #Complete timestamp including millisecond is parsed with the following commented for now line:
0174                         changeDate=datetime.datetime.strptime(token[token.index(":")+2:-1],"%Y/%m/%d %H:%M:%S.%f")
0175                         #Now introduce the check on the timezone:
0176                         #Based on the fact that we have (in Tmin/Tmax) the 1 hr interval of the query time, we can univocally determine the timezone!
0177                         #FIXME:
0178                         #Should not recalculate this at every query result... just store it!
0179                         #deltaTimezone=datetime.timedelta(seconds=((changeDate-Tmin).seconds/3600)*3600) #Using int / as modulo...
0180                         #The above is wrong! I did not realize that Tmin is also in UTC!
0181                         #Need to implement a subclass of tzinfo to handle it properly all the time... to be done later!
0182                         #For now:
0183                         deltaTimezone=datetime.timedelta(seconds=7200) #DST+GMT+1 (not valid in non DST times but ok for Run144086...
0184                         Row.update({'change_date':changeDate+deltaTimezone})
0185                     except:
0186                         print("WEIRD!\n Timestamp had a format YYYY/MM/DD HH:MM:0SS.millisecond (extra 0 tabbing for seconds!)")
0187                         print(line)
0188                         #Approximate time to the second, millisecond seems just to be too much information anyway
0189                         #(one can uncomment following line to do detailed tests if needed for a short time interval)
0190                         #Row.update({'change_date':datetime.datetime.strptime(token[token.index(":")+2:-5],"%Y/%m/%d %H:%M:0%S")})
0191                         #Complete timestamp including millisecond is parsed with the following commented for now line:
0192                         changeDate=datetime.datetime.strptime(token[token.index(":")+2:-1],"%Y/%m/%d %H:%M:0%S.%f")
0193                         #Now introduce the check on the timezone:
0194                         #Based on the fact that we have (in Tmin/Tmax) the 1 hr interval of the query time, we can univocally determine the timezone!
0195                         #FIXME:
0196                         #Should not recalculate this at every query result... just store it!
0197                         #deltaTimezone=datetime.timedelta(seconds=((changeDate-Tmin).seconds/3600)*3600) #Using int / as modulo...
0198                         #The above is wrong! I did not realize that Tmin is also in UTC!
0199                         #Need to implement a subclass of tzinfo to handle it properly all the time... to be done later!
0200                         #For now:
0201                         deltaTimezone=datetime.timedelta(seconds=7200) #DST+GMT+1 (not valid in non DST times but ok for Run144086...
0202                         Row.update({'change_date':changeDate+deltaTimezone})
0203                 if 'ACTUAL_STATUS' in token:
0204                     Row.update({'actual_status':int(token[token.index(":")+2:-1])})
0205                 if 'DPNAME' in token:
0206                     Row.update({'dpname':token[token.index(":")+2:-2]})
0207             if len(Row)>0: #To avoid the case of an empty Row
0208                 QueryResultsDict[(Tmin,Tmax)].append(Row)
0209     ManualO2Olog.close()
0210     return QueryResultsDict 
0211 
0212 class TrkVoltageStatus:    
0213     """
0214     #How about we make a class and instantiate an object TrackerVoltageStatus that we update at each row:
0215     #1-initialize it with a list of detID (from the most complete maps) OFF and ON (the masked ones):
0216     #2-Has a member history, a dictionary, with keeps, by timestamp (key) a list of detIDs with their channel 0,1,2,[3] status (to decide later if we need to already handle here the checks that
0217     #   a-channel 0==1, for example by having a list of detIDs that do not have matching LV channels statuses (easy to check later, for sure it will be not empty during little transition IOVs due to different sub-second timing... we could introduce deadbands later)
0218     #   b-a LV ON/OFF bit
0219     #   c-a HV ON/OFF bit
0220     #   d-add extra (VMON, V0_SETTINGS) info in the query?
0221     #   e-add the power group turning off info in the query to check the effect
0222     #   f-add a RAMPING UP/DOWN flag on top of the conventional O2O ON/OFF?
0223     #3-define an update(row) method that for each row takes the timestamp:
0224     #   a-if timestamp already exist, updates the channel status of the relevant detIDs contained
0225     #   b-if timestamp does not exist, creates an entry for it carrying over the information for the relevant detIDs from previous timestamp and update it with the information in the row
0226     #4-define a getVoltageHistory(start,end) method to dump the list of IOVs, and for each IOV dump a list with the detIDs and their LV/HV status, the number of LV OFF modules and HV OFF modules, (check that there is no module with LV OFF and HV ON... basic check can be done at  the update level)
0227     """
0228     import datetime,sys,copy
0229     def __init__(self,detIDPSUmapfilename="map.txt",mask_mapfilename="mask_map.txt",DetIDAliasFile="StripDetIDAlias.pkl",startTime=datetime.datetime.now(),debug=False):
0230         """Initialize the object with a timestamp for the start time of the query (defaults to datetime.datetime.now()) and a list of modules OFF and ON depending on masking map (for now all OFF)"""
0231         self.debug=debug
0232         try:
0233             mapfile=open(detIDPSUmapfilename,'r')
0234             maplines=mapfile.readlines()
0235             mapfile.close()
0236             #Extract the detIDs from the map:
0237             self.detIDs=[int(line.split()[0]) for line in maplines]
0238             #Check number of detIDs
0239             #FIXME PIXEL:
0240             #Add a flag for pixels (name of the map?) to implement the check for either detector
0241             if len(list(set(self.detIDs)))!=15148:
0242                 print("There is an issue with the map provided: not reporting 15148 unique detIDs!")
0243             #Parse the PSU channel part of the map
0244             #This one does not list all channels, but follows this convention:
0245             #-detids that are HV mapped list channel002 or channel003
0246             #-detids that are NOT HV mapped list channel000
0247             #-detids that are connected to HV1/HV2 crosstalking PSUs list channel999
0248             self.PSUChannelListed=[line.split()[1] for line in maplines]
0249             #In any case all PSU are listed so we can extract them this way:
0250             self.PSUs=list(set([PSUChannel[:-10] for PSUChannel in self.PSUChannelListed]))
0251             #Check number of PSUs:
0252             if len(self.PSUs)!=1944:
0253                 print("There is an issue with the map provided: not reporting 1944 unique Power Supplies!")
0254             #Building now a list of all PSUChannels
0255             #(since queries to the DB will report results in DPNAMEs, i.e. PSU channels
0256             self.PSUChannels=[]
0257             for PSU in self.PSUs:
0258                 self.PSUChannels.append(PSU+"channel000")
0259                 self.PSUChannels.append(PSU+"channel001")
0260                 self.PSUChannels.append(PSU+"channel002")
0261                 self.PSUChannels.append(PSU+"channel003")
0262             #This part should be unnecessary, since we do not mask any detid anymore...
0263             #But we leave it since the possibility is in the O2O code itself still.
0264             try:
0265                 maskfile=open(mask_mapfilename,'r')
0266                 self.masked_detIDs=[int(detID.rstrip()) for detID in maskfile]
0267                 maskfile.close()
0268             except:
0269                 self.masked_detIDs=[] #empty list of detIDs to be "masked"
0270                 print("No mask map was provided, assuming to start from a complete Tracker OFF state")
0271             try:
0272                 DetIDAlias=open(DetIDAliasFile,"rb")
0273                 DetIDAliasDict=pickle.load(DetIDAlias)
0274                 DetIDAlias.close()
0275                 self.DetIDAliasDict=DetIDAliasDict
0276             except:
0277                 self.DetIDAliasDict={}
0278                 print("No valid detID-Alias map pickle file was provided!")
0279                 print("The TkrVoltageStatus object could not be initialized properly!")
0280                 print("Please use an existing detIDPSUChannel mapfilename as argument!")
0281                 sys.exit()
0282         except:
0283             print("Could not find detID-PSUchannel map to initialize the detIDs")
0284             print("The TkrVoltageStatus object could not be initialized properly! Please use an existing detIDPSUChannel mapfilename as argument!")
0285             sys.exit()
0286         #With the information from the map build the 2 map dictionaries of interest:
0287         #DetID->PSUChannelListed (same convention as the map)
0288         self.DetIDPSUmap={}
0289         for detID in self.detIDs:
0290             self.DetIDPSUmap.update({detID:self.PSUChannelListed[self.detIDs.index(detID)]})
0291         #PSUChannel->DetID map:
0292         self.PSUDetIDmap={}
0293         #This map is a bit more complicated than the DetIDPSUmap (that was just a dictionary form of the given detIDPSUChannel map passed as argument):
0294         #Here we make a 1 to 1 map for all the PSUChannels to all detIDs connected, so for a given PSU:
0295         #1-all detIDs listed as channel000 and as channel999 will be listed for all PSUchannels (000/001/002/003)
0296         #2-all detIDs listed as channel002 will be listed for channel002 and for channel000/001
0297         #3-all detIDs listed as channel003 will be listed for channel003 and for channel000/001
0298         #NOTE:
0299         #For Unmapped and Crosstalking channel even though we list all the detids as above (for all channels of the given PSU), we will handle them separately
0300         #when determining whether the detids are going ON or OFF... see below.
0301         for PSUChannel in self.PSUChannels:
0302             if PSUChannel.endswith("0") or PSUChannel.endswith("1"): #Report all channels that match at the PSU level!
0303                 self.PSUDetIDmap.update({PSUChannel:[detid for detid in self.DetIDPSUmap.keys() if PSUChannel[:-3] in self.DetIDPSUmap[detid]]}) #PSU matching (picks all detids by PSU) 
0304             else: #For channel002 and channel003 list in this map ONLY the actual mapped ones, the unmapped or crosstalking are listed in the corresponding PSUDetIDUnmappedMap and PSUDetIDCrosstalkingMap (see below).
0305                 self.PSUDetIDmap.update({PSUChannel:[detid for detid in self.DetIDPSUmap.keys() if (PSUChannel in self.DetIDPSUmap[detid])]})
0306 
0307         #Separate PSU-based maps for unmapped and crosstalking detids:
0308         self.PSUDetIDUnmappedMap={}
0309         UnmappedPSUs=list(set([psuchannel[:-10] for psuchannel in self.DetIDPSUmap.values() if psuchannel.endswith("000")]))
0310         self.PSUDetIDCrosstalkingMap={}
0311         CrosstalkingPSUs=list(set([psuchannel[:-10] for psuchannel in self.DetIDPSUmap.values() if psuchannel.endswith("999")]))
0312         for PSU in self.PSUs:
0313             if PSU in UnmappedPSUs:
0314                 self.PSUDetIDUnmappedMap.update({PSU:[detid for detid in self.DetIDPSUmap.keys() if (self.DetIDPSUmap[detid].endswith("000") and PSU in self.DetIDPSUmap[detid])]})
0315             if PSU in CrosstalkingPSUs:
0316                 self.PSUDetIDCrosstalkingMap.update({PSU:[detid for detid in self.DetIDPSUmap.keys() if (self.DetIDPSUmap[detid].endswith("999") and PSU in self.DetIDPSUmap[detid])]})
0317         #Need also the list of PSU channels that are unmapped and crosstalking with their status!
0318         #Since the state for those detIDs is determined by the knowledge of the other PSU channel, we need a dictionary for both that keeps their "last value" at all times.
0319         self.UnmappedPSUChannelStatus={}
0320         #Initialize the dictionary with all the unmapped (only HV is relevant, channel002/003) PSU channels set to 0 (off).
0321         for PSU in self.PSUDetIDUnmappedMap.keys():
0322             self.UnmappedPSUChannelStatus.update({PSU+"channel002":0})
0323             self.UnmappedPSUChannelStatus.update({PSU+"channel003":0})
0324         self.CrosstalkingPSUChannelStatus={}
0325         #Initialize the dictionary with all the crosstalking (only HV is relevant, channel002/003) PSU channels set to 0 (off).
0326         for PSU in self.PSUDetIDCrosstalkingMap.keys():
0327             self.CrosstalkingPSUChannelStatus.update({PSU+"channel002":0})
0328             self.CrosstalkingPSUChannelStatus.update({PSU+"channel003":0})
0329         
0330         #Now let's initialize the object (data member) we will use to keep track of the tracker status:
0331         
0332         #Make the startTime of the initialization an attribute of the object:
0333         self.startTime=startTime
0334         #Basic structure of the internal history dictionary, based on query results
0335         self.PSUChannelHistory={startTime:range(len(self.PSUChannels))}
0336         #NOTE:
0337         #Using the indeces to the self.PSUChannels list of PSUChannels to spare memory...
0338         #Strategy:
0339         #keep a timestamped list of indeces to PSUChannels
0340         #Dynamically fetch the PSUChannels (or corresponding detIDs via PSUDetIDmap dict) on the fly whey querying the status.
0341         
0342         #For Masked DetIDs set them always ON (since they would be detIDs that we don't know to with PSU to associate them to
0343         #This should NOT BE THE CASE any longer, since all detIDs are now mapped at least to a PSU
0344         #TOBEIMPLEMENTED:
0345         #(some missing HV channel, but we treat those differently: turn them off in case either HV1/HV2 is OFF)
0346         #(some having cross-talking PSUs, we treat those differently: turn them OFF only if both HV1/HV2 are OFF)
0347         for detID in self.masked_detIDs:
0348             #For each detID get the PSUChannelListed, then remove last digit (the channel number that can be 0,2 or 3)
0349             #and then make sure to remove all channels 000,001,002,003 
0350             self.PSUChannelHistory[startTime].remove(index(detIDPSUmap[detid][-1]+'0')) #Channel000
0351             self.PSUChannelHistory[startTime].remove(index(detIDPSUmap[detid][-1]+'1')) #Channel001
0352             self.PSUChannelHistory[startTime].remove(index(detIDPSUmap[detid][-1]+'2')) #Channel002
0353             self.PSUChannelHistory[startTime].remove(index(detIDPSUmap[detid][-1]+'3')) #Channel003
0354         #Let's try to keep the one following dict that reports the number of detIDs with HV OFF and LV OFF at a given timestamp...
0355         #Not sure even just this dict makes sense... but it is the most used information, quite minimal compare to the status of each detID
0356         #and it provides a global IOV for the whole tracker
0357         #NOTE:
0358         #it won't track the case of a channel going down at the exact time another goes up!
0359         #self.TkNumberOfDetIDsOff={startTime:(len([PSUChannel[i] for i in self.PSUChannelHistory]),len(self.detIDs))} 
0360 
0361     def updateO2OQuery(self,row):
0362         """Function to automatically handle the updating the of history dictionary
0363         with the results rows from the O2O query (PSUchannel name based).
0364         Note that row can be an actual row from SQLAlchemy from direct online DB querying
0365         but also a dictionary extracted parsing ManualO2O.log files.
0366         """
0367         #NOTE:May need to introduce IOV reduction at this level,
0368         #but I'd prefer to handle it separately so to conserve the original data for validation purposes.
0369         #For new IOV create a new entry in the dict and initialize it to the last IOV status
0370         #since only change is reported with STATUS_CHANGE queries.
0371 
0372         #First check if we care about this channel (could be a Pixel channel while we want only Strip channels or viceversa):
0373         if row['dpname'] not in self.PSUChannels:
0374             if self.debug:
0375                 print("Discarding results row since it is not in the currently considered map:\n%s"%row['dpname'])
0376             return        
0377         
0378         #NOTE:
0379         #Actually use the getIOV to allow for "asynchronous" updating:
0380         lastTimeStamp,nextTimeStamp=self.getIOV(row['change_date'],self.PSUChannelHistory)
0381         #Print a warning when the timeStamp is not the last one in the history!
0382         if self.debug:
0383             if row['change_date'] not in self.PSUChannelHistory.keys():
0384                 if nextTimeStamp!="Infinity":
0385                     print("WARNING! Asynchronous updating of the Tracker Voltage Status")
0386                     print("WARNING! Inserting an IOV between %s, and %s existing timestamps!"%(lastTimeStamp,nextTimeStamp))
0387                     print("The state will be correctly updated (if necessary) starting from the state at %s"%lastTimeStamp)
0388                 else:
0389                     print("Appending one more IOV to the PSUChannelHistory dictionary")
0390             else:
0391                 print("Updating already present IOV with timestamp %s"%row['change_date'])
0392         #The fact that we edit the lasttimestamp takes care of updating existing IOVs the same way as for new ones...
0393         #Update the internal dictionary modifying the last time stamp state...
0394         #Case of channel being ON (it's only ON if CAEN reports 1)
0395         PSUChannelOFFList=self.PSUChannelHistory[lastTimeStamp][:]
0396         if row['actual_status']==1: #Switching from string to int here to make sure it will be compatible with SQLAlchemy query for testing with direct DB access.
0397             try:
0398                 PSUChannelOFFList.remove(self.PSUChannels.index(row['dpname']))
0399                 self.PSUChannelHistory.update({row['change_date']:PSUChannelOFFList})
0400             except:
0401                 if self.debug:
0402                     print("Found a channel that turned ON but as already ON apparently!")
0403                     print(row)
0404         else: #Case of channel being OFF (it's considered OFF in any non 1 state, ramping, error, off)
0405             PSUChannelOFFList.append(self.PSUChannels.index(row['dpname']))
0406             self.PSUChannelHistory.update({row['change_date']:list(set(PSUChannelOFFList))})
0407         
0408         #First handle the detID based dict:
0409         #Use the map to translate the DPNAME (PSUChannel) into a detid via internal map:
0410         #CAREFUL... need to protect against pixel PSU channels being reported by the query!
0411         #try:
0412         #    detIDList=self.PSUDetIDmap[row['dpname']]
0413         #except:
0414         #    #FIXME: develop the pixel testing part here later...
0415         #    print "PSU channel (DPNAME) reported in not part of the wanted subdetector (only Pixel or Strips at one time)"
0416         #    print "DPNAME=%s"%row['dpname']
0417         #    detIDList=[] #Set it to empty list...
0418         ##print detID, type(detID)
0419         #for detID in detIDList:
0420         #    #Get the previous list of channel statuses for the relevant detID:
0421         #    ModuleStatus=self.history[row['change_date']][detID]
0422         #    #Update it with the current query result row:
0423         #    #DISTINGUISH LV and HV case:
0424         #    if row['dpname'].endswith('channel000') or row['dpname'].endswith('channel001'):
0425         #        ModuleStatus[0]=row['actual_status'] #{detid:[LV,HV]} convention is ModuleStatus[0]=0 is LV OFF
0426         #    else:
0427         #        ModuleStatus[1]=row['actual_status']
0428         #    #Update the history dictionary:
0429         #    self.history[row['change_date']].update({detID:ModuleStatus})
0430         ##Add a check for the possibility of having a second IOV when only channel000/channel001 changed,
0431         ##i.e. the status did not change as a consequence for the detID based history dictionary
0432         ##(this does not affect the historyPSUChannel dictionary by definition)
0433         #if self.history[row['change_date']]==self.history[TimeStamps['detID'][0]] and row['change_date']!=TimeStamps['detID'][0]: #This last condition is just in case something funky happens, to avoid removing good IOVs
0434         #    if self.debug:
0435         #        print "Found that the update for timestamp %s, does not change the detid status w.r.t timestamp %s"%(row['change_date'],TimeStamps['detID'][0])
0436         #        print "Eliminating the temporarily created entry in the history dictionary (it will still be logged in the historyPSUChannel dictionary"
0437         #    self.history.pop(row['change_date'])
0438         ##Then handle the PSUChannel based one:
0439         #self.historyPSUChannel[row['change_date']].update({row['dpname']:row['actual_status']})
0440     
0441                     
0442     def updateDetIDQuery(self,row):
0443         """Function to automatically handle the updating of the history dictionary
0444         with the result rows from detID based query
0445         """
0446         #FIXME:
0447         #this will have to be developed once the detID query is ready and being used in the Tracker DCS O2O
0448         if row['change_date'] not in self.history.keys():#New timestamp (i.e. new IOV)
0449             #For new IOV create a new entry in the dict and initialize it to the last IOV status
0450             #since only change is reported with STATUS_CHANGE queries.
0451             lastTimeStamp=sorted(self.history.keys()).pop(0)
0452             self.history.update({row['change_date']:self.history[lastTimeStamp]})
0453         #Get the previous list of channel statuses for the relevant detID:            
0454         channelStatus=self.history[row['change_date']][row['detid']]
0455         #Update it with the current query result row:
0456         channelStatus[row['channel']]=row['voltage_status']
0457         #Update the history dictionary:
0458         self.history[row['change_date']].update({row['detid']:channelStatus})
0459         
0460         
0461     #Define the getters:
0462 
0463     #First the TIMESTAMP based getters:
0464     
0465     def getPSUChannelsOff(self,timestamp):
0466         """
0467         Function that given a timestamp, returns the list of PSUChannels OFF, the list of PSUChannels ON and the IOV (start/stop timestamps) of the reported status.
0468         NOTE: No distinction between HV and LV is made since all PSUChannels (000/001 and 002/003) are reported.
0469         """
0470         StartTime,StopTime=self.getIOV(timestamp,self.PSUChannelHistory)
0471         PSUChannelsOFF=map(lambda x: self.PSUChannels[x],self.PSUChannelHistory[StartTime])
0472         PSUChannelsON=list(set(self.PSUChannels).difference(set(PSUChannelsOFF)))
0473         return PSUChannelsOFF,PSUChannelsON,StartTime,StopTime
0474     
0475     def getDetIDsHVOff(self,timestamp):
0476         """
0477         Function that given a timestamp, returns the list of detIDs with HV OFF, the list of detIDs with HV ON and the IOV (start/stop times) of the reported status.
0478         """
0479         StartTime,StopTime=self.getIOV(timestamp,self.PSUChannelHistory)
0480         DetIDsHVOFF=[]
0481         #A little too nasty python oneliner:
0482         #[DetIDsHVOFF.extend(i) for i in map(lambda x: self.PSUDetIDmap[self.PSUChannels[x]],[index for index in self.PSUChannelHistory[StartTime] if (self.PSUChannels[index].endswith("2") or self.PSUChannels[index].endswith("3"))])]
0483         #It actually does not work, since we need to consider unmapped and crosstalking channels and handle them differently, let's see:
0484         for index in self.PSUChannelHistory[StartTime]:
0485             #First check only for HV channels!
0486             if self.PSUChannels[index].endswith("2") or self.PSUChannels[index].endswith("3") :
0487                 #Then check HV MAPPED channels:
0488                 if self.PSUChannels[index] in self.PSUDetIDmap.keys():
0489                     #print "Extending the list of DetIdsHVOff with the positively matched detids:",self.PSUDetIDmap[self.PSUChannels[index]]
0490                     DetIDsHVOFF.extend(self.PSUDetIDmap[self.PSUChannels[index]])
0491                 #Furthermore check the unmapped channels:
0492                 if self.PSUChannels[index][:-10] in self.PSUDetIDUnmappedMap.keys():
0493                     #To turn OFF unmapped channels there is no need to check the "other" channel:
0494                     #print "Extending the list of DetIdsHVOff with the HV unmapped (PSU-)matched detids:",self.PSUDetIDUnmappedMap[self.PSUChannels[index][:-10]]
0495                     DetIDsHVOFF.extend(self.PSUDetIDUnmappedMap[self.PSUChannels[index][:-10]])
0496                 #Further check the crosstalking channels:
0497                 if self.PSUChannels[index][:-10] in self.PSUDetIDCrosstalkingMap.keys():
0498                     #To turn OFF crosstalking channels we need to check that the other channel is OFF too!
0499                     if (self.PSUChannels.index(self.PSUChannels[index][:-1]+"2") in self.PSUChannelHistory[StartTime]) and (self.PSUChannels.index(self.PSUChannels[index][:-1]+"3") in self.PSUChannelHistory[StartTime]):
0500                         #print "Extending the list of DetIdsHVOff with the HV-CROSSTALKING (PSU-)matched detids:",self.PSUDetIDCrosstalkingMap[self.PSUChannels[index][:-10]]
0501                         DetIDsHVOFF.extend(self.PSUDetIDCrosstalkingMap[self.PSUChannels[index][:-10]])
0502         DetIDsHVON=list(set(self.detIDs).difference(set(DetIDsHVOFF)))
0503         return list(set(DetIDsHVOFF)),DetIDsHVON,StartTime,StopTime
0504     
0505     def getDetIDsLVOff(self,timestamp):
0506         """
0507         Function that given a timestamp, returns the list of detIDs with LV OFF, the list of detIDs with LV ON and the IOV (start/stop times) of the reported status.
0508         """
0509         #Note that channels with LV OFF naturally should have HV OFF too!
0510         StartTime,StopTime=self.getIOV(timestamp,self.PSUChannelHistory)
0511         DetIDsLVOFF=[]
0512         for detids in map(lambda x: self.PSUDetIDmap[self.PSUChannels[x]],[index for index in self.PSUChannelHistory[StartTime] if (self.PSUChannels[index].endswith("0") or self.PSUChannels[index].endswith("1"))]):
0513             DetIDsLVOFF.extend(detids)
0514         DetIDsLVON=list(set(self.detIDs).difference(set(DetIDsLVOFF)))
0515         return list(set(DetIDsLVOFF)),DetIDsLVON,StartTime,StopTime
0516 
0517     def getAliasesHVOff(self,timestamp):
0518         """
0519         Function that given a timestamp, returns the list of (PSU) Aliases with HV OFF, the list of (PSU) Aliases with HV ON and the IOV (start/stop times) of the reported status. 
0520         """
0521         DetIDsHVOFF,DetIDsHVON,StartTime,StopTime=self.getDetIDsHVOff(timestamp)
0522         AliasesHVOFF=list(set([list(self.DetIDAliasDict[detid])[0] for detid in DetIDsHVOFF])) #FIXME: check on fixing the StripDetIDAlias.pkl dictionary... no need of a set as a result! actually need to check if there is more than one element, that would be an ERROR!
0523         AliasesHVON=list(set([list(self.DetIDAliasDict[detid])[0] for detid in DetIDsHVON]))
0524         return AliasesHVOFF,AliasesHVON,StartTime,StopTime
0525 
0526     def getAliasesLVOff(self,timestamp):
0527         """
0528         Function that given a timestamp, returns the list of (PSU) Aliases with HV OFF, the list of (PSU) Aliases with HV ON and the IOV (start/stop times) of the reported status.
0529         """
0530         DetIDsLVOFF,DetIDsLVON,StartTime,StopTime=self.getDetIDsLVOff(timestamp)
0531         AliasesLVOFF=list(set([list(self.DetIDAliasDict[detid])[0] for detid in DetIDsLVOFF])) #FIXME: check on fixing the StripDetIDAlias.pkl dictionary... no need of a set as a result! actually need to check if there is more than one element, that would be an ERROR!
0532         AliasesLVON=list(set([list(self.DetIDAliasDict[detid])[0] for detid in DetIDsLVON]))
0533         return AliasesLVOFF,AliasesLVON,StartTime,StopTime
0534 
0535     #Then the Value vs. time getters:
0536 
0537     def getDetIDsHVOffVsTime(self):
0538         """
0539         Function that returns the number of DetIDs with HV OFF vs time for all IOVs available in the PSUChannelHistory.
0540         The results is one list of tuples [(datetime.datetime,#DetIDsHVOFF),...] and the timestamp of the last entry processed in the PSUChannelHistory.
0541         This is important, so that the user can check until when in time the last tuple in the list is valid until (closing the last IOV).
0542         This can be easily used to
0543         -Do spot checks by IOV: picking a timestamp inside one of the IOVs one can use the getDetIDsHVOff function to get a list of the detIDs ON/OFF.
0544         -Plot the graph of number of HV OFF (or ON by complementing to 15148) vs time using pyRoot
0545         """
0546         #Loop over the timestamps:
0547         DetIDsHVOffVsTime=[]
0548         PreviousTimeStamp=None
0549         PreviousDetIDsHVOFF=[]
0550         for timestamp in sorted(self.PSUChannelHistory.keys()):
0551             DetIDsHVOFF=self.getDetIDsHVOff(timestamp)[0] #Use the first returned value (DetIDsHVOFF) from getDetIDsHVOff(timestamp)!
0552             #Check with respect to previous IOV, using set functionality, so that only relevant IOVs where there was an actual HV change are reported!
0553             if PreviousTimeStamp:
0554                 if DetIDsHVOFF!=PreviousDetIDsHVOFF: #If there is a previous timestamp to compare to, look for differences before reporting!
0555                     DetIDsHVOffVsTime.append((timestamp,len(DetIDsHVOFF)))
0556             else: #First IOV start... nothing to compare to:
0557                 DetIDsHVOffVsTime.append((timestamp,len(DetIDsHVOFF)))
0558             PreviousTimeStamp=timestamp
0559             PreviousDetIDsHVOFF=DetIDsHVOFF
0560         LastTimeStamp=PreviousTimeStamp
0561         return DetIDsHVOffVsTime,PreviousTimeStamp 
0562 
0563     def getDetIDsLVOffVsTime(self):
0564         """
0565         Function that returns the number of DetIDs with LV OFF vs time for all IOVs available in the PSUChannelHistory.
0566         The results is one list of tuples [(datetime.datetime,#DetIDsLVOFF),...], basically 1 entry per IOV (change in the LV channels only),
0567         and the timestamp of the last entry processed in the PSUChannelHistory. This timestamp is important, so that the user can check until when in time the last tuple in the list is valid until (closing the last IOV).
0568         This can be easily used to
0569         -Do spot checks by IOV: picking a timestamp inside one of the IOVs one can use the getDetIDsHVOff function to get a list of the detIDs ON/OFF.
0570         -Plot the graph of number of HV OFF (or ON by complementing to 15148) vs time using pyRoot
0571         """
0572         #Loop over the timestamps:
0573         DetIDsLVOffVsTime=[]
0574         PreviousTimeStamp=None
0575         for timestamp in sorted(self.PSUChannelHistory.keys()):
0576             DetIDsLVOFF=set(self.getDetIDsLVOff(timestamp)[0]) #Use the first returned value (DetIDsLVOFF) from getDetIDsLVOff(timestamp)!Use set() to be able to compare the sets with != otherwise for lists, the order also makes them different!
0577             #Check with respect to previous IOV, using set functionality, so that only relevant IOVs where there was an actual LV change are reported!
0578             if PreviousTimeStamp:
0579                 if DetIDsLVOFF!=PreviousDetIDsLVOFF: #If there is a previous timestamp to compare to, look for differences before reporting! 
0580                     DetIDsLVOffVsTime.append((timestamp,len(DetIDsLVOFF)))
0581             else: #First IOV start... nothing to compare to:
0582                 DetIDsLVOffVsTime.append((timestamp,len(DetIDsLVOFF)))
0583             PreviousTimeStamp=timestamp
0584             PreviousDetIDsLVOFF=DetIDsLVOFF
0585         LastTimeStamp=PreviousTimeStamp
0586         return DetIDsLVOffVsTime,LastTimeStamp
0587     
0588     #Data massagers:
0589     def getArraysForTimePlots(self,TimeStampsValuesTuple):
0590         """
0591         Function that given a tuple with (datetime.datetime,values) returns the arrays ready to the plotting vs time function (basically renders data histogram-like adding extra data points).
0592         """
0593         import array, time
0594 
0595         #Initialize the lists of timestamps (to be doubled to get the histogram/IOV look)
0596         Timestamps=[]
0597         Values=[]
0598         #Loop over the list of tuples with (timestamp,numberofdetidswithLVOFF):
0599         for item in sorted(TimeStampsValuesTuple):
0600            #FIXME:
0601            #NEED to check the effect/use of tzinfo to avoid issues with UTC/localtime DST etc.
0602            #Will have to massage the datetime.datetime object in a proper way not necessary now.
0603            #Also need to decide if we want add the milliseconds into the "timestamps for plotting in root... not straightforward so will implement only if needed. For now know that the approximation is by truncation for now!
0604            timestamp=int(time.mktime(item[0].timetuple())) #Need to get first a time tuple, then translate it into a unix timestamp (seconds since epoc). This means no resolution under 1 second by default
0605            if item==TimeStampsValuesTuple[0]: #First item does not need duplication
0606                Timestamps.append(timestamp)
0607                Value=item[1]
0608                Values.append(Value)
0609            else: #Add twice each timestamp except the first one (to make a histogram looking graph)
0610                Timestamps.append(timestamp)
0611                Values.append(Value) #Input previous value with new timestamp
0612                Value=item[1]
0613                Timestamps.append(timestamp)
0614                Values.append(Value) #Input new value with new timestamp
0615         #Need to render the two lists as arrays for pyROOT
0616         TimestampsArray=array.array('i',Timestamps)
0617         ValuesArray=array.array('i',Values)
0618 
0619         return TimestampsArray,ValuesArray
0620 
0621     def getReducedArraysForTimePlots(self,TimeStamps,Values):
0622         """
0623         Implement IOV reduction based on timestamp delta, following O2O algorithm.
0624         """
0625         
0626         return
0627     
0628     #The plotters:
0629     def plotGraphSeconds(self,TimeArray,ValuesArray,GraphTitle="Graph",YTitle="Number of channels",GraphFilename="test.gif"):
0630         """
0631         Function that given an array with timestamps (massaged to introduce a second timestamp for each value to produce an histogram-looking plot) and a corresponding array with values to be plotted, a title, a Y axis title and a plot filename, produces with pyROOT a time plot and saves it locally.
0632         The function can be used for cumulative plots (number of channels with HV/LV OFF/ON vs time) or for individual (single detID HV/LV status vs time) plots.
0633         """
0634         import ROOT
0635         canvas=ROOT.TCanvas()
0636         graph=ROOT.TGraph(len(TimeArray),TimeArray,ValuesArray)
0637         graph.GetXaxis().SetTimeDisplay(1)
0638         graph.GetXaxis().SetLabelOffset(0.02)
0639         #Set the time format for the X Axis labels depending on the total time interval of the plot!
0640         TotalTimeIntervalSecs=TimeArray[-1]-TimeArray[0]
0641         if TotalTimeIntervalSecs <= 120: #When zooming into less than 2 mins total interval report seconds too
0642             graph.GetXaxis().SetTimeFormat("#splitline{   %d/%m}{%H:%M:%S}")
0643         elif 120 < TotalTimeIntervalSecs < 6400: #When zooming into less than 2 hrs total interval report minutes too
0644             graph.GetXaxis().SetTimeFormat("#splitline{%d/%m}{%H:%M}")
0645         else: #When plotting more than 2 hrs only report the date and hour of day
0646             graph.GetXaxis().SetTimeFormat("#splitline{%d/%m}{  %H}")
0647         graph.GetYaxis().SetTitle(YTitle)
0648         graph.GetYaxis().SetTitleOffset(1.4)
0649         graph.SetTitle(GraphTitle)
0650         graph.Draw("APL")
0651         canvas.SaveAs(GraphFilename)
0652         print("Saved graph as %s"%GraphFilename)
0653         return
0654     def plotPSUChannelvsTime(self,TimeArray,ValuesArray,GraphTitle="PSUChannelGraph",YTitle="Channel HV Status",GraphFilename="PSUChannel.gif"):
0655         """
0656         Function that given an array with timestamps (massaged to introduce a second timestamp for each value to produce an histogram-looking plot) and a corresponding array with values to be plotted, a title, a Y axis title and a plot filename, produces with pyROOT a time plot and saves it locally.
0657         The function can be used for cumulative plots (number of channels with HV/LV OFF/ON vs time) or for individual (single detID HV/LV status vs time) plots.
0658         """
0659         import ROOT
0660         canvas=ROOT.TCanvas()
0661         graph=ROOT.TGraph(len(TimeArray),TimeArray,ValuesArray)
0662         graph.GetXaxis().SetTimeDisplay(1)
0663         graph.GetXaxis().SetLabelOffset(0.02)
0664         #Set the time format for the X Axis labels depending on the total time interval of the plot!
0665         TotalTimeIntervalSecs=TimeArray[-1]-TimeArray[0]
0666         if TotalTimeIntervalSecs <= 120: #When zooming into less than 2 mins total interval report seconds too
0667             graph.GetXaxis().SetTimeFormat("#splitline{   %d/%m}{%H:%M:%S}")
0668         elif 120 < TotalTimeIntervalSecs < 6400: #When zooming into less than 2 hrs total interval report minutes too
0669             graph.GetXaxis().SetTimeFormat("#splitline{%d/%m}{%H:%M}")
0670         else: #When plotting more than 2 hrs only report the date and hour of day
0671             graph.GetXaxis().SetTimeFormat("#splitline{%d/%m}{  %H}")
0672         graph.GetYaxis().SetTitle(YTitle)
0673         graph.GetYaxis().SetTitleOffset(1.4)
0674         graph.SetTitle(GraphTitle)
0675         graph.Draw("APL")
0676         canvas.SaveAs(GraphFilename)
0677         print("Saved graph as %s"%GraphFilename)
0678         return
0679     
0680     def plotHVOFFvsTime(self):
0681         """
0682         """
0683         return
0684     
0685     def plotLVOFFvsTime(self):
0686         """
0687         """
0688         return
0689 
0690     def plotDetIDHVOFFVsTime(self,detID):
0691         """
0692         """
0693         return
0694     def plotDetIDLVOFFVsTime(self,detID):
0695         """
0696         """
0697         return
0698 
0699     def plotDetIDHistory(self):
0700         return
0701 
0702     def getIOV(self,timeStamp,HistoryDict):
0703         """
0704         Function that given a timeStamp return the TimeStampStart and TimeStampStop of the IOV in the wanted HistoryDict (assumed to have timestamps as keys) that contains the timestamp.
0705         This can be used by all functions that need to access HistoryDict by timeStamp.
0706         """        
0707         
0708         TimeStamps=HistoryDict.keys()[:]
0709         #Add the wanted timeStamp to the list of TimeStamps (if it is not there already!)
0710         if timeStamp not in TimeStamps:
0711             TimeStamps.append(timeStamp)
0712             #Sort the list with the timestamp we added, so it will fall in between the wanted StartTime and EndTime.
0713             TimeStamps.sort()            
0714             TimeStampStart=TimeStamps[TimeStamps.index(timeStamp)-1]
0715             #Remember to remove it now, so that TimeStamps is still the list of available timestamps!
0716             TimeStamps.remove(timeStamp)
0717         else:#If the timeStamp matches one in the file, then the StartTime is that timestamp ;)
0718             TimeStamps.sort() #this is still needed since the keys of a dictionary are not automatically sorted, but we assume TimeStamps to be sorted to spot TimeStampStop...
0719             TimeStampStart=timeStamp
0720         #For the TimeStampStop check the case of hitting the last IOV that is valid until infinity...
0721         if len(TimeStamps)>TimeStamps.index(TimeStampStart)+1:
0722             TimeStampStop=TimeStamps[TimeStamps.index(TimeStampStart)+1]
0723         else:
0724             TimeStampStop="Infinity"
0725         if self.debug:
0726             print("TimeStamp %s falls into IOV starting at %s and ending at %s"%(timeStamp,TimeStampStart,TimeStampStop))
0727         return (TimeStampStart,TimeStampStop)
0728 
0729     def getIOVsInTimeInterval(self,StartTime,StopTime,HistoryDict):
0730         """
0731         Function that returns the IOV timestamps (ordered) contained in a given interval.
0732         """
0733         #Copy and sort the timestamps in a list
0734         TimeStamps=sorted(HistoryDict.keys()[:])
0735         IOVsInTimeInterval=[]
0736         #loop over them:
0737         for timestamp in TimeStamps:
0738             if timestamp>=StartTime and timestamp<=StopTime: #Pick only the timestamps inside the time interval specified
0739                 if self.debug:
0740                     print("Found timestamp %s in the wanted interval [%s,%s]"%(timestamp,StartTime,StopTime))
0741                 IOVsInTimeInterval.append(timestamp)
0742         return IOVsInTimeInterval
0743 
0744     def getReducedIOVs (self,StartTime,StopTime,HistoryDict,deltaT,maxIOVLength):
0745         """
0746         Apply the reduction algorithm to the timeintervals and return them so that one can test the reduction (and do plots more easily).
0747         """
0748         deltaTime=datetime.timedelta(seconds=deltaT)
0749         maxSequenceLength=datetime.timedelta(seconds=maxIOVLength)
0750         #Copy and sort the timestamps in a list:
0751         TimeStamps=sorted(HistoryDict.keys()[:])
0752         ReducedIOVs=TimeStamps[:]
0753         PreviousTimestamp=TimeStamps[0] 
0754         SequenceStart=TimeStamps[0]  #Initialization irrelevant see loop
0755         SequenceOn=False
0756         #Loop over timestamps:
0757         #Note:Leave the ramp-up/down logic for later (an other function), for now we just do IOV reduction: don't care if we are ramping up or down!
0758         for timestamp in TimeStamps[1:]: #Skip the first timestamp!(since we initialize PreviousTimeStamp to the first timestamp!)
0759             #Check only within the interval specified:
0760             if timestamp>=StartTime and timestamp<=StopTime:
0761                 #Check if the timestamp is within the wanted deltaT from previous timestamp
0762                 if timestamp-PreviousTimestamp<=deltaTime:
0763                     #Check if there is already an ongoing sequence of IOVs:
0764                     if SequenceOn:
0765                         #Check that this timestamp is not farther away than the maximum IOV sequence
0766                         if (timestamp-SequenceStart)<=maxSequenceLength and timestamp!=TimeStamps[-1]:#need to handle the last time stamp differently!
0767                             if self.debug:
0768                                 print("Eliminating timestamp %s since it is %s (<=%s)seconds from previous timestamp %s and %s (<=%s) seconds from the IOV sequence start %s!"%(timestamp,timestamp-PreviousTimestamp,deltaTime,PreviousTimestamp,timestamp-SequenceStart,maxSequenceLength,SequenceStart))
0769                             ReducedIOVs.remove(timestamp)
0770                         elif timestamp==TimeStamps[-1]: #special case of last timestamp in the list of input timestamps!
0771                             if self.debug:
0772                                 print("###Terminating the IOV sequence started with %s, since current timestamp %s is the last one in the input list of timestamps to REDUCE!"%(SequenceStart,timestamp))
0773                         else:
0774                             #Terminate the sequence (keep the timestamp):
0775                             SequenceOn=False
0776                             #Reappend the last item of the sequence (that was automatically removed not knowing it would be the last of the sequence):
0777                             ReducedIOVs.append(PreviousTimestamp)
0778                             #Re-order the list via sort():
0779                             ReducedIOVs.sort()
0780                             if self.debug:
0781                                 print("###Terminating the IOV sequence started with %s, since current timestamp %s is %s seconds (>%s) away from the IOV sequence starting timestamp (%s). Re-appending the last timestamp in the sequence %s."%(SequenceStart,timestamp,timestamp-SequenceStart,maxIOVLength,PreviousTimestamp,PreviousTimestamp))
0782                     else:
0783                         #Start the sequence
0784                         SequenceOn=True
0785                         #Save the first timestamp of the sequence (it will be used to check sequence length)
0786                         if self.debug:
0787                             print("@@@Starting a new IOV sequence with previous timestamp %s (current being %s)"%(PreviousTimestamp,timestamp))
0788                         #Still get rid of the current (second in the sequence) IOV:
0789                         ReducedIOVs.remove(timestamp)
0790                         SequenceStart=PreviousTimestamp
0791                 else:
0792                     #Check if there is already an ongoing sequence of IOVs:
0793                     if SequenceOn:
0794                         #Terminate the sequence since timestamp is further than deltaT from previous timestamp:
0795                         SequenceOn=False
0796                         #Reappend the last item of the sequence (that was automatically removed not knowing it would be the last of the sequence):
0797                         ReducedIOVs.append(PreviousTimestamp)
0798                         #Re-order the list via sort():
0799                         ReducedIOVs.sort()
0800                         if self.debug:
0801                             print("$$$Terminating the IOV sequence started with %s, since current timestamp %s is %s seconds (>%s) away from the previous timestamp (%s) in the sequence. Re-appending the last timestamp in the sequence %s."%(SequenceStart,timestamp,timestamp-PreviousTimestamp,deltaT,PreviousTimestamp,PreviousTimestamp))
0802                                                             
0803             else:
0804                 #The following is conventional of course:
0805                 #Since we are outside the wanted time interval we should still kick out the timestamp,
0806                 #Basically we will only report a reduce sequence of timestamps within a certain interval
0807                 ReducedIOVs.remove(timestamp)
0808                 #If there is an ongoing sequence that stretches outside the StopTime:
0809                 if SequenceOn:
0810                     #Terminate sequence (should check the results if they finish with a timestamp>StopTime):
0811                     SequenceOn=False
0812                     #Reappend the last item of the sequence (that was automatically removed not knowing it would be the last of the sequence):
0813                     ReducedIOVs.append(PreviousTimestamp)
0814                     #Re-order the list via sort():
0815                     ReducedIOVs.sort()
0816                     if self.debug:
0817                         print("^^^ Terminating the IOV sequence started with %s, since current timestamp %s is no more in the wanted time interval under investigation ([%s,%s]). Sequence might have been continuing beyond the StopTime %s limit! Re-appending the last timestamp in the sequence %s."%(SequenceStart,timestamp,StartTime,StopTime,StopTime,PreviousTimestamp))
0818             PreviousTimestamp=timestamp
0819         return ReducedIOVs
0820 
0821                     
0822 #Messing around...
0823 TkStatus=TrkVoltageStatus(detIDPSUmapfilename=os.path.join(os.getenv('CMSSW_BASE'),'src/CalibTracker/SiStripDCS/data/StripPSUDetIDMap_FromJan132010_Crosstalk.dat'),DetIDAliasFile=os.path.join(os.getenv('CMSSW_BASE'),'src/CalibTracker/SiStripDCS/data/StripDetIDAlias.pkl'),startTime=datetime.datetime(2010,8,27,12,00,00),debug=True)
0824 
0825 #row1={'change_date':datetime.datetime(2010,8,27,10,37,54,732),'actual_status':1,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel001"}
0826 #
0827 #row2={'change_date':datetime.datetime(2010,8,27,11,37,54,732),'actual_status':0,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel001"}
0828 #
0829 #row3={'change_date':datetime.datetime(2010,8,27,12,37,54,732),'actual_status':1,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel001"}
0830 #
0831 #row4={'change_date':datetime.datetime(2010,8,27,10,37,54,732),'actual_status':1,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel002"}
0832 #
0833 #row5={'change_date':datetime.datetime(2010,8,27,11,37,54,732),'actual_status':0,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel002"}
0834 #
0835 #row6={'change_date':datetime.datetime(2010,8,27,12,37,54,732),'actual_status':1,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel002"}
0836 #
0837 #row7={'change_date':datetime.datetime(2010,8,27,11,37,54,732),'actual_status':0,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel002"}
0838 #
0839 #row8={'change_date':datetime.datetime(2010,8,27,10,37,54,735),'actual_status':1,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_1/branchController04/easyCrate0/easyBoard05/channel002"}
0840 #
0841 ##Initialization IOV:
0842 #print "detID IOVs",TkStatus.history.keys()
0843 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0844 #
0845 #TkStatus.updateO2OQuery(row1)
0846 #print "detID IOVs",TkStatus.history.keys()
0847 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0848 #
0849 #TkStatus.updateO2OQuery(row2)
0850 #print "detID IOVs",TkStatus.history.keys()
0851 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0852 #
0853 #TkStatus.updateO2OQuery(row3)
0854 #print "detID IOVs",TkStatus.history.keys()
0855 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0856 #
0857 #TkStatus.updateO2OQuery(row4)
0858 #print "detID IOVs",TkStatus.history.keys()
0859 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0860 #
0861 #TkStatus.updateO2OQuery(row5)
0862 #print "detID IOVs",TkStatus.history.keys()
0863 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0864 #
0865 #TkStatus.updateO2OQuery(row6)
0866 #print "detID IOVs",TkStatus.history.keys()
0867 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0868 #
0869 #TkStatus.updateO2OQuery(row7)
0870 #print "detID IOVs",TkStatus.history.keys()
0871 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0872 #
0873 #TkStatus.updateO2OQuery(row8)
0874 #print "detID IOVs",TkStatus.history.keys()
0875 #print "PSUChannel IOVs",TkStatus.historyPSUChannel.keys()
0876 #
0877 #TkStatus.getHVOnModules(datetime.datetime(2010, 8, 27, 10, 00, 54, 732))
0878 
0879 #Test Crosstalking channels handling:
0880 #rowCross1={'change_date':datetime.datetime(2010,8,27,10,37,54,732),'actual_status':1,'dpname':"cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel002"}
0881 #rowCross2={'change_date':datetime.datetime(2010,8,27,10,37,57,732),'actual_status':1,'dpname':"cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel003"}
0882 #rowCross3={'change_date':datetime.datetime(2010,8,27,10,38,00,732),'actual_status':0,'dpname':"cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel002"}
0883 #rowCross4={'change_date':datetime.datetime(2010,8,27,10,38,05,732),'actual_status':0,'dpname':"cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel003"}
0884 #print "Starting number of PSUChannels OFF is %s and of DetIDs ON is %s"%(len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,10,30,00),TkStatus.PSUChannelHistory)[0]]),len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010,8,27,10,30,00),TkStatus.PSUChannelHistory)[0])[1]))                                                                         
0885 #TkStatus.updateO2OQuery(rowCross1)
0886 #print "Turning ON HV crosstalking channel cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel002 number of channels OFF is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,10,37,55),TkStatus.PSUChannelHistory)[0]])
0887 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010,8,27,10,37,55),TkStatus.PSUChannelHistory)[0])[1])
0888 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010,8,27,10,37,55),TkStatus.PSUChannelHistory)[0])[1]
0889 #TkStatus.updateO2OQuery(rowCross2)
0890 #print "Turning ON HV crosstalking channel cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel003 number of channels OFF is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,10,37,58),TkStatus.PSUChannelHistory)[0]])
0891 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 10, 37, 58),TkStatus.PSUChannelHistory)[0])[1])
0892 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 10, 37, 58),TkStatus.PSUChannelHistory)[0])[1]
0893 #TkStatus.updateO2OQuery(rowCross3)
0894 #print "Turning OFF HV crosstalking channel cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel002 number of channels off is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,10,38,02),TkStatus.PSUChannelHistory)[0]])
0895 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 10, 38, 02),TkStatus.PSUChannelHistory)[0])[1])
0896 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 10, 38, 02),TkStatus.PSUChannelHistory)[0])[1]
0897 #TkStatus.updateO2OQuery(rowCross4)
0898 #print "Turning OFF HV crosstalking channel cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel003 number of channels off is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,10,38,07),TkStatus.PSUChannelHistory)[0]])
0899 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 10, 38, 07),TkStatus.PSUChannelHistory)[0])[1])
0900 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 10, 38, 07),TkStatus.PSUChannelHistory)[0])[1]
0901 #
0902 ##Test Unmapped channels handling:
0903 #rowUnmap1={'change_date':datetime.datetime(2010,8,27,11,37,54,732),'actual_status':1,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_2/branchController05/easyCrate1/easyBoard07/channel002"}
0904 #rowUnmap2={'change_date':datetime.datetime(2010,8,27,11,37,57,732),'actual_status':1,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_2/branchController05/easyCrate1/easyBoard07/channel003"}
0905 #rowUnmap3={'change_date':datetime.datetime(2010,8,27,11,38,00,732),'actual_status':0,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_2/branchController05/easyCrate1/easyBoard07/channel002"}
0906 #rowUnmap4={'change_date':datetime.datetime(2010,8,27,11,38,05,732),'actual_status':0,'dpname':"cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_2/branchController05/easyCrate1/easyBoard07/channel003"}
0907 #print "Starting number of PSUChannels OFF is %s and of DetIDs ON is %s"%(len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,11,30,00),TkStatus.PSUChannelHistory)[0]]),len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010,8,27,11,30,00),TkStatus.PSUChannelHistory)[0])[1]))                                                                         
0908 #TkStatus.updateO2OQuery(rowUnmap1)
0909 #print "Turning ON HV crosstalking channel cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_2/branchController05/easyCrate1/easyBoard07/channel002  number of channels OFF is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,11,37,55),TkStatus.PSUChannelHistory)[0]])
0910 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010,8,27,11,37,55),TkStatus.PSUChannelHistory)[0])[1])
0911 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010,8,27,11,37,55),TkStatus.PSUChannelHistory)[0])[1]
0912 #TkStatus.updateO2OQuery(rowUnmap2)
0913 #print "Turning ON HV crosstalking channel cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_2/branchController05/easyCrate1/easyBoard07/channel003 number of channels OFF is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,11,37,58),TkStatus.PSUChannelHistory)[0]])
0914 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 11, 37, 58),TkStatus.PSUChannelHistory)[0])[1])
0915 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 11, 37, 58),TkStatus.PSUChannelHistory)[0])[1]
0916 #TkStatus.updateO2OQuery(rowUnmap3)
0917 #print "Turning OFF HV crosstalking channel cms_trk_dcs_02:CAEN/CMS_TRACKER_SY1527_2/branchController05/easyCrate1/easyBoard07/channel002 number of channels off is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,11,38,02),TkStatus.PSUChannelHistory)[0]])
0918 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 11, 38, 02),TkStatus.PSUChannelHistory)[0])[1])
0919 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 11, 38, 02),TkStatus.PSUChannelHistory)[0])[1]
0920 #TkStatus.updateO2OQuery(rowUnmap4)
0921 #print "Turning OFF HV crosstalking channel cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_9/branchController04/easyCrate0/easyBoard09/channel003 number of channels off is %s"%len(TkStatus.PSUChannelHistory[TkStatus.getIOV(datetime.datetime(2010,8,27,11,38,07),TkStatus.PSUChannelHistory)[0]])
0922 #print "and related number of detIDs listed as ON is %s"%len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 11, 38, 07),TkStatus.PSUChannelHistory)[0])[1])
0923 #print TkStatus.getDetIDsHVOff(TkStatus.getIOV(datetime.datetime(2010, 8, 27, 11, 38, 07),TkStatus.PSUChannelHistory)[0])[1]
0924 #
0925 #Test function that parses ManualO2O.log to get by timeinterval the results of each O2O query
0926 QueryResults=GetQueryResults("ManualO2O.log")
0927 #QueryResultsPickle=open("QueryResultsDict.pkl","wb")
0928 #pickle.dump(QueryResults,QueryResultsPickle)
0929 
0930 #Now that we have this we can do:
0931 counter=0
0932 hours=72 #introduced to test only a few hours, setting it to 1000 to process ALL IOVS. 
0933 for interval in sorted(QueryResults.keys()):
0934     counter+=1
0935     if counter<hours: #Hours
0936         print("Updating TkStatus with query results for time interval %s to %s"%interval)
0937         for row in QueryResults[interval]:
0938             TkStatus.updateO2OQuery(row)
0939             print(len(TkStatus.PSUChannelHistory))
0940 if counter-hours>0:
0941     print("Number of intervals skipped %s"%(counter-hours))
0942 #Dump the PSUChannelHistory dictionary!
0943 #TkHistoryPickle=open("TkPSUChannelHistory.pkl","wb")
0944 #pickle.dump(TkStatus.PSUChannelHistory,TkHistoryPickle)
0945 #TkHistoryPickle.close()
0946 #
0947 #for timestamp in sorted(TkStatus.PSUChannelHistory.keys()):
0948 #    if len(TkStatus.PSUChannelHistory[timestamp])<4000:
0949 #        print len(TkStatus.PSUChannelHistory[timestamp]),timestamp                     
0950 
0951 #Test getters as I implement them:
0952 #ChannelsOFF,ChannelsON,Start,Stop=TkStatus.getPSUChannelsOff(datetime.datetime(2010,8,27,10,38,26,700000))
0953 #print "IOV: %s -> %s"%(Start,Stop)
0954 #print "Number of PSU channels reported OFF: %s"%len(ChannelsOFF)
0955 #print "Number of PSU channels reported ON: %s"%len(ChannelsON)
0956 #DetIDsHVOFF,DetIDsHVON,Start,Stop=TkStatus.getDetIDsHVOff(datetime.datetime(2010,8,27,10,38,26,700000))
0957 #print "IOV: %s -> %s"%(Start,Stop)
0958 #print "Number of detID reported with HV OFF: %s"%len(DetIDsHVOFF)
0959 #print "Number of detID reported with HV ON: %s"%len(DetIDsHVON)
0960 #DetIDsLVOFF,DetIDsLVON,Start,Stop=TkStatus.getDetIDsLVOff(datetime.datetime(2010,8,27,10,38,26,700000))
0961 #print "IOV: %s -> %s"%(Start,Stop)
0962 #print "Number of detID reported with LV OFF: %s"%len(DetIDsLVOFF)
0963 #print "Number of detID reported LV ON: %s"%len(DetIDsLVON)
0964 #AliasesHVOFF,AliasesHVON,Start,Stop=TkStatus.getAliasesHVOff(datetime.datetime(2010,8,27,10,38,26,700000))
0965 #print "IOV: %s -> %s"%(Start,Stop)
0966 #print "Number of aliases reported with HV OFF: %s"%len(AliasesHVOFF)
0967 #print "Number of aliases reported HV ON: %s"%len(AliasesHVON)
0968 #AliasesLVOFF,AliasesLVON,Start,Stop=TkStatus.getAliasesLVOff(datetime.datetime(2010,8,27,10,38,26,700000))
0969 #print "IOV: %s -> %s"%(Start,Stop)
0970 #print "Number of aliases reported with LV OFF: %s"%len(AliasesLVOFF)
0971 #print "Number of aliases reported LV ON: %s"%len(AliasesLVON)
0972 
0973 #Check the number of channels off in the Run144086 IOV:
0974 #DetIDsHVOFF,DetIDsHVON,Start,Stop=TkStatus.getDetIDsHVOff(datetime.datetime(2010,8,29,8,0,0,0))
0975 #print "IOV: %s -> %s"%(Start,Stop)
0976 #print "Number of detID reported with HV OFF: %s"%len(DetIDsHVOFF)
0977 #print "Number of detID reported with HV ON: %s"%len(DetIDsHVON)
0978 #DetIDsLVOFF,DetIDsLVON,Start,Stop=TkStatus.getDetIDsLVOff(datetime.datetime(2010,8,29,8,0,0,0))
0979 #print "IOV: %s -> %s"%(Start,Stop)
0980 #print "Number of detID reported with LV OFF: %s"%len(DetIDsLVOFF)
0981 #print "Number of detID reported LV ON: %s"%len(DetIDsLVON)
0982 
0983 #Now implement the following test:
0984 #-Read in all the logs (either the DetVOffReaderDebug or the ones produced by the MakeTkMaps
0985 #-Extract the IOV from the filename
0986 #-Get the corresponding IOVs list from the PSUChannelHistory
0987 #-Print out the number of DetIDs with LV ON/OFF (and of crosstalking and unmapped detids separately)
0988 #-DetID comparison for each IOV (picking the stable one in PSUChannelHistory)
0989 #-Implement the reduction as a method of TrkVoltageStatus that takes as argument deltaTmin and  MaxIOVlength
0990 #and returns the lists of HV and LV channels off (IOVs are for any change of either HV or LV PSU channels)
0991 #-Check the reduction result (number of IOVs, suppression during ramp up/down, ...)
0992 #-DetID comparison for each IOV
0993 #-Implement the direct query check of the query results and report anomalies (old results at the interface of the 1-hour queries:
0994 #   -Get the query dumped by Coral by turning on verbosity and off
0995 #   -Pick the query from the code too
0996 #-Mapping part, integrate results of dictionaries from HV mapping code from Ben, dump all relevant dictionaries
0997 #Cross-check stuff with cabling info?
0998 #Check time changes and effect DST etc timezone issues and plotting limitations...
0999 #Add number of ramp up/downs plots minima/maxima of HV/LV OFF/ON
1000 #Look out for individual channels going down (trips) instead of full ramping
1001 #Talk to Online guys about cross-talking and HV mapping
1002 #Look into the query with the turn off commands and into giving the map for the new table to Robert
1003 #Talk with Frank about extra info needed
1004 #Look into Pixel integration extras
1005 
1006 
1007 ## #This next step is REALLY SLOW if there are thousands of IOVs to handle!
1008 ## 
1009 ## #Turn debug off to avoid a lot of printouts from getIOV:
1010 ## TkStatus.debug=False
1011 ## DetIDsHVOffVsTime,LastHVTimeStamp=TkStatus.getDetIDsHVOffVsTime()
1012 ## TkStatus.debug=True
1013 ## print "There are %s timestamps for DetIDs HV Off. The last timestamp (end of last IOV) is %s"%(len(DetIDsHVOffVsTime),LastHVTimeStamp)
1014 ## 
1015 ## #Turn debug off to avoid a lot of printouts from getIOV:
1016 ## TkStatus.debug=False
1017 ## DetIDsLVOffVsTime,LastLVTimeStamp=TkStatus.getDetIDsLVOffVsTime()
1018 ## TkStatus.debug=True
1019 ## print "There are %s timestamps for DetIDs LV Off. The last timestamp (end of last IOV) is %s"%(len(DetIDsLVOffVsTime),LastLVTimeStamp)
1020 ## 
1021 ## #Prepare the data for LV/HV OFF Graph plotting with PyROOT:
1022 ## import array, time
1023 ## 
1024 ## #Initialize the lists of timestamps (to be doubled to get the histogram/IOV look)
1025 ## LVOFFTimestamps=[]
1026 ## NumberOfLVOFFChannels=[]
1027 ## #Loop over the list of tuples with (timestamp,numberofdetidswithLVOFF):
1028 ## for item in sorted(DetIDsLVOffVsTime):
1029 ##     #FIXME:
1030 ##     #NEED to check the effect/use of tzinfo to avoid issues with UTC/localtime DST etc.
1031 ##     #Will have to massage the datetime.datetime object in a proper way not necessary now.
1032 ##     #Also need to decide if we want add the milliseconds into the "timestamps for plotting in root... not straightforward so will implement only if needed. For now know that the approximation is by truncation for now!
1033 ##     timestamp=int(time.mktime(item[0].timetuple())) #Need to get first a time tuple, then translate it into a unix timestamp (seconds since epoc). This means no resolution under 1 second by default
1034 ##     if item==DetIDsLVOffVsTime[0]: #First item does not need duplication
1035 ##         LVOFFTimestamps.append(timestamp)
1036 ##         LVOff=item[1]
1037 ##         NumberOfLVOFFChannels.append(LVOff)
1038 ##     else: #Add twice each timestamp except the first one (to make a histogram looking graph)
1039 ##         LVOFFTimestamps.append(timestamp)
1040 ##         NumberOfLVOFFChannels.append(LVOff) #Input previous value with new timestamp
1041 ##         LVOff=item[1]
1042 ##         LVOFFTimestamps.append(timestamp)
1043 ##         NumberOfLVOFFChannels.append(LVOff) #Input new value with new timestamp
1044 ##         
1045 ## LVOFFTimestampsArray=array.array('i',LVOFFTimestamps)
1046 ## NumberOfLVOFFChannelsArray=array.array('i',NumberOfLVOFFChannels)
1047 ## 
1048 ## #Initialize the lists of timestamps (to be doubled to get the histogram/IOV look)
1049 ## HVOFFTimestamps=[]
1050 ## NumberOfHVOFFChannels=[]
1051 ## #Loop over the list of tuples with (timestamp,numberofdetidswithHVOFF):
1052 ## for item in sorted(DetIDsHVOffVsTime):
1053 ##     timestamp=int(time.mktime(item[0].timetuple()))
1054 ##     if item==DetIDsHVOffVsTime[0]: #First item does not need duplication
1055 ##         HVOFFTimestamps.append(timestamp)
1056 ##         HVOff=item[1]
1057 ##         NumberOfHVOFFChannels.append(HVOff)
1058 ##     else: #Add twice each timestamp except the first one (to make a histogram looking graph)
1059 ##         HVOFFTimestamps.append(timestamp)
1060 ##         NumberOfHVOFFChannels.append(HVOff) #Input previous value with new timestamp
1061 ##         HVOff=item[1]
1062 ##         HVOFFTimestamps.append(timestamp)
1063 ##         NumberOfHVOFFChannels.append(HVOff) #Input new value with new timestamp
1064 ##         
1065 ## HVOFFTimestampsArray=array.array('i',HVOFFTimestamps) #NEED TO USE DOUBLES if we want the microseconds!!!! or it screws up approximating the timestamp
1066 ## NumberOfHVOFFChannelsArray=array.array('i',NumberOfHVOFFChannels)
1067 ## 
1068 ## #Testing the plotting function
1069 ## TkStatus.plotGraphSeconds(LVOFFTimestampsArray,NumberOfLVOFFChannelsArray,GraphTitle="Modules with LV OFF",YTitle="Number of modules with LV OFF",GraphFilename="Test2.gif")
1070 ## #TkStatus.plotGraphSeconds(LVOFFTimestampsArray,NumberOfLVOFFChannelsArray,"Modules with LV OFF","Number of modules with LV OFF","Test2.gif")
1071 
1072 #Develop the Validation based on the TkStatus object on one side and the actual output of the O2O reader (CheckAllIOVs) in the current dir:
1073 
1074 #FIXME:
1075 #Issues to look into:
1076 #1-first IOV not starting from beginning of the query: that should always be the case!
1077 #2-Reduction seems to not be doing the right thing...
1078 #3-timestamp timezone translation...
1079 
1080 #Get the ReducedIOVsTimestamps (using the TkVoltage.getReducedIOVs() using the same deltaT=15s, and maxIOVSequenceLength=120s):
1081 ReducedIOVsTimestamps=TkStatus.getReducedIOVs(datetime.datetime(2010, 8, 27, 12, 0),datetime.datetime(2010, 8, 29, 17, 45),TkStatus.PSUChannelHistory,15,120)
1082 #Print out for debugging the reduced timestamps with their index (to see when a sequence is present) and the number of LV/HV channels OFF for the given timestamp in the TkStatus object!
1083 print("Dumping ReducedIOVsTimestamps and the corresponding timestamp index and number of HV/LV channels off:")
1084 for timestamp in ReducedIOVsTimestamps:
1085     TkStatus.debug=False
1086     print(timestamp,sorted(TkStatus.PSUChannelHistory.keys()).index(timestamp),len(TkStatus.getDetIDsHVOff(timestamp)[0]), len(TkStatus.getDetIDsLVOff(timestamp)[0]))
1087 
1088 #Following function will be moved inside the TkVoltageStatus class once it's perfected:
1089 def ReducedIOVsHistory(ReducedIOVsTimestamps):
1090     """
1091     Function that given a list of reduced IOVs timestamps (output of getReducedIOVs()), analyses the timestamps to identify the IOV sequences and treats them differently when ramping-up or ramping-down, to return a dictionary that has timestamp as a key and the list of LV and HV channels OFF, that can be compared with the content of the CheckAllIOVs output DetVOffReaderDebug logs for validation.
1092     """
1093     
1094     AllTimestamps=sorted(TkStatus.PSUChannelHistory.keys())
1095     PreviousTimestampIndex=-1
1096     ReducedIOVsDict={}
1097     for timestamp in ReducedIOVsTimestamps:
1098         if AllTimestamps.index(timestamp)!=(PreviousTimestampIndex+1): #Sequence end!
1099             #Get the current timestamp LVOff channels:
1100             LVOffEnd=TkStatus.getDetIDsLVOff(timestamp)[0]
1101             #and the LV Off ones for the previous timestamp (beginning of the sequence):
1102             LVOffStart=TkStatus.getDetIDsLVOff(AllTimestamps[PreviousTimestampIndex])[0]
1103             #Get the current timestamp HVOff channels:
1104             HVOffEnd=TkStatus.getDetIDsHVOff(timestamp)[0]
1105             #and the HV Off ones for the previous timestamp (beginning of the sequence):
1106             HVOffStart=TkStatus.getDetIDsHVOff(AllTimestamps[PreviousTimestampIndex])[0]
1107             #This can be complicated... let's go for the same approach as the official O2O reduction
1108             #We can just test if the other conditions not captured by the ifs ever happen!
1109             #Turning OFF case:
1110             if len(LVOffEnd)>len(LVOffStart) or len(HVOffEnd)>len(HVOffStart): 
1111                 ReducedIOVsDict.update({AllTimestamps[PreviousTimestampIndex]:(TkStatus.getDetIDsHVOff(timestamp)[0],TkStatus.getDetIDsLVOff(timestamp)[0])}) #use the LVOff/HVOff form the last element in the sequence and set the first element of the sequence to it!
1112             #Turning (Staying) ON case:
1113             #Nothing special to do (same as not a sequence)! We're happy with having thrown away all the intermediate timestamps, and keep the validity of the first timestamp of the sequence throughout the sequence:
1114         #For all timestamps reported (if they are a start of a sequence they will be "overwritten" once we process the end of the sequence timestamp) in particular also the end of a ramp-up sequence does not need no special treatement:
1115         #Actually check if the LV Off or HVOff are the same as the previous timestamp: if they are do nothing, if they are not then add an IOV...
1116         if set(TkStatus.getDetIDsHVOff(timestamp)[0])!=set(TkStatus.getDetIDsHVOff(AllTimestamps[PreviousTimestampIndex])[0]) or set(TkStatus.getDetIDsLVOff(timestamp)[0])!=set(TkStatus.getDetIDsLVOff(AllTimestamps[PreviousTimestampIndex])[0]):
1117             ReducedIOVsDict.update({timestamp:(TkStatus.getDetIDsHVOff(timestamp)[0],TkStatus.getDetIDsLVOff(timestamp)[0])})
1118         PreviousTimestampIndex=AllTimestamps.index(timestamp)
1119     return ReducedIOVsDict
1120 
1121 #Now using the ReducedIOVs timestamps we can get the actual ReducedIOVs using the same algorithm as the O2O (implemented in the function ReducedIOVsHistory):
1122 ValidationReducedIOVsHistory=ReducedIOVsHistory(ReducedIOVsTimestamps)
1123 #Print out for debugging the timestamp, the number of HV/LV channels off from the TkStatus object for each of the REDUCED timestamps (AGAIN)
1124 #for timestamp in ReducedIOVsTimestamps:
1125 #    print timestamp,len(TkStatus.getDetIDsHVOff(timestamp)[0]),len(TkStatus.getDetIDsLVOff(timestamp)[0])
1126 
1127 #Print out for debugging the timestamp, the number of HV/LV channels OFF from the ValidationReducedIOVsHistory object directly!
1128 #i=0
1129 print("Dumping ValidationReducedIOVsHistory contents:")
1130 for timestamp in sorted(ValidationReducedIOVsHistory.keys()):
1131     print(timestamp, len(ValidationReducedIOVsHistory[timestamp][0]),len(ValidationReducedIOVsHistory[timestamp][1]))#,sorted(O2OReducedIOVs.keys())[i],len(O2OReducedIOVs[sorted(O2OReducedIOVs.keys())[i]][0]),len(O2OReducedIOVs[sorted(O2OReducedIOVs.keys())[i]][1])
1132 #    i=i+1
1133     
1134 #for i in range(42):
1135 #    print sorted(ValidationReducedIOVsHistory.keys())[i],sorted(O2OReducedIOVs.keys())[i]
1136 
1137 #Now extract the DetVOffInfo from the logfiles directly and then we can compare!
1138 
1139 #Cut&Paste of a quick and dirty script that reads the CheckAllIOVS.py output (ReaderDebug) logs and produces an O2OData dictionary
1140 #that has timestamps (from filename) as keys and a tuple (HVOff,LVOff) as value, where HVOff and LVOff are lists of detids that have HV/LV off respectively.
1141 #The union of the two would be the total number of detids listed as OFF (in either way, OFF-OFF or OFF-ON)
1142 #Can implement a direct IOV comparison using the reduction above (need to be careful on the matching of IOVs)
1143 #import datetime
1144 def ExtractDetVOffInfo(directory=os.getcwd()):
1145     """
1146     Function that given a directory (defaults to the local one in case no dir indicated), parses all local DetVOffReaderDebug*.log files and returna a dictionary with timestamps for keys and a tuple with the list of LV and HV channels OFF (LVOff,HVOff). 
1147     """
1148     ls=os.listdir(directory)
1149     TimesLogs=[]
1150     O2OData={}
1151     for log in ls:
1152         if "DetVOffReaderDebug__FROM" in log:
1153             (start,end)=log[:-4].split("FROM_")[1].split("_TO_")
1154             TimeStamp=datetime.datetime.strptime(start.replace("__","_0"),"%a_%b_%d_%H_%M_%S_%Y")
1155             #print start,TimeStamp
1156             file=open(log,'r')
1157             filelines=file.readlines()
1158             file.close()
1159             LVOff=[]
1160             HVOff=[]
1161             for line in filelines:
1162                 #print line
1163                 if "OFF" in line:
1164                     detid,hv,lv=line.split()
1165                     #print line,detid,hv,lv
1166                     if hv=="OFF":
1167                         HVOff.append(int(detid))
1168                     if lv=="OFF":
1169                         LVOff.append(int(detid))
1170                         
1171                     O2OData.update({TimeStamp:(HVOff,LVOff)})
1172     return O2OData
1173 
1174 #Extract the O2O Reduced IOVs data from the logfiles in the current directory 
1175 O2OReducedIOVs=ExtractDetVOffInfo()
1176 #Print out for debugging the timestamp, the number of HV/LV  channels OFF reported by the O2O
1177 print("Dumping the O2OReducedIOVs contents:")
1178 for timestamp in sorted(O2OReducedIOVs.keys()):
1179     print(timestamp, len(O2OReducedIOVs[timestamp][0]),len(O2OReducedIOVs[timestamp][1]))#,len(TkStatus.getDetIDsHVOff(TkStatus.getIOV(timestamp,TkStatus.PSUChannelHistory)[0])[0]), len(TkStatus.getDetIDsLVOff(TkStatus.getIOV(timestamp,TkStatus.PSUChannelHistory)[0])[0])
1180     #Compare the actual detids after doing the reduction the way we want to do it!
1181     
1182 # len(TkStatus.getDetIDsHVOff(sorted(TkStatus.PSUChannelHistory.keys())[-1])[0])
1183 
1184 
1185 #Now compare the reduced histories:
1186 def CompareReducedDetIDs(FirstDict,SecondDict):
1187     """
1188     Function that given 2 Dictionaries (key=timestamp, value=(LVOff,HVOff)) loops through the first one and compared the content of its IOVs with the IOV in the other dict that overlaps with it.
1189     """
1190     DifferenceDict={}
1191     for timestamp in sorted(FirstDict.keys()):
1192         if timestamp.replace(microsecond=0) in SecondDict.keys():
1193             secondtimestamp=timestamp.replace(microsecond=0)
1194             print("Timestamp %s is present in both Dictionaries!"%timestamp)
1195         else:
1196             secondtimestamps=sorted(SecondDict.keys())
1197             secondtimestamps.append(timestamp)
1198             secondtimestamps.sort()
1199             if secondtimestamps.index(timestamp)!=0:#To avoid wrapping up to the end of the list of timestamps!!!
1200                 secondtimestamp=secondtimestamps[secondtimestamps.index(timestamp)-1]
1201             else:#Default to the earliest timestamp in the second dictionary...
1202                 secondtimestamp=secondtimestamps[secondtimestamps.index(timestamp)+1]
1203             print("Comparing the IOV with timestamp %s (1st dict) with IOV with timestamp %s (2nd dict)"%(timestamp,secondtimestamp)) 
1204         if set(map(lambda x:int(x),FirstDict[timestamp][0]))!=set(map(lambda x:int(x),SecondDict[secondtimestamp][0])) or set(map(lambda x:int(x),FirstDict[timestamp][1]))!=set(map(lambda x:int(x),SecondDict[secondtimestamp][1])): #Change!
1205             if len(set(FirstDict[timestamp][0]))<=len(set(SecondDict[secondtimestamp][0])):
1206                 differenceHV=set(map(lambda x:int(x),SecondDict[secondtimestamp][0]))-set(map(lambda x:int(x),FirstDict[timestamp][0]))
1207             else:
1208             #elif len(set(SecondDict[secondtimestamp][0]))<len(set(FirstDict[timestamp][0])):
1209                 differenceHV=set(map(lambda x:int(x),FirstDict[timestamp][0]))-set(map(lambda x:int(x),SecondDict[secondtimestamp][0]))
1210             #else:
1211             #    print "SCREAM! Something weird going on one of the two should be a subset of the other!"
1212             #    differenceLV=set([])
1213             #    differenceHV=set([])
1214             if len(set(FirstDict[timestamp][1]))<=len(set(SecondDict[secondtimestamp][1])):
1215                 differenceLV=set(map(lambda x:int(x),SecondDict[secondtimestamp][1]))-set(map(lambda x:int(x),FirstDict[timestamp][1]))
1216             else:
1217             #elif set(SecondDict[secondtimestamp][1]).issubset(set(FirstDict[timestamp][1])):
1218                 differenceLV=set(map(lambda x:int(x),FirstDict[timestamp][1]))-set(map(lambda x:int(x),SecondDict[secondtimestamp][1]))
1219             #else:
1220             #    print "SCREAM! Something weird going on one of the two should be a subset of the other!"
1221             #    differenceLV=set([])
1222             #    differenceHV=set([])
1223             DifferenceDict.update({(timestamp,secondtimestamp):(differenceHV,differenceLV)})
1224             print("Difference in timestamp %s (corresponding to %s):"%(timestamp,secondtimestamp))
1225             #print "LV OFF:"
1226             #for LVChannel in differenceLV:
1227             #    print LVChannel
1228             #print "HV OFF:"
1229             #for HVChannel in differenceHV:
1230             #    print HVChannel
1231         else:
1232             print("Timestamp %s is identical in both dictionaries"%timestamp)
1233     return DifferenceDict
1234 Comparison=CompareReducedDetIDs(ValidationReducedIOVsHistory,O2OReducedIOVs)
1235 print("Dumping the results of the comparisons of the two dictionaries:")
1236 for timestamps in sorted(Comparison.keys()):
1237     print(timestamps, Comparison[timestamps])
1238     if Comparison[timestamps][0]:
1239         print("HV:")
1240         if Comparison[timestamps][0].issubset(set(O2OReducedIOVs[timestamps[1]][0])):
1241             print("Only in O2O Dict!")
1242         else:
1243             print("Only in Validation Dict!")
1244         for detid in Comparison[timestamps][0]:
1245             print(detid,TkStatus.DetIDAliasDict[detid])
1246     if Comparison[timestamps][1]:
1247         print("LV:")
1248         if Comparison[timestamps][1].issubset(set(O2OReducedIOVs[timestamps[1]][1])):
1249             print("Only in O2O Dict!")
1250         else:
1251             print("Only in Validation Dict!")
1252         for detid in Comparison[timestamps][1]:
1253             print(detid,TkStatus.DetIDAliasDict[detid])
1254 
1255 
1256 #Add a check with the query using sqlalchemy
1257 #Add a check for channels that never show updates in the queries (even when several ramp cycles happened)
1258 #Add a check that inside the sequence change is consistently up or down
1259 
1260 #Check of the direct DB query:
1261 #Need to first run another python script (using the local python of cmstrko2ovm02 that has sqlalchemy, cx-oracle etc) that dumps a pkl with a list of the result rows of the O2O query....
1262 
1263 TkStatusFromQuery=TrkVoltageStatus(detIDPSUmapfilename=os.path.join(os.getenv('CMSSW_BASE'),'src/CalibTracker/SiStripDCS/data/StripPSUDetIDMap_FromJan132010_Crosstalk.dat'),DetIDAliasFile=os.path.join(os.getenv('CMSSW_BASE'),'src/CalibTracker/SiStripDCS/data/StripDetIDAlias.pkl'),startTime=datetime.datetime(2010,8,27,10,00,00),debug=True)
1264 DBQueryPickle=open('/afs/cern.ch/user/g/gbenelli/scratch0/O2OValidation/QueryResults.pkl','rb')
1265 DBQueryResults=[]
1266 DBQueryResults=pickle.load(DBQueryPickle)
1267 DBQueryPickle.close()
1268 for row in DBQueryResults:
1269     TkStatusFromQuery.updateO2OQuery(row)
1270     print(len(TkStatusFromQuery.PSUChannelHistory))
1271 
1272 delta=datetime.timedelta(seconds=7200)
1273 counter=0
1274 DifferingTimestamps=[]
1275 for timestamp in sorted(TkStatus.PSUChannelHistory.keys()):
1276     if timestamp-delta!=sorted(TkStatusFromQuery.PSUChannelHistory.keys())[counter]:
1277         print(timestamp, sorted(TkStatusFromQuery.PSUChannelHistory.keys())[counter])
1278         DifferingTimestamps.append(timestamp,sorted(TkStatusFromQuery.PSUChannelHistory.keys())[counter])
1279     counter=counter+1
1280 if DifferingTimestamps:
1281     print("There are %s timestamps that are different in the 2 TkVoltageStatus objects!"%len(DifferingTimestamps))
1282 
1283 #Test issue with channel000 and channel001 lagging:
1284 #LVChannelsHistory={}
1285 #for timestamp in sorted(TkStatus.PSUChannelHistory.keys()):
1286 #    LVOFF=[psuchannel for psuchannel in TkStatus.getPSUChannelsOff(timestamp) if (psuchannel.endswith('0') or psuchannel.endswith('1'))]
1287 #    for channel in LVOFF:
1288 LVChannelHistory={}
1289 HVChannelHistory={}
1290 for interval in QueryResults.keys():
1291     for row in QueryResults[interval]:
1292         if row['dpname'].endswith('0') or row['dpname'].endswith('1'):
1293             if row['dpname'][:-1] not in LVChannelHistory.keys():
1294                 LVChannelHistory.update({row['dpname'][:-1]:[(row['change_date'],row['actual_status'])]})
1295             else:
1296                 LVChannelHistory[row['dpname'][:-1]].append((row['change_date'],row['actual_status']))
1297         else:
1298             if row['dpname'] not in HVChannelHistory.keys():
1299                 HVChannelHistory.update({row['dpname']:[(row['change_date'],row['actual_status'])]})
1300             else:
1301                 HVChannelHistory[row['dpname']].append((row['change_date'],row['actual_status']))
1302         
1303         #if row['change_date']==datetime.datetime(2010, 8, 28, 22, 52, 8, 994000):
1304         #    print row
1305         #if row['change_date']==datetime.datetime(2010, 8, 28, 22, 51, 8, 994000):
1306         #    print row
1307         if row['change_date']>datetime.datetime(2010, 8, 28, 22, 44) and row['change_date']<datetime.datetime(2010, 8, 28, 23, 5,37):
1308             print(row)    
1309             
1310         
1311 
1312 for row in sorted(HVChannelHistory['cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_6/branchController02/easyCrate3/easyBoard15/channel002']):
1313     print(row[0],row[1])
1314 for row in sorted(HVChannelHistory['cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_6/branchController02/easyCrate3/easyBoard15/channel003']):
1315     print(row[0],row[1])
1316 print(TkStatus.PSUChannelHistory[sorted(TkStatus.PSUChannelHistory.keys())[sorted(TkStatus.PSUChannelHistory.keys()).index(datetime.datetime(2010, 8, 28, 22, 52, 8, 994000))-1]])
1317 a=TkStatus.getIOV(datetime.datetime(2010, 8, 28, 22, 51, 16),TkStatus.PSUChannelHistory)
1318 
1319 #PLOTTTING!
1320 import array, time
1321 TOBminus_1_3_1_4_HV1_Status=HVChannelHistory['cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_6/branchController02/easyCrate3/easyBoard15/channel002']
1322 #Select time range via:
1323 TOBminus_1_3_1_4_HV1_Status=[item for item in HVChannelHistory['cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_6/branchController02/easyCrate3/easyBoard15/channel002'] if item[0]>datetime.datetime(2010,8,28,22) and item[0]<datetime.datetime(2010,8,29,2)]
1324 TOBTimestamps=[]
1325 TOBHVStatus=[]
1326 for item in TOBminus_1_3_1_4_HV1_Status:
1327     timestamp=int(time.mktime(item[0].timetuple()))
1328     if item==TOBminus_1_3_1_4_HV1_Status[0]: #First item does not need duplication
1329         TOBTimestamps.append(timestamp)
1330         HVStatus=int(item[1]==1)
1331         TOBHVStatus.append(HVStatus)
1332     else: #Add twice each timestamp except the first one (to make a histogram looking graph)
1333         TOBTimestamps.append(timestamp)
1334         TOBHVStatus.append(HVStatus) #Input previous value with new timestamp
1335         HVStatus=int(item[1]==1)
1336         TOBTimestamps.append(timestamp)
1337         TOBHVStatus.append(HVStatus) #Input new value with new timestamp
1338 
1339 
1340 #TOBTimestamps=map(lambda x: int(time.mktime(x[0].timetuple())),TOBminus_1_3_1_4_HV1_Status)
1341 #TOBHVStatus=map(lambda y: int(y[1]==1),TOBminus_1_3_1_4_HV1_Status)
1342 ##print "FIRST the original arrays:"
1343 ###Duplication of timestamps...
1344 ##for timestamp in [item[0] for item in TOBminus_1_3_1_4_HV1_Status]:
1345 ##    print timestamp,TOBHVStatus[TOBTimestamps.index(timestamp)]
1346 ##    TOBTimestamps.insert(TOBTimestamps.index(timestamp)+1,timestamp)
1347 ##    TOBHVStatus.insert(TOBTimestamps.index(timestamp)+1,TOBHVStatus[TOBTimestamps.index(timestamp)-1])
1348 ##print 'NOW "duplicated" arrays'
1349 ##for timestamp in TOBTimestamps:
1350 ##    print timestamp, TOBHVStatus[TOBTimestamps.index(timestamp)]
1351 ##        
1352 TOBTimestampsArray=array.array('i',TOBTimestamps)
1353 TOBHVStatusArray=array.array('i',TOBHVStatus)
1354 
1355 TkStatus.plotGraphSeconds(TOBTimestampsArray,TOBHVStatusArray,GraphTitle="PSUChannelGraph",YTitle="Channel HV Status",GraphFilename="PSUChannel.gif")
1356 TOBminus_1_3_1_4_HV1_Status=HVChannelHistory['cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_6/branchController02/easyCrate3/easyBoard15/channel003']
1357 #Select time range via:
1358 TOBminus_1_3_1_4_HV1_Status=[item for item in HVChannelHistory['cms_trk_dcs_04:CAEN/CMS_TRACKER_SY1527_6/branchController02/easyCrate3/easyBoard15/channel003'] if item[0]>datetime.datetime(2010,8,28,22) and item[0]<datetime.datetime(2010,8,29,2)]
1359 TOBTimestamps=[]
1360 TOBHVStatus=[]
1361 for item in TOBminus_1_3_1_4_HV1_Status:
1362     timestamp=int(time.mktime(item[0].timetuple()))
1363     if item==TOBminus_1_3_1_4_HV1_Status[0]: #First item does not need duplication
1364         TOBTimestamps.append(timestamp)
1365         HVStatus=int(item[1]==1)
1366         TOBHVStatus.append(HVStatus)
1367     else: #Add twice each timestamp except the first one (to make a histogram looking graph)
1368         TOBTimestamps.append(timestamp)
1369         TOBHVStatus.append(HVStatus) #Input previous value with new timestamp
1370         HVStatus=int(item[1]==1)
1371         TOBTimestamps.append(timestamp)
1372         TOBHVStatus.append(HVStatus) #Input new value with new timestamp
1373 
1374 
1375 #TOBTimestamps=map(lambda x: int(time.mktime(x[0].timetuple())),TOBminus_1_3_1_4_HV1_Status)
1376 #TOBHVStatus=map(lambda y: int(y[1]==1),TOBminus_1_3_1_4_HV1_Status)
1377 ##print "FIRST the original arrays:"
1378 ###Duplication of timestamps...
1379 ##for timestamp in [item[0] for item in TOBminus_1_3_1_4_HV1_Status]:
1380 ##    print timestamp,TOBHVStatus[TOBTimestamps.index(timestamp)]
1381 ##    TOBTimestamps.insert(TOBTimestamps.index(timestamp)+1,timestamp)
1382 ##    TOBHVStatus.insert(TOBTimestamps.index(timestamp)+1,TOBHVStatus[TOBTimestamps.index(timestamp)-1])
1383 ##print 'NOW "duplicated" arrays'
1384 ##for timestamp in TOBTimestamps:
1385 ##    print timestamp, TOBHVStatus[TOBTimestamps.index(timestamp)]
1386 ##        
1387 TOBTimestampsArray=array.array('i',TOBTimestamps)
1388 TOBHVStatusArray=array.array('i',TOBHVStatus)
1389 
1390 TkStatus.plotGraphSeconds(TOBTimestampsArray,TOBHVStatusArray,GraphTitle="PSUChannelGraph",YTitle="Channel HV Status",GraphFilename="PSUChannelHV2.gif")
1391 def plotPSUChannelvsTime(self,TimeArray,ValuesArray,GraphTitle="PSUChannelGraph",YTitle="Channel HV Status",GraphFilename="PSUChannel.gif"):
1392     """
1393     Function that given an array with timestamps (massaged to introduce a second timestamp for each value to produce an histogram-looking plot) and a corresponding array with values to be plotted, a title, a Y axis title and a plot filename, produces with pyROOT a time plot and saves it locally.
1394     The function can be used for cumulative plots (number of channels with HV/LV OFF/ON vs time) or for individual (single detID HV/LV status vs time) plots.
1395     """
1396     import ROOT
1397     canvas=ROOT.TCanvas()
1398     graph=ROOT.TGraph(len(TimeArray),TimeArray,ValuesArray)
1399     graph.GetXaxis().SetTimeDisplay(1)
1400     graph.GetXaxis().SetLabelOffset(0.02)
1401     #Set the time format for the X Axis labels depending on the total time interval of the plot!
1402     TotalTimeIntervalSecs=TimeArray[-1]-TimeArray[0]
1403     if TotalTimeIntervalSecs <= 120: #When zooming into less than 2 mins total interval report seconds too
1404         graph.GetXaxis().SetTimeFormat("#splitline{   %d/%m}{%H:%M:%S}")
1405     elif 120 < TotalTimeIntervalSecs < 6400: #When zooming into less than 2 hrs total interval report minutes too
1406         graph.GetXaxis().SetTimeFormat("#splitline{%d/%m}{%H:%M}")
1407     else: #When plotting more than 2 hrs only report the date and hour of day
1408         graph.GetXaxis().SetTimeFormat("#splitline{%d/%m}{  %H}")
1409     graph.GetYaxis().SetTitle(YTitle)
1410     graph.GetYaxis().SetTitleOffset(1.4)
1411     graph.SetTitle(GraphTitle)
1412     graph.Draw("APL")
1413     canvas.SaveAs(GraphFilename)
1414     print("Saved graph as %s"%GraphFilename)
1415     return
1416 
1417 
1418 ReducedIOVsTimestampsTEST=TkStatus.getReducedIOVs(datetime.datetime(2010, 8, 27, 12, 0),datetime.datetime(2010, 8, 29, 17, 45),TkStatus.PSUChannelHistory,2,90)
1419 #Print out for debugging the reduced timestamps with their index (to see when a sequence is present) and the number of LV/HV channels OFF for the given timestamp in the TkStatus object!
1420 print("Dumping ReducedIOVsTimestamps and the corresponding timestamp index and number of HV/LV channels off:")
1421 for timestamp in ReducedIOVsTimestampsTEST:
1422     TkStatus.debug=False
1423     print(timestamp,sorted(TkStatus.PSUChannelHistory.keys()).index(timestamp),len(TkStatus.getDetIDsHVOff(timestamp)[0]), len(TkStatus.getDetIDsLVOff(timestamp)[0]))
1424 ValidationReducedIOVsHistoryTEST=ReducedIOVsHistory(ReducedIOVsTimestampsTEST)
1425 print("Dumping ValidationReducedIOVsHistory contents:")
1426 for timestamp in sorted(ValidationReducedIOVsHistoryTEST.keys()):
1427     print(timestamp, len(ValidationReducedIOVsHistoryTEST[timestamp][0]),len(ValidationReducedIOVsHistoryTEST[timestamp][1]))#,sorted(O2OReducedIOVs.keys())[i],len(O2OReducedIOVs[sorted(O2OReducedIOVs.keys())[i]][0]),len(O2OR