File indexing completed on 2024-11-26 02:34:10
0001
0002
0003 import sys
0004 import math
0005 from ROOT import gROOT, TClass, TFile
0006 from copy import deepcopy
0007 from scipy import signal
0008
0009 gROOT.SetBatch()
0010
0011 class InefficientDeadROCs:
0012
0013
0014 def __TraverseDirTree(self, dir):
0015
0016 for obj in dir.GetListOfKeys():
0017 if not obj.IsFolder():
0018 if obj.ReadObjectAny(TClass.GetClass("TH2")):
0019 th1 = deepcopy(obj.ReadObj())
0020 name = th1.GetName()
0021 if name.startswith(self.lookForStr):
0022
0023 newName = name.split(self.lookForStr)[1]
0024 th1.SetName(newName)
0025
0026
0027 layer = 0
0028
0029 if newName.startswith("B"):
0030 layer = "B" + ((newName.split("_LYR"))[1])[0]
0031 else:
0032 layer = ((newName.split("_D"))[1])[0]
0033 if newName.startswith("FPix_Bm"):
0034 layer = "-" + layer
0035 layer = "F" + layer
0036
0037 if layer in self.dicOfModuleHistograms:
0038 self.dicOfModuleHistograms[layer].append(th1)
0039 else:
0040 self.dicOfModuleHistograms.update({layer : [th1]})
0041 else:
0042 self.__TraverseDirTree(obj.ReadObj())
0043
0044 def __init__(self, inputDQMName, outputFileName, noiseOutputFileName, dirs):
0045
0046 self.inputFileName = inputDQMName
0047 self.outputFileName = outputFileName
0048 self.noiseOutputFileName = noiseOutputFileName
0049 self.dirs = dirs
0050
0051 self.lookForStr = "digi_occupancy_per_col_per_row_"
0052
0053 self.rocMaxCol = 52
0054 self.rocMaxRow = 80
0055 self.rocsInRow = 8
0056 self.rocsInCol = 2
0057
0058 self.inputFile = TFile(self.inputFileName)
0059 self.dicOfModuleHistograms = {}
0060
0061
0062 self.pixelNoisynessTh = 6
0063 self.rocOccupancyTh = 200
0064
0065 self.barrelNoisyColumnTh = 1.35
0066 self.barrelNoisyColumnTh2 = 4.5
0067 self.endcapNoisyColumnTh = 1.5
0068
0069 self.barrelInefficientDColTh = 8
0070 self.endcapInefficientDColTh = 30
0071
0072
0073
0074 if self.inputFile.IsOpen():
0075 print("%s opened successfully!" % (self.inputFileName))
0076
0077 for dir in self.dirs:
0078 self.__TraverseDirTree(self.inputFile.Get(dir))
0079 print("Histograms to read: %d" % (len(self.dicOfModuleHistograms)))
0080
0081 self.detDict = {}
0082
0083 else:
0084 print("Unable to open file %s" % (self.inputFileName))
0085
0086 def __lmsExp(self, data, xMin, xMax):
0087 meanOfX = (xMax + xMin) * 0.5
0088 meanOfY = sum( [math.log(data[i]) for i in range(len(data))] ) / len(data)
0089
0090 D = 0
0091 for i in range(xMin, xMax + 1):
0092 D = D + (i - meanOfX)**2
0093
0094
0095 a = 0
0096 for i in range(len(data)):
0097 a = a + math.log(data[i]) * (xMin + i - meanOfX)
0098 a = a/D
0099
0100 lnb = meanOfY - a * meanOfX
0101
0102 return a, math.exp(lnb)
0103
0104 def __lmsLin(self, data, xMin, xMax):
0105 meanOfX = (xMax + xMin) * 0.5
0106 meanOfY = sum(data) / len(data)
0107
0108 D = 0
0109 for i in range(xMin, xMax + 1):
0110 D = D + (i - meanOfX)**2
0111
0112 a = 0
0113 for i in range(len(data)):
0114 a = a + data[i] * (xMin + i - meanOfX)
0115 a = a/D
0116
0117 b = meanOfY - a * meanOfX
0118
0119 return a, b, D
0120
0121 def __customMedianFilter(self, array, radius = 2):
0122
0123 filtered = [0 for i in range(len(array))]
0124 currArray = []
0125 for i in range(len(array)):
0126 if i - radius < 0:
0127 currArray = array[0 : i + radius + 1]
0128 elif i + radius + 1 >= len(array):
0129 currArray = array[i - radius : ]
0130
0131 currArray.sort()
0132 filtered[i] = currArray[len(currArray) // 2]
0133
0134 return filtered
0135
0136 def __getROCData(self, hist, startPixel, endPixel, row, repeatFilter = 3, filterKernelSize = 5):
0137 pixelArr = []
0138 columnsWithSuspiciouslyNoisyPixels = []
0139
0140 for x in range(startPixel, endPixel):
0141
0142 columnPixels = [hist.GetBinContent(x, y + 1) for y in range(row * self.rocMaxRow, (row + 1) * self.rocMaxRow)]
0143 columnSum = sum(columnPixels)
0144
0145 columnMean = columnSum / len(columnPixels)
0146 for i in range(len(columnPixels)):
0147 if columnPixels[i] > self.pixelNoisynessTh * columnMean:
0148
0149
0150 columnsWithSuspiciouslyNoisyPixels.append(x)
0151
0152
0153 break
0154
0155 pixelArr.append(columnSum)
0156
0157 if len(pixelArr) == 0:
0158 return None, None, None
0159
0160 medFiltRes, sciPyMedFiltRes = deepcopy(pixelArr), deepcopy(pixelArr)
0161 for i in range(repeatFilter):
0162 sciPyMedFiltRes = signal.medfilt(sciPyMedFiltRes, filterKernelSize)
0163 medFiltRes = self.__customMedianFilter(medFiltRes, filterKernelSize // 2)
0164
0165 return pixelArr, medFiltRes, columnsWithSuspiciouslyNoisyPixels, sciPyMedFiltRes
0166
0167 def __getPixelArrWithRemovedDrops(self, pixelArr, medFiltRes):
0168 return [ (pixelArr[i] if pixelArr[i] > medFiltRes[i] else medFiltRes[i]) if 0 < i < len(pixelArr) - 1 else min(medFiltRes) for i in range(len(pixelArr))]
0169
0170 def __normalizeArray(self, pixelArr):
0171 c_min, c_max = min(pixelArr), max(pixelArr)
0172 if c_min != c_max:
0173 c_diff_inv = 1.0 / (c_max - c_min)
0174 return [ (pixelArr[i] - c_min) * c_diff_inv for i in range(len(pixelArr))]
0175 return [0 for i in range(len(pixelArr))]
0176
0177 def __setNormalizedArrayZeroInThePoint(self, pixelArr, pt):
0178 c_diff_inv = 1.0 / (1.0 - pt)
0179
0180 return [ (pixelArr[i] - pt) * c_diff_inv for i in range(len(pixelArr))]
0181
0182 def __determineBarrelNoise(self, noiseFile, columnsWithSuspiciouslyNoisyPixels, histName, meanOfPixels, maxMed, val, pos, rocCol, rocRow):
0183 noisyROC = False;
0184 if meanOfPixels < self.rocOccupancyTh:
0185
0186 noisyROC = True
0187 else:
0188 th = self.barrelNoisyColumnTh * maxMed
0189 if val > th:
0190 if pos not in columnsWithSuspiciouslyNoisyPixels:
0191 rocNum, xCoordInROC = self.__convertCoordinatesFromHistToROCSpace(histName, pos, rocRow)
0192 noiseFile.write("%s\t(x, row)->[rocNum, xRoc]\t(%d, %d)->[%d, %d];\t{VAL, TH}\t{%f, %f}\n" % (histName, pos, rocRow+1, rocNum, xCoordInROC, val, th))
0193
0194 return 1, noisyROC
0195
0196
0197
0198 return 0, noisyROC
0199
0200 def __determineBarrelNoise2(self, noiseFile, columnsWithSuspiciouslyNoisyPixels, histName, meanOfPixels, normMeanOfPixels, normVal, pos, rocCol, rocRow):
0201 noisyROC = False;
0202 if meanOfPixels < self.rocOccupancyTh:
0203
0204 noisyROC = True
0205 else:
0206 th = self.barrelNoisyColumnTh2 * normMeanOfPixels
0207 if normVal > th:
0208 if pos not in columnsWithSuspiciouslyNoisyPixels:
0209 rocNum, xCoordInROC = self.__convertCoordinatesFromHistToROCSpace(histName, pos, rocRow)
0210 noiseFile.write("%s\t(x, row)->[rocNum, xRoc]\t(%d, %d)->[%d, %d];\t{NORMVAL, TH}\t{%f, %f}\n" % (histName, pos, rocRow+1, rocNum, xCoordInROC, normVal, th))
0211
0212 return 1, noisyROC
0213
0214
0215
0216 return 0, noisyROC
0217
0218 def __determineEndcapNoise(self, noiseFile, columnsWithSuspiciouslyNoisyPixels, histName, meanOfPixels, linVal, val, pos, rocCol, rocRow):
0219 noisyROC = False;
0220 if meanOfPixels < self.rocOccupancyTh:
0221
0222 noisyROC = True
0223
0224 else:
0225 th = self.endcapNoisyColumnTh * linVal
0226 if val > th:
0227 if pos not in columnsWithSuspiciouslyNoisyPixels:
0228 rocNum, xCoordInROC = self.__convertCoordinatesFromHistToROCSpace(histName, pos, rocRow)
0229 noiseFile.write("%s\t(x, row)->[rocNum, xRoc]\t(%d, %d)->[%d, %d];\t{VAL, TH}\t{%f, %f}\n" % (histName, pos, rocRow+1, rocNum, xCoordInROC, val, th))
0230
0231 return 1, noisyROC
0232
0233
0234
0235 return 0, noisyROC
0236
0237 def __convertCoordinatesFromHistToROCSpace(self, histName, histXpos, histRow):
0238 tempXROC = (histXpos / self.rocMaxCol)
0239 tempYROC = histRow
0240
0241 tempXCoordInROC = histXpos % self.rocMaxCol
0242
0243 realXROC, realYROC = tempXROC, tempYROC
0244 xCoordInROC = tempXCoordInROC
0245
0246 rocNum = 0
0247
0248 if histName.find("BPix_Bp") != -1:
0249 realYROC = 1 - tempYROC
0250 if realYROC == 1:
0251 rocNum = 15 - realXROC
0252 xCoordInROC = self.rocMaxCol - 1 - xCoordInROC
0253 else:
0254 rocNum = realXROC
0255 else:
0256 realXROC = 7 - tempXROC
0257 if realYROC == 1:
0258 rocNum = 15 - realXROC
0259 else:
0260 rocNum = realXROC
0261 xCoordInROC = self.rocMaxCol - 1 - xCoordInROC
0262
0263 return rocNum, xCoordInROC
0264
0265
0266 def __determineBarrelDColInefficiencyAndNoise(self, medFiltRes, histName, pixelArr, pixelArrWithoutDrops, startPixel, rocCol, rocRow, outputFile, columnsWithSuspiciouslyNoisyPixels, noiseFile):
0267 meanOfPixels = sum(medFiltRes) / len(medFiltRes)
0268 maxMed = max(medFiltRes)
0269 minMed = min(medFiltRes)
0270
0271 normMeanOfPixels = sum(pixelArrWithoutDrops) / len(pixelArrWithoutDrops)
0272
0273
0274 doubleDeadCols = 0
0275 noisyColsNum = 0
0276 noisyROC = 0
0277
0278
0279 for i in range(1, len(pixelArr) - 2):
0280
0281 bin1valDiff = minMed - pixelArr[i + 0]
0282 bin2valDiff = minMed - pixelArr[i + 1]
0283
0284 bin0valDiff = minMed - pixelArr[i - 1]
0285 bin3valDiff = minMed - pixelArr[i + 2]
0286
0287
0288 currentDoubleBinThreshold = math.sqrt(meanOfPixels) * self.barrelInefficientDColTh
0289
0290 if bin1valDiff > currentDoubleBinThreshold and bin2valDiff > currentDoubleBinThreshold and not bin3valDiff > currentDoubleBinThreshold and not bin0valDiff > currentDoubleBinThreshold:
0291
0292 doubleColInRoc = ((i + startPixel) % (self.rocMaxCol)) // 2 + 1
0293 doubleDeadCols = doubleDeadCols + 1
0294
0295
0296 rocNum, xCoordInROC = self.__convertCoordinatesFromHistToROCSpace(histName, startPixel + i, rocRow)
0297 outputFile.write("%s\t(x, row)->[rocNum, doubleXPixelColInROC]\t(%d, %d)->[%d, %d];\t{MIN - VAL, TH}\t{%f, %f}\n" % (histName, startPixel + i, rocRow + 1, rocNum, xCoordInROC / 2, bin1valDiff, currentDoubleBinThreshold))
0298
0299
0300 if noisyROC == True:
0301 continue
0302
0303
0304
0305
0306
0307
0308
0309
0310 res = self.__determineBarrelNoise2(noiseFile, columnsWithSuspiciouslyNoisyPixels, histName, meanOfPixels, normMeanOfPixels, pixelArrWithoutDrops[i], startPixel + i, rocCol, rocRow)
0311 noisyColsNum, noisyROC = noisyColsNum + res[0], res[1]
0312 if i == len(pixelArr) - 3:
0313 res = self.__determineBarrelNoise2(noiseFile, columnsWithSuspiciouslyNoisyPixels, histName, meanOfPixels, normMeanOfPixels, pixelArrWithoutDrops[i + 1], startPixel + i + 1, rocCol, rocRow)
0314 noisyColsNum, noisyROC = noisyColsNum + res[0], res[1]
0315
0316
0317
0318 return doubleDeadCols, noisyColsNum
0319
0320 def __determineEndcapDColInefficiencyAndNoise(self, medFiltRes, histName, pixelArr, startPixel, rocCol, rocRow, outputFile, columnsWithSuspiciouslyNoisyPixels, noiseFile):
0321 doubleDeadCols = 0
0322 noisyColsNum = 0
0323 noisyROC = 0
0324
0325 useLin = True
0326
0327 a, b, D = self.__lmsLin(medFiltRes, startPixel, len(medFiltRes) + startPixel)
0328
0329 meanOfPixels = sum(medFiltRes) / len(medFiltRes)
0330
0331
0332 for i in range(1, len(pixelArr) - 2):
0333
0334 if useLin == True:
0335 linVal1 = a * (i + startPixel + 0) + b
0336 linVal2 = a * (i + startPixel + 1) + b
0337
0338 linVal0 = a * (i + startPixel - 1) + b
0339 linVal3 = a * (i + startPixel + 2) + b
0340 else:
0341 linVal1 = b * math.exp(a * (i + startPixel + 0))
0342 linVal2 = b * math.exp(a * (i + startPixel + 1))
0343
0344 linVal0 = b * math.exp(a * (i + startPixel - 1))
0345 linVal3 = b * math.exp(a * (i + startPixel + 2))
0346
0347 bin1valDiff = linVal1 - pixelArr[i + 0]
0348 bin2valDiff = linVal2 - pixelArr[i + 1]
0349
0350 bin0valDiff = linVal0 - pixelArr[i - 1]
0351 bin3valDiff = linVal3 - pixelArr[i + 2]
0352
0353 try:
0354 currentDoubleBinThreshold = math.sqrt((linVal1 + linVal2) * 0.5) * self.endcapInefficientDColTh
0355 except:
0356
0357 continue
0358
0359 if bin1valDiff > currentDoubleBinThreshold and bin2valDiff > currentDoubleBinThreshold and not bin3valDiff > currentDoubleBinThreshold and not bin0valDiff > currentDoubleBinThreshold:
0360
0361 doubleColInRoc = ((i + startPixel) % (self.rocMaxCol)) // 2 + 1
0362 doubleDeadCols = doubleDeadCols + 1
0363
0364
0365 rocNum, xCoordInROC = self.__convertCoordinatesFromHistToROCSpace(histName, startPixel + i, rocRow)
0366 outputFile.write("%s\t(x, row)->[rocNum, doubleXPixelColInROC]\t(%d, %d)->[%d, %d];\t{LIN(x) - VAL, TH}\t{%f, %f}\n" % (histName, startPixel + i, rocRow + 1, rocNum, xCoordInROC / 2, bin1valDiff, currentDoubleBinThreshold))
0367
0368
0369
0370 if noisyROC == True:
0371 continue
0372
0373 res = self.__determineEndcapNoise(noiseFile, columnsWithSuspiciouslyNoisyPixels, histName, meanOfPixels, linVal1, pixelArr[i], i + startPixel, rocCol, rocRow)
0374 noisyColsNum, noisyROC = noisyColsNum + res[0], res[1]
0375 if i == len(pixelArr) - 3:
0376 res = self.__determineEndcapNoise(noiseFile, columnsWithSuspiciouslyNoisyPixels, histName, meanOfPixels, linVal2, pixelArr[i + 1], i + 1 + startPixel, rocCol, rocRow)
0377 noisyColsNum, noisyROC = noisyColsNum + res[0], res[1]
0378
0379 return doubleDeadCols, noisyColsNum
0380
0381 def ReadHistograms(self):
0382 doubleDeadCols, noisyColsNum = 0, 0
0383
0384 with open(self.noiseOutputFileName, "w") as noiseFile:
0385 with open(self.outputFileName, "w") as outputFile:
0386 for layer in self.dicOfModuleHistograms:
0387
0388 doubleDeadColsInLayer, noisyColsNumInLayer = 0, 0
0389
0390 outputFile.write("-> " + layer + "\n\n")
0391 noiseFile.write("-> " + layer + "\n\n")
0392
0393 for hist in self.dicOfModuleHistograms[layer]:
0394 for row in range(2):
0395 for rocNum in range(self.rocsInRow):
0396 startPixel = rocNum * self.rocMaxCol + 1
0397 endPixel = (rocNum + 1) * self.rocMaxCol + 1
0398
0399 rocCol = rocNum + 1
0400
0401 pixelArr, medFiltRes, columnsWithSuspiciouslyNoisyPixels, sciPyMedFiltRes = self.__getROCData(hist, startPixel, endPixel, row, 3, 5)
0402
0403 if pixelArr == None:
0404 continue
0405
0406
0407
0408
0409
0410
0411 if "F" not in layer:
0412
0413 pixelArrWithoutDrops = self.__getPixelArrWithRemovedDrops(pixelArr, sciPyMedFiltRes)
0414 pixelArrWithoutDropsNormalized = self.__normalizeArray(pixelArrWithoutDrops)
0415
0416
0417
0418
0419 result = self.__determineBarrelDColInefficiencyAndNoise(medFiltRes, hist.GetName(), pixelArr, pixelArrWithoutDropsNormalized, startPixel, rocCol, row, outputFile, columnsWithSuspiciouslyNoisyPixels, noiseFile)
0420 else:
0421 result = self.__determineEndcapDColInefficiencyAndNoise(medFiltRes, hist.GetName(), pixelArr, startPixel, rocCol, row, outputFile, columnsWithSuspiciouslyNoisyPixels, noiseFile)
0422
0423 doubleDeadCols, doubleDeadColsInLayer = doubleDeadCols + result[0], doubleDeadColsInLayer + result[0]
0424 noisyColsNum, noisyColsNumInLayer = noisyColsNum + result[1], noisyColsNumInLayer + result[1]
0425
0426 outputFile.write("\n\tTOTAL IN LAYER/DISK: %d\n\n" % (doubleDeadColsInLayer))
0427 noiseFile.write("\n\tTOTAL IN LAYER/DISK: %d\n\n" % (noisyColsNumInLayer))
0428
0429 print("Number of inefficient double columns: %d"%(doubleDeadCols))
0430 print("Number of noisy cols: %d"%(noisyColsNum))
0431
0432
0433
0434 for i in range(1, len(sys.argv), 1):
0435 if i == 1:
0436 inputFileName = sys.argv[i]
0437
0438 runNum = ((inputFileName.split("/")[-1].split("."))[0].split("_R000"))[1]
0439 print("Run number: %s"%(runNum))
0440 baseRootDir = ["DQMData/Run " + runNum + "/PixelPhase1/Run summary/Phase1_MechanicalView"]
0441 print(baseRootDir[0])
0442 outputFileName = "inefficientDPixelColumns.txt"
0443 noiseOutputFileName = "noisyPixelColumns.txt"
0444
0445 readerObj = InefficientDeadROCs(inputFileName, outputFileName, noiseOutputFileName, baseRootDir)
0446 readerObj.ReadHistograms()