File indexing completed on 2024-11-06 06:06:09
0001 import copy
0002 import os
0003
0004 def DMR(config, validationDir):
0005
0006 jobs = []
0007 dmrType = "single"
0008
0009
0010 IOVs = {}
0011
0012
0013 isDataMerged = {}
0014
0015
0016 if not dmrType in config["validations"]["DMR"]:
0017 raise Exception("No 'single' key word in config for DMR")
0018
0019 for singleName in config["validations"]["DMR"][dmrType]:
0020 aux_IOV = config["validations"]["DMR"][dmrType][singleName]["IOV"]
0021 if not isinstance(aux_IOV, list) and aux_IOV.endswith(".txt"):
0022 config["validations"]["DMR"][dmrType][singleName]["IOV"] = []
0023 with open(aux_IOV, 'r') as IOVfile:
0024 for line in IOVfile.readlines():
0025 if len(line) != 0: config["validations"]["DMR"][dmrType][singleName]["IOV"].append(int(line))
0026 for IOV in config["validations"]["DMR"][dmrType][singleName]["IOV"]:
0027
0028 if singleName not in IOVs.keys():
0029 IOVs[singleName] = []
0030 if IOV not in IOVs[singleName]:
0031 IOVs[singleName].append(IOV)
0032
0033 for alignment in config["validations"]["DMR"][dmrType][singleName]["alignments"]:
0034
0035 workDir = "{}/DMR/{}/{}/{}/{}".format(validationDir, dmrType, singleName, alignment, IOV)
0036
0037
0038 local = {}
0039 local["output"] = "{}/{}/DMR/{}/{}/{}/{}".format(config["LFS"], config["name"], dmrType, alignment, singleName, IOV)
0040 local["alignment"] = copy.deepcopy(config["alignments"][alignment])
0041 local["validation"] = copy.deepcopy(config["validations"]["DMR"][dmrType][singleName])
0042 local["validation"].pop("alignments")
0043 local["validation"]["IOV"] = IOV
0044 if "dataset" in local["validation"]:
0045 local["validation"]["dataset"] = local["validation"]["dataset"].format(IOV)
0046 if "goodlumi" in local["validation"]:
0047 local["validation"]["goodlumi"] = local["validation"]["goodlumi"].format(IOV)
0048
0049
0050 job = {
0051 "name": "DMR_{}_{}_{}_{}".format(dmrType, alignment, singleName, IOV),
0052 "dir": workDir,
0053 "exe": "cmsRun",
0054 "cms-config": "{}/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/DMR_cfg.py".format(os.environ["CMSSW_BASE"]),
0055 "run-mode": "Condor",
0056 "dependencies": [],
0057 "config": local,
0058 }
0059
0060 jobs.append(job)
0061
0062
0063 if "merge" in config["validations"]["DMR"]:
0064
0065 mergeJobs = []
0066 dmrType = "merge"
0067
0068
0069 for mergeName in config["validations"]["DMR"][dmrType]:
0070
0071 singlesMC = []
0072 for singleName in config["validations"]["DMR"][dmrType][mergeName]['singles']:
0073 if len(IOVs[singleName]) == 1 and int(IOVs[singleName][0]) == 1: singlesMC.append(singleName)
0074 isMConly = (len(singlesMC) == len(config["validations"]["DMR"][dmrType][mergeName]['singles']))
0075 if isMConly:
0076 isDataMerged[mergeName] = 0
0077 elif len(singlesMC) == 0:
0078 isDataMerged[mergeName] = 1
0079 else:
0080 isDataMerged[mergeName] = -1
0081
0082
0083 for iname,singleName in enumerate(config["validations"]["DMR"][dmrType][mergeName]['singles']):
0084 isMC = (singleName in singlesMC)
0085 if isMConly and iname > 0: continue
0086 elif isMConly: singlesMC.pop(singlesMC.index(singleName))
0087
0088 for IOV in IOVs[singleName]:
0089 if isMC and not isMConly: continue
0090
0091
0092 workDir = "{}/DMR/{}/{}/{}".format(validationDir, dmrType, mergeName, IOV)
0093
0094
0095 local = {}
0096
0097 job = {
0098 "name": "DMR_{}_{}_{}".format(dmrType, mergeName, IOV),
0099 "dir": workDir,
0100 "exe": "DMRmerge",
0101 "run-mode": "Condor",
0102 "dependencies": [],
0103 "config": local,
0104 }
0105
0106
0107 for alignment in config["alignments"]:
0108 idxIncrement = 0
0109 local.setdefault("alignments", {})
0110 if alignment in config["validations"]["DMR"]["single"][singleName]["alignments"]:
0111 local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
0112 local["alignments"][alignment]['index'] = config["validations"]["DMR"]["single"][singleName]["alignments"].index(alignment)
0113 for singleMCname in singlesMC:
0114 if alignment in config["validations"]["DMR"]["single"][singleMCname]["alignments"]:
0115 local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
0116 local["alignments"][alignment]['index'] = len(config["validations"]["DMR"]["single"][singleName]["alignments"])
0117 local["alignments"][alignment]['index'] += idxIncrement + config["validations"]["DMR"]["single"][singleMCname]["alignments"].index(alignment)
0118 idxIncrement += len(config["validations"]["DMR"]["single"][singleMCname]["alignments"])
0119 local["validation"] = copy.deepcopy(config["validations"]["DMR"][dmrType][mergeName])
0120 local["validation"]["IOV"] = IOV
0121 if "customrighttitle" in local["validation"].keys():
0122 if "IOV" in local["validation"]["customrighttitle"]:
0123 local["validation"]["customrighttitle"] = local["validation"]["customrighttitle"].replace("IOV",str(IOV))
0124 local["output"] = "{}/{}/DMR/{}/{}/{}".format(config["LFS"], config["name"], dmrType, mergeName, IOV)
0125
0126
0127 if "style" in config.keys():
0128 if "DMR" in config['style'].keys():
0129 if dmrType in config['style']['DMR'].keys():
0130 local["style"] = copy.deepcopy(config["style"]["DMR"][dmrType])
0131 if "Rlabel" in local["style"] and "customrighttitle" in local["validation"].keys():
0132 print("WARNING: custom right label is overwritten by global settings")
0133
0134
0135 for singleJob in jobs:
0136
0137 _alignment, _singleName, _singleIOV = singleJob["name"].split("_")[2:]
0138 if _singleName in config["validations"]["DMR"][dmrType][mergeName]["singles"]:
0139 if int(_singleIOV) == IOV or (int(_singleIOV) == 1 and _singleName in singlesMC):
0140 local["alignments"][_alignment]["file"] = singleJob["config"]["output"]
0141 job["dependencies"].append(singleJob["name"])
0142
0143
0144 mergeJobs.append(job)
0145
0146
0147 jobs.extend(mergeJobs)
0148
0149 if "trends" in config["validations"]["DMR"]:
0150
0151
0152 trendJobs = []
0153 dmrType = "trends"
0154
0155 for trendName in config["validations"]["DMR"][dmrType]:
0156
0157
0158 workDir = "{}/DMR/{}/{}".format(validationDir, dmrType, trendName)
0159
0160
0161 local = {}
0162 job = {
0163 "name": "DMR_{}_{}".format(dmrType, trendName),
0164 "dir": workDir,
0165 "exe": "DMRtrends",
0166 "run-mode": "Condor",
0167 "dependencies": [],
0168 "config": local,
0169 }
0170
0171
0172 mergesDATA = []
0173 for mergeName in config["validations"]["DMR"][dmrType][trendName]["merges"]:
0174
0175 if isDataMerged[mergeName] < 0:
0176 raise Exception("Trend jobs cannot process merge jobs containing both DATA and MC objects.")
0177 elif isDataMerged[mergeName] == 1:
0178 mergesDATA.append(mergeName)
0179 else:
0180 if "doUnitTest" in config["validations"]["DMR"][dmrType][trendName].keys() and config["validations"]["DMR"][dmrType][trendName]["doUnitTest"]:
0181 local.setdefault("alignments", {})
0182 continue
0183 else:
0184 raise Exception("Trend jobs are not implemented for treating MC.")
0185
0186
0187 trendIOVs = []
0188 _mergeFiles = []
0189 for mergeName in mergesDATA:
0190 for iname,singleName in enumerate(config["validations"]["DMR"]['merge'][mergeName]['singles']):
0191 trendIOVs += [IOV for IOV in IOVs[singleName]]
0192
0193 for alignment in config["alignments"]:
0194 local.setdefault("alignments", {})
0195 if alignment in config["validations"]["DMR"]["single"][singleName]["alignments"]:
0196 local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
0197 local["alignments"][alignment]['index'] = config["validations"]["DMR"]["single"][singleName]["alignments"].index(alignment)
0198 _mergeFiles.append("{}/{}/DMR/{}/{}/{}".format(config["LFS"], config["name"], "merge", mergeName, "{}"))
0199 trendIOVs.sort()
0200 local["validation"] = copy.deepcopy(config["validations"]["DMR"][dmrType][trendName])
0201 if len(_mergeFiles) == 1:
0202 local["validation"]["mergeFile"] = _mergeFiles[0]
0203 else:
0204 local["validation"]["mergeFile"] = _mergeFiles
0205 local["validation"]["IOV"] = trendIOVs
0206 local["output"] = "{}/{}/DMR/{}/{}/".format(config["LFS"], config["name"], dmrType, trendName)
0207 if "style" in config.keys() and "trends" in config["style"].keys():
0208 local["style"] = copy.deepcopy(config["style"])
0209 if "DMR" in local["style"].keys(): local["style"].pop("DMR")
0210 if "CMSlabel" in config["style"]["trends"].keys(): local["style"]["CMSlabel"] = config["style"]["trends"]["CMSlabel"]
0211 if "Rlabel" in config["style"]["trends"].keys():
0212 local["style"]["trends"].pop("Rlabel")
0213 local["style"]["trends"]["TitleCanvas"] = config["style"]["trends"]["Rlabel"]
0214 else:
0215 raise Exception("You want to create 'trends' jobs, but there are no 'lines' section in the config for pixel updates!")
0216
0217
0218 for mergeName in mergesDATA:
0219 for mergeJob in mergeJobs:
0220 alignment, mergeJobName, mergeIOV = mergeJob["name"].split("_")[1:]
0221 if mergeJobName == mergeName and int(mergeIOV) in trendIOVs:
0222 job["dependencies"].append(mergeJob["name"])
0223
0224 trendJobs.append(job)
0225
0226 jobs.extend(trendJobs)
0227
0228 if "averaged" in config["validations"]["DMR"]:
0229
0230
0231 avpJobs = []
0232 dmrType = "averaged"
0233 for avpName in config["validations"]["DMR"][dmrType]:
0234
0235 workDir = "{}/DMR/{}/{}".format(validationDir, dmrType, avpName)
0236 output = "{}/{}/DMR/{}/{}".format(config["LFS"], config["name"], dmrType, avpName)
0237
0238
0239 mergesDATA = []
0240 mergesMC = []
0241 for mergeName in config["validations"]["DMR"][dmrType][avpName]["merges"]:
0242
0243 if isDataMerged[mergeName] < 0:
0244 raise Exception("Average jobs cannot process merge jobs containing both DATA and MC objects.")
0245 elif isDataMerged[mergeName] == 1:
0246 mergesDATA.append(mergeName)
0247 else:
0248 mergesMC.append(mergeName)
0249 if "moduleFilterFile" not in config["validations"]["DMR"]["merge"][mergeName]:
0250 raise Exception("Value for 'moduleFilterFile' is required for the 'merge' step if the 'averaged' step is requested.")
0251 if "maxBadLumiPixel" not in config["validations"]["DMR"]["merge"][mergeName]:
0252 print("WARNING: Default value for the 'maxBadLumiPixel' is used.")
0253 if "maxBadLumiStrip" not in config["validations"]["DMR"]["merge"][mergeName]:
0254 print("WARNING: Default value for the 'maxBadLumiStrip' is used.")
0255
0256 for singleName in config["validations"]["DMR"]["merge"][mergeName]['singles']:
0257 if "maxEntriesPerModuleForDmr" not in config["validations"]["DMR"]["single"][singleName]:
0258 raise Exception("Value for 'maxEntriesPerModuleForDmr' is required for the 'single' step if the 'averaged' step is requested.")
0259
0260 lumiPerRun = []
0261 lumiPerIoV = []
0262 lumiMC = []
0263 if len(mergesDATA) > 0:
0264 if "lumiPerRun" in config["validations"]["DMR"][dmrType][avpName].keys():
0265 for lumifile in config["validations"]["DMR"][dmrType][avpName]['lumiPerRun']:
0266 if lumifile.split(".")[-1] in ["txt","csv"]:
0267 lumiPerRun.append(lumifile)
0268 if "lumiPerIoV" in config["validations"]["DMR"][dmrType][avpName].keys():
0269 for lumifile in config["validations"]["DMR"][dmrType][avpName]['lumiPerIoV']:
0270 if lumifile.split(".")[-1] in ["txt","csv"]:
0271 lumiPerIoV.append(lumifile)
0272 if len(lumiPerRun) == 0 and len(lumiPerIoV) == 0:
0273 raise Exception("No lumi per run/IoV file found or not specified in .csv/.txt format.")
0274 if len(mergesMC) > 0:
0275 if 'lumiMC' in config["validations"]["DMR"][dmrType][avpName].keys():
0276 lumiMC = config["validations"]["DMR"][dmrType][avpName]['lumiMC']
0277
0278
0279 plotJob = {}
0280 plotJob['workdir'] = "{}/{}".format(workDir,"plots")
0281 plotJob['output'] = "{}/{}".format(output,"plots")
0282 plotJob['inputData'] = []
0283 plotJob['inputMC'] = []
0284 plotJob['dependencies'] = []
0285
0286
0287 for mergeName in mergesDATA:
0288
0289 workDirMerge = "{}/{}".format(workDir, mergeName)
0290 outputMerge = "{}/{}".format(output, mergeName)
0291
0292
0293 local = {}
0294 local["type"] = "DMR"
0295 local["mode"] = "merge"
0296 local["isData"] = True
0297 local["isMC"] = False
0298
0299
0300 for alignment in config["alignments"]:
0301 local.setdefault("alignments", {})
0302 for singleName in config["validations"]["DMR"]["merge"][mergeName]["singles"]:
0303 if alignment in config["validations"]["DMR"]["single"][singleName]['alignments']:
0304 local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
0305 local["validation"] = copy.deepcopy(config["validations"]["DMR"][dmrType][avpName])
0306 local["validation"]["mergeFile"] = "{}/{}/DMR/{}/{}/{}".format(config["LFS"], config["name"], "merge", mergeName, "{}")
0307 local["validation"]["lumiPerRun"] = lumiPerRun
0308 local["validation"]["lumiPerIoV"] = lumiPerIoV
0309 local["validation"]["lumiMC"] = lumiMC
0310 local["validation"]["firstFromNext"] = []
0311
0312
0313 IOVsPerMergeStep = []
0314 for singleName in config["validations"]["DMR"]["merge"][mergeName]["singles"]:
0315 for IOV in IOVs[singleName]:
0316 if IOV not in IOVsPerMergeStep:
0317 IOVsPerMergeStep.append(IOV)
0318 IOVsPerMergeStep.sort()
0319
0320
0321 extra_part = 0
0322 maxfiles = int(config["validations"]["DMR"][dmrType][avpName]['maxfiles'])
0323 if len(IOVsPerMergeStep)%maxfiles >= 2:
0324 extra_part = 1
0325 parts = extra_part+len(IOVsPerMergeStep)//maxfiles
0326
0327 subJob = {'name' : [], 'output' : [], 'lumiPerFile' : []}
0328 for ipart in range(0,parts):
0329
0330 workDirSub = workDirMerge+"_"+str(ipart)
0331 outputSub = outputMerge+"_"+str(ipart)
0332
0333
0334 IOVGroup = []
0335 lastIndex = 0
0336 for iIOV,IOV in enumerate(IOVsPerMergeStep):
0337 if (iIOV//maxfiles == ipart) or (ipart == parts-1 and iIOV//maxfiles > ipart):
0338 IOVGroup.append(IOV)
0339 lastIndex = iIOV
0340 firstFromNext = []
0341 if lastIndex != len(IOVsPerMergeStep)-1:
0342 firstFromNext.append(IOVsPerMergeStep[lastIndex+1])
0343
0344
0345 _local = copy.deepcopy(local)
0346 _local["output"] = outputSub
0347 _local["validation"]["IOV"] = IOVGroup
0348 _local["validation"]["firstFromNext"] = firstFromNext
0349 job = {
0350 "name": "DMR_{}_{}_{}".format(dmrType, avpName, mergeName+"_"+str(ipart)),
0351 "dir": workDirSub,
0352 "exe": "mkLumiAveragedPlots.py",
0353 "run-mode": "Condor",
0354 "dependencies": [],
0355 "config": _local,
0356 }
0357 subJob['output'].append(outputSub)
0358 subJob['name'].append("DMR_{}_{}_{}".format(dmrType, avpName, mergeName+"_"+str(ipart)))
0359 subJob['lumiPerFile'].append(os.path.join(outputSub,"lumiPerFile.csv"))
0360 if parts == 1:
0361 plotJob['inputData'].append(outputSub)
0362 plotJob['dependencies'].append(job['name'])
0363
0364
0365 for mergeJob in mergeJobs:
0366 alignment, mergeJobName, mergeIOV = mergeJob["name"].split("_")[1:]
0367 if mergeJobName == mergeName and int(mergeIOV) in IOVGroup:
0368 job["dependencies"].append(mergeJob["name"])
0369
0370
0371
0372
0373 avpJobs.append(job)
0374
0375
0376 if parts > 1:
0377 localFinalize = copy.deepcopy(local)
0378 localFinalize['mode'] = "finalize"
0379 localFinalize['output'] = outputMerge
0380 localFinalize["validation"]["IOV"] = []
0381 localFinalize["validation"]["mergeFile"] = subJob['output']
0382 localFinalize["validation"]["lumiPerRun"] = []
0383 localFinalize["validation"]["lumiPerIoV"] = subJob['lumiPerFile']
0384 job = {
0385 "name": "DMR_{}_{}_{}".format(dmrType, avpName, mergeName+"_finalize"),
0386 "dir": workDirMerge,
0387 "exe": "mkLumiAveragedPlots.py",
0388 "run-mode": "Condor",
0389 "dependencies": subJob['name'],
0390 "config": localFinalize,
0391 }
0392 avpJobs.append(job)
0393 plotJob['inputData'].append(outputMerge)
0394 plotJob['dependencies'].append(job['name'])
0395
0396
0397 if len(mergesMC) != 0:
0398
0399 workDirMerge = "{}/{}".format(workDir, "MC")
0400 outputMerge = "{}/{}".format(output, "MC")
0401
0402
0403 local = {}
0404 local["type"] = "DMR"
0405 local["mode"] = "merge"
0406 local["isData"] = False
0407 local["isMC"] = True
0408 local["output"] = outputMerge
0409
0410
0411 local["validation"] = copy.deepcopy(config["validations"]["DMR"][dmrType][avpName])
0412 local["validation"]["mergeFile"] = []
0413 for mergeName in mergesMC:
0414 for alignment in config["alignments"]:
0415 local.setdefault("alignments", {})
0416 for singleName in config["validations"]["DMR"]["merge"][mergeName]["singles"]:
0417 if alignment in config["validations"]["DMR"]["single"][singleName]['alignments']:
0418 local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
0419 local["validation"]["mergeFile"].append("{}/{}/DMR/{}/{}/{}".format(config["LFS"], config["name"], "merge", mergeName, "{}"))
0420 local["validation"]["lumiPerRun"] = lumiPerRun
0421 local["validation"]["lumiPerIoV"] = lumiPerIoV
0422 local["validation"]["lumiMC"] = lumiMC
0423 local["validation"]["IOV"] = [1]
0424
0425
0426 job = {
0427 "name": "DMR_{}_{}_{}".format(dmrType, avpName, mergeName+"_MC"),
0428 "dir": workDirMerge,
0429 "exe": "mkLumiAveragedPlots.py",
0430 "run-mode": "Condor",
0431 "dependencies": [],
0432 "config": local,
0433 }
0434 plotJob['inputMC'].append(outputMerge)
0435 plotJob['dependencies'].append(job['name'])
0436
0437
0438 for mergeJob in mergeJobs:
0439 alignment, mergeJobName, mergeIOV = mergeJob["name"].split("_")[1:]
0440 if mergeJobName in mergesMC:
0441 job["dependencies"].append(mergeJob["name"])
0442
0443
0444 avpJobs.append(job)
0445
0446
0447 if len(plotJob['inputData'])+len(plotJob['inputMC']) > 0:
0448 local = {}
0449 local["type"] = "DMR"
0450 local["mode"] = "plot"
0451 local["isData"] = True if len(plotJob['inputData']) > 0 else False
0452 local["isMC"] = True if len(plotJob['inputMC']) > 0 else False
0453 local["output"] = plotJob['output']
0454 local["plot"] = { "inputData" : plotJob['inputData'],
0455 "inputMC" : plotJob['inputMC'],
0456 "alignments" : [],
0457 "objects" : [],
0458 "labels" : [],
0459 "colors" : [],
0460 "styles" : [],
0461 "useFit" : True,
0462 "useFitError" : False,
0463 "showMean" : True,
0464 "showMeanError" : False,
0465 "showRMS" : False,
0466 "showRMSError" : False}
0467
0468 for mergeName in mergesDATA+mergesMC:
0469 for singleName in config["validations"]["DMR"]["merge"][mergeName]["singles"]:
0470 for alignment in config["validations"]["DMR"]["single"][singleName]['alignments']:
0471 if alignment in config['alignments'] and alignment not in local["plot"]["alignments"]:
0472 local["plot"]["alignments"].append(alignment)
0473 objectName = config["alignments"][alignment]["title"].replace(" ","_")
0474 if objectName not in local["plot"]["objects"]:
0475 local["plot"]["objects"].append(objectName)
0476 local["plot"]["labels"].append(config["alignments"][alignment]["title"])
0477 local["plot"]["colors"].append(config["alignments"][alignment]["color"])
0478 local["plot"]["styles"].append(config["alignments"][alignment]["style"])
0479
0480 for extraKey in ["objects","labels","colors","styles","useFit","useFitError","showMean","showMeamError","showRMS","showRMSError"]:
0481 if extraKey in config["validations"]["DMR"][dmrType][avpName].keys():
0482 local["plot"][extraKey] = config["validations"]["DMR"][dmrType][avpName][extraKey]
0483
0484 if "style" in config.keys():
0485 if "DMR" in config['style'].keys():
0486 if dmrType in config['style']['DMR'].keys():
0487 local["plotGlobal"] = copy.deepcopy(config["style"]['DMR'][dmrType])
0488
0489
0490 job = {
0491 "name": "DMR_{}_{}_{}".format(dmrType, avpName, "plot"),
0492 "dir": plotJob['workdir'],
0493 "exe": "mkLumiAveragedPlots.py",
0494 "run-mode": "Condor",
0495 "dependencies": plotJob['dependencies'],
0496 "config": local,
0497 }
0498 avpJobs.append(job)
0499
0500
0501 jobs.extend(avpJobs)
0502
0503 return jobs