python/TkAlAllInOneTool/PV.py

0001 import copy
0002 import os
0003 import pprint
0004
0005 def PV(config, validationDir):
0006     ##List with all jobs
0007     jobs, singleJobs = [], []
0008     pvType = "single"
0009
0010     ##Dictionary of lists of all IOVs (can be different per each single job)
0011     IOVs = {}
0012
0013     ##Auxilliary dictionary of isData flags per each merged job
0014     isDataMerged = {}
0015
0016     ##Start with single PV jobs
0017     if not pvType in config["validations"]["PV"]:
0018         raise Exception("No 'single' key word in config for PV")
0019
0020     for singleName in config["validations"]["PV"][pvType]:
0021         #print("Reading singleName = {}".format(singleName))
0022         aux_IOV = config["validations"]["PV"][pvType][singleName]["IOV"]
0023         if not isinstance(aux_IOV, list) and aux_IOV.endswith(".txt"):
0024             config["validations"]["PV"][pvType][singleName]["IOV"] = []
0025             with open(aux_IOV, 'r') as IOVfile:
0026                 for line in IOVfile.readlines():
0027                     if len(line) != 0: config["validations"]["PV"][pvType][singleName]["IOV"].append(int(line))
0028         for IOV in config["validations"]["PV"][pvType][singleName]["IOV"]:
0029             ##Save IOV to loop later for merge jobs
0030             if singleName not in IOVs.keys():
0031                 IOVs[singleName] = []
0032             if IOV not in IOVs[singleName]:
0033                 IOVs[singleName].append(IOV)
0034
0035             for alignment in config["validations"]["PV"][pvType][singleName]["alignments"]:
0036                 ##Work directory for each IOV
0037                 workDir = "{}/PV/{}/{}/{}/{}".format(validationDir, pvType, singleName, alignment, IOV)
0038
0039                 ##Write local config
0040                 local = {}
0041                 local["output"] = "{}/{}/PV/{}/{}/{}/{}".format(config["LFS"], config["name"], pvType, alignment, singleName, IOV)
0042                 local["alignment"] = copy.deepcopy(config["alignments"][alignment])
0043                 local["alignment"]["name"] = alignment
0044                 local["validation"] = copy.deepcopy(config["validations"]["PV"][pvType][singleName])
0045                 local["validation"].pop("alignments")
0046                 local["validation"]["IOV"] = IOV
0047                 if "dataset" in local["validation"]:
0048                     local["validation"]["dataset"] = local["validation"]["dataset"].format(IOV)
0049                 if "goodlumi" in local["validation"]:
0050                     local["validation"]["goodlumi"] = local["validation"]["goodlumi"].format(IOV)
0051
0052                 ##Write job info
0053                 job = {
0054                     "name": "PV_{}_{}_{}_{}".format(pvType, alignment, singleName, IOV),
0055                     "dir": workDir,
0056                     "exe": "cmsRun",
0057                     "cms-config": "{}/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/PV_cfg.py".format(os.environ["CMSSW_BASE"]),
0058                     "run-mode": "Condor",
0059                     "dependencies": [],
0060                     "config": local,
0061                 }
0062
0063                 #jobs.append(job)
0064                 singleJobs.append(job)
0065
0066     jobs.extend(singleJobs)
0067
0068     ##Do merge PV if wished
0069     if "merge" in config["validations"]["PV"]:
0070         ##List with merge jobs, will be expanded to jobs after looping
0071         mergeJobs = []
0072         pvType = "merge"
0073
0074         ##Loop over all merge jobs/IOVs which are wished
0075         for mergeName in config["validations"]["PV"][pvType]:
0076             ##Search for MC single(s)
0077             singlesMC = []
0078             for singleName in config["validations"]["PV"][pvType][mergeName]['singles']:
0079                 if len(IOVs[singleName]) == 1 and int(IOVs[singleName][0]) == 1: singlesMC.append(singleName)
0080             isMConly = (len(singlesMC) == len(config["validations"]["PV"][pvType][mergeName]['singles']))
0081             if isMConly:
0082                 isDataMerged[mergeName] = 0
0083             elif len(singlesMC) == 0:
0084                 isDataMerged[mergeName] = 1
0085             else:
0086                 isDataMerged[mergeName] = -1
0087
0088             ##Loop over singles
0089             for iname,singleName in enumerate(config["validations"]["PV"][pvType][mergeName]['singles']):
0090                 isMC = (singleName in singlesMC)
0091                 if isMConly and iname > 0: continue #special case for MC only comparison
0092                 elif isMConly: singlesMC.pop(singlesMC.index(singleName))
0093
0094                 for IOV in IOVs[singleName]:
0095                     if isMC and not isMConly: continue #ignore IOV=1 as it is automatically added to each DATA IOV unless MC only comparison
0096                     #print("mergeName = {}".format(mergeName))
0097
0098                     ##Work directory for each IOV
0099                     workDir = "{}/PV/{}/{}/{}".format(validationDir, pvType, mergeName, IOV) #Different (DATA) single jobs must contain different set of IOVs
0100
0101                     ##Write job info
0102                     local = {}
0103
0104                     job = {
0105                         "name": "PV_{}_{}_{}".format(pvType, mergeName, IOV),
0106                         "dir": workDir,
0107                         "exe": "PVmerge",
0108                         "run-mode": "Condor",
0109                         "dependencies": [],
0110                         "config": local,
0111                     }
0112
0113                     ##Deep copy necessary things from global config + assure plot order
0114                     for alignment in config["alignments"]:
0115                         idxIncrement = 0
0116                         local.setdefault("alignments", {})
0117                         if alignment in config["validations"]["PV"]["single"][singleName]["alignments"]: #Cover all DATA validations
0118                             local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
0119                             local["alignments"][alignment]['index'] = config["validations"]["PV"]["single"][singleName]["alignments"].index(alignment)
0120                             local["alignments"][alignment]['isMC'] = False
0121                         for singleMCname in singlesMC:
0122                             if alignment in config["validations"]["PV"]["single"][singleMCname]["alignments"]: #Add MC objects
0123                                 local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
0124                                 local["alignments"][alignment]['index']  = len(config["validations"]["PV"]["single"][singleName]["alignments"])
0125                                 local["alignments"][alignment]['index'] += idxIncrement + config["validations"]["PV"]["single"][singleMCname]["alignments"].index(alignment)
0126                                 local["alignments"][alignment]['isMC'] = True
0127                             idxIncrement += len(config["validations"]["PV"]["single"][singleMCname]["alignments"])
0128                     local["validation"] = copy.deepcopy(config["validations"]["PV"][pvType][mergeName])
0129                     local["validation"]["IOV"] = IOV
0130                     if "customrighttitle" in local["validation"].keys():
0131                         if "IOV" in local["validation"]["customrighttitle"]:
0132                             local["validation"]["customrighttitle"] = local["validation"]["customrighttitle"].replace("IOV",str(IOV))
0133                     local["output"] = "{}/{}/PV/{}/{}/{}".format(config["LFS"], config["name"], pvType, mergeName, IOV)
0134
0135                     ##Add global plotting options
0136                     if "style" in config.keys():
0137                         if "PV" in config['style'].keys():
0138                             if pvType in config['style']['PV'].keys():
0139                                 local["style"] = copy.deepcopy(config["style"]["PV"][pvType])
0140                                 if "Rlabel" in local["style"] and "customrighttitle" in local["validation"].keys():
0141                                     print("WARNING: custom right label is overwritten by global settings")
0142
0143                     ##Loop over all single jobs
0144                     for singleJob in jobs:
0145                         ##Get single job info and append to merge job if requirements fullfilled
0146                         _alignment, _singleName, _singleIOV = singleJob["name"].split("_")[2:]
0147                         if _singleName in config["validations"]["PV"][pvType][mergeName]["singles"]:
0148                             if int(_singleIOV) == IOV or (int(_singleIOV) == 1 and _singleName in singlesMC): #matching DATA job or any MC single job
0149                                 local["alignments"][_alignment]["file"] = singleJob["config"]["output"]
0150                                 job["dependencies"].append(singleJob["name"])
0151
0152                     mergeJobs.append(job)
0153
0154         jobs.extend(mergeJobs)
0155
0156     if "trends" in config["validations"]["PV"]:
0157
0158         ##List with merge jobs, will be expanded to jobs after looping
0159         trendJobs = []
0160         pvType = "trends"
0161
0162         for trendName in config["validations"]["PV"][pvType]:
0163             #print("trendName = {}".format(trendName))
0164
0165             ##Work directory for each IOV
0166             workDir = "{}/PV/{}/{}".format(validationDir, pvType, trendName)
0167
0168             ##Write general job info
0169             local = {}
0170
0171             job = {
0172                 "name": "PV_{}_{}".format(pvType, trendName),
0173                 "dir": workDir,
0174                 "exe": "PVtrends",
0175                 "run-mode": "Condor",
0176                 "dependencies": [],
0177                 "config": local,
0178             }
0179
0180             ##Loop over singles
0181             if 'merges' in config["validations"]["PV"][pvType][trendName].keys()\
0182                or 'singles' not in config["validations"]["PV"][pvType][trendName].keys():
0183                 raise Exception("Specify list of \'singles\' to run PV trends.")
0184                 #TODO: possible also to run over merges for consistency with DMR jobs
0185             trendIOVs = [] #TODO: allow different IOV list for each single job?
0186             alignmentList = []
0187             for iname, singleName in enumerate(config["validations"]["PV"][pvType][trendName]["singles"]):
0188                 isMC = (len(IOVs[singleName]) == 1 and int(IOVs[singleName][0]) == 1)
0189                 if isMC:
0190                     raise Exception("Trend jobs are not implemented for treating MC.")
0191                 if iname == 0:
0192                     trendIOVs = IOVs[singleName]
0193                 else:
0194                     for IOV in IOVs[singleName]:
0195                         if IOV not in trendIOVs or (len(IOVs[singleName]) != len(trendIOVs)):
0196                             raise Exception("List of IOVs must be the same for each single job.")
0197                 for alignment in config["validations"]["PV"]["single"][singleName]["alignments"]:
0198                     if alignment not in alignmentList and alignment in config["alignments"]:
0199                         local.setdefault("alignments", {})
0200                         alignmentList.append(alignment)
0201                         local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
0202                         local["alignments"][alignment]["file"] = "{}/{}/PV/{}/{}/{}/{}".format(config["LFS"], config["name"], "single", alignment, singleName, "{}")
0203             trendIOVs.sort()
0204             local["validation"] = copy.deepcopy(config["validations"]["PV"][pvType][trendName])
0205             local["validation"]["IOV"] = trendIOVs
0206             if "label" in config["validations"]["PV"][pvType][trendName]:
0207                 local["validation"]["label"] = copy.deepcopy(config["validations"]["PV"][pvType][trendName]["label"])
0208             local["output"] = "{}/{}/PV/{}/{}/".format(config["LFS"], config["name"], pvType, trendName)
0209             if "style" in config.keys() and "trends" in config["style"].keys():
0210                 local["style"] = copy.deepcopy(config["style"])
0211                 if "PV" in local["style"].keys(): local["style"].pop("PV")
0212                 if "CMSlabel" in config["style"]["trends"].keys(): local["style"]["CMSlabel"] = config["style"]["trends"]["CMSlabel"]
0213                 if "Rlabel" in config["style"]["trends"].keys():
0214                     local["style"]["trends"].pop("Rlabel")
0215                     local["style"]["trends"]["TitleCanvas"] = config["style"]["trends"]["Rlabel"]
0216             else:
0217                 raise Exception("You want to create 'trends' jobs, but there are no 'lines' section in the config for pixel updates!")
0218
0219             #Loop over all single jobs
0220             for singleJob in singleJobs:
0221                 #Get single job info and append to job if requirements fullfilled
0222                 alignment, singleName, singleIOV = singleJob["name"].split("_")[2:]
0223
0224                 if singleName in config["validations"]["PV"][pvType][trendName]["singles"]\
0225                   and int(singleIOV) in trendIOVs:
0226                     job["dependencies"].append(singleJob["name"])
0227
0228             trendJobs.append(job)
0229
0230         jobs.extend(trendJobs)
0231
0232     return jobs