python/TkAlAllInOneTool/MTS.py

0001 import copy
0002 import os
0003
0004 def MTS(config, validationDir):
0005     ##List with all jobs
0006     jobs = []
0007     mtsType = "single"
0008
0009     ##Dictionary of lists of all IOVs (can be different per each single job)
0010     IOVs = {}
0011
0012     ##Auxilliary dictionary of isData flags per each merged job
0013     isDataMerged = {}
0014
0015     ##Start with single MTS jobs
0016     if not mtsType in config["validations"]["MTS"]:
0017         raise Exception("No 'single' key word in config for MTS")
0018
0019     for singleName in config["validations"]["MTS"][mtsType]:
0020         aux_IOV = config["validations"]["MTS"][mtsType][singleName]["IOV"]
0021         if not isinstance(aux_IOV, list) and aux_IOV.endswith(".txt"):
0022             config["validations"]["MTS"][mtsType][singleName]["IOV"] = []
0023             with open(aux_IOV, 'r') as IOVfile:
0024                 for line in IOVfile.readlines():
0025                     if len(line) != 0: config["validations"]["MTS"][mtsType][singleName]["IOV"].append(int(line))
0026         for IOV in config["validations"]["MTS"][mtsType][singleName]["IOV"]:
0027             ##Save IOV to loop later for merge jobs
0028             if singleName not in IOVs.keys():
0029                 IOVs[singleName] = []
0030             if IOV not in IOVs[singleName]:
0031                 IOVs[singleName].append(IOV)
0032
0033             for alignment in config["validations"]["MTS"][mtsType][singleName]["alignments"]:
0034                 ##Work directory for each IOV
0035                 workDir = "{}/MTS/{}/{}/{}/{}".format(validationDir, mtsType, singleName, alignment, IOV)
0036
0037                 ##Write local config
0038                 local = {}
0039                 local["output"] = "{}/{}/MTS/{}/{}/{}/{}".format(config["LFS"], config["name"], mtsType, alignment, singleName, IOV)
0040                 local["alignment"] = copy.deepcopy(config["alignments"][alignment])
0041                 local["alignment"]["name"] = alignment
0042                 local["validation"] = copy.deepcopy(config["validations"]["MTS"][mtsType][singleName])
0043                 local["validation"].pop("alignments")
0044                 local["validation"]["IOV"] = IOV
0045                 if "dataset" in local["validation"]:
0046                     local["validation"]["dataset"] = local["validation"]["dataset"].format(IOV)
0047                 if "goodlumi" in local["validation"]:
0048                     local["validation"]["goodlumi"] = local["validation"]["goodlumi"].format(IOV)
0049
0050                 ##Write job info
0051                 job = {
0052                     "name": "MTS_{}_{}_{}_{}".format(mtsType, alignment, singleName, IOV),
0053                     "dir": workDir,
0054                     "exe": "cmsRun",
0055                     "cms-config": "{}/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/MTS_cfg.py".format(os.environ["CMSSW_BASE"]),
0056                     "run-mode": "Condor",
0057                     "dependencies": [],
0058                     "config": local,
0059                 }
0060
0061                 jobs.append(job)
0062
0063     ##Do merge MTS if wished
0064     if "merge" in config["validations"]["MTS"]:
0065         ##List with merge jobs, will be expanded to jobs after looping
0066         mergeJobs = []
0067         pvType = "merge"
0068
0069         ##Loop over all merge jobs/IOVs which are wished
0070         for mergeName in config["validations"]["MTS"][pvType]:
0071             ##Loop over singles
0072             for iname,singleName in enumerate(config["validations"]["MTS"][pvType][mergeName]['singles']):
0073                 for IOV in IOVs[singleName]:
0074
0075                     ##Work directory for each IOV
0076                     workDir = "{}/MTS/{}/{}/{}".format(validationDir, pvType, mergeName, IOV) #Different (DATA) single jobs must contain different set of IOVs
0077
0078                     ##Write job info
0079                     local = {}
0080
0081                     job = {
0082                         "name": "MTS_{}_{}_{}".format(pvType, mergeName, IOV),
0083                         "dir": workDir,
0084                         "exe": "MTSmerge",
0085                         "run-mode": "Condor",
0086                         "dependencies": [],
0087                         "config": local,
0088                     }
0089
0090                     ##Deep copy necessary things from global config + assure plot order
0091                     for alignment in config["alignments"]:
0092                         local.setdefault("alignments", {})
0093                         if alignment in config["validations"]["MTS"]["single"][singleName]["alignments"]: #Cover all DATA validations
0094                             local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
0095                             local["alignments"][alignment]['index'] = config["validations"]["MTS"]["single"][singleName]["alignments"].index(alignment)
0096                             local["alignments"][alignment]['isMC'] = False
0097                     local["validation"] = copy.deepcopy(config["validations"]["MTS"][pvType][mergeName])
0098                     local["validation"]["IOV"] = IOV
0099                     if "customrighttitle" in local["validation"].keys():
0100                         if "IOV" in local["validation"]["customrighttitle"]:
0101                             local["validation"]["customrighttitle"] = local["validation"]["customrighttitle"].replace("IOV",str(IOV))
0102                     local["output"] = "{}/{}/MTS/{}/{}/{}".format(config["LFS"], config["name"], pvType, mergeName, IOV)
0103
0104                     ##Add global plotting options
0105                     if "style" in config.keys():
0106                         if "MTS" in config['style'].keys():
0107                             if pvType in config['style']['MTS'].keys():
0108                                 local["style"] = copy.deepcopy(config["style"]["MTS"][pvType])
0109                                 if "Rlabel" in local["style"] and "customrighttitle" in local["validation"].keys():
0110                                     print("WARNING: custom right label is overwritten by global settings")
0111
0112                     ##Loop over all single jobs
0113                     for singleJob in jobs:
0114                         ##Get single job info and append to merge job if requirements fullfilled
0115                         _alignment, _singleName, _singleIOV = singleJob["name"].split("_")[2:]
0116                         if _singleName in config["validations"]["MTS"][pvType][mergeName]["singles"]:
0117                             if (int(_singleIOV) == IOV): #matching DATA job or any MC single job
0118                                 local["alignments"][_alignment]["file"] = singleJob["config"]["output"]
0119                                 job["dependencies"].append(singleJob["name"])
0120
0121                     mergeJobs.append(job)
0122
0123         jobs.extend(mergeJobs)
0124
0125     return jobs