python/TkAlAllInOneTool/SplitV.py

0001 import copy
0002 import os
0003
0004 def SplitV(config, validationDir):
0005     ##List with all jobs
0006     jobs = []
0007     SplitVType = "single"
0008
0009     ##List with all wished IOVs
0010     IOVs = []
0011
0012     ##Start with single SplitV jobs
0013     if not SplitVType in config["validations"]["SplitV"]:
0014         raise Exception("No 'single' key word in config for SplitV")
0015
0016     for singleName in config["validations"]["SplitV"][SplitVType]:
0017         for IOV in config["validations"]["SplitV"][SplitVType][singleName]["IOV"]:
0018             ##Save IOV to loop later for merge jobs
0019             if not IOV in IOVs:
0020                 IOVs.append(IOV)
0021
0022             for alignment in config["validations"]["SplitV"][SplitVType][singleName]["alignments"]:
0023                 ##Work directory for each IOV
0024                 workDir = "{}/SplitV/{}/{}/{}/{}".format(validationDir, SplitVType, singleName, alignment, IOV)
0025
0026                 ##Write local config
0027                 local = {}
0028                 local["output"] = "{}/{}/SplitV/{}/{}/{}/{}".format(config["LFS"], config["name"], SplitVType, alignment, singleName, IOV)
0029                 local["alignment"] = copy.deepcopy(config["alignments"][alignment])
0030                 local["validation"] = copy.deepcopy(config["validations"]["SplitV"][SplitVType][singleName])
0031                 local["validation"].pop("alignments")
0032                 local["validation"]["IOV"] = IOV
0033                 if "dataset" in local["validation"]:
0034                     local["validation"]["dataset"] = local["validation"]["dataset"].format(IOV)
0035                 if "goodlumi" in local["validation"]:
0036                     local["validation"]["goodlumi"] = local["validation"]["goodlumi"].format(IOV)
0037
0038                 ##Write job info
0039                 job = {
0040                     "name": "SplitV_{}_{}_{}_{}".format(SplitVType, alignment, singleName, IOV),
0041                     "dir": workDir,
0042                     "exe": "cmsRun",
0043                     "cms-config": "{}/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/SplitV_cfg.py".format(os.environ["CMSSW_BASE"]),
0044                     "run-mode": "Condor",
0045                     "dependencies": [],
0046                     "config": local,
0047                 }
0048
0049                 jobs.append(job)
0050
0051     ##Do merge SplitV if wished
0052     if "merge" in config["validations"]["SplitV"]:
0053         ##List with merge jobs, will be expanded to jobs after looping
0054         mergeJobs = []
0055         SplitVType = "merge"
0056
0057         ##Loop over all merge jobs/IOVs which are wished
0058         for mergeName in config["validations"]["SplitV"][SplitVType]:
0059             for IOV in IOVs:
0060                 ##Work directory for each IOV
0061                 workDir = "{}/SplitV/{}/{}/{}".format(validationDir, SplitVType, mergeName, IOV)
0062
0063                 ##Write job info
0064                 local = {}
0065
0066                 job = {
0067                     "name": "SplitV_{}_{}_{}".format(SplitVType, mergeName, IOV),
0068                     "dir": workDir,
0069                     "exe": "SplitVmerge",
0070                     "run-mode": "Condor",
0071                     "dependencies": [],
0072                     "config": local,
0073                 }
0074
0075                 for alignment in config["alignments"]:
0076                     ##Deep copy necessary things from global config
0077                     local.setdefault("alignments", {})
0078                     if alignment in config["validations"]["SplitV"]["single"][mergeName]["alignments"]:
0079                         local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
0080                 local["validation"] = copy.deepcopy(config["validations"]["SplitV"][SplitVType][mergeName])
0081                 local["output"] = "{}/{}/SplitV/{}/{}/{}".format(config["LFS"], config["name"], SplitVType, mergeName, IOV)
0082
0083                 ##Loop over all single jobs
0084                 for singleJob in jobs:
0085                     ##Get single job info and append to merge job if requirements fullfilled
0086                     alignment, singleName, singleIOV = singleJob["name"].split("_")[2:]
0087
0088                     if int(singleIOV) == IOV and singleName in config["validations"]["SplitV"][SplitVType][mergeName]["singles"]:
0089                         local["alignments"][alignment]["file"] = singleJob["config"]["output"]
0090                         job["dependencies"].append(singleJob["name"])
0091
0092                 mergeJobs.append(job)
0093
0094         jobs.extend(mergeJobs)
0095
0096     return jobs