|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
import copy
import os
def MTS(config, validationDir):
##List with all jobs
jobs = []
mtsType = "single"
##Dictionary of lists of all IOVs (can be different per each single job)
IOVs = {}
##Auxilliary dictionary of isData flags per each merged job
isDataMerged = {}
##Start with single MTS jobs
if not mtsType in config["validations"]["MTS"]:
raise Exception("No 'single' key word in config for MTS")
for singleName in config["validations"]["MTS"][mtsType]:
aux_IOV = config["validations"]["MTS"][mtsType][singleName]["IOV"]
if not isinstance(aux_IOV, list) and aux_IOV.endswith(".txt"):
config["validations"]["MTS"][mtsType][singleName]["IOV"] = []
with open(aux_IOV, 'r') as IOVfile:
for line in IOVfile.readlines():
if len(line) != 0: config["validations"]["MTS"][mtsType][singleName]["IOV"].append(int(line))
for IOV in config["validations"]["MTS"][mtsType][singleName]["IOV"]:
##Save IOV to loop later for merge jobs
if singleName not in IOVs.keys():
IOVs[singleName] = []
if IOV not in IOVs[singleName]:
IOVs[singleName].append(IOV)
for alignment in config["validations"]["MTS"][mtsType][singleName]["alignments"]:
##Work directory for each IOV
workDir = "{}/MTS/{}/{}/{}/{}".format(validationDir, mtsType, singleName, alignment, IOV)
##Write local config
local = {}
local["output"] = "{}/{}/MTS/{}/{}/{}/{}".format(config["LFS"], config["name"], mtsType, alignment, singleName, IOV)
local["alignment"] = copy.deepcopy(config["alignments"][alignment])
local["alignment"]["name"] = alignment
local["validation"] = copy.deepcopy(config["validations"]["MTS"][mtsType][singleName])
local["validation"].pop("alignments")
local["validation"]["IOV"] = IOV
if "dataset" in local["validation"]:
local["validation"]["dataset"] = local["validation"]["dataset"].format(IOV)
if "goodlumi" in local["validation"]:
local["validation"]["goodlumi"] = local["validation"]["goodlumi"].format(IOV)
##Write job info
job = {
"name": "MTS_{}_{}_{}_{}".format(mtsType, alignment, singleName, IOV),
"dir": workDir,
"exe": "cmsRun",
"cms-config": "{}/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/MTS_cfg.py".format(os.environ["CMSSW_BASE"]),
"run-mode": "Condor",
"dependencies": [],
"config": local,
}
jobs.append(job)
##Do merge MTS if wished
if "merge" in config["validations"]["MTS"]:
##List with merge jobs, will be expanded to jobs after looping
mergeJobs = []
pvType = "merge"
##Loop over all merge jobs/IOVs which are wished
for mergeName in config["validations"]["MTS"][pvType]:
##Loop over singles
for iname,singleName in enumerate(config["validations"]["MTS"][pvType][mergeName]['singles']):
for IOV in IOVs[singleName]:
##Work directory for each IOV
workDir = "{}/MTS/{}/{}/{}".format(validationDir, pvType, mergeName, IOV) #Different (DATA) single jobs must contain different set of IOVs
##Write job info
local = {}
job = {
"name": "MTS_{}_{}_{}".format(pvType, mergeName, IOV),
"dir": workDir,
"exe": "MTSmerge",
"run-mode": "Condor",
"dependencies": [],
"config": local,
}
##Deep copy necessary things from global config + assure plot order
for alignment in config["alignments"]:
local.setdefault("alignments", {})
if alignment in config["validations"]["MTS"]["single"][singleName]["alignments"]: #Cover all DATA validations
local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
local["alignments"][alignment]['index'] = config["validations"]["MTS"]["single"][singleName]["alignments"].index(alignment)
local["alignments"][alignment]['isMC'] = False
local["validation"] = copy.deepcopy(config["validations"]["MTS"][pvType][mergeName])
local["validation"]["IOV"] = IOV
if "customrighttitle" in local["validation"].keys():
if "IOV" in local["validation"]["customrighttitle"]:
local["validation"]["customrighttitle"] = local["validation"]["customrighttitle"].replace("IOV",str(IOV))
local["output"] = "{}/{}/MTS/{}/{}/{}".format(config["LFS"], config["name"], pvType, mergeName, IOV)
##Add global plotting options
if "style" in config.keys():
if "MTS" in config['style'].keys():
if pvType in config['style']['MTS'].keys():
local["style"] = copy.deepcopy(config["style"]["MTS"][pvType])
if "Rlabel" in local["style"] and "customrighttitle" in local["validation"].keys():
print("WARNING: custom right label is overwritten by global settings")
##Loop over all single jobs
for singleJob in jobs:
##Get single job info and append to merge job if requirements fullfilled
_alignment, _singleName, _singleIOV = singleJob["name"].split("_")[2:]
if _singleName in config["validations"]["MTS"][pvType][mergeName]["singles"]:
if (int(_singleIOV) == IOV): #matching DATA job or any MC single job
local["alignments"][_alignment]["file"] = singleJob["config"]["output"]
job["dependencies"].append(singleJob["name"])
mergeJobs.append(job)
jobs.extend(mergeJobs)
return jobs
|