python/TkAlAllInOneTool/GCP.py

0001 import os
0002 import copy
0003
0004 def GCP(config, validationDir):
0005     ## List with all jobs
0006     jobs = []
0007
0008     ## Get unit test flag
0009     doUnitTest = False
0010     if "doUnitTest" in config["validations"]["GCP"].keys():
0011         doUnitTest = config["validations"]["GCP"]["doUnitTest"]
0012
0013     ## Main loop
0014     for comparison in config["validations"]["GCP"]["compare"]:
0015         for ali_pair in config["validations"]["GCP"]["compare"][comparison]:
0016             ref_name  = copy.deepcopy(config["validations"]["GCP"]["compare"][comparison][ali_pair]["reference"])
0017             comp_name = copy.deepcopy(config["validations"]["GCP"]["compare"][comparison][ali_pair]["compared"])
0018             IOVpair_list = []
0019             IOVali_list = []
0020
0021             # Construct pairs from IOVlist
0022             IOV_list = []
0023             if "IOVlist" in config["validations"]["GCP"]["compare"][comparison][ali_pair]: IOV_list = copy.deepcopy(config["validations"]["GCP"]["compare"][comparison][ali_pair]["IOVlist"])
0024             IOV_list.sort()
0025             for idx,IOV in enumerate(IOV_list):
0026                 if ref_name == comp_name:
0027                     IOV_pair = str(IOV)+'_vs_'+str(IOV_list[0])
0028                     IOV_ali_r = ref_name+'_'+str(IOV_list[0])
0029                     IOV_ali_c = comp_name+'_'+str(IOV)
0030                 else:
0031                     IOV_pair = str(IOV)+'_vs_'+str(IOV)
0032                     IOV_ali_r = ref_name+'_'+str(IOV)
0033                     IOV_ali_c = comp_name+'_'+str(IOV)
0034                 if IOV_pair not in IOVpair_list: IOVpair_list.append(IOV_pair)
0035                 if IOV_ali_r not in IOVali_list: IOVali_list.append(IOV_ali_r)
0036                 if IOV_ali_c not in IOVali_list: IOVali_list.append(IOV_ali_c)
0037
0038             # Read explicit pairs from IOVpairs
0039             pair_list = []
0040             if "IOVpairs" in config["validations"]["GCP"]["compare"][comparison][ali_pair]: pair_list = copy.deepcopy(config["validations"]["GCP"]["compare"][comparison][ali_pair]["IOVpairs"])
0041             for IOV_p in pair_list:
0042                 IOV_pair = str(IOV_p[0])+'_vs_'+str(IOV_p[1])
0043                 IOV_ali_r = ref_name+'_'+str(IOV_p[1])
0044                 IOV_ali_c = comp_name+'_'+str(IOV_p[0])
0045                 if IOV_pair not in IOVpair_list: IOVpair_list.append(IOV_pair)
0046                 if IOV_ali_r not in IOVali_list: IOVali_list.append(IOV_ali_r)
0047                 if IOV_ali_c not in IOVali_list: IOVali_list.append(IOV_ali_c)
0048
0049             # GCP Ntuple job preps
0050             for IOV_ali in IOVali_list:
0051                 ali = IOV_ali.split('_')[0]
0052                 IOV = int(IOV_ali.split('_')[1])
0053                 workDir = "{}/GCP/{}/{}/{}".format(validationDir, comparison, 'Ntuples', IOV_ali)
0054
0055                 # local config
0056                 local = {}
0057                 local["output"] = "{}/{}/{}/{}/{}".format(config["LFS"], config["name"], comparison, 'Ntuples', IOV_ali)
0058                 local["alignments"] = copy.deepcopy(config["alignments"][ali])
0059                 local["validation"] = {}
0060                 local["validation"]['GCP'] = copy.deepcopy(config["validations"]["GCP"][comparison])
0061                 local["validation"]['GCP']['doUnitTest'] = doUnitTest
0062                 local["validation"]['IOV'] = IOV
0063
0064                 # job info
0065                 job = {
0066                     "name": "GCP_{}_Ntuple_{}".format(comparison, IOV_ali),
0067                     "dir": workDir,
0068                     "exe": "cmsRun",
0069                     "cms-config": "{}/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/GCP_Ntuples_cfg.py".format(os.environ["CMSSW_BASE"]),
0070                     "run-mode": "Condor",
0071                     "dependencies": [],
0072                     "config": local,
0073                     "flavour": "espresso",
0074                 }
0075
0076                 # Ntuple jobs might appear multiple times, only add if not there yet
0077                 already_there = False
0078                 for j in jobs:
0079                     if j["name"] == job["name"]:
0080                         already_there = True
0081                         break
0082
0083                 if not already_there: jobs.append(job)
0084
0085             # Comparison job preps
0086             for IOV_pair in IOVpair_list:
0087                 ref_IOV  = int(IOV_pair.split('_vs_')[1])
0088                 comp_IOV = int(IOV_pair.split('_vs_')[0])
0089
0090                 # local config
0091                 local = {}
0092                 local["output"] = "{}/{}/{}/{}/{}".format(config["LFS"], config["name"], comparison, ali_pair, IOV_pair)
0093                 local["alignments"] = {}
0094                 local["alignments"]["ref"]  = copy.deepcopy(config["alignments"][ref_name])
0095                 local["alignments"]["comp"] = copy.deepcopy(config["alignments"][comp_name])
0096                 local["validation"] = {}
0097                 local["validation"]['GCP'] = copy.deepcopy(config["validations"]["GCP"][comparison])
0098                 local["validation"]['GCP']['doUnitTest'] = doUnitTest
0099                 local["validation"]["IOVref"] = ref_IOV
0100                 local["validation"]["ALIref"] = ref_name
0101                 local["validation"]["IOVcomp"] = comp_IOV
0102                 local["validation"]["ALIcomp"] = comp_name
0103
0104                 # dependancies
0105                 parents = []
0106                 for j in jobs:
0107                     if not comparison in j['name']: continue
0108                     if not 'Ntuple' in j['name']: continue
0109                     if ref_name in j['name'] and str(ref_IOV) in j['name']:
0110                         parents.append(j['name'])
0111                         local["input_ref"] = j['config']['output']
0112                     if comp_name in j['name'] and str(comp_IOV) in j['name']:
0113                         parents.append(j['name'])
0114                         local["input_comp"] = j['config']['output']
0115
0116                 # Comparison jobs
0117                 for step in ['GCPtree', 'GCPcpp', 'GCPpython']:
0118                     workDir = "{}/GCP/{}/{}/{}/{}".format(validationDir, comparison, ali_pair, IOV_pair, step)
0119                     job = {
0120                         "name": "GCP_{}_{}_{}_{}".format(comparison, ali_pair, IOV_pair, step),
0121                         "dir": workDir,
0122                         "run-mode": "Condor",
0123                         "config": local,
0124                         "flavour": "espresso",
0125                     }
0126                     if step == 'GCPtree':
0127                         job['exe'] = 'cmsRun'
0128                         job['cms-config'] = "{}/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/GCP_tree_cfg.py".format(os.environ["CMSSW_BASE"])
0129                         job['dependencies'] = parents
0130                     elif step == 'GCPcpp':
0131                         job['flavour'] = 'microcentury'
0132                         job['exe'] = 'GCP'
0133                         job['dependencies'] = parents + ["GCP_{}_{}_{}_{}".format(comparison, ali_pair, IOV_pair, 'GCPtree')]
0134                     else:
0135                         job['exe'] = 'GCPpyPlots.py'
0136                         job['dependencies'] = parents + ["GCP_{}_{}_{}_{}".format(comparison, ali_pair, IOV_pair, 'GCPtree')]
0137
0138                     jobs.append(job)
0139
0140     return jobs