File indexing completed on 2025-01-09 23:33:05
0001
0002
0003 from future.utils import lmap
0004 import subprocess
0005 import json
0006 import yaml
0007 import os
0008 import argparse
0009 import pprint
0010 import sys
0011 import shutil
0012 import Alignment.OfflineValidation.TkAlAllInOneTool.findAndChange as fnc
0013
0014 import Alignment.OfflineValidation.TkAlAllInOneTool.GCP as GCP
0015 import Alignment.OfflineValidation.TkAlAllInOneTool.DMR as DMR
0016 import Alignment.OfflineValidation.TkAlAllInOneTool.Zmumu as Zmumu
0017 import Alignment.OfflineValidation.TkAlAllInOneTool.PV as PV
0018 import Alignment.OfflineValidation.TkAlAllInOneTool.SplitV as SplitV
0019 import Alignment.OfflineValidation.TkAlAllInOneTool.JetHT as JetHT
0020 import Alignment.OfflineValidation.TkAlAllInOneTool.DiMuonV as DiMuonV
0021 import Alignment.OfflineValidation.TkAlAllInOneTool.MTS as MTS
0022 import Alignment.OfflineValidation.TkAlAllInOneTool.PixBary as PixBary
0023 import Alignment.OfflineValidation.TkAlAllInOneTool.GenericV as GenericV
0024
0025
0026 def parser():
0027
0028 """ Parse user input """
0029
0030 parser = argparse.ArgumentParser(description = "AllInOneTool for validation of the tracker alignment", formatter_class=argparse.RawTextHelpFormatter)
0031 parser.add_argument("config", metavar='config', type=str, action="store", help="Global AllInOneTool config (json/yaml format)")
0032 parser.add_argument("-d", "--dry", action = "store_true", help ="Set up everything, but don't run anything")
0033 parser.add_argument("-v", "--verbose", action = "store_true", help ="Enable standard output stream")
0034 parser.add_argument("-e", "--example", action = "store_true", help ="Print example of config in JSON format")
0035 parser.add_argument("-f", "--force", action = "store_true", help ="Force creation of enviroment, possible overwritten old configuration")
0036 parser.add_argument("-j", "--job-flavour", action = "store", default = "workday", choices = ["espresso", "microcentury", "longlunch", "workday", "tomorrow", "testmatch", "nextweek"], help ="Job flavours for HTCondor at CERN, default is 'workday'")
0037
0038 return parser.parse_args()
0039
0040
0041 def check_proxy():
0042
0043 """Check if GRID proxy has been initialized."""
0044
0045 try:
0046 with open(os.devnull, "w") as dump:
0047 subprocess.check_call(["voms-proxy-info", "--exists"],
0048 stdout = dump, stderr = dump)
0049 except subprocess.CalledProcessError:
0050 return False
0051 return True
0052
0053
0054 def forward_proxy(rundir):
0055
0056 """Forward proxy to location visible from the batch system.
0057 Arguments:
0058 - `rundir`: directory for storing the forwarded proxy
0059 Return:
0060 - Full path to the forwarded proxy
0061 """
0062
0063 if not check_proxy():
0064 print("Please create proxy via 'voms-proxy-init -voms cms'.")
0065 sys.exit(1)
0066
0067
0068 proxyName = "{}/.user_proxy".format(rundir)
0069 localProxy = subprocess.check_output(["voms-proxy-info", "--path"]).strip()
0070 shutil.copyfile(localProxy, proxyName)
0071
0072
0073 return proxyName
0074
0075
0076
0077 def updateConfigurationFile(configurationFile, updateInstructions):
0078
0079 """Update a template configuration file with custom configuration
0080 Arguments:
0081 - configurationFile: File name for the configuration file that will be updated
0082 - updateInstructions: A dictionary defining the updated configuration with keys "overwrite", "remove", "add" and "addBefore" each containing a list with the instructions on what should be replaced, removed or added.
0083 """
0084
0085
0086 with open(configurationFile,"r") as inputFile:
0087 fileContent = inputFile.readlines()
0088
0089
0090 if "overwrite" in updateInstructions:
0091
0092 for instruction in updateInstructions["overwrite"]:
0093
0094 decodeInstruction = instruction.split("|")
0095 if(len(decodeInstruction) > 1):
0096 lineToReplace = decodeInstruction[0]
0097 newInstruction = instruction[instruction.index("|")+1:]
0098 else:
0099 lineToReplace = instruction.split()[0]
0100 newInstruction = instruction
0101
0102 lineOverwritten = False
0103 for iLine in range(0,len(fileContent)):
0104 if fileContent[iLine].startswith(lineToReplace):
0105 fileContent[iLine] = newInstruction
0106 if not fileContent[iLine].endswith("\n"):
0107 fileContent[iLine] = fileContent[iLine] + "\n"
0108 lineOverwritten = True
0109 break
0110
0111
0112 if not lineOverwritten:
0113 fileContent.append(newInstruction)
0114 if not fileContent[-1].endswith("\n"):
0115 fileContent[-1] = fileContent[-1] + "\n"
0116
0117
0118 if "remove" in updateInstructions:
0119 for instruction in updateInstructions["remove"]:
0120 for iLine in range(0,len(fileContent)):
0121 if fileContent[iLine].startswith(instruction):
0122 fileContent.pop(iLine)
0123 break
0124
0125
0126 if "add" in updateInstructions:
0127 for instruction in updateInstructions["add"]:
0128 categories = instruction.split(".")
0129 if len(categories) > 2:
0130 category = categories[1]
0131 else:
0132 category = "nonExistent"
0133 previousCategory = ""
0134 lineFound = False
0135
0136
0137 for iLine in range(0,len(fileContent)):
0138 if fileContent[iLine] == "\n" and previousCategory == category:
0139 fileContent.insert(iLine, instruction)
0140 if not fileContent[iLine].endswith("\n"):
0141 fileContent[iLine] = fileContent[iLine] + "\n"
0142 lineFound = True
0143 break
0144 elif fileContent[iLine] == "\n":
0145 previousCategory = ""
0146 else:
0147 newCategories = fileContent[iLine].split(".")
0148 if len(newCategories) > 2:
0149 previousCategory = newCategories[1]
0150 else:
0151 previousCategory = ""
0152
0153
0154 if not lineFound:
0155 fileContent.append(instruction)
0156 if not fileContent[-1].endswith("\n"):
0157 fileContent[-1] = fileContent[-1] + "\n"
0158
0159
0160 if "addBefore" in updateInstructions:
0161 for instruction in updateInstructions["addBefore"]:
0162 lineBefore = instruction.split("|")[0]
0163 newInstruction = instruction[instruction.index("|")+1:]
0164 lineFound = False
0165 for iLine in range(0,len(fileContent)):
0166 if fileContent[iLine].startswith(lineBefore):
0167 fileContent.insert(iLine,newInstruction)
0168 if not fileContent[iLine].endswith("\n"):
0169 fileContent[iLine] = fileContent[iLine] + "\n"
0170 lineFound = True
0171 break
0172
0173
0174
0175 if not lineFound:
0176 fileContent.append(newInstruction)
0177 if not fileContent[-1].endswith("\n"):
0178 fileContent[-1] = fileContent[-1] + "\n"
0179
0180
0181 with open(configurationFile,"w") as outputFile:
0182 outputFile.writelines(fileContent)
0183
0184
0185
0186 def main():
0187
0188
0189
0190 if not check_proxy():
0191 print("Grid proxy is required in most use cases of the tool.")
0192 print("Please create a proxy via 'voms-proxy-init -voms cms'.")
0193 sys.exit(1)
0194
0195
0196 args = parser()
0197
0198
0199 if args.example:
0200 with open("{}/src/Alignment/OfflineValidation/bin/example.yaml".format(os.environ["CMSSW_BASE"]), "r") as exampleFile:
0201 config = yaml.load(exampleFile, Loader=yaml.Loader)
0202 pprint.pprint(config, width=30)
0203 sys.exit(0)
0204
0205
0206 with open(args.config, "r") as configFile:
0207 if args.verbose:
0208 print("Read AllInOne config: '{}'".format(args.config))
0209
0210 if args.config.split(".")[-1] == "json":
0211 config = json.load(configFile)
0212
0213 elif args.config.split(".")[-1] == "yaml":
0214 config = yaml.load(configFile, Loader=yaml.Loader)
0215
0216 else:
0217 raise Exception("Unknown config extension '{}'. Please use json/yaml format!".format(args.config.split(".")[-1]))
0218
0219
0220
0221 for path in fnc.find_and_change(list(), config):
0222 if args.verbose and ("." in str(path) or "/" in str(path)):
0223 print("Digesting path: "+str(path))
0224
0225
0226 if os.path.isdir(config["name"]) and not args.force:
0227 raise Exception("Validation directory '{}' already exists! Please choose another name for your directory.".format(config["name"]))
0228
0229 validationDir = os.path.abspath(config["name"])
0230 exeDir = "{}/executables".format(validationDir)
0231 cmsconfigDir = "{}/cmsConfigs".format(validationDir)
0232
0233 subprocess.call(["mkdir", "-p", validationDir] + ((["-v"] if args.verbose else [])))
0234 subprocess.call(["mkdir", "-p", exeDir] + (["-v"] if args.verbose else []))
0235 subprocess.call(["mkdir", "-p", cmsconfigDir] + (["-v"] if args.verbose else []))
0236
0237
0238 subprocess.call(["cp", "-f", args.config, validationDir] + (["-v"] if args.verbose else []))
0239
0240
0241 crabTemplateFile = fnc.digest_path("$CMSSW_BASE/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/templates/crabTemplate.py")
0242 condorTemplateFile = fnc.digest_path("$CMSSW_BASE/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/templates/condorTemplate.submit")
0243 executableTempleteFile = fnc.digest_path("$CMSSW_BASE/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/templates/executableTemplate.sh")
0244
0245
0246
0247 jobs = []
0248
0249
0250 for validation in config["validations"]:
0251 if validation == "GCP":
0252 jobs.extend(GCP.GCP(config, validationDir))
0253
0254 elif validation == "DMR":
0255 jobs.extend(DMR.DMR(config, validationDir))
0256
0257 elif validation == "Zmumu":
0258 jobs.extend(Zmumu.Zmumu(config, validationDir))
0259
0260 elif validation == "PV":
0261 jobs.extend(PV.PV(config, validationDir))
0262
0263 elif validation == "SplitV":
0264 jobs.extend(SplitV.SplitV(config, validationDir))
0265
0266 elif validation == "JetHT":
0267 jobs.extend(JetHT.JetHT(config, validationDir))
0268
0269 elif validation == "DiMuonV":
0270 jobs.extend(DiMuonV.DiMuonV(config, validationDir))
0271
0272 elif validation == "MTS":
0273 jobs.extend(MTS.MTS(config, validationDir))
0274
0275 elif validation == "PixBary":
0276 jobs.extend(PixBary.PixBary(config, validationDir, args.verbose))
0277
0278 elif validation == "Generic":
0279 jobs.extend(GenericV.GenericV(config, validationDir))
0280
0281 else:
0282 raise Exception("Unknown validation method: {}".format(validation))
0283
0284
0285 subprocess.call(["mkdir", "-p", "{}/DAG/".format(validationDir)] + (["-v"] if args.verbose else []))
0286
0287 with open("{}/DAG/dagFile".format(validationDir), "w") as dag:
0288 for job in jobs:
0289
0290 subprocess.call(["mkdir", "-p", job["dir"]] + (["-v"] if args.verbose else []))
0291 subprocess.call(["mkdir", "-p", job["config"]["output"]] + (["-v"] if args.verbose else []))
0292 subprocess.call(["mkdir", "-p", "{}/condor".format(job["dir"])] + (["-v"] if args.verbose else []))
0293 subprocess.call(["ln", "-fs", job["config"]["output"], "{}/output".format(job["dir"])] + (["-v"] if args.verbose else []))
0294
0295
0296 crabConfigurationFile = "{}/crabConfiguration.py".format(job["dir"])
0297 subprocess.call(["cp", crabTemplateFile, crabConfigurationFile] + (["-v"] if args.verbose else []))
0298 condorSubmitFile = "{}/condor.sub".format(job["dir"])
0299 subprocess.call(["cp", condorTemplateFile, condorSubmitFile] + (["-v"] if args.verbose else []))
0300 executableFile = "{}/run.sh".format(job["dir"])
0301 subprocess.call(["cp", executableTempleteFile, executableFile] + (["-v"] if args.verbose else []))
0302
0303
0304 if args.verbose:
0305 print("Forwarding grid proxy to directory {}".format(job["dir"]))
0306 myProxy = forward_proxy(job["dir"])
0307
0308
0309 subprocess.call("cp -f $(which {}) {}".format(job["exe"], exeDir) + (" -v" if args.verbose else ""), shell = True)
0310 subprocess.call(["ln", "-fs", "{}/{}".format(exeDir, job["exe"]), job["dir"]] + (["-v"] if args.verbose else []))
0311 if "cms-config" in job:
0312 cmsConfig = job["cms-config"].split("/")[-1]
0313
0314 subprocess.call(["cp", "-f", job["cms-config"], "{}/{}".format(cmsconfigDir, cmsConfig)] + (["-v"] if args.verbose else []))
0315 subprocess.call(["ln", "-fs", "{}/{}".format(cmsconfigDir, cmsConfig), "{}/validation_cfg.py".format(job["dir"])] + (["-v"] if args.verbose else []))
0316
0317
0318 with open("{}/validation.json".format(job["dir"]), "w") as jsonFile:
0319 if args.verbose:
0320 print("Write local json config: '{}'".format("{}/validation.json".format(job["dir"])))
0321
0322 json.dump(job["config"], jsonFile, indent=4)
0323
0324
0325 executableCustomization = {"overwrite": [], "addBefore": []}
0326
0327 executableCustomization["overwrite"].append("export X509|export X509_USER_PROXY={}".format(myProxy))
0328 executableCustomization["overwrite"].append("cd workDir|cd {}".format(job["dir"]))
0329
0330
0331 if "exeArguments" in job:
0332 executableCustomization["overwrite"].append("./cmsRun|./{} {}".format(job["exe"], job["exeArguments"]))
0333 else:
0334 executableCustomization["overwrite"].append("./cmsRun|./{} {}validation.json".format(job["exe"], "validation_cfg.py config=" if "cms-config" in job else ""))
0335
0336
0337 if "nCondorJobs" in job:
0338 executableCustomization["addBefore"].append("./{}|JOBNUMBER=${{1:--1}}".format(job["exe"]))
0339
0340
0341 updateConfigurationFile(executableFile, executableCustomization)
0342
0343
0344 subprocess.call(["chmod", "a+rx", executableFile] + (["-v"] if args.verbose else []))
0345
0346
0347 condorSubmitCustomization = {"overwrite": [], "addBefore": []}
0348
0349
0350 condorSubmitCustomization["addBefore"].append('+JobFlavour|+environment = "CMSSW_BASE={}"'.format(fnc.digest_path("$CMSSW_BASE")))
0351
0352
0353 condorSubmitCustomization["overwrite"].append('+JobFlavour = "{}"'.format(args.job_flavour if not 'flavour' in job else job['flavour']))
0354
0355
0356 if "nCondorJobs" in job:
0357 condorSubmitCustomization["addBefore"].append("output|arguments = $(ProcID)")
0358 condorSubmitCustomization["overwrite"].append("output = condor/condor$(ProcID).out")
0359 condorSubmitCustomization["overwrite"].append("error = condor/condor$(ProcID).err")
0360 condorSubmitCustomization["overwrite"].append("log = condor/condor$(ProcID).log")
0361 condorSubmitCustomization["overwrite"].append("queue {}".format(job["nCondorJobs"]))
0362
0363
0364 updateConfigurationFile(condorSubmitFile, condorSubmitCustomization)
0365
0366
0367 dag.write("JOB {} condor.sub DIR {}\n".format(job["name"], job["dir"]))
0368
0369 if job["dependencies"]:
0370 dag.write("\n")
0371 dag.write("PARENT {} CHILD {}".format(" ".join(job["dependencies"]), job["name"]))
0372
0373 dag.write("\n\n")
0374
0375
0376 if "crabCustomConfiguration" in job["config"]:
0377 updateConfigurationFile(crabConfigurationFile, job["config"]["crabCustomConfiguration"])
0378
0379
0380 if args.verbose:
0381 print("DAGman config has been written: '{}'".format("{}/DAG/dagFile".format(validationDir)))
0382
0383
0384 if args.dry:
0385 print("Enviroment is set up. If you want to submit everything, call 'condor_submit_dag {}/DAG/dagFile'".format(validationDir))
0386
0387 else:
0388 subprocess.call(["condor_submit_dag", "{}/DAG/dagFile".format(validationDir)])
0389
0390
0391 if __name__ == "__main__":
0392
0393 main()