OfflineValidation/scripts/validateAlignments.py

0001 #!/usr/bin/env python3
0002 #test execute: export CMSSW_BASE=/tmp/CMSSW && ./validateAlignments.py -c defaultCRAFTValidation.ini,test.ini -n -N test
0003 from future.utils import lmap
0004 import subprocess
0005 import json
0006 import yaml
0007 import os
0008 import argparse
0009 import pprint
0010 import sys
0011 import shutil
0012 import Alignment.OfflineValidation.TkAlAllInOneTool.findAndChange as fnc
0013
0014 import Alignment.OfflineValidation.TkAlAllInOneTool.GCP as GCP
0015 import Alignment.OfflineValidation.TkAlAllInOneTool.DMR as DMR
0016 import Alignment.OfflineValidation.TkAlAllInOneTool.Zmumu as Zmumu
0017 import Alignment.OfflineValidation.TkAlAllInOneTool.PV as PV
0018 import Alignment.OfflineValidation.TkAlAllInOneTool.SplitV as SplitV
0019 import Alignment.OfflineValidation.TkAlAllInOneTool.JetHT as JetHT
0020 import Alignment.OfflineValidation.TkAlAllInOneTool.DiMuonV as DiMuonV
0021 import Alignment.OfflineValidation.TkAlAllInOneTool.MTS as MTS
0022 import Alignment.OfflineValidation.TkAlAllInOneTool.PixBary as PixBary
0023 import Alignment.OfflineValidation.TkAlAllInOneTool.GenericV as GenericV
0024
0025 ##############################################
0026 def parser():
0027 ##############################################
0028     """ Parse user input """
0029
0030     parser = argparse.ArgumentParser(description = "AllInOneTool for validation of the tracker alignment", formatter_class=argparse.RawTextHelpFormatter)
0031     parser.add_argument("config", metavar='config', type=str, action="store", help="Global AllInOneTool config (json/yaml format)")
0032     parser.add_argument("-d", "--dry", action = "store_true", help ="Set up everything, but don't run anything")
0033     parser.add_argument("-v", "--verbose", action = "store_true", help ="Enable standard output stream")
0034     parser.add_argument("-e", "--example", action = "store_true", help ="Print example of config in JSON format")
0035     parser.add_argument("-f", "--force", action = "store_true", help ="Force creation of enviroment, possible overwritten old configuration")
0036     parser.add_argument("-j", "--job-flavour", action = "store", default = "workday", choices = ["espresso", "microcentury", "longlunch", "workday", "tomorrow", "testmatch", "nextweek"], help ="Job flavours for HTCondor at CERN, default is 'workday'")
0037
0038     return parser.parse_args()
0039
0040 ##############################################
0041 def check_proxy():
0042 ##############################################
0043     """Check if GRID proxy has been initialized."""
0044
0045     try:
0046         with open(os.devnull, "w") as dump:
0047             subprocess.check_call(["voms-proxy-info", "--exists"],
0048                                   stdout = dump, stderr = dump)
0049     except subprocess.CalledProcessError:
0050         return False
0051     return True
0052
0053 ##############################################
0054 def forward_proxy(rundir):
0055 ##############################################
0056     """Forward proxy to location visible from the batch system.
0057     Arguments:
0058     - `rundir`: directory for storing the forwarded proxy
0059     Return:
0060     - Full path to the forwarded proxy
0061     """
0062
0063     if not check_proxy():
0064         print("Please create proxy via 'voms-proxy-init -voms cms'.")
0065         sys.exit(1)
0066
0067     ## Move the proxy to the run directory
0068     proxyName = "{}/.user_proxy".format(rundir)
0069     localProxy = subprocess.check_output(["voms-proxy-info", "--path"]).strip()
0070     shutil.copyfile(localProxy, proxyName)
0071
0072     ## Return the path to the forwarded proxy
0073     return proxyName
0074
0075
0076 ##############################################
0077 def updateConfigurationFile(configurationFile, updateInstructions):
0078 ##############################################
0079     """Update a template configuration file with custom configuration
0080     Arguments:
0081     - configurationFile: File name for the configuration file that will be updated
0082     - updateInstructions: A dictionary defining the updated configuration with keys "overwrite", "remove", "add" and "addBefore" each containing a list with the instructions on what should be replaced, removed or added.
0083     """
0084
0085     # Read the original configuration file
0086     with open(configurationFile,"r") as inputFile:
0087         fileContent = inputFile.readlines()
0088
0089     # Perform all overwrite operations to the configuration file. First string where the substring before the first space matches with the replacing string is overwritten. If a character "|" is included in the instruction, the subtring before that is used to search for the overwritten line instead. If no such string is found, add the instruction to the end of the file.
0090     if "overwrite" in updateInstructions:
0091
0092         for instruction in updateInstructions["overwrite"]:
0093
0094             decodeInstruction = instruction.split("|")
0095             if(len(decodeInstruction) > 1):
0096                 lineToReplace = decodeInstruction[0]
0097                 newInstruction = instruction[instruction.index("|")+1:]
0098             else:
0099                 lineToReplace = instruction.split()[0]
0100                 newInstruction = instruction
0101
0102             lineOverwritten = False
0103             for iLine in range(0,len(fileContent)):
0104                 if fileContent[iLine].startswith(lineToReplace):
0105                     fileContent[iLine] = newInstruction
0106                     if not fileContent[iLine].endswith("\n"):
0107                         fileContent[iLine] = fileContent[iLine] + "\n"
0108                     lineOverwritten = True
0109                     break
0110
0111             # If did not find a line to overwrite, add the instruction to the end of the file
0112             if not lineOverwritten:
0113                 fileContent.append(newInstruction)
0114                 if not fileContent[-1].endswith("\n"):
0115                     fileContent[-1] = fileContent[-1] + "\n"
0116
0117     # Perform all remove operations to the configuration file. First string that starst with the instruction will be removed from the configuration file.
0118     if "remove" in updateInstructions:
0119         for instruction in updateInstructions["remove"]:
0120             for iLine in range(0,len(fileContent)):
0121                 if fileContent[iLine].startswith(instruction):
0122                     fileContent.pop(iLine)
0123                     break
0124
0125     # Perform all add operations to the configuration file. The instruction is added to the matching CRAB configuration section. If one is not found, it is added to the end of the file.
0126     if "add" in updateInstructions:
0127         for instruction in updateInstructions["add"]:
0128             categories = instruction.split(".")
0129             if len(categories) > 2:
0130                 category = categories[1]
0131             else:
0132                 category = "nonExistent"
0133             previousCategory = ""
0134             lineFound = False
0135
0136             # First try to add the line to a correct section in CRAB configuration
0137             for iLine in range(0,len(fileContent)):
0138                 if fileContent[iLine] == "\n" and previousCategory == category:
0139                     fileContent.insert(iLine, instruction)
0140                     if not fileContent[iLine].endswith("\n"):
0141                         fileContent[iLine] = fileContent[iLine] + "\n"
0142                     lineFound = True
0143                     break
0144                 elif fileContent[iLine] == "\n":
0145                     previousCategory = ""
0146                 else:
0147                     newCategories = fileContent[iLine].split(".")
0148                     if len(newCategories) > 2:
0149                         previousCategory = newCategories[1]
0150                     else:
0151                         previousCategory = ""
0152
0153             # If the correct section is not found, add the new line to the end of the file
0154             if not lineFound:
0155                 fileContent.append(instruction)
0156                 if not fileContent[-1].endswith("\n"):
0157                     fileContent[-1] = fileContent[-1] + "\n"
0158
0159     # Perform all addBefore operations to the configuration file. This adds an instruction to the configuration file just before a line that starts with a string defined before the '|' character. If one is not found, the line is added to the end of the file.
0160     if "addBefore" in updateInstructions:
0161         for instruction in updateInstructions["addBefore"]:
0162             lineBefore = instruction.split("|")[0]
0163             newInstruction = instruction[instruction.index("|")+1:]
0164             lineFound = False
0165             for iLine in range(0,len(fileContent)):
0166                 if fileContent[iLine].startswith(lineBefore):
0167                     fileContent.insert(iLine,newInstruction)
0168                     if not fileContent[iLine].endswith("\n"):
0169                         fileContent[iLine] = fileContent[iLine] + "\n"
0170                     lineFound = True
0171                     break
0172
0173
0174             # If the searched line is not found, add the new line to the end of the file
0175             if not lineFound:
0176                 fileContent.append(newInstruction)
0177                 if not fileContent[-1].endswith("\n"):
0178                     fileContent[-1] = fileContent[-1] + "\n"
0179
0180     # Write the updates to the configuration file
0181     with open(configurationFile,"w") as outputFile:
0182         outputFile.writelines(fileContent)
0183
0184
0185 ##############################################
0186 def main():
0187 ##############################################
0188
0189     ## Before doing anything, check that grip proxy exists
0190     if not check_proxy():
0191         print("Grid proxy is required in most use cases of the tool.")
0192         print("Please create a proxy via 'voms-proxy-init -voms cms'.")
0193         sys.exit(1)
0194
0195     ##Read parser arguments
0196     args = parser()
0197
0198     ##Print example config which is in Aligment/OfflineValidation/bin if wished
0199     if args.example:
0200         with open("{}/src/Alignment/OfflineValidation/bin/example.yaml".format(os.environ["CMSSW_BASE"]), "r") as exampleFile:
0201             config = yaml.load(exampleFile, Loader=yaml.Loader)
0202             pprint.pprint(config, width=30)
0203             sys.exit(0)
0204
0205     ##Read in AllInOne config dependent on what format you choose
0206     with open(args.config, "r") as configFile:
0207         if args.verbose:
0208             print("Read AllInOne config: '{}'".format(args.config))
0209
0210         if args.config.split(".")[-1] == "json":
0211             config = json.load(configFile)
0212
0213         elif args.config.split(".")[-1] == "yaml":
0214             config = yaml.load(configFile, Loader=yaml.Loader)
0215
0216         else:
0217             raise Exception("Unknown config extension '{}'. Please use json/yaml format!".format(args.config.split(".")[-1]))
0218
0219     ##Check for all paths in configuration and attempt to "digest" them
0220     ##As a bonus, all ROOT colors are turned to the integer value
0221     for path in fnc.find_and_change(list(), config):
0222         if args.verbose and ("." in str(path) or "/" in str(path)):
0223             print("Digesting path: "+str(path))
0224
0225     ##Create working directory
0226     if os.path.isdir(config["name"]) and not args.force:
0227         raise Exception("Validation directory '{}' already exists! Please choose another name for your directory.".format(config["name"]))
0228
0229     validationDir = os.path.abspath(config["name"])
0230     exeDir = "{}/executables".format(validationDir)
0231     cmsconfigDir =  "{}/cmsConfigs".format(validationDir)
0232
0233     subprocess.call(["mkdir", "-p", validationDir] + ((["-v"] if args.verbose else [])))
0234     subprocess.call(["mkdir", "-p", exeDir] + (["-v"] if args.verbose else []))
0235     subprocess.call(["mkdir", "-p", cmsconfigDir] + (["-v"] if args.verbose else []))
0236
0237     ##Copy AllInOne config in working directory in json/yaml format
0238     subprocess.call(["cp", "-f", args.config, validationDir] + (["-v"] if args.verbose else []))
0239
0240     ## Define the template files
0241     crabTemplateFile = fnc.digest_path("$CMSSW_BASE/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/templates/crabTemplate.py")
0242     condorTemplateFile = fnc.digest_path("$CMSSW_BASE/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/templates/condorTemplate.submit")
0243     executableTempleteFile = fnc.digest_path("$CMSSW_BASE/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/templates/executableTemplate.sh")
0244
0245
0246     ##List with all jobs
0247     jobs = []
0248
0249     ##Check in config for all validation and create jobs
0250     for validation in config["validations"]:
0251         if validation == "GCP":
0252             jobs.extend(GCP.GCP(config, validationDir))
0253
0254         elif validation == "DMR":
0255             jobs.extend(DMR.DMR(config, validationDir))
0256
0257         elif validation == "Zmumu":
0258             jobs.extend(Zmumu.Zmumu(config, validationDir))
0259
0260         elif validation == "PV":
0261             jobs.extend(PV.PV(config, validationDir))
0262
0263         elif validation == "SplitV":
0264             jobs.extend(SplitV.SplitV(config, validationDir))
0265
0266         elif validation == "JetHT":
0267             jobs.extend(JetHT.JetHT(config, validationDir))
0268
0269         elif validation == "DiMuonV":
0270             jobs.extend(DiMuonV.DiMuonV(config, validationDir))
0271
0272         elif validation == "MTS":
0273             jobs.extend(MTS.MTS(config, validationDir))
0274
0275         elif validation == "PixBary":
0276             jobs.extend(PixBary.PixBary(config, validationDir, args.verbose))
0277
0278         elif validation == "Generic":
0279             jobs.extend(GenericV.GenericV(config, validationDir))
0280
0281         else:
0282             raise Exception("Unknown validation method: {}".format(validation))
0283
0284     ##Create dir for DAG file and loop over all jobs
0285     subprocess.call(["mkdir", "-p", "{}/DAG/".format(validationDir)] + (["-v"] if args.verbose else []))
0286
0287     with open("{}/DAG/dagFile".format(validationDir), "w") as dag:
0288         for job in jobs:
0289             ##Create job dir, output dir
0290             subprocess.call(["mkdir", "-p", job["dir"]] + (["-v"] if args.verbose else []))
0291             subprocess.call(["mkdir", "-p", job["config"]["output"]] + (["-v"] if args.verbose else []))
0292             subprocess.call(["mkdir", "-p", "{}/condor".format(job["dir"])] + (["-v"] if args.verbose else []))
0293             subprocess.call(["ln", "-fs", job["config"]["output"], "{}/output".format(job["dir"])] + (["-v"] if args.verbose else []))
0294
0295             ## Copy the template files to the job directory
0296             crabConfigurationFile = "{}/crabConfiguration.py".format(job["dir"])
0297             subprocess.call(["cp", crabTemplateFile, crabConfigurationFile] + (["-v"] if args.verbose else []))
0298             condorSubmitFile = "{}/condor.sub".format(job["dir"])
0299             subprocess.call(["cp", condorTemplateFile, condorSubmitFile] + (["-v"] if args.verbose else []))
0300             executableFile = "{}/run.sh".format(job["dir"])
0301             subprocess.call(["cp", executableTempleteFile, executableFile] + (["-v"] if args.verbose else []))
0302
0303             ## Forward the proxy to the job directory
0304             if args.verbose:
0305                 print("Forwarding grid proxy to directory {}".format(job["dir"]))
0306             myProxy = forward_proxy(job["dir"])
0307
0308             ##Create symlink for executable/python cms config if needed
0309             subprocess.call("cp -f $(which {}) {}".format(job["exe"], exeDir) + (" -v" if args.verbose else ""), shell = True)
0310             subprocess.call(["ln", "-fs", "{}/{}".format(exeDir, job["exe"]), job["dir"]] + (["-v"] if args.verbose else []))
0311             if "cms-config" in job:
0312                 cmsConfig = job["cms-config"].split("/")[-1]
0313
0314                 subprocess.call(["cp", "-f", job["cms-config"], "{}/{}".format(cmsconfigDir, cmsConfig)] + (["-v"] if args.verbose else []))
0315                 subprocess.call(["ln", "-fs", "{}/{}".format(cmsconfigDir, cmsConfig), "{}/validation_cfg.py".format(job["dir"])] + (["-v"] if args.verbose else []))
0316
0317             ##Write local config file
0318             with open("{}/validation.json".format(job["dir"]), "w") as jsonFile:
0319                 if args.verbose:
0320                     print("Write local json config: '{}'".format("{}/validation.json".format(job["dir"])))
0321
0322                 json.dump(job["config"], jsonFile, indent=4)
0323
0324             ## Customize the executable template file for this specific job
0325             executableCustomization = {"overwrite": [], "addBefore": []}
0326
0327             executableCustomization["overwrite"].append("export X509|export X509_USER_PROXY={}".format(myProxy)) # Define the proxy location
0328             executableCustomization["overwrite"].append("cd workDir|cd {}".format(job["dir"])) # Define the work directory for this job
0329
0330             # Option the give free arguments to the executable
0331             if "exeArguments" in job:
0332                 executableCustomization["overwrite"].append("./cmsRun|./{} {}".format(job["exe"], job["exeArguments"])) # Define the correct executable for this job
0333             else: # Default arguments
0334                 executableCustomization["overwrite"].append("./cmsRun|./{} {}validation.json".format(job["exe"], "validation_cfg.py config=" if "cms-config" in job else "")) # Define the correct executable for this job
0335
0336             # Option to include the condor job number given as a command line argument
0337             if "nCondorJobs" in job:
0338                 executableCustomization["addBefore"].append("./{}|JOBNUMBER=${{1:--1}}".format(job["exe"]))
0339
0340             # Do the manual configuration on top of the executable file
0341             updateConfigurationFile(executableFile, executableCustomization)
0342
0343             # Give the correct access rights for the executable
0344             subprocess.call(["chmod", "a+rx", executableFile] + (["-v"] if args.verbose else []))
0345
0346             ## Customize the condor submit file for this specific job
0347             condorSubmitCustomization = {"overwrite": [], "addBefore": []}
0348
0349             ## Hack to solve condor dagman issue with passing environmental variables
0350             condorSubmitCustomization["addBefore"].append('+JobFlavour|+environment = "CMSSW_BASE={}"'.format(fnc.digest_path("$CMSSW_BASE")))
0351
0352             # Take given flavour for the job, except if overwritten in job config
0353             condorSubmitCustomization["overwrite"].append('+JobFlavour = "{}"'.format(args.job_flavour if not 'flavour' in job else job['flavour']))
0354
0355             # If condor job array is sent, add job ID information to submit file
0356             if "nCondorJobs" in job:
0357                 condorSubmitCustomization["addBefore"].append("output|arguments = $(ProcID)")
0358                 condorSubmitCustomization["overwrite"].append("output = condor/condor$(ProcID).out")
0359                 condorSubmitCustomization["overwrite"].append("error  = condor/condor$(ProcID).err")
0360                 condorSubmitCustomization["overwrite"].append("log    = condor/condor$(ProcID).log")
0361                 condorSubmitCustomization["overwrite"].append("queue {}".format(job["nCondorJobs"]))
0362
0363             # Do the customization for the condor submit file
0364             updateConfigurationFile(condorSubmitFile, condorSubmitCustomization)
0365
0366             ##Write command in dag file
0367             dag.write("JOB {} condor.sub DIR {}\n".format(job["name"], job["dir"]))
0368
0369             if job["dependencies"]:
0370                 dag.write("\n")
0371                 dag.write("PARENT {} CHILD {}".format(" ".join(job["dependencies"]), job["name"]))
0372
0373             dag.write("\n\n")
0374
0375             ## If there is custom crab configuration defined, modify the crab template file based on that
0376             if "crabCustomConfiguration" in job["config"]:
0377                 updateConfigurationFile(crabConfigurationFile, job["config"]["crabCustomConfiguration"])
0378
0379
0380     if args.verbose:
0381         print("DAGman config has been written: '{}'".format("{}/DAG/dagFile".format(validationDir)))
0382
0383     ##Call submit command if not dry run
0384     if args.dry:
0385         print("Enviroment is set up. If you want to submit everything, call 'condor_submit_dag {}/DAG/dagFile'".format(validationDir))
0386
0387     else:
0388         subprocess.call(["condor_submit_dag", "{}/DAG/dagFile".format(validationDir)])
0389
0390 ##############################################
0391 if __name__ == "__main__":
0392 ##############################################
0393     main()