OfflineValidation/scripts/validateAlignments.py

0001 #!/usr/bin/env python3
0002 #test execute: export CMSSW_BASE=/tmp/CMSSW && ./validateAlignments.py -c defaultCRAFTValidation.ini,test.ini -n -N test
0003 from __future__ import print_function
0004 from future.utils import lmap
0005 import subprocess
0006 import json
0007 import yaml
0008 import os
0009 import argparse
0010 import pprint
0011 import sys
0012 import shutil
0013 import Alignment.OfflineValidation.TkAlAllInOneTool.findAndChange as fnc
0014
0015 import Alignment.OfflineValidation.TkAlAllInOneTool.GCP as GCP
0016 import Alignment.OfflineValidation.TkAlAllInOneTool.DMR as DMR
0017 import Alignment.OfflineValidation.TkAlAllInOneTool.Zmumu as Zmumu
0018 import Alignment.OfflineValidation.TkAlAllInOneTool.PV as PV
0019 import Alignment.OfflineValidation.TkAlAllInOneTool.SplitV as SplitV
0020 import Alignment.OfflineValidation.TkAlAllInOneTool.JetHT as JetHT
0021
0022 ##############################################
0023 def parser():
0024 ##############################################
0025     """ Parse user input """
0026
0027     parser = argparse.ArgumentParser(description = "AllInOneTool for validation of the tracker alignment", formatter_class=argparse.RawTextHelpFormatter)
0028     parser.add_argument("config", metavar='config', type=str, action="store", help="Global AllInOneTool config (json/yaml format)")
0029     parser.add_argument("-d", "--dry", action = "store_true", help ="Set up everything, but don't run anything")
0030     parser.add_argument("-v", "--verbose", action = "store_true", help ="Enable standard output stream")
0031     parser.add_argument("-e", "--example", action = "store_true", help ="Print example of config in JSON format")
0032     parser.add_argument("-f", "--force", action = "store_true", help ="Force creation of enviroment, possible overwritten old configuration")
0033     parser.add_argument("-j", "--job-flavour", action = "store", default = "workday", choices = ["espresso", "microcentury", "longlunch", "workday", "tomorrow", "testmatch", "nextweek"], help ="Job flavours for HTCondor at CERN, default is 'workday'")
0034
0035     return parser.parse_args()
0036
0037 ##############################################
0038 def check_proxy():
0039 ##############################################
0040     """Check if GRID proxy has been initialized."""
0041
0042     try:
0043         with open(os.devnull, "w") as dump:
0044             subprocess.check_call(["voms-proxy-info", "--exists"],
0045                                   stdout = dump, stderr = dump)
0046     except subprocess.CalledProcessError:
0047         return False
0048     return True
0049
0050 ##############################################
0051 def forward_proxy(rundir):
0052 ##############################################
0053     """Forward proxy to location visible from the batch system.
0054     Arguments:
0055     - `rundir`: directory for storing the forwarded proxy
0056     Return:
0057     - Full path to the forwarded proxy
0058     """
0059
0060     if not check_proxy():
0061         print("Please create proxy via 'voms-proxy-init -voms cms'.")
0062         sys.exit(1)
0063
0064     ## Move the proxy to the run directory
0065     proxyName = "{}/.user_proxy".format(rundir)
0066     localProxy = subprocess.check_output(["voms-proxy-info", "--path"]).strip()
0067     shutil.copyfile(localProxy, proxyName)
0068
0069     ## Return the path to the forwarded proxy
0070     return proxyName
0071
0072
0073 ##############################################
0074 def updateConfigurationFile(configurationFile, updateInstructions):
0075 ##############################################
0076     """Update a template configuration file with custom configuration
0077     Arguments:
0078     - configurationFile: File name for the configuration file that will be updated
0079     - updateInstructions: A dictionary defining the updated configuration with keys "overwrite", "remove", "add" and "addBefore" each containing a list with the instructions on what should be replaced, removed or added.
0080     """
0081
0082     # Read the original configuration file
0083     with open(configurationFile,"r") as inputFile:
0084         fileContent = inputFile.readlines()
0085
0086     # Perform all overwrite operations to the configuration file. First string where the substring before the first space matches with the replacing string is overwritten. If a character "|" is included in the instruction, the subtring before that is used to search for the overwritten line instead. If no such string is found, add the instruction to the end of the file.
0087     if "overwrite" in updateInstructions:
0088
0089         for instruction in updateInstructions["overwrite"]:
0090
0091             decodeInstruction = instruction.split("|")
0092             if(len(decodeInstruction) > 1):
0093                 lineToReplace = decodeInstruction[0]
0094                 newInstruction = instruction[instruction.index("|")+1:]
0095             else:
0096                 lineToReplace = instruction.split()[0]
0097                 newInstruction = instruction
0098
0099             lineOverwritten = False
0100             for iLine in range(0,len(fileContent)):
0101                 if fileContent[iLine].startswith(lineToReplace):
0102                     fileContent[iLine] = newInstruction
0103                     if not fileContent[iLine].endswith("\n"):
0104                         fileContent[iLine] = fileContent[iLine] + "\n"
0105                     lineOverwritten = True
0106                     break
0107
0108             # If did not find a line to overwrite, add the instruction to the end of the file
0109             if not lineOverwritten:
0110                 fileContent.append(newInstruction)
0111                 if not fileContent[-1].endswith("\n"):
0112                     fileContent[-1] = fileContent[-1] + "\n"
0113
0114     # Perform all remove operations to the configuration file. First string that starst with the instruction will be removed from the configuration file.
0115     if "remove" in updateInstructions:
0116         for instruction in updateInstructions["remove"]:
0117             for iLine in range(0,len(fileContent)):
0118                 if fileContent[iLine].startswith(instruction):
0119                     fileContent.pop(iLine)
0120                     break
0121
0122     # Perform all add operations to the configuration file. The instruction is added to the matching CRAB configuration section. If one is not found, it is added to the end of the file.
0123     if "add" in updateInstructions:
0124         for instruction in updateInstructions["add"]:
0125             categories = instruction.split(".")
0126             if len(categories) > 2:
0127                 category = categories[1]
0128             else:
0129                 category = "nonExistent"
0130             previousCategory = ""
0131             lineFound = False
0132
0133             # First try to add the line to a correct section in CRAB configuration
0134             for iLine in range(0,len(fileContent)):
0135                 if fileContent[iLine] == "\n" and previousCategory == category:
0136                     fileContent.insert(iLine, instruction)
0137                     if not fileContent[iLine].endswith("\n"):
0138                         fileContent[iLine] = fileContent[iLine] + "\n"
0139                     lineFound = True
0140                     break
0141                 elif fileContent[iLine] == "\n":
0142                     previousCategory = ""
0143                 else:
0144                     newCategories = fileContent[iLine].split(".")
0145                     if len(newCategories) > 2:
0146                         previousCategory = newCategories[1]
0147                     else:
0148                         previousCategory = ""
0149
0150             # If the correct section is not found, add the new line to the end of the file
0151             if not lineFound:
0152                 fileContent.append(instruction)
0153                 if not fileContent[-1].endswith("\n"):
0154                     fileContent[-1] = fileContent[-1] + "\n"
0155
0156     # Perform all addBefore operations to the configuration file. This adds an instruction to the configuration file just before a line that starts with a string defined before the '|' character. If one is not found, the line is added to the end of the file.
0157     if "addBefore" in updateInstructions:
0158         for instruction in updateInstructions["addBefore"]:
0159             lineBefore = instruction.split("|")[0]
0160             newInstruction = instruction[instruction.index("|")+1:]
0161             lineFound = False
0162             for iLine in range(0,len(fileContent)):
0163                 if fileContent[iLine].startswith(lineBefore):
0164                     fileContent.insert(iLine,newInstruction)
0165                     if not fileContent[iLine].endswith("\n"):
0166                         fileContent[iLine] = fileContent[iLine] + "\n"
0167                     lineFound = True
0168                     break
0169
0170
0171             # If the searched line is not found, add the new line to the end of the file
0172             if not lineFound:
0173                 fileContent.append(newInstruction)
0174                 if not fileContent[-1].endswith("\n"):
0175                     fileContent[-1] = fileContent[-1] + "\n"
0176
0177     # Write the updates to the configuration file
0178     with open(configurationFile,"w") as outputFile:
0179         outputFile.writelines(fileContent)
0180
0181
0182 ##############################################
0183 def main():
0184 ##############################################
0185
0186     ## Before doing anything, check that grip proxy exists
0187     if not check_proxy():
0188         print("Grid proxy is required in most use cases of the tool.")
0189         print("Please create a proxy via 'voms-proxy-init -voms cms'.")
0190         sys.exit(1)
0191
0192     ##Read parser arguments
0193     args = parser()
0194
0195     ##Print example config which is in Aligment/OfflineValidation/bin if wished
0196     if args.example:
0197         with open("{}/src/Alignment/OfflineValidation/bin/example.yaml".format(os.environ["CMSSW_BASE"]), "r") as exampleFile:
0198             config = yaml.load(exampleFile, Loader=yaml.Loader)
0199             pprint.pprint(config, width=30)
0200             sys.exit(0)
0201
0202     ##Read in AllInOne config dependent on what format you choose
0203     with open(args.config, "r") as configFile:
0204         if args.verbose:
0205             print("Read AllInOne config: '{}'".format(args.config))
0206
0207         if args.config.split(".")[-1] == "json":
0208             config = json.load(configFile)
0209
0210         elif args.config.split(".")[-1] == "yaml":
0211             config = yaml.load(configFile, Loader=yaml.Loader)
0212
0213         else:
0214             raise Exception("Unknown config extension '{}'. Please use json/yaml format!".format(args.config.split(".")[-1]))
0215
0216     ##Check for all paths in configuration and attempt to "digest" them
0217     for path in fnc.find_and_change(list(), config):
0218         if args.verbose and ("." in str(path) or "/" in str(path)):
0219             print("Digesting path: "+str(path))
0220
0221     ##Create working directory
0222     if os.path.isdir(config["name"]) and not args.force:
0223         raise Exception("Validation directory '{}' already exists! Please choose another name for your directory.".format(config["name"]))
0224
0225     validationDir = os.path.abspath(config["name"])
0226     exeDir = "{}/executables".format(validationDir)
0227     cmsconfigDir =  "{}/cmsConfigs".format(validationDir)
0228
0229     subprocess.call(["mkdir", "-p", validationDir] + ((["-v"] if args.verbose else [])))
0230     subprocess.call(["mkdir", "-p", exeDir] + (["-v"] if args.verbose else []))
0231     subprocess.call(["mkdir", "-p", cmsconfigDir] + (["-v"] if args.verbose else []))
0232
0233     ##Copy AllInOne config in working directory in json/yaml format
0234     subprocess.call(["cp", "-f", args.config, validationDir] + (["-v"] if args.verbose else []))
0235
0236     ## Define the template files
0237     crabTemplateFile = fnc.digest_path("$CMSSW_BASE/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/templates/crabTemplate.py")
0238     condorTemplateFile = fnc.digest_path("$CMSSW_BASE/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/templates/condorTemplate.submit")
0239     executableTempleteFile = fnc.digest_path("$CMSSW_BASE/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/templates/executableTemplate.sh")
0240
0241
0242     ##List with all jobs
0243     jobs = []
0244
0245     ##Check in config for all validation and create jobs
0246     for validation in config["validations"]:
0247         if validation == "GCP":
0248             jobs.extend(GCP.GCP(config, validationDir))
0249
0250         elif validation == "DMR":
0251             jobs.extend(DMR.DMR(config, validationDir))
0252
0253         elif validation == "Zmumu":
0254             jobs.extend(Zmumu.Zmumu(config, validationDir))
0255
0256         elif validation == "PV":
0257             jobs.extend(PV.PV(config, validationDir))
0258
0259         elif validation == "SplitV":
0260             jobs.extend(SplitV.SplitV(config, validationDir))
0261
0262         elif validation == "JetHT":
0263             jobs.extend(JetHT.JetHT(config, validationDir))
0264
0265         else:
0266             raise Exception("Unknown validation method: {}".format(validation))
0267
0268     ##Create dir for DAG file and loop over all jobs
0269     subprocess.call(["mkdir", "-p", "{}/DAG/".format(validationDir)] + (["-v"] if args.verbose else []))
0270
0271     with open("{}/DAG/dagFile".format(validationDir), "w") as dag:
0272         for job in jobs:
0273             ##Create job dir, output dir
0274             subprocess.call(["mkdir", "-p", job["dir"]] + (["-v"] if args.verbose else []))
0275             subprocess.call(["mkdir", "-p", job["config"]["output"]] + (["-v"] if args.verbose else []))
0276             subprocess.call(["mkdir", "-p", "{}/condor".format(job["dir"])] + (["-v"] if args.verbose else []))
0277             subprocess.call(["ln", "-fs", job["config"]["output"], "{}/output".format(job["dir"])] + (["-v"] if args.verbose else []))
0278
0279             ## Copy the template files to the job directory
0280             crabConfigurationFile = "{}/crabConfiguration.py".format(job["dir"])
0281             subprocess.call(["cp", crabTemplateFile, crabConfigurationFile] + (["-v"] if args.verbose else []))
0282             condorSubmitFile = "{}/condor.sub".format(job["dir"])
0283             subprocess.call(["cp", condorTemplateFile, condorSubmitFile] + (["-v"] if args.verbose else []))
0284             executableFile = "{}/run.sh".format(job["dir"])
0285             subprocess.call(["cp", executableTempleteFile, executableFile] + (["-v"] if args.verbose else []))
0286
0287             ## Forward the proxy to the job directory
0288             if args.verbose:
0289                 print("Forwarding grid proxy to directory {}".format(job["dir"]))
0290             myProxy = forward_proxy(job["dir"])
0291
0292             ##Create symlink for executable/python cms config if needed
0293             subprocess.call("cp -f $(which {}) {}".format(job["exe"], exeDir) + (" -v" if args.verbose else ""), shell = True)
0294             subprocess.call(["ln", "-fs", "{}/{}".format(exeDir, job["exe"]), job["dir"]] + (["-v"] if args.verbose else []))
0295             if "cms-config" in job:
0296                 cmsConfig = job["cms-config"].split("/")[-1]
0297
0298                 subprocess.call(["cp", "-f", job["cms-config"], "{}/{}".format(cmsconfigDir, cmsConfig)] + (["-v"] if args.verbose else []))
0299                 subprocess.call(["ln", "-fs", "{}/{}".format(cmsconfigDir, cmsConfig), "{}/validation_cfg.py".format(job["dir"])] + (["-v"] if args.verbose else []))
0300
0301             ##Write local config file
0302             with open("{}/validation.json".format(job["dir"]), "w") as jsonFile:
0303                 if args.verbose:
0304                     print("Write local json config: '{}'".format("{}/validation.json".format(job["dir"])))
0305
0306                 json.dump(job["config"], jsonFile, indent=4)
0307
0308             ## Customize the executable template file for this specific job
0309             executableCustomization = {"overwrite": [], "addBefore": []}
0310
0311             executableCustomization["overwrite"].append("export X509|export X509_USER_PROXY={}".format(myProxy)) # Define the proxy location
0312             executableCustomization["overwrite"].append("cd workDir|cd {}".format(job["dir"])) # Define the work directory for this job
0313
0314             # Option the give free arguments to the executable
0315             if "exeArguments" in job:
0316                 executableCustomization["overwrite"].append("./cmsRun|./{} {}".format(job["exe"], job["exeArguments"])) # Define the correct executable for this job
0317             else: # Default arguments
0318                 executableCustomization["overwrite"].append("./cmsRun|./{} {}validation.json".format(job["exe"], "validation_cfg.py config=" if "cms-config" in job else "")) # Define the correct executable for this job
0319
0320             # Option to include the condor job number given as a command line argument
0321             if "nCondorJobs" in job:
0322                 executableCustomization["addBefore"].append("./{}|JOBNUMBER=${{1:--1}}".format(job["exe"]))
0323
0324             # Do the manual configuration on top of the executable file
0325             updateConfigurationFile(executableFile, executableCustomization)
0326
0327             # Give the correct access rights for the executable
0328             subprocess.call(["chmod", "a+rx", executableFile] + (["-v"] if args.verbose else []))
0329
0330             ## Customize the condor submit file for this specific job
0331             condorSubmitCustomization = {"overwrite": [], "addBefore": []}
0332
0333             # Take given flavour for the job, except if overwritten in job config
0334             condorSubmitCustomization["overwrite"].append('+JobFlavour = "{}"'.format(args.job_flavour if not 'flavour' in job else job['flavour']))
0335
0336             # If condor job array is sent, add job ID information to submit file
0337             if "nCondorJobs" in job:
0338                 condorSubmitCustomization["addBefore"].append("output|arguments = $(ProcID)")
0339                 condorSubmitCustomization["overwrite"].append("output = condor/condor$(ProcID).out")
0340                 condorSubmitCustomization["overwrite"].append("error  = condor/condor$(ProcID).err")
0341                 condorSubmitCustomization["overwrite"].append("log    = condor/condor$(ProcID).log")
0342                 condorSubmitCustomization["overwrite"].append("queue {}".format(job["nCondorJobs"]))
0343
0344             # Do the customization for the condor submit file
0345             updateConfigurationFile(condorSubmitFile, condorSubmitCustomization)
0346
0347             ##Write command in dag file
0348             dag.write("JOB {} condor.sub DIR {}\n".format(job["name"], job["dir"]))
0349
0350             if job["dependencies"]:
0351                 dag.write("\n")
0352                 dag.write("PARENT {} CHILD {}".format(" ".join(job["dependencies"]), job["name"]))
0353
0354             dag.write("\n\n")
0355
0356             ## If there is custom crab configuration defined, modify the crab template file based on that
0357             if "crabCustomConfiguration" in job["config"]:
0358                 updateConfigurationFile(crabConfigurationFile, job["config"]["crabCustomConfiguration"])
0359
0360
0361     if args.verbose:
0362         print("DAGman config has been written: '{}'".format("{}/DAG/dagFile".format(validationDir)))
0363
0364     ##Call submit command if not dry run
0365     if args.dry:
0366         print("Enviroment is set up. If you want to submit everything, call 'condor_submit_dag {}/DAG/dagFile'".format(validationDir))
0367
0368     else:
0369         subprocess.call(["condor_submit_dag", "{}/DAG/dagFile".format(validationDir)])
0370
0371 ##############################################
0372 if __name__ == "__main__":
0373 ##############################################
0374     main()