OfflineValidation/scripts/validateAlignments.py

0001 #!/usr/bin/env python3
0002 #test execute: export CMSSW_BASE=/tmp/CMSSW && ./validateAlignments.py -c defaultCRAFTValidation.ini,test.ini -n -N test
0003 from __future__ import print_function
0004 from future.utils import lmap
0005 import subprocess
0006 import json
0007 import yaml
0008 import os
0009 import argparse
0010 import pprint
0011 import sys
0012 import shutil
0013 import Alignment.OfflineValidation.TkAlAllInOneTool.findAndChange as fnc
0014
0015 import Alignment.OfflineValidation.TkAlAllInOneTool.GCP as GCP
0016 import Alignment.OfflineValidation.TkAlAllInOneTool.DMR as DMR
0017 import Alignment.OfflineValidation.TkAlAllInOneTool.Zmumu as Zmumu
0018 import Alignment.OfflineValidation.TkAlAllInOneTool.PV as PV
0019 import Alignment.OfflineValidation.TkAlAllInOneTool.SplitV as SplitV
0020 import Alignment.OfflineValidation.TkAlAllInOneTool.JetHT as JetHT
0021 import Alignment.OfflineValidation.TkAlAllInOneTool.DiMuonV as DiMuonV
0022 import Alignment.OfflineValidation.TkAlAllInOneTool.MTS as MTS
0023
0024 ##############################################
0025 def parser():
0026 ##############################################
0027     """ Parse user input """
0028
0029     parser = argparse.ArgumentParser(description = "AllInOneTool for validation of the tracker alignment", formatter_class=argparse.RawTextHelpFormatter)
0030     parser.add_argument("config", metavar='config', type=str, action="store", help="Global AllInOneTool config (json/yaml format)")
0031     parser.add_argument("-d", "--dry", action = "store_true", help ="Set up everything, but don't run anything")
0032     parser.add_argument("-v", "--verbose", action = "store_true", help ="Enable standard output stream")
0033     parser.add_argument("-e", "--example", action = "store_true", help ="Print example of config in JSON format")
0034     parser.add_argument("-f", "--force", action = "store_true", help ="Force creation of enviroment, possible overwritten old configuration")
0035     parser.add_argument("-j", "--job-flavour", action = "store", default = "workday", choices = ["espresso", "microcentury", "longlunch", "workday", "tomorrow", "testmatch", "nextweek"], help ="Job flavours for HTCondor at CERN, default is 'workday'")
0036
0037     return parser.parse_args()
0038
0039 ##############################################
0040 def check_proxy():
0041 ##############################################
0042     """Check if GRID proxy has been initialized."""
0043
0044     try:
0045         with open(os.devnull, "w") as dump:
0046             subprocess.check_call(["voms-proxy-info", "--exists"],
0047                                   stdout = dump, stderr = dump)
0048     except subprocess.CalledProcessError:
0049         return False
0050     return True
0051
0052 ##############################################
0053 def forward_proxy(rundir):
0054 ##############################################
0055     """Forward proxy to location visible from the batch system.
0056     Arguments:
0057     - `rundir`: directory for storing the forwarded proxy
0058     Return:
0059     - Full path to the forwarded proxy
0060     """
0061
0062     if not check_proxy():
0063         print("Please create proxy via 'voms-proxy-init -voms cms'.")
0064         sys.exit(1)
0065
0066     ## Move the proxy to the run directory
0067     proxyName = "{}/.user_proxy".format(rundir)
0068     localProxy = subprocess.check_output(["voms-proxy-info", "--path"]).strip()
0069     shutil.copyfile(localProxy, proxyName)
0070
0071     ## Return the path to the forwarded proxy
0072     return proxyName
0073
0074
0075 ##############################################
0076 def updateConfigurationFile(configurationFile, updateInstructions):
0077 ##############################################
0078     """Update a template configuration file with custom configuration
0079     Arguments:
0080     - configurationFile: File name for the configuration file that will be updated
0081     - updateInstructions: A dictionary defining the updated configuration with keys "overwrite", "remove", "add" and "addBefore" each containing a list with the instructions on what should be replaced, removed or added.
0082     """
0083
0084     # Read the original configuration file
0085     with open(configurationFile,"r") as inputFile:
0086         fileContent = inputFile.readlines()
0087
0088     # Perform all overwrite operations to the configuration file. First string where the substring before the first space matches with the replacing string is overwritten. If a character "|" is included in the instruction, the subtring before that is used to search for the overwritten line instead. If no such string is found, add the instruction to the end of the file.
0089     if "overwrite" in updateInstructions:
0090
0091         for instruction in updateInstructions["overwrite"]:
0092
0093             decodeInstruction = instruction.split("|")
0094             if(len(decodeInstruction) > 1):
0095                 lineToReplace = decodeInstruction[0]
0096                 newInstruction = instruction[instruction.index("|")+1:]
0097             else:
0098                 lineToReplace = instruction.split()[0]
0099                 newInstruction = instruction
0100
0101             lineOverwritten = False
0102             for iLine in range(0,len(fileContent)):
0103                 if fileContent[iLine].startswith(lineToReplace):
0104                     fileContent[iLine] = newInstruction
0105                     if not fileContent[iLine].endswith("\n"):
0106                         fileContent[iLine] = fileContent[iLine] + "\n"
0107                     lineOverwritten = True
0108                     break
0109
0110             # If did not find a line to overwrite, add the instruction to the end of the file
0111             if not lineOverwritten:
0112                 fileContent.append(newInstruction)
0113                 if not fileContent[-1].endswith("\n"):
0114                     fileContent[-1] = fileContent[-1] + "\n"
0115
0116     # Perform all remove operations to the configuration file. First string that starst with the instruction will be removed from the configuration file.
0117     if "remove" in updateInstructions:
0118         for instruction in updateInstructions["remove"]:
0119             for iLine in range(0,len(fileContent)):
0120                 if fileContent[iLine].startswith(instruction):
0121                     fileContent.pop(iLine)
0122                     break
0123
0124     # Perform all add operations to the configuration file. The instruction is added to the matching CRAB configuration section. If one is not found, it is added to the end of the file.
0125     if "add" in updateInstructions:
0126         for instruction in updateInstructions["add"]:
0127             categories = instruction.split(".")
0128             if len(categories) > 2:
0129                 category = categories[1]
0130             else:
0131                 category = "nonExistent"
0132             previousCategory = ""
0133             lineFound = False
0134
0135             # First try to add the line to a correct section in CRAB configuration
0136             for iLine in range(0,len(fileContent)):
0137                 if fileContent[iLine] == "\n" and previousCategory == category:
0138                     fileContent.insert(iLine, instruction)
0139                     if not fileContent[iLine].endswith("\n"):
0140                         fileContent[iLine] = fileContent[iLine] + "\n"
0141                     lineFound = True
0142                     break
0143                 elif fileContent[iLine] == "\n":
0144                     previousCategory = ""
0145                 else:
0146                     newCategories = fileContent[iLine].split(".")
0147                     if len(newCategories) > 2:
0148                         previousCategory = newCategories[1]
0149                     else:
0150                         previousCategory = ""
0151
0152             # If the correct section is not found, add the new line to the end of the file
0153             if not lineFound:
0154                 fileContent.append(instruction)
0155                 if not fileContent[-1].endswith("\n"):
0156                     fileContent[-1] = fileContent[-1] + "\n"
0157
0158     # Perform all addBefore operations to the configuration file. This adds an instruction to the configuration file just before a line that starts with a string defined before the '|' character. If one is not found, the line is added to the end of the file.
0159     if "addBefore" in updateInstructions:
0160         for instruction in updateInstructions["addBefore"]:
0161             lineBefore = instruction.split("|")[0]
0162             newInstruction = instruction[instruction.index("|")+1:]
0163             lineFound = False
0164             for iLine in range(0,len(fileContent)):
0165                 if fileContent[iLine].startswith(lineBefore):
0166                     fileContent.insert(iLine,newInstruction)
0167                     if not fileContent[iLine].endswith("\n"):
0168                         fileContent[iLine] = fileContent[iLine] + "\n"
0169                     lineFound = True
0170                     break
0171
0172
0173             # If the searched line is not found, add the new line to the end of the file
0174             if not lineFound:
0175                 fileContent.append(newInstruction)
0176                 if not fileContent[-1].endswith("\n"):
0177                     fileContent[-1] = fileContent[-1] + "\n"
0178
0179     # Write the updates to the configuration file
0180     with open(configurationFile,"w") as outputFile:
0181         outputFile.writelines(fileContent)
0182
0183
0184 ##############################################
0185 def main():
0186 ##############################################
0187
0188     ## Before doing anything, check that grip proxy exists
0189     if not check_proxy():
0190         print("Grid proxy is required in most use cases of the tool.")
0191         print("Please create a proxy via 'voms-proxy-init -voms cms'.")
0192         sys.exit(1)
0193
0194     ##Read parser arguments
0195     args = parser()
0196
0197     ##Print example config which is in Aligment/OfflineValidation/bin if wished
0198     if args.example:
0199         with open("{}/src/Alignment/OfflineValidation/bin/example.yaml".format(os.environ["CMSSW_BASE"]), "r") as exampleFile:
0200             config = yaml.load(exampleFile, Loader=yaml.Loader)
0201             pprint.pprint(config, width=30)
0202             sys.exit(0)
0203
0204     ##Read in AllInOne config dependent on what format you choose
0205     with open(args.config, "r") as configFile:
0206         if args.verbose:
0207             print("Read AllInOne config: '{}'".format(args.config))
0208
0209         if args.config.split(".")[-1] == "json":
0210             config = json.load(configFile)
0211
0212         elif args.config.split(".")[-1] == "yaml":
0213             config = yaml.load(configFile, Loader=yaml.Loader)
0214
0215         else:
0216             raise Exception("Unknown config extension '{}'. Please use json/yaml format!".format(args.config.split(".")[-1]))
0217
0218     ##Check for all paths in configuration and attempt to "digest" them
0219     ##As a bonus, all ROOT colors are turned to the integer value
0220     for path in fnc.find_and_change(list(), config):
0221         if args.verbose and ("." in str(path) or "/" in str(path)):
0222             print("Digesting path: "+str(path))
0223
0224     ##Create working directory
0225     if os.path.isdir(config["name"]) and not args.force:
0226         raise Exception("Validation directory '{}' already exists! Please choose another name for your directory.".format(config["name"]))
0227
0228     validationDir = os.path.abspath(config["name"])
0229     exeDir = "{}/executables".format(validationDir)
0230     cmsconfigDir =  "{}/cmsConfigs".format(validationDir)
0231
0232     subprocess.call(["mkdir", "-p", validationDir] + ((["-v"] if args.verbose else [])))
0233     subprocess.call(["mkdir", "-p", exeDir] + (["-v"] if args.verbose else []))
0234     subprocess.call(["mkdir", "-p", cmsconfigDir] + (["-v"] if args.verbose else []))
0235
0236     ##Copy AllInOne config in working directory in json/yaml format
0237     subprocess.call(["cp", "-f", args.config, validationDir] + (["-v"] if args.verbose else []))
0238
0239     ## Define the template files
0240     crabTemplateFile = fnc.digest_path("$CMSSW_BASE/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/templates/crabTemplate.py")
0241     condorTemplateFile = fnc.digest_path("$CMSSW_BASE/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/templates/condorTemplate.submit")
0242     executableTempleteFile = fnc.digest_path("$CMSSW_BASE/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/templates/executableTemplate.sh")
0243
0244
0245     ##List with all jobs
0246     jobs = []
0247
0248     ##Check in config for all validation and create jobs
0249     for validation in config["validations"]:
0250         if validation == "GCP":
0251             jobs.extend(GCP.GCP(config, validationDir))
0252
0253         elif validation == "DMR":
0254             jobs.extend(DMR.DMR(config, validationDir))
0255
0256         elif validation == "Zmumu":
0257             jobs.extend(Zmumu.Zmumu(config, validationDir))
0258
0259         elif validation == "PV":
0260             jobs.extend(PV.PV(config, validationDir))
0261
0262         elif validation == "SplitV":
0263             jobs.extend(SplitV.SplitV(config, validationDir))
0264
0265         elif validation == "JetHT":
0266             jobs.extend(JetHT.JetHT(config, validationDir))
0267         elif validation == "DiMuonV":
0268             jobs.extend(DiMuonV.DiMuonV(config, validationDir))
0269         elif validation == "MTS":
0270             jobs.extend(MTS.MTS(config, validationDir))
0271         else:
0272             raise Exception("Unknown validation method: {}".format(validation))
0273
0274     ##Create dir for DAG file and loop over all jobs
0275     subprocess.call(["mkdir", "-p", "{}/DAG/".format(validationDir)] + (["-v"] if args.verbose else []))
0276
0277     with open("{}/DAG/dagFile".format(validationDir), "w") as dag:
0278         for job in jobs:
0279             ##Create job dir, output dir
0280             subprocess.call(["mkdir", "-p", job["dir"]] + (["-v"] if args.verbose else []))
0281             subprocess.call(["mkdir", "-p", job["config"]["output"]] + (["-v"] if args.verbose else []))
0282             subprocess.call(["mkdir", "-p", "{}/condor".format(job["dir"])] + (["-v"] if args.verbose else []))
0283             subprocess.call(["ln", "-fs", job["config"]["output"], "{}/output".format(job["dir"])] + (["-v"] if args.verbose else []))
0284
0285             ## Copy the template files to the job directory
0286             crabConfigurationFile = "{}/crabConfiguration.py".format(job["dir"])
0287             subprocess.call(["cp", crabTemplateFile, crabConfigurationFile] + (["-v"] if args.verbose else []))
0288             condorSubmitFile = "{}/condor.sub".format(job["dir"])
0289             subprocess.call(["cp", condorTemplateFile, condorSubmitFile] + (["-v"] if args.verbose else []))
0290             executableFile = "{}/run.sh".format(job["dir"])
0291             subprocess.call(["cp", executableTempleteFile, executableFile] + (["-v"] if args.verbose else []))
0292
0293             ## Forward the proxy to the job directory
0294             if args.verbose:
0295                 print("Forwarding grid proxy to directory {}".format(job["dir"]))
0296             myProxy = forward_proxy(job["dir"])
0297
0298             ##Create symlink for executable/python cms config if needed
0299             subprocess.call("cp -f $(which {}) {}".format(job["exe"], exeDir) + (" -v" if args.verbose else ""), shell = True)
0300             subprocess.call(["ln", "-fs", "{}/{}".format(exeDir, job["exe"]), job["dir"]] + (["-v"] if args.verbose else []))
0301             if "cms-config" in job:
0302                 cmsConfig = job["cms-config"].split("/")[-1]
0303
0304                 subprocess.call(["cp", "-f", job["cms-config"], "{}/{}".format(cmsconfigDir, cmsConfig)] + (["-v"] if args.verbose else []))
0305                 subprocess.call(["ln", "-fs", "{}/{}".format(cmsconfigDir, cmsConfig), "{}/validation_cfg.py".format(job["dir"])] + (["-v"] if args.verbose else []))
0306
0307             ##Write local config file
0308             with open("{}/validation.json".format(job["dir"]), "w") as jsonFile:
0309                 if args.verbose:
0310                     print("Write local json config: '{}'".format("{}/validation.json".format(job["dir"])))
0311
0312                 json.dump(job["config"], jsonFile, indent=4)
0313
0314             ## Customize the executable template file for this specific job
0315             executableCustomization = {"overwrite": [], "addBefore": []}
0316
0317             executableCustomization["overwrite"].append("export X509|export X509_USER_PROXY={}".format(myProxy)) # Define the proxy location
0318             executableCustomization["overwrite"].append("cd workDir|cd {}".format(job["dir"])) # Define the work directory for this job
0319
0320             # Option the give free arguments to the executable
0321             if "exeArguments" in job:
0322                 executableCustomization["overwrite"].append("./cmsRun|./{} {}".format(job["exe"], job["exeArguments"])) # Define the correct executable for this job
0323             else: # Default arguments
0324                 executableCustomization["overwrite"].append("./cmsRun|./{} {}validation.json".format(job["exe"], "validation_cfg.py config=" if "cms-config" in job else "")) # Define the correct executable for this job
0325
0326             # Option to include the condor job number given as a command line argument
0327             if "nCondorJobs" in job:
0328                 executableCustomization["addBefore"].append("./{}|JOBNUMBER=${{1:--1}}".format(job["exe"]))
0329
0330             # Do the manual configuration on top of the executable file
0331             updateConfigurationFile(executableFile, executableCustomization)
0332
0333             # Give the correct access rights for the executable
0334             subprocess.call(["chmod", "a+rx", executableFile] + (["-v"] if args.verbose else []))
0335
0336             ## Customize the condor submit file for this specific job
0337             condorSubmitCustomization = {"overwrite": [], "addBefore": []}
0338
0339             ## Hack to solve condor dagman issue with passing environmental variables
0340             condorSubmitCustomization["addBefore"].append('+JobFlavour|+environment = "CMSSW_BASE={}"'.format(fnc.digest_path("$CMSSW_BASE")))
0341
0342             # Take given flavour for the job, except if overwritten in job config
0343             condorSubmitCustomization["overwrite"].append('+JobFlavour = "{}"'.format(args.job_flavour if not 'flavour' in job else job['flavour']))
0344
0345             # If condor job array is sent, add job ID information to submit file
0346             if "nCondorJobs" in job:
0347                 condorSubmitCustomization["addBefore"].append("output|arguments = $(ProcID)")
0348                 condorSubmitCustomization["overwrite"].append("output = condor/condor$(ProcID).out")
0349                 condorSubmitCustomization["overwrite"].append("error  = condor/condor$(ProcID).err")
0350                 condorSubmitCustomization["overwrite"].append("log    = condor/condor$(ProcID).log")
0351                 condorSubmitCustomization["overwrite"].append("queue {}".format(job["nCondorJobs"]))
0352
0353             # Do the customization for the condor submit file
0354             updateConfigurationFile(condorSubmitFile, condorSubmitCustomization)
0355
0356             ##Write command in dag file
0357             dag.write("JOB {} condor.sub DIR {}\n".format(job["name"], job["dir"]))
0358
0359             if job["dependencies"]:
0360                 dag.write("\n")
0361                 dag.write("PARENT {} CHILD {}".format(" ".join(job["dependencies"]), job["name"]))
0362
0363             dag.write("\n\n")
0364
0365             ## If there is custom crab configuration defined, modify the crab template file based on that
0366             if "crabCustomConfiguration" in job["config"]:
0367                 updateConfigurationFile(crabConfigurationFile, job["config"]["crabCustomConfiguration"])
0368
0369
0370     if args.verbose:
0371         print("DAGman config has been written: '{}'".format("{}/DAG/dagFile".format(validationDir)))
0372
0373     ##Call submit command if not dry run
0374     if args.dry:
0375         print("Enviroment is set up. If you want to submit everything, call 'condor_submit_dag {}/DAG/dagFile'".format(validationDir))
0376
0377     else:
0378         subprocess.call(["condor_submit_dag", "{}/DAG/dagFile".format(validationDir)])
0379
0380 ##############################################
0381 if __name__ == "__main__":
0382 ##############################################
0383     main()