Common/python/HltComparatorCreateWorkflow.py

0001 #!/usr/bin/env python
0002 # Original Author: James Jackson
0003
0004 # MakeValidationConfig.py
0005 #   Makes CMSSW config for prompt validation of a given run number,
0006 #   HLT Key or HLT Config
0007
0008 from __future__ import print_function
0009 from optparse import OptionParser
0010 import os, time, re
0011
0012 jobHash = "%s_%s_%s" % (os.getuid(), os.getpid(), int(time.time()))
0013
0014 usage = '%prog [options]. \n\t-a and -k required, -h for help.'
0015 parser = OptionParser(usage)
0016 parser.add_option("-a", "--analysis", dest="analysis",
0017                   help="analysis configuration file")
0018 # not yet implemented
0019 #parser.add_option("-r", "--run", dest="run",
0020 #                  help="construct config for runnumber RUN", metavar="RUN")
0021 parser.add_option("-k", "--hltkey", dest="hltkey",
0022                   help="ignore RunRegistry and force the use of HLT key KEY",
0023                   metavar="KEY")
0024 parser.add_option("-c", "--hltcff", dest="hltcff",
0025                   help="use the config fragment CFF to define HLT configuration",
0026                   metavar="CFF")
0027 parser.add_option("-f", "--frontier", dest="frontier",
0028                   help="frontier connection string to use, defaults to frontier://FrontierProd/CMS_COND_21X_GLOBALTAG",
0029                   default="frontier://FrontierProd/CMS_COND_31X_GLOBALTAG")
0030
0031 # Parse options and perform sanity checks
0032 (options, args) = parser.parse_args()
0033 #if options.run == None and options.hltkey == None and options.hltcff == None:
0034 if options.hltkey == None and options.hltcff == None:
0035    parser.error("I don't have all the required options.")
0036    raise SystemExit(
0037       "Please specify one of --hltkey (-k) or --hltcff (-s)")
0038 if options.hltkey != None and options.hltcff != None:
0039    raise SystemExit("Please only specify --hltkey (-k) or --hltcff (-c)")
0040 # if options.run != None and options.hltkey != None:
0041 #    raise SystemExit("Please only specify --run (-r) or --hltkey (-k)")
0042 # if options.run != None and options.hltcff != None:
0043 #    raise SystemExit("Please only specify --run (-r) or --hltcff (-c)")
0044 if options.analysis == None:
0045    raise SystemExit(
0046       "Please specify an analysis configuration: -a or --analysis")
0047
0048 # Checks the runtime environment is suitable
0049 def CheckEnvironment():
0050    cwd = os.getcwd()
0051    t = cwd.split("/")
0052    if t[-1] != "python":
0053       raise SystemExit("Must run from a Module/Package/python directory")
0054
0055 # Converts an online HLT configuration into offline suitable format
0056 def ConvertHltOnlineToOffline(config, frontierString):
0057    # Replace the frontier string
0058    onlineFrontier = re.search('"(frontier:.*)"', config)
0059    if not onlineFrontier:
0060       print("WARNING: Could not find Frontier string in HLT configuration. Will ignore.")
0061    else:
0062       config = config.replace(onlineFrontier.group(1), frontierString)
0063
0064    # Replace the global tag
0065    config = config.replace("H::All", "P::All")
0066
0067    # print config -- debugging
0068    # Remove unwanted PSets
0069    config = RemovePSet(config, "MessageLogger")
0070 #   config = RemovePSet(config, "DQMStore")
0071    config = RemovePSet(config, "DQM")
0072    config = RemovePSet(config, "FUShmDQMOutputService")
0073
0074    return config
0075
0076 def RemovePSet(config, pset):
0077    startLoc = config.find(pset)
0078    started = False
0079    count = 0
0080    curLoc = startLoc
0081    endLoc = 0
0082
0083    # Find starting parenthesis
0084    while not started:
0085       if config[curLoc] == "(":
0086          started = True
0087          count = 1
0088       curLoc += 1
0089
0090    # Find end parenthesis
0091    while endLoc == 0:
0092       if config[curLoc] == "(":
0093          count += 1
0094       elif config[curLoc] == ")":
0095          count -= 1
0096       if count == 0:
0097          endLoc = curLoc
0098       curLoc += 1
0099
0100    config = config.replace(config[startLoc:endLoc + 1], "")
0101
0102    return config
0103
0104 # Fetches the HLT configuration from ConfDB
0105 def GetHltConfiguration(hltKey, frontierString):
0106    # Format the config path for include
0107    cwd = os.getcwd()
0108    t = cwd.split("/")
0109    module = t[-3]
0110    package = t[-2]
0111
0112    # Get the HLT config
0113    config = os.popen2('wget "http://cms-project-confdb-hltdev.web.cern.ch/cms-project-confdb-hltdev/get.jsp?dbName=ORCOFF&configName=%s&cff=&nooutput=&format=Python" -O- -o /dev/null' % hltKey)[1].read()
0114    #config = os.popen2("edmConfigFromDB --debug --configName %s --orcoff --format Python --nooutput --cff" % hltKey)[1].read()
0115    configName = "JobHLTConfig_%s_cff.py" % jobHash
0116
0117    # Perform online --> offline conversion
0118    config = ConvertHltOnlineToOffline(config, frontierString)
0119
0120    # Write the config
0121    f = open(configName, "w")
0122    f.write(config)
0123    f.close()
0124
0125    return 'process.load("%s.%s.%s")' % (module, package, configName.split(".")[0])
0126
0127 # Fetches the HLT key from ConfDB - turned off in options for now
0128 def GetHltKeyForRun(run):
0129    raise SystemExit("Not implemented yet")
0130
0131 # Formats an HLT CFF path into a suitable python include statement
0132 def FormatHltCff(cffPath):
0133    pathParts = cffPath.split(".")
0134    if not re.match("^[_A-Za-z0-9]*\.[_A-Za-z0-9]*\.[_A-Za-z0-9]*$", cffPath):
0135       raise SystemExit("Expected cff in form Package.Module.configName_cff")
0136    return 'process.load("%s")' % cffPath
0137
0138 # Returns python code to compile an HLT configuration
0139 def GetHltCompileCode(subsystem, package, hltConfig):
0140    tmpCode = compileCode.replace("SUBSYSTEM", subsystem)
0141    tmpCode = tmpCode.replace("PACKAGE", package)
0142    tmpCode = tmpCode.replace("CONFIG", hltConfig + "c")
0143    return tmpCode
0144
0145 # Prepares the analysis configuration
0146 def CreateAnalysisConfig(analysis, hltInclude):
0147    anaName = "JobAnalysisConfig_%s_cfg.py" % jobHash
0148    f = open(analysis)
0149    g = open(anaName, "w")
0150    g.write(f.read())
0151    g.write("\n")
0152    g.write(hltInclude)
0153    g.close()
0154    f.close()
0155    return anaName
0156
0157 # Quick sanity check of the environment
0158 CheckEnvironment()
0159
0160 # Get the required HLT config snippet
0161 hltConfig = None
0162 if options.hltkey != None:
0163    hltConfig = GetHltConfiguration(options.hltkey, options.frontier)
0164 elif options.hltcff != None:
0165    hltConfig = FormatHltCff(options.hltcff)
0166 else:
0167    hltKey = GetHltKeyForRun(0)
0168    hltConfig = GetHltConfiguration(hltKey)
0169
0170 # Prepare the analysis configuration
0171 anaConfig = CreateAnalysisConfig(options.analysis, hltConfig)
0172
0173 if options.hltcff:
0174     print("Using HLT configuration:           %s" % hltConfig)
0175 else:
0176     print("Created HLT configuration:         %s" % hltConfig)
0177 print("Created analysis configuration:    %s" % anaConfig)
0178