Common/python/HltComparatorCreateWorkflow.py

0001 #!/usr/bin/env python
0002 # Original Author: James Jackson
0003
0004 # MakeValidationConfig.py
0005 #   Makes CMSSW config for prompt validation of a given run number,
0006 #   HLT Key or HLT Config
0007
0008 from optparse import OptionParser
0009 import os, time, re
0010
0011 jobHash = "%s_%s_%s" % (os.getuid(), os.getpid(), int(time.time()))
0012
0013 usage = '%prog [options]. \n\t-a and -k required, -h for help.'
0014 parser = OptionParser(usage)
0015 parser.add_option("-a", "--analysis", dest="analysis",
0016                   help="analysis configuration file")
0017 # not yet implemented
0018 #parser.add_option("-r", "--run", dest="run",
0019 #                  help="construct config for runnumber RUN", metavar="RUN")
0020 parser.add_option("-k", "--hltkey", dest="hltkey",
0021                   help="ignore RunRegistry and force the use of HLT key KEY",
0022                   metavar="KEY")
0023 parser.add_option("-c", "--hltcff", dest="hltcff",
0024                   help="use the config fragment CFF to define HLT configuration",
0025                   metavar="CFF")
0026 parser.add_option("-f", "--frontier", dest="frontier",
0027                   help="frontier connection string to use, defaults to frontier://FrontierProd/CMS_COND_21X_GLOBALTAG",
0028                   default="frontier://FrontierProd/CMS_COND_31X_GLOBALTAG")
0029
0030 # Parse options and perform sanity checks
0031 (options, args) = parser.parse_args()
0032 #if options.run == None and options.hltkey == None and options.hltcff == None:
0033 if options.hltkey == None and options.hltcff == None:
0034    parser.error("I don't have all the required options.")
0035    raise SystemExit(
0036       "Please specify one of --hltkey (-k) or --hltcff (-s)")
0037 if options.hltkey != None and options.hltcff != None:
0038    raise SystemExit("Please only specify --hltkey (-k) or --hltcff (-c)")
0039 # if options.run != None and options.hltkey != None:
0040 #    raise SystemExit("Please only specify --run (-r) or --hltkey (-k)")
0041 # if options.run != None and options.hltcff != None:
0042 #    raise SystemExit("Please only specify --run (-r) or --hltcff (-c)")
0043 if options.analysis == None:
0044    raise SystemExit(
0045       "Please specify an analysis configuration: -a or --analysis")
0046
0047 # Checks the runtime environment is suitable
0048 def CheckEnvironment():
0049    cwd = os.getcwd()
0050    t = cwd.split("/")
0051    if t[-1] != "python":
0052       raise SystemExit("Must run from a Module/Package/python directory")
0053
0054 # Converts an online HLT configuration into offline suitable format
0055 def ConvertHltOnlineToOffline(config, frontierString):
0056    # Replace the frontier string
0057    onlineFrontier = re.search('"(frontier:.*)"', config)
0058    if not onlineFrontier:
0059       print("WARNING: Could not find Frontier string in HLT configuration. Will ignore.")
0060    else:
0061       config = config.replace(onlineFrontier.group(1), frontierString)
0062
0063    # Replace the global tag
0064    config = config.replace("H::All", "P::All")
0065
0066    # print config -- debugging
0067    # Remove unwanted PSets
0068    config = RemovePSet(config, "MessageLogger")
0069 #   config = RemovePSet(config, "DQMStore")
0070    config = RemovePSet(config, "DQM")
0071    config = RemovePSet(config, "FUShmDQMOutputService")
0072
0073    return config
0074
0075 def RemovePSet(config, pset):
0076    startLoc = config.find(pset)
0077    started = False
0078    count = 0
0079    curLoc = startLoc
0080    endLoc = 0
0081
0082    # Find starting parenthesis
0083    while not started:
0084       if config[curLoc] == "(":
0085          started = True
0086          count = 1
0087       curLoc += 1
0088
0089    # Find end parenthesis
0090    while endLoc == 0:
0091       if config[curLoc] == "(":
0092          count += 1
0093       elif config[curLoc] == ")":
0094          count -= 1
0095       if count == 0:
0096          endLoc = curLoc
0097       curLoc += 1
0098
0099    config = config.replace(config[startLoc:endLoc + 1], "")
0100
0101    return config
0102
0103 # Fetches the HLT configuration from ConfDB
0104 def GetHltConfiguration(hltKey, frontierString):
0105    # Format the config path for include
0106    cwd = os.getcwd()
0107    t = cwd.split("/")
0108    module = t[-3]
0109    package = t[-2]
0110
0111    # Get the HLT config
0112    config = os.popen2('wget "http://cms-project-confdb-hltdev.web.cern.ch/cms-project-confdb-hltdev/get.jsp?dbName=ORCOFF&configName=%s&cff=&nooutput=&format=Python" -O- -o /dev/null' % hltKey)[1].read()
0113    #config = os.popen2("edmConfigFromDB --debug --configName %s --orcoff --format Python --nooutput --cff" % hltKey)[1].read()
0114    configName = "JobHLTConfig_%s_cff.py" % jobHash
0115
0116    # Perform online --> offline conversion
0117    config = ConvertHltOnlineToOffline(config, frontierString)
0118
0119    # Write the config
0120    f = open(configName, "w")
0121    f.write(config)
0122    f.close()
0123
0124    return 'process.load("%s.%s.%s")' % (module, package, configName.split(".")[0])
0125
0126 # Fetches the HLT key from ConfDB - turned off in options for now
0127 def GetHltKeyForRun(run):
0128    raise SystemExit("Not implemented yet")
0129
0130 # Formats an HLT CFF path into a suitable python include statement
0131 def FormatHltCff(cffPath):
0132    pathParts = cffPath.split(".")
0133    if not re.match("^[_A-Za-z0-9]*\\.[_A-Za-z0-9]*\\.[_A-Za-z0-9]*$", cffPath):
0134       raise SystemExit("Expected cff in form Package.Module.configName_cff")
0135    return 'process.load("%s")' % cffPath
0136
0137 # Returns python code to compile an HLT configuration
0138 def GetHltCompileCode(subsystem, package, hltConfig):
0139    tmpCode = compileCode.replace("SUBSYSTEM", subsystem)
0140    tmpCode = tmpCode.replace("PACKAGE", package)
0141    tmpCode = tmpCode.replace("CONFIG", hltConfig + "c")
0142    return tmpCode
0143
0144 # Prepares the analysis configuration
0145 def CreateAnalysisConfig(analysis, hltInclude):
0146    anaName = "JobAnalysisConfig_%s_cfg.py" % jobHash
0147    f = open(analysis)
0148    g = open(anaName, "w")
0149    g.write(f.read())
0150    g.write("\n")
0151    g.write(hltInclude)
0152    g.close()
0153    f.close()
0154    return anaName
0155
0156 # Quick sanity check of the environment
0157 CheckEnvironment()
0158
0159 # Get the required HLT config snippet
0160 hltConfig = None
0161 if options.hltkey != None:
0162    hltConfig = GetHltConfiguration(options.hltkey, options.frontier)
0163 elif options.hltcff != None:
0164    hltConfig = FormatHltCff(options.hltcff)
0165 else:
0166    hltKey = GetHltKeyForRun(0)
0167    hltConfig = GetHltConfiguration(hltKey)
0168
0169 # Prepare the analysis configuration
0170 anaConfig = CreateAnalysisConfig(options.analysis, hltConfig)
0171
0172 if options.hltcff:
0173     print("Using HLT configuration:           %s" % hltConfig)
0174 else:
0175     print("Created HLT configuration:         %s" % hltConfig)
0176 print("Created analysis configuration:    %s" % anaConfig)
0177