Tools/python/makeHLTPrescaleTable.py

0001 #!/usr/bin/env python3
0002 from sys import stderr, exit
0003 import subprocess, os
0004
0005 from optparse import OptionParser
0006 parser = OptionParser(usage=
0007 """
0008 usage: %prog [options] csv_output_file
0009
0010 examples:
0011
0012  %prog out.csv
0013
0014      produces a table of ALL runs and ALL paths (can take quite some time)
0015
0016  %prog --path='*ele*' --path='*photon*' out.csv
0017
0018      select only paths containing 'ele' and 'photon'
0019
0020 """)
0021 parser.add_option("--firstRun",  dest="firstRun",  help="first run", type="int", metavar="RUN", default="1")
0022 parser.add_option("--lastRun",   dest="lastRun",   help="last run",  type="int", metavar="RUN", default="9999999")
0023 parser.add_option("--groupName", dest="groupName", help="select runs of name like NAME", metavar="NAME", default="Collisions%")
0024 parser.add_option("--overwrite", dest="overwrite", help="force overwriting of output CSV file", action="store_true", default=False)
0025
0026 parser.add_option("--path",
0027                  dest="pathPatterns",
0028                  default = [],
0029                  action="append",
0030                  metavar="PATTERN",
0031                  help="restrict paths to PATTERN. Note that this can be a single path name or a pattern " +
0032                       "using wildcards (*,?) similar to those used for selecting multiple files (see " +
0033                       "the documentation of the fnmatch module for details). Note also that this option " +
0034                       "can be specified more than once to select multiple paths or patterns. If this option " +
0035                       "is not specified, all paths are considered. Note that the comparison is done " +
0036                       "in a case-INsensitive manner. " +
0037                       "You may have to escape wildcards (with quotes or backslash) in order to avoid "+
0038                       "expansion by the shell"
0039                  )
0040
0041 # parser.add_option("--jsonOut",   dest="jsonOut",   help="dump prescales in JSON format on FILE", metavar="FILE")
0042 (options, args) = parser.parse_args()
0043 if len(args) != 1:
0044    parser.print_help()
0045    exit(2)
0046
0047 csv_output_file = args[0]
0048
0049 if os.path.exists(csv_output_file) and not options.overwrite:
0050    print("cowardly refusing to overwrite existing output file '" + csv_output_file + "'. Run this script without argument to see options for overriding this check.", file=stderr)
0051    exit(1)
0052
0053 #----------------------------------------------------------------------
0054 def getPrescaleTableFromProcessObject(process):
0055    """ returns a dict of hlt key to vector of prescales
0056    mapping """
0057
0058    retval = {}
0059    for entry in process.PrescaleService.prescaleTable:
0060        retval[entry.pathName.value()] = entry.prescales.value()
0061
0062    return retval
0063
0064 #----------------------------------------------------------------------
0065
0066 def getProcessObjectFromConfDB(hlt_key):
0067    # print >> stderr,"\t%s ..." % hlt_key
0068    cmd = "edmConfigFromDB --orcoff --configName " + hlt_key
0069    # print >> stderr, "cmd=",cmd
0070    res = subprocess.getoutput(cmd)
0071
0072    # potentially dangerous: we're running python code here
0073    # which we get from an external process (confDB).
0074    # we trust that there are no file deletion commands
0075    # in the HLT configurations...
0076
0077    # for some keys, edmConfigFromDB seems to produce error messages.
0078    # just return None in this case
0079    try:
0080        exec(res)
0081    except:
0082        return None
0083
0084    return process
0085
0086 #----------------------------------------------------------------------
0087 from .queryRR import queryRR
0088
0089 #----------------------------------------------------------------------
0090 # main
0091 #----------------------------------------------------------------------
0092
0093 # check whether we have a CMSSW environment initalized
0094 if os.system("which edmConfigFromDB") != 0:
0095    print("could not find the command edmConfigFromDB. Did you initialize your CMSSW runtime environment ?", file=stderr)
0096    exit(1)
0097
0098 runKeys = queryRR(options.firstRun,options.lastRun,options.groupName)
0099
0100 # maps from HLT key to prescale information.
0101 # indexed as: prescaleTable[hlt_key][hlt_path_name]
0102 prescaleTable = {}
0103
0104 # maps from
0105 hlt_path_names_table = {}
0106
0107 # set of all HLT paths seen so far
0108 all_hlt_path_names_seen = set()
0109
0110 runs = sorted(runKeys.keys())
0111
0112 # loop over all hlt keys found
0113
0114 all_hlt_keys_seen = set(runKeys.values())
0115
0116 print("found %d runs and %d HLT menus" % ( len(runKeys), len(all_hlt_keys_seen)), file=stderr)
0117
0118 index = 1
0119
0120 for hlt_key in all_hlt_keys_seen:
0121
0122    print("(%3d/%3d) Querying ConfDB for HLT menu %s" % (index, len(all_hlt_keys_seen) , hlt_key), file=stderr)
0123    process = getProcessObjectFromConfDB(hlt_key)
0124
0125    if process == None:
0126        print("WARNING: unable to retrieve hlt_key '" + hlt_key + "'", file=stderr)
0127        continue
0128
0129    prescaleTable[hlt_key] = getPrescaleTableFromProcessObject(process)
0130
0131    all_path_names = set(process.paths.keys())
0132
0133    # remember which hlt paths were in this menu
0134    hlt_path_names_table[hlt_key] = all_path_names
0135
0136    # add this configuration's HLT paths to the list
0137    # of overall path names seen
0138    all_hlt_path_names_seen.update(all_path_names)
0139
0140    index += 1
0141
0142 # end of loop over all HLT keys
0143
0144 # make sure the list of all HLT path names ever seen is sorted
0145 all_hlt_path_names_seen = sorted(all_hlt_path_names_seen)
0146
0147 #--------------------
0148 # filter paths if required by the user
0149 if len(options.pathPatterns) > 0:
0150    import fnmatch
0151
0152    tmp = []
0153
0154    for path in all_hlt_path_names_seen:
0155
0156        for pattern in options.pathPatterns:
0157            if fnmatch.fnmatch(path.lower(), pattern.lower()):
0158
0159                # accept this path
0160                tmp.append(path)
0161                break
0162
0163    all_hlt_path_names_seen = tmp
0164
0165 # sanity check
0166
0167 if len(all_hlt_path_names_seen) == 0:
0168    print("no HLT paths found, exiting", file=stderr)
0169    exit(1)
0170
0171 #--------------------
0172 # now that we know all path names of all runs, produce the CSV
0173 import csv
0174
0175 previous_hlt_key = None
0176
0177 fout = open(csv_output_file,"w")
0178
0179 csv_writer = csv.writer(fout,delimiter=";")
0180
0181 csv_writer.writerow(['Table of HLT prescale factors'])
0182 csv_writer.writerow([])
0183 csv_writer.writerow(['Explanation:'])
0184 csv_writer.writerow(['number(s) = prescale factor(s), HLT path present in this menu'])
0185 csv_writer.writerow(['empty = HLT path NOT present in this menu'])
0186 csv_writer.writerow(['0 = HLT path present in this menu but prescale factor is zero'])
0187 csv_writer.writerow(['U = could not retrieve menu for this HLT key from confDB'])
0188 csv_writer.writerow([])
0189 csv_writer.writerow([])
0190
0191 # write the header line
0192 column_names = [ 'run','' ]
0193 column_names.extend(all_hlt_path_names_seen)
0194 csv_writer.writerow(column_names)
0195
0196 csv_writer.writerow([])
0197
0198 for run in runs:
0199    hlt_key = runKeys[run]
0200
0201    if hlt_key == previous_hlt_key:
0202        # the hlt key is the same as for the previous run
0203        # just reuse the existing contents of the variable 'values'
0204        # (apart from the run number)
0205        values[0] = run
0206        csv_writer.writerow(values)
0207        continue
0208
0209    # HLT key has changed
0210
0211    # insert a line with the menu's name
0212    #
0213    # check whether we actually could determine the menu
0214    if hlt_key not in hlt_path_names_table:
0215        # could previously not retrieve the python
0216        # configuration for this key
0217        #
0218        # put some warnings in the output table
0219
0220        csv_writer.writerow([hlt_key, "COULD NOT RETRIEVE MENU FROM CONFDB"])
0221
0222        values = [ run , '' ]
0223        values.extend(len(all_hlt_path_names_seen) * [ "U" ])
0224
0225        csv_writer.writerow(values)
0226
0227        previous_hlt_key = hlt_key
0228        continue
0229
0230    # everything seems ok for this key
0231
0232    csv_writer.writerow([hlt_key])
0233
0234    # now put together the list of prescales
0235    values = [ run , '' ]
0236
0237    # find out which HLT keys were present and which prescale factors
0238    # they had
0239    for hlt_path in all_hlt_path_names_seen:
0240
0241        if hlt_path in hlt_path_names_table[hlt_key]:
0242            # this HLT path was present in this menu
0243            # check whether there was an entry in the prescale
0244            # table
0245
0246            # get the prescale factors (list) or just a list with 1
0247            # if this path was not present in the prescale table
0248            # for this menu
0249            prescales = prescaleTable[hlt_key].get(hlt_path, [ 1 ] )
0250
0251            # merge the values to one comma separated string
0252            # print "prescales=",prescales
0253            values.append(",".join([str(x) for x in prescales]))
0254
0255        else:
0256            # path not found for this hlt key. append
0257            # an empty string for this column
0258            values.append('')
0259
0260    # end loop over hlt paths
0261
0262    csv_writer.writerow(values)
0263
0264    previous_hlt_key = hlt_key
0265
0266 # end loop over runs
0267
0268 fout.close()
0269
0270 print("created CSV file",csv_output_file,". Field delimiter is semicolon.", file=stderr)
0271