Tools/python/makeHLTPrescaleTable.py

0001 #!/usr/bin/env python3
0002 from __future__ import print_function
0003 from __future__ import absolute_import
0004 from sys import stderr, exit
0005 import subprocess, os
0006
0007 from optparse import OptionParser
0008 parser = OptionParser(usage=
0009 """
0010 usage: %prog [options] csv_output_file
0011
0012 examples:
0013
0014  %prog out.csv
0015
0016      produces a table of ALL runs and ALL paths (can take quite some time)
0017
0018  %prog --path='*ele*' --path='*photon*' out.csv
0019
0020      select only paths containing 'ele' and 'photon'
0021
0022 """)
0023 parser.add_option("--firstRun",  dest="firstRun",  help="first run", type="int", metavar="RUN", default="1")
0024 parser.add_option("--lastRun",   dest="lastRun",   help="last run",  type="int", metavar="RUN", default="9999999")
0025 parser.add_option("--groupName", dest="groupName", help="select runs of name like NAME", metavar="NAME", default="Collisions%")
0026 parser.add_option("--overwrite", dest="overwrite", help="force overwriting of output CSV file", action="store_true", default=False)
0027
0028 parser.add_option("--path",
0029                  dest="pathPatterns",
0030                  default = [],
0031                  action="append",
0032                  metavar="PATTERN",
0033                  help="restrict paths to PATTERN. Note that this can be a single path name or a pattern " +
0034                       "using wildcards (*,?) similar to those used for selecting multiple files (see " +
0035                       "the documentation of the fnmatch module for details). Note also that this option " +
0036                       "can be specified more than once to select multiple paths or patterns. If this option " +
0037                       "is not specified, all paths are considered. Note that the comparison is done " +
0038                       "in a case-INsensitive manner. " +
0039                       "You may have to escape wildcards (with quotes or backslash) in order to avoid "+
0040                       "expansion by the shell"
0041                  )
0042
0043 # parser.add_option("--jsonOut",   dest="jsonOut",   help="dump prescales in JSON format on FILE", metavar="FILE")
0044 (options, args) = parser.parse_args()
0045 if len(args) != 1:
0046    parser.print_help()
0047    exit(2)
0048
0049 csv_output_file = args[0]
0050
0051 if os.path.exists(csv_output_file) and not options.overwrite:
0052    print("cowardly refusing to overwrite existing output file '" + csv_output_file + "'. Run this script without argument to see options for overriding this check.", file=stderr)
0053    exit(1)
0054
0055 #----------------------------------------------------------------------
0056 def getPrescaleTableFromProcessObject(process):
0057    """ returns a dict of hlt key to vector of prescales
0058    mapping """
0059
0060    retval = {}
0061    for entry in process.PrescaleService.prescaleTable:
0062        retval[entry.pathName.value()] = entry.prescales.value()
0063
0064    return retval
0065
0066 #----------------------------------------------------------------------
0067
0068 def getProcessObjectFromConfDB(hlt_key):
0069    # print >> stderr,"\t%s ..." % hlt_key
0070    cmd = "edmConfigFromDB --orcoff --configName " + hlt_key
0071    # print >> stderr, "cmd=",cmd
0072    res = subprocess.getoutput(cmd)
0073
0074    # potentially dangerous: we're running python code here
0075    # which we get from an external process (confDB).
0076    # we trust that there are no file deletion commands
0077    # in the HLT configurations...
0078
0079    # for some keys, edmConfigFromDB seems to produce error messages.
0080    # just return None in this case
0081    try:
0082        exec(res)
0083    except:
0084        return None
0085
0086    return process
0087
0088 #----------------------------------------------------------------------
0089 from .queryRR import queryRR
0090
0091 #----------------------------------------------------------------------
0092 # main
0093 #----------------------------------------------------------------------
0094
0095 # check whether we have a CMSSW environment initalized
0096 if os.system("which edmConfigFromDB") != 0:
0097    print("could not find the command edmConfigFromDB. Did you initialize your CMSSW runtime environment ?", file=stderr)
0098    exit(1)
0099
0100 runKeys = queryRR(options.firstRun,options.lastRun,options.groupName)
0101
0102 # maps from HLT key to prescale information.
0103 # indexed as: prescaleTable[hlt_key][hlt_path_name]
0104 prescaleTable = {}
0105
0106 # maps from
0107 hlt_path_names_table = {}
0108
0109 # set of all HLT paths seen so far
0110 all_hlt_path_names_seen = set()
0111
0112 runs = sorted(runKeys.keys())
0113
0114 # loop over all hlt keys found
0115
0116 all_hlt_keys_seen = set(runKeys.values())
0117
0118 print("found %d runs and %d HLT menus" % ( len(runKeys), len(all_hlt_keys_seen)), file=stderr)
0119
0120 index = 1
0121
0122 for hlt_key in all_hlt_keys_seen:
0123
0124    print("(%3d/%3d) Querying ConfDB for HLT menu %s" % (index, len(all_hlt_keys_seen) , hlt_key), file=stderr)
0125    process = getProcessObjectFromConfDB(hlt_key)
0126
0127    if process == None:
0128        print("WARNING: unable to retrieve hlt_key '" + hlt_key + "'", file=stderr)
0129        continue
0130
0131    prescaleTable[hlt_key] = getPrescaleTableFromProcessObject(process)
0132
0133    all_path_names = set(process.paths.keys())
0134
0135    # remember which hlt paths were in this menu
0136    hlt_path_names_table[hlt_key] = all_path_names
0137
0138    # add this configuration's HLT paths to the list
0139    # of overall path names seen
0140    all_hlt_path_names_seen.update(all_path_names)
0141
0142    index += 1
0143
0144 # end of loop over all HLT keys
0145
0146 # make sure the list of all HLT path names ever seen is sorted
0147 all_hlt_path_names_seen = sorted(all_hlt_path_names_seen)
0148
0149 #--------------------
0150 # filter paths if required by the user
0151 if len(options.pathPatterns) > 0:
0152    import fnmatch
0153
0154    tmp = []
0155
0156    for path in all_hlt_path_names_seen:
0157
0158        for pattern in options.pathPatterns:
0159            if fnmatch.fnmatch(path.lower(), pattern.lower()):
0160
0161                # accept this path
0162                tmp.append(path)
0163                break
0164
0165    all_hlt_path_names_seen = tmp
0166
0167 # sanity check
0168
0169 if len(all_hlt_path_names_seen) == 0:
0170    print("no HLT paths found, exiting", file=stderr)
0171    exit(1)
0172
0173 #--------------------
0174 # now that we know all path names of all runs, produce the CSV
0175 import csv
0176
0177 previous_hlt_key = None
0178
0179 fout = open(csv_output_file,"w")
0180
0181 csv_writer = csv.writer(fout,delimiter=";")
0182
0183 csv_writer.writerow(['Table of HLT prescale factors'])
0184 csv_writer.writerow([])
0185 csv_writer.writerow(['Explanation:'])
0186 csv_writer.writerow(['number(s) = prescale factor(s), HLT path present in this menu'])
0187 csv_writer.writerow(['empty = HLT path NOT present in this menu'])
0188 csv_writer.writerow(['0 = HLT path present in this menu but prescale factor is zero'])
0189 csv_writer.writerow(['U = could not retrieve menu for this HLT key from confDB'])
0190 csv_writer.writerow([])
0191 csv_writer.writerow([])
0192
0193 # write the header line
0194 column_names = [ 'run','' ]
0195 column_names.extend(all_hlt_path_names_seen)
0196 csv_writer.writerow(column_names)
0197
0198 csv_writer.writerow([])
0199
0200 for run in runs:
0201    hlt_key = runKeys[run]
0202
0203    if hlt_key == previous_hlt_key:
0204        # the hlt key is the same as for the previous run
0205        # just reuse the existing contents of the variable 'values'
0206        # (apart from the run number)
0207        values[0] = run
0208        csv_writer.writerow(values)
0209        continue
0210
0211    # HLT key has changed
0212
0213    # insert a line with the menu's name
0214    #
0215    # check whether we actually could determine the menu
0216    if hlt_key not in hlt_path_names_table:
0217        # could previously not retrieve the python
0218        # configuration for this key
0219        #
0220        # put some warnings in the output table
0221
0222        csv_writer.writerow([hlt_key, "COULD NOT RETRIEVE MENU FROM CONFDB"])
0223
0224        values = [ run , '' ]
0225        values.extend(len(all_hlt_path_names_seen) * [ "U" ])
0226
0227        csv_writer.writerow(values)
0228
0229        previous_hlt_key = hlt_key
0230        continue
0231
0232    # everything seems ok for this key
0233
0234    csv_writer.writerow([hlt_key])
0235
0236    # now put together the list of prescales
0237    values = [ run , '' ]
0238
0239    # find out which HLT keys were present and which prescale factors
0240    # they had
0241    for hlt_path in all_hlt_path_names_seen:
0242
0243        if hlt_path in hlt_path_names_table[hlt_key]:
0244            # this HLT path was present in this menu
0245            # check whether there was an entry in the prescale
0246            # table
0247
0248            # get the prescale factors (list) or just a list with 1
0249            # if this path was not present in the prescale table
0250            # for this menu
0251            prescales = prescaleTable[hlt_key].get(hlt_path, [ 1 ] )
0252
0253            # merge the values to one comma separated string
0254            # print "prescales=",prescales
0255            values.append(",".join([str(x) for x in prescales]))
0256
0257        else:
0258            # path not found for this hlt key. append
0259            # an empty string for this column
0260            values.append('')
0261
0262    # end loop over hlt paths
0263
0264    csv_writer.writerow(values)
0265
0266    previous_hlt_key = hlt_key
0267
0268 # end loop over runs
0269
0270 fout.close()
0271
0272 print("created CSV file",csv_output_file,". Field delimiter is semicolon.", file=stderr)
0273