Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-11-26 02:34:21

0001 #!/usr/bin/env python3 
0002 from sys import stderr, exit
0003 import subprocess, os
0004 
0005 from optparse import OptionParser
0006 parser = OptionParser(usage=
0007 """
0008 usage: %prog [options] csv_output_file
0009 
0010 examples:
0011 
0012  %prog out.csv
0013 
0014      produces a table of ALL runs and ALL paths (can take quite some time)
0015 
0016  %prog --path='*ele*' --path='*photon*' out.csv
0017 
0018      select only paths containing 'ele' and 'photon'
0019 
0020 """)
0021 parser.add_option("--firstRun",  dest="firstRun",  help="first run", type="int", metavar="RUN", default="1")
0022 parser.add_option("--lastRun",   dest="lastRun",   help="last run",  type="int", metavar="RUN", default="9999999")
0023 parser.add_option("--groupName", dest="groupName", help="select runs of name like NAME", metavar="NAME", default="Collisions%")
0024 parser.add_option("--overwrite", dest="overwrite", help="force overwriting of output CSV file", action="store_true", default=False)
0025 
0026 parser.add_option("--path",
0027                  dest="pathPatterns",
0028                  default = [],
0029                  action="append",
0030                  metavar="PATTERN",
0031                  help="restrict paths to PATTERN. Note that this can be a single path name or a pattern " +
0032                       "using wildcards (*,?) similar to those used for selecting multiple files (see " +
0033                       "the documentation of the fnmatch module for details). Note also that this option " +
0034                       "can be specified more than once to select multiple paths or patterns. If this option " +
0035                       "is not specified, all paths are considered. Note that the comparison is done " +
0036                       "in a case-INsensitive manner. " +
0037                       "You may have to escape wildcards (with quotes or backslash) in order to avoid "+
0038                       "expansion by the shell"
0039                  )
0040 
0041 # parser.add_option("--jsonOut",   dest="jsonOut",   help="dump prescales in JSON format on FILE", metavar="FILE")
0042 (options, args) = parser.parse_args()
0043 if len(args) != 1:
0044    parser.print_help()
0045    exit(2)
0046 
0047 csv_output_file = args[0]
0048 
0049 if os.path.exists(csv_output_file) and not options.overwrite:
0050    print("cowardly refusing to overwrite existing output file '" + csv_output_file + "'. Run this script without argument to see options for overriding this check.", file=stderr)
0051    exit(1)
0052 
0053 #----------------------------------------------------------------------
0054 def getPrescaleTableFromProcessObject(process):
0055    """ returns a dict of hlt key to vector of prescales
0056    mapping """
0057 
0058    retval = {}
0059    for entry in process.PrescaleService.prescaleTable:
0060        retval[entry.pathName.value()] = entry.prescales.value()
0061 
0062    return retval    
0063 
0064 #----------------------------------------------------------------------
0065 
0066 def getProcessObjectFromConfDB(hlt_key):
0067    # print >> stderr,"\t%s ..." % hlt_key
0068    cmd = "edmConfigFromDB --orcoff --configName " + hlt_key
0069    # print >> stderr, "cmd=",cmd
0070    res = subprocess.getoutput(cmd)
0071 
0072    # potentially dangerous: we're running python code here
0073    # which we get from an external process (confDB).
0074    # we trust that there are no file deletion commands
0075    # in the HLT configurations...
0076 
0077    # for some keys, edmConfigFromDB seems to produce error messages.
0078    # just return None in this case
0079    try:
0080        exec(res)
0081    except:
0082        return None
0083 
0084    return process
0085 
0086 #----------------------------------------------------------------------
0087 from .queryRR import queryRR
0088 
0089 #----------------------------------------------------------------------
0090 # main
0091 #----------------------------------------------------------------------
0092 
0093 # check whether we have a CMSSW environment initalized
0094 if os.system("which edmConfigFromDB") != 0:
0095    print("could not find the command edmConfigFromDB. Did you initialize your CMSSW runtime environment ?", file=stderr)
0096    exit(1)
0097 
0098 runKeys = queryRR(options.firstRun,options.lastRun,options.groupName)
0099 
0100 # maps from HLT key to prescale information. 
0101 # indexed as: prescaleTable[hlt_key][hlt_path_name]
0102 prescaleTable = {}
0103 
0104 # maps from 
0105 hlt_path_names_table = {}
0106 
0107 # set of all HLT paths seen so far
0108 all_hlt_path_names_seen = set()
0109 
0110 runs = sorted(runKeys.keys())
0111 
0112 # loop over all hlt keys found
0113 
0114 all_hlt_keys_seen = set(runKeys.values())
0115 
0116 print("found %d runs and %d HLT menus" % ( len(runKeys), len(all_hlt_keys_seen)), file=stderr)
0117 
0118 index = 1
0119 
0120 for hlt_key in all_hlt_keys_seen:
0121 
0122    print("(%3d/%3d) Querying ConfDB for HLT menu %s" % (index, len(all_hlt_keys_seen) , hlt_key), file=stderr)
0123    process = getProcessObjectFromConfDB(hlt_key)
0124 
0125    if process == None:
0126        print("WARNING: unable to retrieve hlt_key '" + hlt_key + "'", file=stderr)
0127        continue
0128 
0129    prescaleTable[hlt_key] = getPrescaleTableFromProcessObject(process)
0130 
0131    all_path_names = set(process.paths.keys())
0132 
0133    # remember which hlt paths were in this menu
0134    hlt_path_names_table[hlt_key] = all_path_names
0135 
0136    # add this configuration's HLT paths to the list
0137    # of overall path names seen
0138    all_hlt_path_names_seen.update(all_path_names)
0139 
0140    index += 1
0141 
0142 # end of loop over all HLT keys
0143 
0144 # make sure the list of all HLT path names ever seen is sorted
0145 all_hlt_path_names_seen = sorted(all_hlt_path_names_seen)
0146 
0147 #--------------------
0148 # filter paths if required by the user
0149 if len(options.pathPatterns) > 0:
0150    import fnmatch
0151 
0152    tmp = []
0153 
0154    for path in all_hlt_path_names_seen:
0155 
0156        for pattern in options.pathPatterns:
0157            if fnmatch.fnmatch(path.lower(), pattern.lower()):
0158 
0159                # accept this path
0160                tmp.append(path)
0161                break
0162 
0163    all_hlt_path_names_seen = tmp
0164 
0165 # sanity check
0166 
0167 if len(all_hlt_path_names_seen) == 0:
0168    print("no HLT paths found, exiting", file=stderr)
0169    exit(1)
0170 
0171 #--------------------
0172 # now that we know all path names of all runs, produce the CSV
0173 import csv
0174 
0175 previous_hlt_key = None
0176 
0177 fout = open(csv_output_file,"w")
0178 
0179 csv_writer = csv.writer(fout,delimiter=";")
0180 
0181 csv_writer.writerow(['Table of HLT prescale factors'])
0182 csv_writer.writerow([])
0183 csv_writer.writerow(['Explanation:'])
0184 csv_writer.writerow(['number(s) = prescale factor(s), HLT path present in this menu'])
0185 csv_writer.writerow(['empty = HLT path NOT present in this menu'])
0186 csv_writer.writerow(['0 = HLT path present in this menu but prescale factor is zero'])
0187 csv_writer.writerow(['U = could not retrieve menu for this HLT key from confDB'])
0188 csv_writer.writerow([])
0189 csv_writer.writerow([])
0190 
0191 # write the header line
0192 column_names = [ 'run','' ]
0193 column_names.extend(all_hlt_path_names_seen)
0194 csv_writer.writerow(column_names)
0195 
0196 csv_writer.writerow([])
0197 
0198 for run in runs:
0199    hlt_key = runKeys[run]
0200 
0201    if hlt_key == previous_hlt_key:
0202        # the hlt key is the same as for the previous run
0203        # just reuse the existing contents of the variable 'values'
0204        # (apart from the run number)
0205        values[0] = run
0206        csv_writer.writerow(values)
0207        continue
0208 
0209    # HLT key has changed
0210 
0211    # insert a line with the menu's name
0212    #
0213    # check whether we actually could determine the menu
0214    if hlt_key not in hlt_path_names_table:
0215        # could previously not retrieve the python
0216        # configuration for this key 
0217        #
0218        # put some warnings in the output table
0219 
0220        csv_writer.writerow([hlt_key, "COULD NOT RETRIEVE MENU FROM CONFDB"])
0221 
0222        values = [ run , '' ]
0223        values.extend(len(all_hlt_path_names_seen) * [ "U" ])
0224 
0225        csv_writer.writerow(values)
0226 
0227        previous_hlt_key = hlt_key
0228        continue
0229 
0230    # everything seems ok for this key
0231 
0232    csv_writer.writerow([hlt_key])
0233 
0234    # now put together the list of prescales
0235    values = [ run , '' ]
0236 
0237    # find out which HLT keys were present and which prescale factors
0238    # they had
0239    for hlt_path in all_hlt_path_names_seen:
0240 
0241        if hlt_path in hlt_path_names_table[hlt_key]:
0242            # this HLT path was present in this menu
0243            # check whether there was an entry in the prescale
0244            # table
0245 
0246            # get the prescale factors (list) or just a list with 1
0247            # if this path was not present in the prescale table
0248            # for this menu
0249            prescales = prescaleTable[hlt_key].get(hlt_path, [ 1 ] )
0250 
0251            # merge the values to one comma separated string 
0252            # print "prescales=",prescales
0253            values.append(",".join([str(x) for x in prescales]))
0254 
0255        else:
0256            # path not found for this hlt key. append
0257            # an empty string for this column
0258            values.append('')
0259 
0260    # end loop over hlt paths        
0261 
0262    csv_writer.writerow(values)
0263 
0264    previous_hlt_key = hlt_key
0265 
0266 # end loop over runs
0267 
0268 fout.close()
0269 
0270 print("created CSV file",csv_output_file,". Field delimiter is semicolon.", file=stderr)
0271