File indexing completed on 2023-03-17 11:07:38
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016 .PHONY: all list check clean summary help
0017
0018 .SECONDEXPANSION:
0019
0020
0021 PROCESS := HLT
0022 EVENTS := 100
0023
0024
0025 MENUS := 8E29 1E31 GRun
0026
0027 HLT_GRun_CONFIG := /dev/CMSSW_3_3_0/backport/GRun
0028 HLT_GRun_GLOBALTAG := STARTUP31X_V8
0029 HLT_GRun_SOURCE := file:RelVal_DigiL1Raw_8E29.root
0030
0031 HLT_8E29_CONFIG := /dev/CMSSW_3_3_0/backport/8E29
0032 HLT_8E29_GLOBALTAG := STARTUP31X_V8
0033 HLT_8E29_SOURCE := file:RelVal_DigiL1Raw_8E29.root
0034
0035 HLT_1E31_CONFIG := /dev/CMSSW_3_3_0/backport/1E31
0036 HLT_1E31_GLOBALTAG := MC_31X_V9
0037 HLT_1E31_SOURCE := file:RelVal_DigiL1Raw_1E31.root
0038
0039
0040 CMSRUN := cmsRun
0041 GETCONFIG := hltConfigFromDB
0042
0043
0044 ifeq (,$(CMSSW_RELEASE_BASE))
0045 $(error Please configure the cmsRun environment with the 'cmsenv' command)
0046 endif
0047
0048
0049 NORMAL := \033[0m
0050 BOLD := \033[1m
0051 RED := \033[31m
0052 GREEN := \033[32m
0053 YELLOW := \033[33m
0054 BLUE := \033[34m
0055 COLUMN := \033[100G
0056 CLEAR := \033[0K
0057 DONE := $(CLEAR)$(COLUMN)$(GREEN)done$(NORMAL)
0058 WARNING := $(CLEAR)$(COLUMN)$(YELLOW)warning$(NORMAL)
0059 ERROR := $(CLEAR)$(COLUMN)$(RED)error$(NORMAL)
0060
0061
0062 LUMI = $(strip $(word 1, $(subst _, , $@)) )
0063 NAME = $(strip $(subst $(LUMI)_, , $(word 1, $(subst ., , $@))) )
0064 TYPE = $(strip $(word 2, $(subst ., , $@)) )
0065
0066 LIST_OF_GRun_PATHS := $(shell hltConfigFromDB --configName $(HLT_GRun_CONFIG) --nopsets --noedsources --noes --noservices --nooutput --nosequences --nomodules --format python | gawk '/^process\..*(AlCa|HLT)_.* = cms.Path/ { print gensub(/^process\.(.*(AlCa|HLT)_.*) = cms.Path.*/, "\\1", 1) }' | sort)
0067 LIST_OF_GRun_PYS := $(patsubst %, GRun_%.py, $(LIST_OF_GRun_PATHS))
0068 LIST_OF_GRun_LOGS := $(patsubst %, GRun_%.log, $(LIST_OF_GRun_PATHS))
0069 LIST_OF_GRun_DIFFS := $(patsubst %, GRun_%.diff, $(LIST_OF_GRun_PATHS))
0070
0071 LIST_OF_8E29_PATHS := $(shell hltConfigFromDB --configName $(HLT_8E29_CONFIG) --nopsets --noedsources --noes --noservices --nooutput --nosequences --nomodules --format python | gawk '/^process\..*(AlCa|HLT)_.* = cms.Path/ { print gensub(/^process\.(.*(AlCa|HLT)_.*) = cms.Path.*/, "\\1", 1) }' | sort)
0072 LIST_OF_8E29_PYS := $(patsubst %, 8E29_%.py, $(LIST_OF_8E29_PATHS))
0073 LIST_OF_8E29_LOGS := $(patsubst %, 8E29_%.log, $(LIST_OF_8E29_PATHS))
0074 LIST_OF_8E29_DIFFS := $(patsubst %, 8E29_%.diff, $(LIST_OF_8E29_PATHS))
0075
0076 LIST_OF_1E31_PATHS := $(shell hltConfigFromDB --configName $(HLT_1E31_CONFIG) --nopsets --noedsources --noes --noservices --nooutput --nosequences --nomodules --format python | gawk '/^process\..*(AlCa|HLT)_.* = cms.Path/ { print gensub(/^process\.(.*(AlCa|HLT)_.*) = cms.Path.*/, "\\1", 1) }' | sort)
0077 LIST_OF_1E31_PYS := $(patsubst %, 1E31_%.py, $(LIST_OF_1E31_PATHS))
0078 LIST_OF_1E31_LOGS := $(patsubst %, 1E31_%.log, $(LIST_OF_1E31_PATHS))
0079 LIST_OF_1E31_DIFFS := $(patsubst %, 1E31_%.diff, $(LIST_OF_1E31_PATHS))
0080
0081 TABLE_PATHS := $(patsubst %, %_GlobalTable, $(MENUS))
0082 TABLE_PYS := $(patsubst %, %.py, $(TABLE_PATHS))
0083 TABLE_LOGS := $(patsubst %, %.log, $(TABLE_PATHS))
0084
0085 LIST_OF_PATHS := $(sort $(LIST_OF_8E29_PATHS) $(LIST_OF_1E31_PATHS) $(LIST_OF_GRun_PATHS))
0086 LIST_OF_PYS := $(LIST_OF_8E29_PYS) $(LIST_OF_1E31_PYS) $(LIST_OF_GRun_PYS)
0087 LIST_OF_LOGS := $(LIST_OF_8E29_LOGS) $(LIST_OF_1E31_LOGS) $(LIST_OF_GRun_LOGS)
0088 LIST_OF_DIFFS := $(LIST_OF_8E29_DIFFS) $(LIST_OF_1E31_DIFFS) $(LIST_OF_GRun_DIFFS)
0089
0090 .PHONY: $(LIST_OF_PATHS)
0091
0092
0093 .SECONDARY: $(LIST_OF_LOGS)
0094
0095 all: pys logs check summary
0096
0097 pys: $(TABLE_PYS) $(LIST_OF_PYS)
0098
0099 logs: $(TABLE_LOGS) $(LIST_OF_LOGS)
0100
0101 check: $(LIST_OF_DIFFS)
0102
0103 clean:
0104 @rm -f .database_* *.py *.pyc *.log *.single *.master *.diff
0105
0106 list:
0107 @echo "$(LIST_OF_PATHS)"
0108
0109 summary: | check
0110 @echo
0111 @DIFF=`find $(LIST_OF_DIFFS) -not -empty | sed -e's/\.diff$$//'`; \
0112 if [ -z "$$DIFF" ]; then \
0113 echo "No discrepancies found."; \
0114 else \
0115 echo "Found discrepancies in the trigger paths:" ;\
0116 for P in $$DIFF; do echo -e "\t$(RED)$$P$(NORMAL)$(CLEAR)"; done; \
0117 fi
0118 @DIFF=`find $(LIST_OF_DIFFS) -not -empty | sed -e's/\.diff$$//'`; \
0119 PARTIAL=$$(gawk 'FNR==1 { NAME=gensub(/^...._(.*)\.log/,"\\1",1,FILENAME); LUMI=gensub(/^(....)_.*\.log/,"\\1",1,FILENAME); HEADER="TrigReport ---------- Modules in Path: "NAME" ------------"; } $$0 ~ HEADER { while ($$0 !~ /TrigReport.*hltBoolEnd/) getline; if ($$5 == 0) printf "%s_%s\n", LUMI, NAME; nextfile; }' *_*.log); \
0120 PARTIAL=$$(for P in $$PARTIAL; do echo $$DIFF | grep -q $$P || echo $$P; done); \
0121 if [ -n "$$PARTIAL" ]; then \
0122 echo; \
0123 echo "These paths where not fully excercised:"; \
0124 for P in $$PARTIAL; do echo -e "\t$(YELLOW)$$P$(NORMAL)$(CLEAR)"; done; \
0125 echo; \
0126 fi
0127 @MISS=`grep -L 'TrigReport.*hltBoolEnd' $(LIST_OF_LOGS) | sed -e's/\.log$$//'`; \
0128 if [ -n "$$MISS" ]; then \
0129 echo; \
0130 echo "These paths miss the trailing hltBoolEnd filter:"; \
0131 for P in $$MISS; do echo -e "\t$(YELLOW)$$P$(NORMAL)$(CLEAR)"; done; \
0132 echo; \
0133 fi
0134
0135
0136 DB_GRun_NEW:=$(shell echo -e "CONFIG=$(HLT_GRun_CONFIG)\nSOURCE=$(HLT_GRun_SOURCE)\nGLOBALTAG=$(HLT_GRun_GLOBALTAG)" | md5sum | cut -c -32)
0137 DB_GRun_SUM:=$(shell [ -f .database_GRun ] && cat .database_GRun | md5sum | cut -c -32)
0138
0139 ifneq ($(DB_GRun_NEW), $(DB_GRun_SUM))
0140 .database_GRun:
0141 @echo -e "CONFIG=$(HLT_GRun_CONFIG)\nSOURCE=$(HLT_GRun_SOURCE)\nGLOBALTAG=$(HLT_GRun_GLOBALTAG)" > .database_GRun
0142 endif
0143
0144 DB_8E29_NEW:=$(shell echo -e "CONFIG=$(HLT_8E29_CONFIG)\nSOURCE=$(HLT_8E29_SOURCE)\nGLOBALTAG=$(HLT_8E29_GLOBALTAG)" | md5sum | cut -c -32)
0145 DB_8E29_SUM:=$(shell [ -f .database_8E29 ] && cat .database_8E29 | md5sum | cut -c -32)
0146
0147 ifneq ($(DB_8E29_NEW), $(DB_8E29_SUM))
0148 .database_8E29:
0149 @echo -e "CONFIG=$(HLT_8E29_CONFIG)\nSOURCE=$(HLT_8E29_SOURCE)\nGLOBALTAG=$(HLT_8E29_GLOBALTAG)" > .database_8E29
0150 endif
0151
0152 DB_1E31_NEW:=$(shell echo -e "CONFIG=$(HLT_1E31_CONFIG)\nSOURCE=$(HLT_1E31_SOURCE)\nGLOBALTAG=$(HLT_1E31_GLOBALTAG)" | md5sum | cut -c -32)
0153 DB_1E31_SUM:=$(shell [ -f .database_1E31 ] && cat .database_1E31 | md5sum | cut -c -32)
0154
0155 ifneq ($(DB_1E31_NEW), $(DB_1E31_SUM))
0156 .database_1E31:
0157 @echo -e "CONFIG=$(HLT_1E31_CONFIG)\nSOURCE=$(HLT_1E31_SOURCE)\nGLOBALTAG=$(HLT_1E31_GLOBALTAG)" > .database_1E31
0158 endif
0159
0160
0161
0162 $(TABLE_PYS): .database_$$(LUMI)
0163 @echo -e "ConfDB [$(BLUE)$(HLT_$(LUMI)_CONFIG)$(NORMAL)] menu $(BOLD)$(LUMI)_GlobalTable$(NORMAL)$(CLEAR)"
0164 @$(GETCONFIG) --configName $(HLT_$(LUMI)_CONFIG) --input $(HLT_$(LUMI)_SOURCE) --nopsets --nooutput --services -PrescaleService --esmodules -l1GtTriggerMenuXml,-L1GtTriggerMaskAlgoTrigTrivialProducer --format python --paths -OfflineOutput | sed -e's/^process = cms.Process(.*)/process = cms.Process( "$(PROCESS)" )/' -e's/^process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32( $(EVENTS) ) )/process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32( 100 ) )/' > $(LUMI)_GlobalTable.py
0165 @sed -e '/^process.streams/,/^)/d' -e'/^process.datasets/,/^)/d' -i $(LUMI)_GlobalTable.py
0166 @sed -e 's/cms.InputTag( "source" )/cms.InputTag( "rawDataCollector" )/' -i $(LUMI)_GlobalTable.py
0167 @sed -e 's/cms.string( "source" )/cms.string( "rawDataCollector" )/' -i $(LUMI)_GlobalTable.py
0168 @sed -e '/DTUnpackingModule/a\ \ \ \ inputLabel = cms.untracked.InputTag( "rawDataCollector" ),' -i $(LUMI)_GlobalTable.py
0169 @echo -e "process.GlobalTag.connect = 'frontier://FrontierProd/CMS_CONDITIONS'" >> $(LUMI)_GlobalTable.py
0170 @echo -e "process.GlobalTag.globaltag = '$(HLT_$(LUMI)_GLOBALTAG)'" >> $(LUMI)_GlobalTable.py
0171 @echo -e "process.options = cms.untracked.PSet(\n wantSummary = cms.untracked.bool( True )\n)\n" >> $(LUMI)_GlobalTable.py
0172
0173 $(LIST_OF_PYS): .database_$$(LUMI)
0174 @echo -e "ConfDB [$(BLUE)$(HLT_$(LUMI)_CONFIG)$(NORMAL)] path $(BOLD)$(NAME)$(NORMAL)$(CLEAR)"
0175 @$(GETCONFIG) --configName $(HLT_$(LUMI)_CONFIG) --input $(HLT_$(LUMI)_SOURCE) --nopsets --nooutput --services -PrescaleService --esmodules -l1GtTriggerMenuXml,-L1GtTriggerMaskAlgoTrigTrivialProducer --format python --paths HLTriggerFirstPath,$(NAME),HLTriggerFinalPath | sed -e's/^process = cms.Process(.*)/process = cms.Process( "$(PROCESS)" )/' -e's/^process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32( $(EVENTS) ) )/process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32( 100 ) )/' > $@
0176 @sed -e '/^process.streams/,/^)/d' -e'/^process.datasets/,/^)/d' -i $@
0177 @sed -e 's/cms.InputTag( "source" )/cms.InputTag( "rawDataCollector" )/' -i $@
0178 @sed -e 's/cms.string( "source" )/cms.string( "rawDataCollector" )/' -i $@
0179 @sed -e '/DTUnpackingModule/a\ \ \ \ inputLabel = cms.untracked.InputTag( "rawDataCollector" ),' -i $@
0180 @echo -e "process.GlobalTag.connect = 'frontier://FrontierProd/CMS_CONDITIONS'" >> $@
0181 @echo -e "process.GlobalTag.globaltag = '$(HLT_$(LUMI)_GLOBALTAG)'" >> $@
0182 @echo -e "process.options = cms.untracked.PSet(\n wantSummary = cms.untracked.bool( True )\n)\n" >> $@
0183
0184
0185 $(TABLE_LOGS): %.log: %.py
0186 @echo -e -n "running $(LUMI) Global Table$(CLEAR)\r"
0187 @$(CMSRUN) $< >& $@
0188 @echo -e "running $(LUMI) Global Table$(DONE)"
0189
0190 $(LIST_OF_LOGS): %.log: %.py
0191 @echo -e -n "running $(LUMI) trigger path $(NAME)$(CLEAR)\r"
0192 @$(CMSRUN) $< >& $@
0193 @echo -e "running $(LUMI) trigger path $(NAME)$(DONE)"
0194
0195
0196 $(LIST_OF_DIFFS:diff=single): %.single: %.log
0197 @cat $< | gawk 'BEGIN { FOUND=0 } /TrigReport ---------- Modules in Path: $(NAME) ------------/ { FOUND=1; print; getline; print "TrigReport Trig Visited Passed Failed Error Name"; next; } /^$$/ {FOUND=0; next; } // { if (FOUND) printf "TrigReport %3d %10d %10d %10d %10d %s\n",$$2,$$4,$$5,$$6,$$7,$$8 }' > $@
0198
0199 $(LIST_OF_DIFFS:diff=master): %.master: $$(LUMI)_GlobalTable.log
0200 @cat $(LUMI)_GlobalTable.log | gawk 'BEGIN { FOUND=0 } /TrigReport ---------- Modules in Path: $(NAME) ------------/ { FOUND=1; print; getline; print "TrigReport Trig Visited Passed Failed Error Name"; next; } /^$$/ {FOUND=0; next; } // { if (FOUND) printf "TrigReport %3d %10d %10d %10d %10d %s\n",$$2,$$4,$$5,$$6,$$7,$$8 }' > $@
0201
0202
0203 $(LIST_OF_DIFFS): %.diff: %.log $$(LUMI)_GlobalTable.log | %.single %.master
0204 @echo -e -n "checking $(LUMI) trigger path $(NAME)$(CLEAR)\r"
0205 @if diff -w -U 999 $*.master $*.single > $@; then \
0206 PARTIAL=$$(gawk ' \
0207 /TrigReport ---------- Modules in Path: $(NAME) ------------/ \
0208 { \
0209 while ($$0 !~ /TrigReport.*hltBoolEnd/) \
0210 if (getline <= 0) { \
0211 print "$*"; \
0212 exit; \
0213 } \
0214 if ($$5 == 0) \
0215 print "$*"; \
0216 exit; \
0217 } \
0218 ' $<); \
0219 if [ -z "$$PARTIAL" ]; then \
0220 echo -e "checking $(LUMI) trigger path $(NAME)$(DONE)"; \
0221 else \
0222 echo -e "checking $(LUMI) trigger path $(NAME)$(WARNING)"; \
0223 fi \
0224 else \
0225 echo -e "checking $(LUMI) trigger path $(NAME)$(ERROR)"; \
0226 fi
0227
0228
0229 help:
0230 @echo 'This makefile checks that each trigger path gives the same results if run stand-alone or in the global trigger table.'
0231 @echo 'Given a configName, it extracts from ConfDB the global table (*.py_GlobalTable), the list of path, and a configuration'
0232 @echo 'for each path (TriggerName.py).'
0233 @echo 'Then both the trigger table and each paths are run, the corresponding TrigReport is extracted from the log, and compared.'
0234 @echo
0235 @echo 'After running each path, a warning is issued if a path is not fully excersided, i.e. no events pass all the filters.'
0236 @echo 'After comparing with the global table, an error is issued if there are discrepenaices in the TrigReport (at any level).'
0237 @echo
0238 @echo 'Working files:'
0239 @echo ' .database_LUMI running conditions: ConfDB configuration, input dataset, GlobalTag'
0240 @echo ' LUMI_GlobalTable.py configuration for the global table'
0241 @echo ' LUMI_GlobalTable.log output of cmsRun LUMI_GlobalTable.py'
0242 @echo ' TriggerName.py configuration for TriggerName path'
0243 @echo ' TriggerName.log output of cmsRun TriggerName.py'
0244 @echo ' TriggerName.diff differences between TrigReport for TriggerName between global table run and stand-alone run'
0245 @echo
0246 @echo 'Supported targets:'
0247 @echo ' all same as "pys logs diffs summary"'
0248 @echo ' list list all trigger paths'
0249 @echo ' pys extract all (python) configuration files'
0250 @echo ' logs run all configuration files'
0251 @echo ' check extract all differences'
0252 @echo ' summary analyze all *available* diff and log files to print the list of trigger paths with discrepancies and/or with no accepted events'
0253 @echo ' help print a simple description of this tool'
0254 @echo
0255 @echo ' TriggerName run the full chain for trigger TriggerName: extract TriggerName.py, cmsRun logging to TriggerName.log,'
0256 @echo ' if necessary extract and run the global table, and compare the TrigResults.'
0257 @echo
0258 @echo 'Version 1.4.0, 2009.08.19'
0259 @echo 'Andrea Bocci <andrea.bocci@cern.ch>'