Configuration/tables/subtables.sh

0001 #!/bin/bash
0002 #
0003 # utility functions used to generate HLT tables from master table in ConfDB
0004 #
0005
0006 # db-proxy configuration
0007 DBPROXY=""
0008 DBPROXYHOST="localhost"
0009 DBPROXYPORT="8080"
0010 while [[ $# -gt 0 ]]; do
0011   case "$1" in
0012     --dbproxy) DBPROXY="--dbproxy"; shift;;
0013     --dbproxyhost) DBPROXYHOST="$2"; shift; shift;;
0014     --dbproxyport) DBPROXYPORT="$2"; shift; shift;;
0015     *) shift;;
0016   esac
0017 done
0018
0019 # load common HLT functions
0020 if [ -f "$CMSSW_BASE/src/HLTrigger/Configuration/common/utils.sh" ]; then
0021   source "$CMSSW_BASE/src/HLTrigger/Configuration/common/utils.sh"
0022 elif [ -f "$CMSSW_RELEASE_BASE/src/HLTrigger/Configuration/common/utils.sh" ]; then
0023   source "$CMSSW_RELEASE_BASE/src/HLTrigger/Configuration/common/utils.sh"
0024 else
0025   exit 1
0026 fi
0027
0028 CONFDB_TAG="HEAD"
0029
0030 # if set, remove the ConfDB working directory
0031 private=false
0032
0033 function cleanup() {
0034   local TABLES="$@"
0035
0036   # clean up
0037   for TABLE in $TABLES; do
0038     rm -f "${PREFIX}${TABLE}_expanded.txt"
0039   done
0040
0041   if $private; then
0042     rm -rf $workDir
0043   fi
0044 }
0045
0046 function getPathList() {
0047   [ "x${DBPROXY}" = "x" ] || local DBPROXYOPTS="${DBPROXY} --dbproxyhost ${DBPROXYHOST} --dbproxyport ${DBPROXYPORT}"
0048   local DATA=$(hltConfigFromDB --${Vx} --${DB} --cff --configName ${MASTER} \
0049     --noedsources --noes --noservices --nosequences --nomodules ${DBPROXYOPTS})
0050   if echo "${DATA}" | grep -q 'Exhausted Resultset\|CONFIG_NOT_FOUND'; then
0051     echo "Error: $MASTER is not a valid HLT menu"
0052     exit 1
0053   fi
0054   echo "${DATA}" | sed -ne's/ *= *cms.\(Final\|End\)\?Path.*//p'
0055 }
0056
0057 function checkJars() {
0058   local BASE="$1"; shift
0059   local JARS="$@"
0060   for F in "$BASE/$JARS"; do
0061     [ -f "$F" ] || return 1
0062   done
0063   return 0
0064 }
0065
0066 function makeCreateConfig() {
0067   local baseDir="/afs/cern.ch/user/c/confdb/www/${Vx}/lib"
0068   local baseUrl="http://confdb.web.cern.ch/confdb/${Vx}/lib"
0069   local JARS="ojdbc8.jar cmssw-evf-confdb-gui.jar"
0070   workDir="$baseDir"
0071
0072   # try to read the .jar files from AFS, or download them
0073   if checkJars "$baseDir" $JARS; then
0074     # read the .jar fles from AFS
0075     workDir="$baseDir"
0076   else
0077     # try to use $CMSSW_BASE/tmp
0078     mkdir -p "$CMSSW_BASE/tmp/confdb"
0079     if [ -d "$CMSSW_BASE/tmp/confdb" ]; then
0080       workDir="$CMSSW_BASE/tmp/confdb"
0081     else
0082       workDir=$(mktemp -d confdb.XXXXXXXXXX)
0083       private=true
0084     fi
0085     # download the .jar files
0086     for JAR in $JARS; do
0087       # check if the file is already present
0088       if [ -f $workDir/$JAR ]; then
0089         continue
0090       fi
0091       # download to a temporary file and use an atomic move (in case another instance is downloading the same file)
0092       local TMPJAR=$(mktemp -p "$workDir" .${JAR}.XXXXXXXXXX)
0093       curl -s -L "$baseUrl/$JAR" -o "$TMPJAR"
0094       mv -n "$TMPJAR" "$workDir/$JAR"
0095       rm -f "$TMPJAR"
0096     done
0097   fi
0098
0099   CLASSPATH=
0100   for JAR in $JARS; do
0101     CLASSPATH="$CLASSPATH${CLASSPATH:+:}$workDir/$JAR"
0102   done
0103 }
0104
0105 function loadConfiguration() {
0106   case "$1" in
0107     # v1 offline aka "hltdev"
0108     "v1/offline" | "v1/hltdev")
0109       DBHOST="cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch"
0110       [ "x${DBPROXY}" = "x" ] || DBHOST="10.116.96.89,10.116.96.139,10.116.96.105"
0111       DBNAME="cms_cond.cern.ch"
0112       DBUSER="cms_hltdev_writer"
0113       PWHASH="0196d34dd35b04c0f3597dc89fbbe6e2"
0114       ;;
0115     # v2 offline
0116     "v2/offline")
0117       DBHOST="cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch"
0118       [ "x${DBPROXY}" = "x" ] || DBHOST="10.116.96.89,10.116.96.139,10.116.96.105"
0119       DBNAME="cms_cond.cern.ch"
0120       DBUSER="cms_hlt_gdr_w"
0121       PWHASH="0196d34dd35b04c0f3597dc89fbbe6e2"
0122       ;;
0123     # converter=v3*, db=run3
0124     "v3/run3" | "v3-beta/run3" | "v3-test/run3")
0125       DBHOST="cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch"
0126       [ "x${DBPROXY}" = "x" ] || DBHOST="10.116.96.89,10.116.96.139,10.116.96.105"
0127       DBNAME="cms_hlt.cern.ch"
0128       DBUSER="cms_hlt_v3_w"
0129       PWHASH="0196d34dd35b04c0f3597dc89fbbe6e2"
0130       ;;
0131     # converter=v3*, db=dev
0132     "v3/dev" | "v3-beta/dev" | "v3-test/dev")
0133       DBHOST="cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch"
0134       [ "x${DBPROXY}" = "x" ] || DBHOST="10.116.96.89,10.116.96.139,10.116.96.105"
0135       DBNAME="cms_hlt.cern.ch"
0136       DBUSER="cms_hlt_gdrdev_w"
0137       PWHASH="0196d34dd35b04c0f3597dc89fbbe6e2"
0138       ;;
0139     *)
0140       # see https://github.com/fwyzard/hlt-confdb/blob/confdbv2/test/runCreateConfig
0141       echo "Error, unknown database \"$1\", exiting."
0142       exit 1
0143       ;;
0144   esac
0145 }
0146
0147 function runCreateConfig() {
0148   [ "x${DBPROXY}" = "x" ] || local DBPROXYOPTS="-DsocksProxyHost=${DBPROXYHOST} -DsocksProxyPort=${DBPROXYPORT}"
0149   loadConfiguration "$1"
0150   java \
0151     -Djava.security.egd=file:///dev/urandom \
0152     -Doracle.jdbc.timezoneAsRegion=false \
0153     ${DBPROXYOPTS} \
0154     -Xss32M \
0155     -Xmx1024m \
0156     -classpath "${CLASSPATH}" \
0157     confdb.db.ConfDBCreateConfig \
0158     --dbHost "${DBHOST}" \
0159     --dbName "${DBNAME}" \
0160     --dbUser "${DBUSER}" \
0161     --dbPwrd $2 \
0162     --master $3 \
0163     --paths $4 \
0164     --name $5
0165 }
0166
0167 # expands the patterns in "TABLE.txt" into "TABLE_expanded.txt"
0168 function expandSubtable() {
0169   local TABLE="$1"
0170   local LIST="$2"
0171   local FAIL=0
0172
0173   echo "Parsing table: $TABLE ..."
0174   rm -f ${TABLE}_expanded.txt
0175   cat "$TABLE.txt" | while read LINE; do
0176     PATTERN=$(echo $LINE | sed -e's/ *#.*//' -e's/^/\\</' -e's/$/\\>/' -e's/?/./g' -e's/\*/.*/g')
0177     [ "$PATTERN" == "\<\>" ] && continue
0178     echo "$LIST" | grep "$PATTERN" >> "${TABLE}_expanded.txt"
0179     if (( $? != 0 )); then
0180       echo "Error: pattern \"$LINE\" does not match any paths" 1>&2
0181       FAIL=1
0182     fi
0183   done
0184
0185   return $FAIL
0186 }
0187
0188
0189 function readPassword() {
0190   # ask for the ConfDB password, and validate its hash
0191   loadConfiguration "$DATABASE"
0192   PASSWORD=""
0193   read -p "Enter password for DB: " -s PASSWORD
0194   echo
0195
0196   if [ "$(echo "$PASSWORD" | tr 'a-z' 'A-Z' | md5sum | cut -c1-32)" != "$PWHASH" ]; then
0197     echo "Incorrect password, exiting." 1>&2
0198     exit 1
0199   fi
0200 }
0201
0202
0203 function createSubtables() {
0204   local DATABASE="$1"; shift
0205   local MASTER="$1";   shift
0206   local TARGET="$1";   shift
0207   local PREFIX="$1";   shift
0208   local TABLES="$@"
0209
0210   # extract the schema version from the database name
0211   local Vx DB
0212   read Vx DB <<< $(parse_HLT_schema "$DATABASE")
0213   local DATABASE="${Vx}/${DB}"
0214
0215   # dump the requested configuration
0216   echo "ConfDB master: $DATABASE:$MASTER"
0217   echo "Subtables:     $TABLES"
0218   echo "Created under: $DATABASE:$TARGET"
0219
0220   # install a clean up hook
0221   trap "cleanup $TABLES; exit 1" INT TERM EXIT
0222
0223   # expand the wildcards in the path names in each subtables
0224   local LIST=$(getPathList $MASTER)
0225   local FAIL=0
0226   for TABLE in $TABLES; do
0227     expandSubtable "${PREFIX}${TABLE}" "$LIST" || FAIL=1
0228   done
0229   if (( $FAIL )); then
0230     echo "Error: one or more patterns do not match any paths, exiting." 1>&2
0231     exit 1
0232   fi
0233
0234   # ask the user for the database password
0235   readPassword
0236
0237   # make sure the needed scripts are available
0238   makeCreateConfig
0239
0240   # extract each subtable
0241   for TABLE in $TABLES; do
0242     runCreateConfig "$DATABASE" "$PASSWORD" "$MASTER" "${PREFIX}${TABLE}_expanded.txt" $(echo "$TARGET" | sed -e"s|TABLE|$TABLE|")
0243   done
0244
0245   # remove clean up hook, and call explicit cleanup
0246   trap - INT TERM EXIT
0247   cleanup ${TABLES}
0248 }