SherpaInterface/data/PrepareSherpaLibs.sh

0001 #!/bin/bash
0002 #
0003 #  file:        PrepareSherpaLibs.sh
0004 #  description: prepare SHERPA libraries and cross sections for local use
0005 #               generate CMSSW python script
0006 #  uses:        SHERPA datacards, libs and cross sections
0007 #
0008 #  author:      Markus Merschmeyer, Sebastian Thüer
0009 #               III. Physics Institute A, RWTH Aachen University
0010 #  date:        15th August 2012
0011 #  version:     4.2
0012
0013
0014
0015 # +-----------------------------------------------------------------------------------------------+
0016 # function definitions
0017 # +-----------------------------------------------------------------------------------------------+
0018
0019 function print_help() {
0020     echo "" && \
0021     echo "PrepareSherpaLibs version 4.2" && echo && \
0022     echo "options: -i  path       path to SHERPA datacard, library & cross section files" && \
0023     echo "                         can also be in WWW (http://...) or SE (srm://...)" && \
0024     echo "                         -> ( "${datadir}" )" && \
0025     echo "         -p  process    SHERPA dataset/process name ( "${dataset}" )" && \
0026 ##    echo "         -m  mode       CMSSW running mode ( "${imode}" )" && \
0027 ##    echo "                         [ 'PROD'   : for production validation      ]" && \
0028 ##    echo "                         [ 'LOCAL'  : local running of CMSSW         ]" && \
0029 ##    echo "         -a  path       user analysis path inside CMSSW ( "${MYANADIR}" )" && \
0030     echo "         -D  filename   (optional) name of data card file     ( "${cfdc}" )" && \
0031     echo "         -L  filename   (optional) name of library file       ( "${cflb}" )" && \
0032     echo "         -C  filename   (optional) name of cross section file ( "${cfcr}" )" && \
0033     echo "         -G  filename   (optional) name of MI grid file       ( "${cfgr}" )" && \
0034     echo "         -e  filename   (optional) name of extended weight list file ( "${weightfile}" ) " && \
0035     echo "             (example file in GeneratorInterface/SherpaInterface/python/ExtendedSherpaWeights_cfi.py)" && \
0036 ##    echo "         -P  SRM path   (CRAB) SE path for final results" && \
0037 ##    echo "                         -> ( "${MYSRMPATH}" )" && \
0038     echo "         -h             display this help and exit" && echo
0039 }
0040
0041
0042
0043
0044 # function to build a python script for cmsDriver
0045 function build_python_cff() {
0046
0047 ##  imode=$1        # mode (PRODuction, LOCAL, CRAB, ...)
0048   cfffilename=$1  # config file name
0049   process=$2      # process name
0050   checksum=$3     # MD5 checksum
0051
0052   if [ -e ${cfffilename} ]; then rm ${cfffilename}; fi
0053   touch ${cfffilename}
0054
0055   echo "import FWCore.ParameterSet.Config as cms"                          >> ${cfffilename}
0056   echo "import os"                                                         >> ${cfffilename}
0057   if [ ! $weightlist == "" ]; then
0058     echo "from $weightlist import *"                                       >> ${cfffilename}
0059   fi
0060   echo ""                                                                  >> ${cfffilename}
0061   echo "source = cms.Source(\"EmptySource\")"                              >> ${cfffilename}
0062   echo ""                                                                  >> ${cfffilename}
0063   echo "generator = cms.EDFilter(\"SherpaGeneratorFilter\","               >> ${cfffilename}
0064   echo "  maxEventsToPrint = cms.int32(0),"                                >> ${cfffilename}
0065   echo "  filterEfficiency = cms.untracked.double(1.0),"                   >> ${cfffilename}
0066   echo "  crossSection = cms.untracked.double(-1),"                        >> ${cfffilename}
0067   echo "  SherpaProcess = cms.string('"${process}"'),"                     >> ${cfffilename}
0068   echo "  SherpackLocation = cms.string('./'),"                            >> ${cfffilename}
0069   echo "  SherpackChecksum = cms.string('"${checksum}"'),"                 >> ${cfffilename}
0070   echo "  FetchSherpack = cms.bool(False),"                                >> ${cfffilename}
0071 ##  if [ "${imode}" = "PROD" ]; then
0072   echo "  SherpaPath = cms.string('./'),"                                  >> ${cfffilename}
0073   echo "  SherpaPathPiece = cms.string('./'),"                             >> ${cfffilename}
0074 ##  elif [ "${imode}" = "LOCAL" ]; then
0075 ##  echo "  SherpaPath = cms.string(os.getcwd()),"                           >> ${cfffilename}
0076 ##  echo "  SherpaPathPiece = cms.string(os.getcwd()),"                      >> ${cfffilename}
0077 ##  fi
0078   echo "  SherpaResultDir = cms.string('Result'),"                         >> ${cfffilename}
0079   echo "  SherpaDefaultWeight = cms.double(1.0),"                          >> ${cfffilename}
0080   if [ ! $weightlist == "" ]; then
0081       echo "  SherpaWeightsBlock = SherpaWeightsBlock,"                    >> ${cfffilename}
0082   fi
0083   echo "  SherpaParameters = cms.PSet(parameterSets = cms.vstring("        >> ${cfffilename}
0084   fcnt=0
0085   for file in `ls *.dat`; do
0086     let fcnt=${fcnt}+1
0087   done
0088   for file in `ls *.dat`; do
0089     let fcnt=${fcnt}-1
0090     pstnam=`echo ${file} | cut -f1 -d"."`
0091     if [ ${fcnt} -gt 0 ]; then
0092   echo "                             \""${pstnam}"\","    >> ${cfffilename}
0093     else
0094   echo "                             \""${pstnam}"\"),"   >> ${cfffilename}
0095     fi
0096   done
0097   for file in `ls *.dat`; do
0098     pstnam=`echo ${file} | cut -f1 -d"."`
0099   echo "                              "${pstnam}" = cms.vstring(" >> ${cfffilename}
0100     cp ${file} ${file}.tmp1
0101     sed -e 's/[%\!].*//g' < ${file}.tmp1 > ${file}.tmp2            # remove comment lines (beginning with % or !)
0102     mv ${file}.tmp2 ${file}.tmp1
0103     sed -e 's/^[ \t]*//;s/[ \t]*$//' < ${file}.tmp1 > ${file}.tmp2 # remove beginnig & trailing whitespaces
0104     mv ${file}.tmp2 ${file}.tmp1
0105     sed '/^$/d' < ${file}.tmp1 > ${file}.tmp2                      # remove empty lines
0106     mv ${file}.tmp2 ${file}.tmp1
0107     #sed -e 's/^/ /g;s/ (/(/;s/ }/}/' < ${file}.tmp1 > ${file}.tmp2 # add single space in front of parameters
0108     sed -e 's/^/ /g;s/ }/}/' < ${file}.tmp1 > ${file}.tmp2 # add single space in front of parameters
0109     mv ${file}.tmp2 ${file}.tmp1
0110 ###
0111     sed -e 's/\"/\\"/g' < ${file}.tmp1 > ${file}.tmp2              # protect existing '"' by '\"'
0112     mv ${file}.tmp2 ${file}.tmp1
0113 ###
0114     sed -e 's/^/\t\t\t\t"/;s/$/\",/' < ${file}.tmp1 > ${file}.tmp2 # add ([]") and ("') around parameters
0115     mv ${file}.tmp2 ${file}.tmp1
0116     sed -e '$s/\",/\"/' < ${file}.tmp1 > ${file}.tmp2              # fix last line
0117     mv ${file}.tmp2 ${file}.tmp1
0118     cat  ${file}.tmp1                                         >> ${cfffilename}
0119   echo "                                                  )," >> ${cfffilename}
0120     rm ${file}.tmp*
0121   done
0122   echo "                             )"                       >> ${cfffilename}
0123   echo ")"                                                    >> ${cfffilename}
0124   echo ""                                                     >> ${cfffilename}
0125 #  echo "ProducerSourceSequence = cms.Sequence(generator)"     >> ${cfffilename}
0126   echo "ProductionFilterSequence = cms.Sequence(generator)"   >> ${cfffilename}
0127   echo ""                                                     >> ${cfffilename}
0128
0129 #  cat > sherpa_custom_cff.py << EOF
0130 #import FWCore.ParameterSet.Config as cms
0131 #
0132 #def customise(process):
0133 #
0134 #       process.genParticles.abortOnUnknownPDGCode = False
0135 #
0136 #       return(process)
0137 #EOF
0138
0139 }
0140
0141
0142 # function to copy files from different locations (WWW, SE, local)
0143 function file_copy() {
0144 # $1 : target path
0145 # $2 : file name
0146 # $3 : destination path
0147 #
0148 #  srmopt=" -debug -streams_num=1 "
0149   srmopt=" -streams_num=1 "
0150 #
0151   tpath="aaa/"
0152   fname="xxx.yyy"
0153   dpath="bbb/"
0154   if [ $# -ge 1 ]; then
0155     tpath=$1
0156     if [ $# -ge 2 ]; then
0157       fname=$2
0158       if [ $# -ge 3 ]; then
0159         dpath=$3
0160       fi
0161     fi
0162   fi
0163 #
0164   if [ ! "${tpath}" = "${dpath}" ]; then
0165     if [ "${tpath}" = "./" ]; then
0166       tpath=${PWD}
0167     fi
0168     if [ "${dpath}" = "./" ]; then
0169       dpath=${PWD}
0170     fi
0171     cd /tmp
0172 #
0173   TLOC0=`echo ${tpath} | cut -f1 -d "/" | grep -c -i http`
0174   TLOC1=`echo ${tpath} | cut -f1 -d "/" | grep -c -i srm`
0175   if [ ${TLOC0} -gt 0 ]; then    # file is in WWW
0176     echo " <I> retrieving WWW file: "${fname}
0177     echo " <I>  from path: "${tpath}
0178     wget ${tpath}/${fname}
0179   elif [ ${TLOC1} -gt 0 ]; then  # file is in SE
0180     echo " <I> retrieving SE file: "${fname}
0181     echo " <I>  from path: "${tpath}
0182     srmcp ${srmopt} ${tpath}/${fname} file:///${PWD}/${fname}
0183   else                           # local file
0184     echo " <I> copying local file: "${fname}
0185     echo " <I>  from path: "${tpath}
0186     if [ -e ${tpath}/${fname} ]; then
0187       cp ${tpath}/${fname} ./
0188     else
0189       echo " <W> file does not exist, make sure everything is OK"
0190     fi
0191   fi
0192 #
0193   DLOC0=`echo ${dpath} | cut -f1 -d "/" | grep -c -i http`
0194   DLOC1=`echo ${dpath} | cut -f1 -d "/" | grep -c -i srm`
0195   if [ ${DLOC1} -gt 0 ]; then    # file has to go to SE
0196     echo " <I> storing SE file: "${fname}
0197     echo " <I>  to path: "${dpath}
0198     srmcp ${srmopt} file:///${PWD}/${fname} ${dpath}/${fname}
0199   else                           # local file
0200     echo " <I> copying local file: "${fname}
0201     echo " <I>  to path: "${dpath}
0202     if [ -e ${fname} ]; then
0203       cp ${fname} ${dpath}/
0204     else
0205       echo " <W> file does not exist, make sure everything is OK"
0206     fi
0207   fi
0208 #
0209     rm -f ${fname}
0210     cd -
0211   fi
0212 }
0213
0214
0215 function parse_extended_weight() {
0216  if [ -e $CMSSW_BASE/src/$weightlist ]
0217  then
0218      weightlist=$(echo "$weightlist" | sed 's:/python/:.:g' | sed 's:/:.:g' | sed 's:.py::g' )
0219  else
0220      echo '$CMSSW_BASE/src/$weightlist does not exist.'
0221      exit 1
0222  fi
0223 }
0224
0225
0226
0227
0228
0229
0230 # +-----------------------------------------------------------------------------------------------+
0231 # start of the script
0232 # +-----------------------------------------------------------------------------------------------+
0233
0234
0235
0236 # save current path name
0237 HDIR=`pwd`
0238
0239 # 'guess' SHERPA dataset name
0240 ip=1
0241 cproc="XXX"
0242 for file in `ls sherpa_*_libs.tgz`; do
0243   if [ $ip -eq 1 ]; then
0244     cproc=`echo $file | awk -F"sherpa_" '{print $2}' | awk -F"_libs" '{print $1}'`
0245   fi
0246   let ip=$ip+1
0247 done
0248
0249 # dummy setup (if all options are missing)
0250 datadir=${HDIR}                                      # path to SHERPA datacards (libraries)
0251 dataset=${cproc}                                     # SHERPA dataset/process name
0252 if [ -e ${CMSSW_BASE} ]; then
0253   CMSSWDIR=${CMSSW_BASE}                             # CMSSW directory
0254 ##else
0255 ##  if [ ! "${imode}" = "PROD" ];then
0256 ##    echo " <E> \$CMSSW_BASE "${CMSSW_BASE}" does not exist"
0257 ##    echo " <E> stopping..."
0258 ##    exit 1
0259 ##  fi
0260 fi
0261 ##imode="PROD"                                         # CMSSW running mode
0262 ##MYANADIR="A/B"                                       # user analysis directory inside CMSSW
0263 #                                                    # -> CMSSW_X_Y_Z/src/${MYANADIR}/
0264 cfdc=""                                              # custom data card file name
0265 cflb=""                                              # custom library file name
0266 cfcr=""                                              # custom cross section file name
0267 cfgr=""                                              # custom MI grid file name
0268 weightfile=""                                        # sherpa weights ordering file
0269 ##MYSRMPATH="./"                                       # SRM path for storage of results
0270 TDIR=TMP
0271
0272
0273 # get & evaluate options
0274 ##while getopts :i:p:d:m:a:D:L:C:G:P:h OPT
0275 while getopts :i:p:d:D:L:C:G:e:h OPT
0276 do
0277   case $OPT in
0278   i) datadir=$OPTARG ;;
0279   p) dataset=$OPTARG ;;
0280 ##  m) imode=$OPTARG ;;
0281 ##  a) MYANADIR=$OPTARG ;;
0282   D) cfdc=$OPTARG ;;
0283   L) cflb=$OPTARG ;;
0284   C) cfcr=$OPTARG ;;
0285   G) cfgr=$OPTARG ;;
0286   e) weightlist=$OPTARG && parse_extended_weight;;
0287 ##  P) MYSRMPATH=$OPTARG ;;
0288   h) print_help && exit 0 ;;
0289   \?)
0290     shift `expr $OPTIND - 1`
0291     if [ "$1" = "--help" ]; then print_help && exit 0;
0292     else
0293       echo -n "PrepareSherpaLibs: error: unrecognized option "
0294       if [ $OPTARG != "-" ]; then echo "'-$OPTARG'. try '-h'"
0295       else echo "'$1'. try '-h'"
0296       fi
0297       print_help && exit 1
0298     fi
0299     shift 1
0300     OPTIND=1
0301   esac
0302 done
0303
0304 # make sure to use absolute path names...
0305 xpth=`echo ${datadir} | cut -f1 -d"/"`
0306 if [ "$xpth" = "" ] || [ "$xpth" = "." ] || [ "$xpth" = ".." ] || [ "$xpth" = "~" ]; then
0307   echo " <I> fixing data directory: "${datadir}
0308   cd ${datadir} && datadir=`pwd`;  cd ${HDIR}
0309   echo " <I>    to: "${datadir}
0310 fi
0311
0312 # print current options/parameters
0313 echo "  -> data card directory '"${datadir}"'"
0314 echo "  -> dataset name '"${dataset}"'"
0315 ##echo "  -> operation mode: '"${imode}"'"
0316 ##echo "  -> CMSSW user analysis path: '"${MYANADIR}"'"
0317
0318
0319 # set up
0320 ##if [ "${imode}" = "PROD" ] || [ "${imode}" = "GRID" ]; then
0321   MYCMSSWTEST=${HDIR}/${TDIR}
0322   MYCMSSWPYTH=${HDIR}/${TDIR}
0323   MYCMSSWSHPA=${HDIR}/${TDIR}
0324 ##else
0325 ##  MYCMSSWTEST=${CMSSWDIR}/src/${MYANADIR}/test
0326 ##  MYCMSSWPYTH=${CMSSWDIR}/src/${MYANADIR}/python
0327 ##  MYCMSSWSHPA=${CMSSWDIR}/src/${MYANADIR}/test
0328 ##  if [ ! -e ${MYCMSSWTEST} ]; then                            # create user analysis path
0329 ##    mkdir -p ${MYCMSSWTEST}
0330 ##  else
0331 ##    rm -f ${CMSSWDIR}/python/${MYANADIR}/*.py*                # ...clean up
0332 ##  fi
0333 ##  if [ ! -e ${MYCMSSWPYTH} ]; then                            # create 'python' subdirectory
0334 ##    mkdir -p ${MYCMSSWPYTH}
0335 ##  else
0336 ##    rm -f ${MYCMSSWPYTH}/*.py*                                # ...clean up
0337 ##  fi
0338 ##fi
0339
0340
0341 # set SHERPA data file names
0342 cardfile=sherpa_${dataset}_crdE.tgz
0343 libsfile=sherpa_${dataset}_libs.tgz
0344 crssfile=sherpa_${dataset}_crss.tgz
0345 gridfile=sherpa_${dataset}_migr.tgz
0346 if [ ! "${cfdc}" = "" ]; then cardfile=${cfdc}; fi
0347 if [ ! "${cflb}" = "" ]; then libsfile=${cflb}; fi
0348 if [ ! "${cfcr}" = "" ]; then crssfile=${cfcr}; fi
0349 if [ ! "${cfgr}" = "" ]; then gridfile=${cfgr}; fi
0350
0351
0352
0353 ##if [ ! "${imode}" = "CRAB" ]; then
0354
0355 # create dataset directory tree
0356   if [ -e ${MYCMSSWSHPA} ]; then
0357     echo " <W> dataset directory "${MYCMSSWSHPA}" exists"
0358     echo " <W> ...removing..."
0359     rm -rf ${MYCMSSWSHPA}
0360   fi
0361   mkdir -p ${MYCMSSWSHPA}
0362
0363 # get & unpack dataset files, generate .cff and .cfi files
0364   cd ${MYCMSSWSHPA}
0365   file_copy ${datadir} ${cardfile} ${PWD}
0366   file_copy ${datadir} ${libsfile} ${PWD}
0367   file_copy ${datadir} ${crssfile} ${PWD}
0368   file_copy ${datadir} ${gridfile} ${PWD}
0369   if [ -e ${cardfile} ]; then
0370     tar -xzf ${cardfile} && rm ${cardfile}
0371   else
0372     echo " <E> file not found: "${cardfile}
0373     exit 1
0374   fi
0375   if [ -e ${libsfile} ]; then
0376     tar -xzf ${libsfile} && rm ${libsfile}
0377   else
0378     echo " <E> file not found: "${libsfile}
0379     exit 1
0380   fi
0381   if [ -e ${crssfile} ]; then
0382     tar -xzf ${crssfile} && rm ${crssfile}
0383   else
0384     echo " <E> file not found: "${crssfile}
0385     exit 1
0386   fi
0387   if [ -e ${gridfile} ]; then
0388     tar -xzf ${gridfile} && rm ${gridfile}
0389   else
0390     echo " <W> no MI grid file: "${gridfile}
0391   fi
0392   cd -
0393
0394 ##fi
0395
0396
0397 # initialize variables for python file generation
0398 spsummd5=""
0399
0400 ##if [ "${imode}" = "PROD" ] || [ "${imode}" = "LOCAL" ]; then
0401   shpamstfile="sherpa_"${dataset}"_MASTER.tgz"
0402   shpamstmd5s="sherpa_"${dataset}"_MASTER.md5"
0403   shpacfffile="sherpa_"${dataset}"_MASTER_cff.py"
0404
0405   cd ${MYCMSSWSHPA}
0406
0407   tar -czf ${shpamstfile} *
0408 ####
0409   md5sum ${shpamstfile} > ${shpamstmd5s}
0410   spsummd5=`md5sum ${shpamstfile} | cut -f1 -d" "`
0411 ####
0412
0413 ##  build_python_cff ${imode} ${shpacfffile} ${dataset} ${spsummd5}
0414   build_python_cff ${shpacfffile} ${dataset} ${spsummd5}
0415
0416   mv ${shpamstfile} $HDIR
0417   mv ${shpamstmd5s} $HDIR
0418   mv ${shpacfffile} $HDIR
0419
0420   cd $HDIR
0421
0422   rm -rf $TDIR
0423
0424 ##fi
0425
0426
0427 echo " <I>  "
0428 echo " <I>  "
0429 echo " <I>  "
0430 echo " <I>  generated sherpack:"
0431 echo " <I>    "${shpamstfile}
0432 echo " <I>  generated python fragment:"
0433 echo " <I>    "${shpacfffile}
0434 echo " <I>  MD5 checksum file:"
0435 echo " <I>    "${shpamstmd5s}
0436 echo " <I>  "
0437 echo " <I>  "
0438 echo " <I>  "
0439 echo " <I> ATTENTION,"
0440 echo " <I>  please edit the generated python fragment according to your needs:"
0441 echo " <I>  "
0442 echo " <I>  for a local production edit the line"
0443 echo " <I>    SherpackLocation = cms.string('[local path to your sherpack]')"
0444 echo " <I>  and make sure that the SherpaInterface does not try to fetch the sherpack"
0445 echo " <I>    FetchSherpack = cms.bool(False)"
0446 echo " <I>  "
0447 echo " <I>  "
0448 echo " <I>  for a production using the frontier database use"
0449 echo " <I>    SherpackLocation = cms.string('[frontier database path to your sherpack]')"
0450 echo " <I>  and this time make sure that the SherpaInterface fetches the sherpack"
0451 echo " <I>    FetchSherpack = cms.bool(true)"
0452 echo " <I>  "
0453 echo " <I>  "
0454 echo " <I>  for a production with CRAB add the sherpack to the list of additional files"
0455 echo " <I>    additional_input_files = [name of the ..._MASTER.tgz sherpack]"
0456 echo " <I>  make sure that the sherpack location is"
0457 echo " <I>    SherpackLocation = cms.string('./')"
0458 echo " <I>  and make sure that the SherpaInterface does not try to fetch the sherpack"
0459 echo " <I>    FetchSherpack = cms.bool(False)"
0460 echo " <I>  "
0461 echo " <I>  "
0462 echo " <I>  "
0463 echo " <I>  a good way to test the generated python file is to cross-check it with cmsDriver.py:"
0464 echo "       cmsDriver.py A/B/python/"${shpacfffile}" \\"
0465 echo "        -s GEN -n 100 --no_exec --conditions auto:mc --eventcontent RAWSIM"
0466 echo " <I>  "
0467 echo " <I>  "
0468 echo " <I>  "
0469