SiStripMonitorCluster/test/filter_mtcc.sh

0001 #!/bin/sh
0002 #dkcira - 2006.08.20, add more flexibility, change to options instead of add more flexibility, positional parameters
0003
0004
0005 ###############################################################################################################
0006 # users can change these to their default directories that will overwrite command line options
0007 ###############################################################################################################
0008 #DEFAULT_INPUT_DIRECTORY="/castor/cern.ch/user/d/dkcira/MTCC/2006_08_16_090/";
0009 #DEFAULT_CASTOR_OUTPUT_DIRECTORY="/castor/cern.ch/user/d/dkcira/MTCC/2006_08_19_filter"
0010
0011 ###############################################################################################################
0012 # read options
0013 ###############################################################################################################
0014 echo $0 $@; # echo $OPTIND;
0015 OPTERROR=11; # error option to use when exiting
0016 #
0017 show_usage(){
0018   echo ""
0019   echo "Usage:"
0020   echo "`basename $0` -r mtcc_runnumber [ -i which_input_directory -o output_directory_in_castor -h ]";
0021   echo ""
0022   echo " -h"
0023   echo "     print out this message"
0024   echo " -r  mtcc_runnumber"
0025   echo "      run number you want to filter"
0026   echo " -i which_input_directory"
0027   echo "      Set this to castor directory or local directory"
0028   echo "      If DEFAULT_CASTOR_INPUT_DIRECTORY is set in the script, it will ignore this option"
0029   echo " -o output_directory_in_castor :"
0030   echo "      Set this for your output files to be copied to castor."
0031   echo "      If DEFAULT_CASTOR_OUTPUT_DIRECTORY is set in the script, it will ignore this option"
0032   echo ""
0033 }
0034 #
0035 # Exit and complain if no argument given.
0036 if [ -z $1 ]; then show_usage; exit $OPTERROR; fi
0037 #
0038 # echo "OPTIND=$OPTIND"; echo "#=$#";
0039 while getopts ":r:i:o:h" Option
0040 do
0041   case $Option in
0042     r)  RUNNR=$OPTARG; echo "will reconstruct RUNNR=$RUNNR" ;;
0043     i)  WHICH_INPUT_DIRECTORY=$OPTARG; echo "WHICH_INPUT_DIRECTORY=$WHICH_INPUT_DIRECTORY" ;;
0044     o)  WHERE_TO_COPY_OUTPUT=$OPTARG; echo "WHERE_TO_COPY_OUTPUT=$WHERE_TO_COPY_OUTPUT" ;;
0045     h)  show_usage; exit 0;;
0046     *)  echo "No such option -${Option}";;
0047   esac
0048 done
0049 shift $(($OPTIND - 1)) # Decrements the argument pointer so it points to next argument.
0050
0051
0052 ###############################################################################################################
0053 # define functions
0054 ###############################################################################################################
0055 #--- this function has to be called first, obviously
0056 set_shell_variables(){
0057  # this directory must be visible from remote batch machine
0058  DIR_WHERE_TO_EVAL="/afs/cern.ch/user/d/dkcira/scratch0/MTCC/2006_08_14_code_090/CMSSW_0_9_0"
0059  # for online db access. some variables for the oracle tool of cmssw are not set anymore
0060  LOCAL_ORACLE_ADMINDIR="/afs/cern.ch/project/oracle/admin/"
0061  # username to connect to cmsdisk0.cern.ch for asking list of files and copying them
0062  BATCH_USER_NAME=`whoami`
0063  # directory where the job is run or submitted
0064  if [ "${LS_SUBCWD+set}" = set ]; then
0065    LK_WKDIR="${LS_SUBCWD}" # directory where you submit in case of bsub
0066    WWDIR="${WORKDIR}"
0067  else
0068    LK_WKDIR=`pwd`          # directory where you run locally otherwise
0069    WWDIR=`pwd`
0070  fi
0071  # this directory will be created, use '/pool' for production and '/tmp' for testing
0072  MTCC_OUTPUT_DIR="${WWDIR}/mtcc_filter_${RUNNR}"
0073  # template
0074  TEMPLATE_FILTER_CFG="${LK_WKDIR}/template_filter.cfg"
0075  # config files
0076  FILTER_CFG="${MTCC_OUTPUT_DIR}/${RUNNR}_filter.cfg";
0077  # log files
0078  FILTER_LOG="${MTCC_OUTPUT_DIR}/${RUNNR}_filter.log";
0079  # histograms + pool
0080  FILTER_POOL_OUTPUT_FILE="${MTCC_OUTPUT_DIR}/${RUNNR}_filter_rec.root";
0081  DQM_OUTPUT_FILE="${MTCC_OUTPUT_DIR}/${RUNNR}_filter_dqm.root"
0082  # for testing, set to 100000
0083  MAX_FILES_TO_RUN_OVER=100000;
0084  # echo '###########################################################'
0085  # echo 'SHELL VARIABLES'; set; export;
0086  # echo '###########################################################'
0087 }
0088
0089
0090 #--- first general tests
0091 inital_checks_and_settings(){
0092   # need a run
0093   if [ "X$RUNNR" == "X" ]; then
0094    echo "You did not choose a run number. Stopping here!"
0095    exit $OPTERROR;
0096   fi
0097
0098   # need templates
0099   if [ ! -f "$TEMPLATE_FILTER_CFG" ]; then echo "file ${TEMPLATE_FILTER_CFG} does not exist, stopping here"; exit $OPTERROR; fi
0100   if [ -f "$FILTER_CFG" ]; then echo "file ${FILTER_CFG} already exists, stopping here"; exit $OPTERROR; fi
0101
0102   # first choose input directory from command line or default variable
0103   if [ "X$DEFAULT_INPUT_DIRECTORY" != "X" ]; then # if set and not empty
0104          echo "Using input directory from default variable DEFAULT_INPUT_DIRECTORY";
0105          WHICH_INPUT_DIRECTORY="$DEFAULT_INPUT_DIRECTORY";
0106   elif [ -n "$WHICH_INPUT_DIRECTORY" ]; then
0107          echo "Using input directory from command line option.";
0108   else
0109      echo "No input directory. Stopping here!"; exit $OPTERROR;
0110   fi
0111   # then check if castor/local and if exists
0112   if nsls -d "$WHICH_INPUT_DIRECTORY" >& /dev/null # Suppress default output
0113   then
0114     echo "Input directory is castor $WHICH_INPUT_DIRECTORY";
0115     TYPE_INPUT_DIRECTORY=1; # found on castor
0116   elif [ -d $WHICH_INPUT_DIRECTORY ] ; then
0117     # transform relative path name in absolute path name if necessary
0118     if [ "${InputFiles:0:1}" != "/" ] ; then # does not start with "/" so is relative path name
0119       WHICH_INPUT_DIRECTORY="${PWD}/${WHICH_INPUT_DIRECTORY}"; # translate into absolute path name
0120     fi
0121     echo "Input directory is local $WHICH_INPUT_DIRECTORY";
0122     TYPE_INPUT_DIRECTORY=2; # found locally
0123   else
0124     echo "Input directory does not exist $WHICH_INPUT_DIRECTORY"
0125     echo "Stopping here!"; exit $OPTERROR;
0126   fi
0127
0128   # first choose castor directory from command line or default variable
0129   if [ "X${DEFAULT_CASTOR_OUTPUT_DIRECTORY}" != "X" ]; then # if set and not empty
0130          echo "Using castor directory from default variable DEFAULT_CASTOR_OUTPUT_DIRECTORY";
0131          WHERE_TO_COPY_OUTPUT="$DEFAULT_CASTOR_OUTPUT_DIRECTORY";
0132   elif [  -n "$WHERE_TO_COPY_OUTPUT" ]; then
0133          echo "Using castor directory from command line option";
0134   else
0135          echo "Output files will NOT be copied to castor.";
0136          CASTOROUTPUT="no"
0137   fi
0138   # then check if castor directory exists
0139   if [ "$CASTOROUTPUT" != "no" ] ; then
0140    if nsls -d "$WHERE_TO_COPY_OUTPUT" > /dev/null # Suppress default output
0141    then
0142      echo "Using $WHERE_TO_COPY_OUTPUT to copy files to castor";
0143    else
0144      echo "Directory WHERE_TO_COPY_OUTPUT=$WHERE_TO_COPY_OUTPUT does not exist on castor.";
0145      echo "Stopping here!"; exit $OPTERROR;
0146    fi
0147   fi
0148
0149   # document which code you were using
0150   echo "Using code from ${DIR_WHERE_TO_EVAL}";
0151 }
0152
0153 #---
0154 create_output_directory(){
0155   if [ -d "$MTCC_OUTPUT_DIR" ]; then
0156     echo "directory ${MTCC_OUTPUT_DIR} already exists, stopping here"; exit $OPTERROR;
0157   else
0158     echo "creating directory ${MTCC_OUTPUT_DIR}"; mkdir $MTCC_OUTPUT_DIR;
0159   fi
0160 }
0161
0162 #---
0163 get_list_of_input_files(){
0164  echo "getting the list of files corresponding to run ${RUNNR}";
0165  if [ "$TYPE_INPUT_DIRECTORY" == "1"  ] ; then
0166    LIST_OF_DATA_FILES=`nsls $WHICH_INPUT_DIRECTORY | grep "${RUNNR}_rec" | grep '\.root' | head -${MAX_FILES_TO_RUN_OVER}`
0167  elif [ "$TYPE_INPUT_DIRECTORY" == "2"  ] ; then
0168    LIST_OF_DATA_FILES=`ls $WHICH_INPUT_DIRECTORY | grep "${RUNNR}_rec" | grep '\.root' | head -${MAX_FILES_TO_RUN_OVER}`
0169  else
0170   echo "No such TYPE_INPUT_DIRECTORY=$TYPE_INPUT_DIRECTORY. Stopping here!"; exit $OPTERROR;
0171  fi
0172  if [ "X$LIST_OF_DATA_FILES" == "X"  ] ; then echo "No input files found. Stopping here!"; exit $OPTERROR; fi
0173 }
0174
0175 #---
0176 create_filter_config_file(){
0177 # create list with full paths
0178   LIST_WITH_PATH="";
0179   for rfile in $LIST_OF_DATA_FILES
0180   do
0181     if [ "$TYPE_INPUT_DIRECTORY" == "1"  ]; then # castor
0182        LIST_WITH_PATH="${LIST_WITH_PATH},\"rfio:${WHICH_INPUT_DIRECTORY}/${rfile}\""
0183     elif [ "$TYPE_INPUT_DIRECTORY" == "2"  ] ; then # local
0184        LIST_WITH_PATH="${LIST_WITH_PATH},\"file:${WHICH_INPUT_DIRECTORY}/${rfile}\""
0185     else
0186        echo "No such TYPE_INPUT_DIRECTORY=$TYPE_INPUT_DIRECTORY. Stopping here!"; exit $OPTERROR;
0187     fi
0188   done
0189   # remove first comma
0190   LIST_WITH_PATH=`echo $LIST_WITH_PATH | sed 's/\,//'`;
0191   echo "creating $FILTER_CFG";
0192   touch $FILTER_CFG;
0193   cat  "$TEMPLATE_FILTER_CFG" | sed "s@SCRIPT_POOL_OUTPUT_FILE@${FILTER_POOL_OUTPUT_FILE}@" | sed "s@SCRIPT_DQM_OUTPUT_FILE@${DQM_OUTPUT_FILE}@" | sed "s@SCRIPT_LIST_OF_FILES@${LIST_WITH_PATH}@" >>  ${FILTER_CFG}
0194 }
0195
0196 #---
0197 runfilter(){
0198   cd ${DIR_WHERE_TO_EVAL}; eval `scramv1 runtime -sh`;
0199   cd ${MTCC_OUTPUT_DIR};
0200   echo "##### RUNNING THE RECONSTRUCTION USING THE CFG FILE ${FILTER_CFG}"
0201   cat ${FILTER_CFG}
0202   cmsRun ${FILTER_CFG}
0203   echo "filter jobstatus: $?";
0204 }
0205
0206 #---
0207 copy_output_to_castor(){
0208 case $# in
0209 1) OUTPUT_CASTOR_DIR="$1" ;;
0210 *) echo "No output castor directory given, not performing copy_output_to_castor." ;;
0211 esac
0212  # copy (some) output files to castor
0213  # copy (some) output files to castor
0214  if [  "X$OUTPUT_CASTOR_DIR" != "X" ]; then
0215    echo "copying output files to $OUTPUT_CASTOR_DIR";
0216    for ifile in ${MTCC_OUTPUT_DIR}/${RUNNR}_filter*
0217    do
0218     rfcp ${ifile}  ${OUTPUT_CASTOR_DIR}/.
0219    done
0220  fi
0221 }
0222
0223 ###############################################################################################################
0224 # actual execution
0225 ###############################################################################################################
0226 # GENERAL
0227 ls -lh;
0228 set_shell_variables;
0229 inital_checks_and_settings;
0230 create_output_directory;
0231 get_list_of_input_files;
0232 create_filter_config_file;
0233 echo "Running filter. Log file: ${FILTER_LOG}";
0234 time runfilter > ${FILTER_LOG} 2>&1 ;
0235 ls -lh . ${MTCC_OUTPUT_DIR}/ ;
0236
0237 # copy output to CASTOR if the output directory variable is set
0238 if [ -n "WHERE_TO_COPY_OUTPUT" ]; then
0239    copy_output_to_castor "$WHERE_TO_COPY_OUTPUT";
0240 fi
0241 ###############################################################################################################
0242