SiStripMonitorCluster/test/filter_mtcc_reprocessed.sh

0001 #!/bin/sh
0002 #dkcira - 2006.08.20, add more flexibility, change to options instead of add more flexibility, positional parameters
0003
0004 ###############################################################################################################
0005 # users can change these to their default directories that will overwrite command line options
0006 ###############################################################################################################
0007 DEFAULT_INPUT_DIRECTORY="/castor/cern.ch/cms/testbeam/tkmtcc/P5_data/tracker_reprocessing/pass1";
0008 #DEFAULT_CASTOR_OUTPUT_DIRECTORY="/castor/cern.ch/user/d/dkcira/MTCC/2006_08_19_filter"
0009
0010 ###############################################################################################################
0011 # read options
0012 ###############################################################################################################
0013 echo $0 $@; # echo $OPTIND;
0014 OPTERROR=11; # error option to use when exiting
0015 #
0016 show_usage(){
0017   echo ""
0018   echo "Usage:"
0019   echo "`basename $0` -r mtcc_runnumber [ -i which_input_directory -o output_directory_in_castor -h ]";
0020   echo ""
0021   echo " -h"
0022   echo "     print out this message"
0023   echo " -r  mtcc_runnumber"
0024   echo "      run number you want to filter"
0025   echo " -i which_input_directory"
0026   echo "      Set this to castor directory or local directory"
0027 #  echo "      If DEFAULT_CASTOR_INPUT_DIRECTORY is set in the script, it will ignore this option"
0028   echo " -o output_directory_in_castor :"
0029   echo "      Set this for your output files to be copied to castor."
0030 #  echo "      If DEFAULT_CASTOR_OUTPUT_DIRECTORY is set in the script, it will ignore this option"
0031   echo ""
0032 }
0033 #
0034 # Exit and complain if no argument given.
0035 if [ -z $1 ]; then show_usage; exit $OPTERROR; fi
0036 #
0037 # echo "OPTIND=$OPTIND"; echo "#=$#";
0038 while getopts ":r:i:o:h" Option
0039 do
0040   case $Option in
0041     r)  RUNNR=$OPTARG; echo "will reconstruct RUNNR=$RUNNR" ;;
0042     i)  WHICH_INPUT_DIRECTORY=$OPTARG; echo "WHICH_INPUT_DIRECTORY=$WHICH_INPUT_DIRECTORY" ;;
0043     o)  WHERE_TO_COPY_OUTPUT=$OPTARG; echo "WHERE_TO_COPY_OUTPUT=$WHERE_TO_COPY_OUTPUT" ;;
0044     h)  show_usage; exit 0;;
0045     *)  echo "No such option -${Option}";;
0046   esac
0047 done
0048 shift $(($OPTIND - 1)) # Decrements the argument pointer so it points to next argument.
0049
0050
0051 ###############################################################################################################
0052 # define functions
0053 ###############################################################################################################
0054 #--- this function has to be called first, obviously
0055 set_shell_variables(){
0056  # this directory must be visible from remote batch machine
0057  DIR_WHERE_TO_EVAL="/afs/cern.ch/user/v/vciulli/scratch0/MTCC/2006_10_11_102/CMSSW_1_0_2"
0058  # for online db access. some variables for the oracle tool of cmssw are not set anymore
0059  LOCAL_ORACLE_ADMINDIR="/afs/cern.ch/project/oracle/admin/"
0060  # username to connect to cmsdisk0.cern.ch for asking list of files and copying them
0061  BATCH_USER_NAME=`whoami`
0062  # directory where the job is run or submitted
0063  if [ "${LS_SUBCWD+set}" = set ]; then
0064    LK_WKDIR="${LS_SUBCWD}" # directory where you submit in case of bsub
0065    WWDIR="${WORKDIR}"
0066  else
0067    LK_WKDIR=`pwd`          # directory where you run locally otherwise
0068    WWDIR=`pwd`
0069  fi
0070  # this directory will be created, use '/pool' for production and '/tmp' for testing
0071  MTCC_OUTPUT_DIR="${WWDIR}/mtcc_filter_${RUNNR}"
0072  # template
0073  TEMPLATE_FILTER_CFG="${LK_WKDIR}/template_filter_reprocessed.cfg"
0074  # config files
0075  FILTER_CFG="${MTCC_OUTPUT_DIR}/${RUNNR}_filter.cfg";
0076  # log files
0077  FILTER_LOG="${MTCC_OUTPUT_DIR}/${RUNNR}_filter.log";
0078  # histograms + pool
0079  FILTER_POOL_OUTPUT_FILE="${MTCC_OUTPUT_DIR}/${RUNNR}_filter_rec.root";
0080  DQM_OUTPUT_FILE="${MTCC_OUTPUT_DIR}/${RUNNR}_filter_dqm.root"
0081  # for testing, set to 100000
0082  MAX_FILES_TO_RUN_OVER=100000;
0083  # echo '###########################################################'
0084  # echo 'SHELL VARIABLES'; set; export;
0085  # echo '###########################################################'
0086 }
0087
0088
0089 #--- first general tests
0090 inital_checks_and_settings(){
0091   # need a run
0092   if [ "X$RUNNR" == "X" ]; then
0093    echo "You did not choose a run number. Stopping here!"
0094    exit $OPTERROR;
0095   fi
0096
0097   # need templates
0098   if [ ! -f "$TEMPLATE_FILTER_CFG" ]; then echo "file ${TEMPLATE_FILTER_CFG} does not exist, stopping here"; exit $OPTERROR; fi
0099   if [ -f "$FILTER_CFG" ]; then echo "file ${FILTER_CFG} already exists, stopping here"; exit $OPTERROR; fi
0100
0101   # first choose input directory from command line or default variable
0102   if [ -n "$WHICH_INPUT_DIRECTORY" ]; then
0103          echo "Using input directory from command line option.";
0104   elif [ "X$DEFAULT_INPUT_DIRECTORY" != "X" ]; then # if set and not empty
0105          echo "Using input directory from default variable DEFAULT_INPUT_DIRECTORY";
0106          WHICH_INPUT_DIRECTORY="$DEFAULT_INPUT_DIRECTORY";
0107   else
0108      echo "No input directory. Stopping here!"; exit $OPTERROR;
0109   fi
0110   # then check if castor/local and if exists
0111   if nsls -d "$WHICH_INPUT_DIRECTORY" >& /dev/null # Suppress default output
0112   then
0113     echo "Input directory is castor $WHICH_INPUT_DIRECTORY";
0114     TYPE_INPUT_DIRECTORY=1; # found on castor
0115   elif [ -d $WHICH_INPUT_DIRECTORY ] ; then
0116     # transform relative path name in absolute path name if necessary
0117     if [ "${InputFiles:0:1}" != "/" ] ; then # does not start with "/" so is relative path name
0118       WHICH_INPUT_DIRECTORY="${PWD}/${WHICH_INPUT_DIRECTORY}"; # translate into absolute path name
0119     fi
0120     echo "Input directory is local $WHICH_INPUT_DIRECTORY";
0121     TYPE_INPUT_DIRECTORY=2; # found locally
0122   else
0123     echo "Input directory does not exist $WHICH_INPUT_DIRECTORY"
0124     echo "Stopping here!"; exit $OPTERROR;
0125   fi
0126
0127   # first choose castor directory from command line or default variable
0128   if [  -n "$WHERE_TO_COPY_OUTPUT" ]; then
0129          echo "Using castor directory from command line option";
0130   elif [ "X${DEFAULT_CASTOR_OUTPUT_DIRECTORY}" != "X" ]; then # if set and not empty
0131          echo "Using castor directory from default variable DEFAULT_CASTOR_OUTPUT_DIRECTORY";
0132          WHERE_TO_COPY_OUTPUT="$DEFAULT_CASTOR_OUTPUT_DIRECTORY";
0133   else
0134          echo "Output files will NOT be copied to castor.";
0135          CASTOROUTPUT="no"
0136   fi
0137   # then check if castor directory exists
0138   if [ "$CASTOROUTPUT" != "no" ] ; then
0139    if nsls -d "$WHERE_TO_COPY_OUTPUT" > /dev/null # Suppress default output
0140    then
0141      echo "Using $WHERE_TO_COPY_OUTPUT to copy files to castor";
0142    else
0143      echo "Directory WHERE_TO_COPY_OUTPUT=$WHERE_TO_COPY_OUTPUT does not exist on castor.";
0144      echo "Stopping here!"; exit $OPTERROR;
0145    fi
0146   fi
0147
0148   # document which code you were using
0149   echo "Using code from ${DIR_WHERE_TO_EVAL}";
0150 }
0151
0152 #---
0153 create_output_directory(){
0154   if [ -d "$MTCC_OUTPUT_DIR" ]; then
0155     echo "directory ${MTCC_OUTPUT_DIR} already exists, stopping here"; exit $OPTERROR;
0156   else
0157     echo "creating directory ${MTCC_OUTPUT_DIR}"; mkdir $MTCC_OUTPUT_DIR;
0158   fi
0159 }
0160
0161 #---
0162 get_list_of_input_files(){
0163  echo "getting the list of files corresponding to run ${RUNNR}";
0164  if [ "$TYPE_INPUT_DIRECTORY" == "1"  ] ; then
0165    LIST_OF_DATA_FILES=`nsls $WHICH_INPUT_DIRECTORY | grep "${RUNNR}_reco_cluster" | grep '\.root' | head -${MAX_FILES_TO_RUN_OVER}`
0166  elif [ "$TYPE_INPUT_DIRECTORY" == "2"  ] ; then
0167    LIST_OF_DATA_FILES=`ls $WHICH_INPUT_DIRECTORY | grep "${RUNNR}_reco_cluster" | grep '\.root' | head -${MAX_FILES_TO_RUN_OVER}`
0168  else
0169   echo "No such TYPE_INPUT_DIRECTORY=$TYPE_INPUT_DIRECTORY. Stopping here!"; exit $OPTERROR;
0170  fi
0171  if [ "X$LIST_OF_DATA_FILES" == "X"  ] ; then echo "No input files found. Stopping here!"; exit $OPTERROR; fi
0172 }
0173
0174 #---
0175 create_filter_config_file(){
0176 # create list with full paths
0177   LIST_WITH_PATH="";
0178   for rfile in $LIST_OF_DATA_FILES
0179   do
0180     if [ "$TYPE_INPUT_DIRECTORY" == "1"  ]; then # castor
0181        LIST_WITH_PATH="${LIST_WITH_PATH},\"rfio:${WHICH_INPUT_DIRECTORY}/${rfile}\""
0182     elif [ "$TYPE_INPUT_DIRECTORY" == "2"  ] ; then # local
0183        LIST_WITH_PATH="${LIST_WITH_PATH},\"file:${WHICH_INPUT_DIRECTORY}/${rfile}\""
0184     else
0185        echo "No such TYPE_INPUT_DIRECTORY=$TYPE_INPUT_DIRECTORY. Stopping here!"; exit $OPTERROR;
0186     fi
0187   done
0188   # remove first comma
0189   LIST_WITH_PATH=`echo $LIST_WITH_PATH | sed 's/\,//'`;
0190   echo "creating $FILTER_CFG";
0191   touch $FILTER_CFG;
0192   cat  "$TEMPLATE_FILTER_CFG" | sed "s@SCRIPT_POOL_OUTPUT_FILE@${FILTER_POOL_OUTPUT_FILE}@" | sed "s@SCRIPT_DQM_OUTPUT_FILE@${DQM_OUTPUT_FILE}@" | sed "s@SCRIPT_LIST_OF_FILES@${LIST_WITH_PATH}@" >>  ${FILTER_CFG}
0193 }
0194
0195 #---
0196 runfilter(){
0197   cd ${DIR_WHERE_TO_EVAL}; eval `scramv1 runtime -sh`;
0198   cd ${MTCC_OUTPUT_DIR};
0199   echo "##### RUNNING THE RECONSTRUCTION USING THE CFG FILE ${FILTER_CFG}"
0200   cat ${FILTER_CFG}
0201   cmsRun ${FILTER_CFG}
0202   echo "filter jobstatus: $?";
0203 }
0204
0205 #---
0206 copy_output_to_castor(){
0207 case $# in
0208 1) OUTPUT_CASTOR_DIR="$1" ;;
0209 *) echo "No output castor directory given, not performing copy_output_to_castor." ;;
0210 esac
0211  # copy (some) output files to castor
0212  # copy (some) output files to castor
0213  if [  "X$OUTPUT_CASTOR_DIR" != "X" ]; then
0214    echo "copying output files to $OUTPUT_CASTOR_DIR";
0215    for ifile in ${MTCC_OUTPUT_DIR}/${RUNNR}_filter*
0216    do
0217     rfcp ${ifile}  ${OUTPUT_CASTOR_DIR}/.
0218    done
0219  fi
0220 }
0221
0222 ###############################################################################################################
0223 # actual execution
0224 ###############################################################################################################
0225 # GENERAL
0226 ls -lh;
0227 set_shell_variables;
0228 inital_checks_and_settings;
0229 create_output_directory;
0230 get_list_of_input_files;
0231 create_filter_config_file;
0232 echo "Running filter. Log file: ${FILTER_LOG}";
0233 time runfilter > ${FILTER_LOG} 2>&1 ;
0234 ls -lh . ${MTCC_OUTPUT_DIR}/ ;
0235
0236 # copy output to CASTOR if the output directory variable is set
0237 if [ -n "WHERE_TO_COPY_OUTPUT" ]; then
0238    copy_output_to_castor "$WHERE_TO_COPY_OUTPUT";
0239 fi
0240 ###############################################################################################################
0241