scripts/filecollector/aliveCheck2.sh

0001 #! /bin/bash
0002
0003 #export WorkDir=$(dirname $0)
0004 YourEmail=sdutta@cern.ch
0005 #source /nfshome0/cmssw2/scripts/setup.sh
0006
0007 export SCRAM_ARCH=slc5_amd64_gcc462
0008 if [[ ! $HOME =~ /nfshome0/dqm* ]]
0009 then
0010   HOME=/nfshome0/${USER/local/}
0011 fi
0012 if [[ -d ${HOME}/prod || -d ${HOME}/dev ]]
0013 then
0014   source /nfshome0/dqmpro/bin/setup_cmssw.sh
0015   [[ -d ${HOME}/prod ]] && cd ${HOME}/prod || cd ${HOME}/dev
0016   eval `scram runtime -sh`
0017 else
0018   source $WorkDir/env3.sh
0019 fi
0020 export PYTHONPATH=$XPYTHONPATH:$PYTHONPATH
0021 export HOSTNAME=$HOSTNAME
0022 agents_pnames=("fileCollector" "producerFileCleanner")
0023 agents_executables=("/nfshome0/dqmpro/filecollector/fileCollector2.py" "/nfshome0/dqmpro/filecollector/producerFileCleanner.py")
0024 if [[ $USER =~ 'dqmpr.*' ]]
0025 then
0026   agents_parameters=("/home/dqmprolocal/output /home/dqmprolocal/done /dqmdata/dqm/uploads" \
0027                      "/cmsnfshome0/nfshome0/dqmpro/filecollector/RootArchivalAndTransferSystem_cfg.py")
0028 else
0029   agents_parameters=("/home/dqmdevlocal/output /home/dqmdevlocal/done /dqmdata/dqmintegration/upload" \
0030                      "/cmsnfshome0/nfshome0/dqmpro/filecollector/RootArchivalAndTransferSystem_cfg.py")
0031 fi
0032 WorkDir=$( dirname ${agents_executables[0]} )
0033 [[ -e $WorkDir/.start ]] && [[ -e $WorkDir/.stop ]] && rm $WorkDir/.stop
0034 [[ -e $WorkDir/.stop ]] && echo Found stop file not starting the agents && exit 0
0035
0036 msg=
0037 new_line=
0038 for pos in $(seq 0 $(( ${#agents_executables[@]} - 1 ))); do
0039   RUN_STAT=`ps -ef | grep -P "(${agents_executables[$pos]})" | grep -v grep | wc | awk '{print $1}'`
0040   if [ $RUN_STAT -ne 0 ];then
0041     echo ${agents_pnames[$pos]} is running
0042   else
0043     echo ${agents_pnames[$pos]} stopped by unknown reason and restarted now.
0044     TIMETAG=$(date +"%Y%m%d_%H%M%S")
0045     LOG=$WorkDir/log/LOG.${agents_pnames[$pos]}.$HOSTNAME.$TIMETAG
0046     ${agents_executables[$pos]} ${agents_parameters[$pos]} >& $LOG &
0047     date >> $LOG
0048     [[ ! -e $WorkDir/.start ]] &&
0049          echo ${agents_pnames[$pos]} stopped by unknown reason and restarted at $HOSTNAME. >> $LOG ||
0050          echo ${agents_pnames[$pos]} Found .start file, starting
0051     [[ ! -z $msg ]] && new_line="\n"
0052     msg=$msg$new_line${agents_pnames[$pos]}" stopped by unknown reason and restarted now at $HOSTNAME."
0053   fi
0054 done
0055
0056 [[ ! -e $WorkDir/.start && ! -z $msg ]] && echo $msg | mail -s "File Collection Agents not Running" $YourEmail
0057
0058 if [[ -e $WorkDir/.start ]]
0059 then
0060   sleep 10
0061   master=$(cat $WorkDir/.start)
0062   [[ $(hostname -s) == $master ]] && rm $WorkDir/.start
0063 fi