scripts/filecollector/OnlineSync.sh

0001 #!/bin/zsh
0002 BASE_DIR=/dqmdata/offline/repository/data/OnlineData
0003 X509_CERT_DIR=/etc/grid-security/certificates
0004 X509_USER_PROXY=/dqmdata/auth/proxy.pem
0005
0006 #BASE_DIR=~/DQMWorkArea/dqmdata/repository/data/OnlineData
0007 #X509_USER_PROXY=~/.globus/x509up_u44417
0008 #X509_CERT_DIR=/afs/cern.ch/project/gd/LCG-share2/certificates/
0009
0010 Counter=0
0011 log(){
0012   echo $(date +"%F %T") \[OnlineSynchronyzer/$$\] $1
0013 }
0014 rcurl(){
0015   [[ -z $2 ]] && removeTree=0 || removeTree=$2
0016   if [[ $removeTree -eq 0 ]]
0017   then
0018     rootDir=$1
0019     remoteDirBase="dqm/online/data/browse"
0020   else
0021     [[ ${(w)#${(SI:$removeTree:)1/\// }} -gt 1 ]] && rootDir=${${(s: :SI:$removeTree:)1/\// }[2]} || rootDir=""
0022     [[ ${(w)#${(SI:$removeTree:)1/\// }} -gt 1 ]] && remoteDirBase="dqm/online/data/browse"/${${(s: :SI:$removeTree:)1/\// }[1]} || remoteDirBase="dqm/online/data/browse"/$1
0023   fi
0024   host="https://cmsweb.cern.ch"
0025   [[ -e $rootDir/index.html ]] && dirTimeStamp=$(date -d "$(curl -A "OnlineSyncDev/0.1" -s -I \
0026        --capath $X509_CERT_DIR/ \
0027        --key $X509_USER_PROXY \
0028        --cert $X509_USER_PROXY \
0029        $host/$remoteDirBase/$rootDir/ | grep "Last-Modified:.*$" | sed -e "s|Last-Modified: ||" )" +%s) || dirTimeStamp=$(date +%s)
0030   if [[ ! -e $rootDir/index.html || $dirTimeStamp -gt $(stat $rootDir/index.html -c %Y) ]]
0031   then
0032     echo Fetching $rootDir content
0033     curl -A "OnlineSyncDev/0.1" -s -R -o $rootDir/index.html --create-dirs \
0034          --capath $X509_CERT_DIR/ \
0035          --key $X509_USER_PROXY \
0036          --cert $X509_USER_PROXY \
0037          $host/$remoteDirBase/$rootDir/
0038   fi
0039   dirs=($(<$rootDir/index.html | egrep -oe "<tr><td><a.*</a>" | egrep -o "href='([^']*)'" | egrep -v "\.root" | sed -re "s/(href='\/|\/'$)//g" | sort -r ))
0040   files=($(<$rootDir/index.html | egrep -oe "<tr><td><a.*</a>" | egrep -o "href='([^']*)'" | egrep  "\.root" |  sed -re "s/(href='\/|'$)//g" | sort -r ))
0041   for d in $dirs
0042   do
0043     rcurl ${d/dqm\/online\/data\/browse\//} $removeTree
0044   done
0045   for f in $files
0046   do
0047     [[ -e $rootDir/$(basename $f) ]] && fTimeStamp=$(date -d "$(curl -A "OnlineSyncDev/0.1" -s -I \
0048        --capath $X509_CERT_DIR/ \
0049        --key $X509_USER_PROXY \
0050        --cert $X509_USER_PROXY \
0051        $host/$remoteDirBase/$rootDir/$(basename $f ) | grep "Last-Modified:.*$" | sed -e "s|Last-Modified: ||" )" +%s) || fTimeStamp=$(date +%s)
0052     if [[ ! -e $rootDir/$(basename $f) || $fTimeStamp -gt $(stat $rootDir/$(basename $f) -c %Y) ]]
0053     then
0054       curl -A "OnlineSyncDev/0.1" -s -R -o $rootDir/$(basename $f) --create-dirs \
0055          --capath $X509_CERT_DIR/ \
0056          --key $X509_USER_PROXY \
0057          --cert $X509_USER_PROXY \
0058          $host/$f && log "INFO: Successfully downloaded "$PWD/$rootDir/$(basename $f) || \
0059          log "ERROR: Unable to download "$PWD/$rootDir/$(basename $f)
0060     fi
0061
0062   done
0063
0064 }
0065 while [ 1 ]
0066 do
0067   if [[ $Counter -gt 7 ]]
0068   then
0069     Counter=0
0070     #log "INFO: Starting daily full tree synchronization"
0071     #cd $BASE_DIR/original
0072     #[[ $(pwd) == $BASE_DIR/original ]] && wget -q -e robots=off --mirror -np -nH --cut-dirs=5 -T120 \
0073     #   --ca-dir /etc/grid-security/certificates \
0074     #   --private-key /dqmdata/auth/proxy.pem \
0075     #   --certificate /dqmdata/auth/proxy.pem \
0076     #   https://cmsweb.cern.ch/dqm/online/data/browse/Original/ || (log "ERROR: Could not go to $BASE_DIR/original";exit)
0077     #cd $BASE_DIR/merged
0078     #[[ $(pwd) == $BASE_DIR/merged ]] && wget -q -e robots=off --mirror -np -nH --cut-dirs=5 -T120 \
0079     #   --ca-dir /etc/grid-security/certificates \
0080     #   --private-key /dqmdata/auth/proxy.pem \
0081     #   --certificate /dqmdata/auth/proxy.pem \
0082     #   https://cmsweb.cern.ch/dqm/online/data/browse/Merged/ || (log "ERROR: Could not go to $BASE_DIR/Merged";exit)
0083     #log "INFO: Finished daily full tree synchronization"wget
0084     continue
0085   fi
0086   Counter=$(( $Counter + 1 ))
0087   cd $BASE_DIR/original
0088   latestLocalDir=$(find $BASE_DIR/original -maxdepth 1 -type d -exec basename {} \; | sort -n | tail -n 1)
0089   latestDir=$(curl -A "OnlineSyncDev/0.1" \
0090     --capath $X509_CERT_DIR --key $X509_USER_PROXY \
0091     --cert $X509_USER_PROXY \
0092     https://cmsweb.cern.ch/dqm/online/data/browse/Original/ 2>&1 \
0093     | egrep -oe "<tr><td><a.*</a>" | egrep -o "'>.*<" | egrep -o "[0-9]{5}xxxx" |sort -r | head -n 1)
0094   [[ -z $latestDir ]] && continue
0095   dirs=({${latestLocalDir/xxxx/}..${latestDir/xxxx/}}xxxx)
0096   #log "INFO: Starting partial synchronization of original files"
0097   for d in $dirs
0098   do
0099     #log "INFO: Synchronizing  https://cmsweb.cern.ch/dqm/online/data/browse/Original/$d"
0100     rcurl Original/$d 1
0101   done
0102   #log "INFO: Finished partial synchronization of original files"
0103
0104   cd $BASE_DIR/merged
0105   latestLocalDir=$(find $BASE_DIR/merged -maxdepth 1 -type d -exec basename {} \; | sort -n | tail -n 1)
0106   latestDir=$(curl -A "OnlineSyncDev/0.1" \
0107     --capath $X509_CERT_DIR --key $X509_USER_PROXY \
0108     --cert $X509_USER_PROXY \
0109     https://cmsweb.cern.ch/dqm/online/data/browse/Merged/ 2>&1|
0110     egrep -oe "<tr><td><a.*</a>" | egrep -o "'>.*<" | egrep -o "[0-9]{5}xxxx" |sort -r | head -n 1)
0111   dirs=({${latestLocalDir/xxxx/}..${latestDir/xxxx/}}xxxx)
0112   log "INFO: Starting partial synchronization of merged files"
0113   if [[ X$latestDir != X ]]
0114   then
0115     for d in $dirs
0116     do
0117       log "INFO: Synchronizing  https://cmsweb.cern.ch/dqm/online/data/browse/Merged/$d"
0118       rcurl Merged/$d 1
0119     done
0120     log "INFO: Finished partial synchronization of merged files"
0121   fi
0122   sleep $(( 3600 * 4 ))
0123 done