File indexing completed on 2024-04-06 12:07:43
0001
0002 BASE_DIR=/dqmdata/offline/repository/data/OnlineData
0003 X509_CERT_DIR=/etc/grid-security/certificates
0004 X509_USER_PROXY=/dqmdata/auth/proxy.pem
0005
0006
0007
0008
0009
0010 Counter=0
0011 log(){
0012 echo $(date +"%F %T") \[OnlineSynchronyzer/$$\] $1
0013 }
0014 rcurl(){
0015 [[ -z $2 ]] && removeTree=0 || removeTree=$2
0016 if [[ $removeTree -eq 0 ]]
0017 then
0018 rootDir=$1
0019 remoteDirBase="dqm/online/data/browse"
0020 else
0021 [[ ${(w)
0022 [[ ${(w)
0023 fi
0024 host="https://cmsweb.cern.ch"
0025 [[ -e $rootDir/index.html ]] && dirTimeStamp=$(date -d "$(curl -A "OnlineSyncDev/0.1" -s -I \
0026 --capath $X509_CERT_DIR/ \
0027 --key $X509_USER_PROXY \
0028 --cert $X509_USER_PROXY \
0029 $host/$remoteDirBase/$rootDir/ | grep "Last-Modified:.*$" | sed -e "s|Last-Modified: ||" )" +%s) || dirTimeStamp=$(date +%s)
0030 if [[ ! -e $rootDir/index.html || $dirTimeStamp -gt $(stat $rootDir/index.html -c %Y) ]]
0031 then
0032 echo Fetching $rootDir content
0033 curl -A "OnlineSyncDev/0.1" -s -R -o $rootDir/index.html --create-dirs \
0034 --capath $X509_CERT_DIR/ \
0035 --key $X509_USER_PROXY \
0036 --cert $X509_USER_PROXY \
0037 $host/$remoteDirBase/$rootDir/
0038 fi
0039 dirs=($(<$rootDir/index.html | egrep -oe "<tr><td><a.*</a>" | egrep -o "href='([^']*)'" | egrep -v "\.root" | sed -re "s/(href='\/|\/'$)//g" | sort -r ))
0040 files=($(<$rootDir/index.html | egrep -oe "<tr><td><a.*</a>" | egrep -o "href='([^']*)'" | egrep "\.root" | sed -re "s/(href='\/|'$)//g" | sort -r ))
0041 for d in $dirs
0042 do
0043 rcurl ${d/dqm\/online\/data\/browse\//} $removeTree
0044 done
0045 for f in $files
0046 do
0047 [[ -e $rootDir/$(basename $f) ]] && fTimeStamp=$(date -d "$(curl -A "OnlineSyncDev/0.1" -s -I \
0048 --capath $X509_CERT_DIR/ \
0049 --key $X509_USER_PROXY \
0050 --cert $X509_USER_PROXY \
0051 $host/$remoteDirBase/$rootDir/$(basename $f ) | grep "Last-Modified:.*$" | sed -e "s|Last-Modified: ||" )" +%s) || fTimeStamp=$(date +%s)
0052 if [[ ! -e $rootDir/$(basename $f) || $fTimeStamp -gt $(stat $rootDir/$(basename $f) -c %Y) ]]
0053 then
0054 curl -A "OnlineSyncDev/0.1" -s -R -o $rootDir/$(basename $f) --create-dirs \
0055 --capath $X509_CERT_DIR/ \
0056 --key $X509_USER_PROXY \
0057 --cert $X509_USER_PROXY \
0058 $host/$f && log "INFO: Successfully downloaded "$PWD/$rootDir/$(basename $f) || \
0059 log "ERROR: Unable to download "$PWD/$rootDir/$(basename $f)
0060 fi
0061
0062 done
0063
0064 }
0065 while [ 1 ]
0066 do
0067 if [[ $Counter -gt 7 ]]
0068 then
0069 Counter=0
0070
0071
0072
0073
0074
0075
0076
0077
0078
0079
0080
0081
0082
0083
0084 continue
0085 fi
0086 Counter=$(( $Counter + 1 ))
0087 cd $BASE_DIR/original
0088 latestLocalDir=$(find $BASE_DIR/original -maxdepth 1 -type d -exec basename {} \; | sort -n | tail -n 1)
0089 latestDir=$(curl -A "OnlineSyncDev/0.1" \
0090 --capath $X509_CERT_DIR --key $X509_USER_PROXY \
0091 --cert $X509_USER_PROXY \
0092 https://cmsweb.cern.ch/dqm/online/data/browse/Original/ 2>&1 \
0093 | egrep -oe "<tr><td><a.*</a>" | egrep -o "'>.*<" | egrep -o "[0-9]{5}xxxx" |sort -r | head -n 1)
0094 [[ -z $latestDir ]] && continue
0095 dirs=({${latestLocalDir/xxxx/}..${latestDir/xxxx/}}xxxx)
0096
0097 for d in $dirs
0098 do
0099
0100 rcurl Original/$d 1
0101 done
0102
0103
0104 cd $BASE_DIR/merged
0105 latestLocalDir=$(find $BASE_DIR/merged -maxdepth 1 -type d -exec basename {} \; | sort -n | tail -n 1)
0106 latestDir=$(curl -A "OnlineSyncDev/0.1" \
0107 --capath $X509_CERT_DIR --key $X509_USER_PROXY \
0108 --cert $X509_USER_PROXY \
0109 https://cmsweb.cern.ch/dqm/online/data/browse/Merged/ 2>&1|
0110 egrep -oe "<tr><td><a.*</a>" | egrep -o "'>.*<" | egrep -o "[0-9]{5}xxxx" |sort -r | head -n 1)
0111 dirs=({${latestLocalDir/xxxx/}..${latestDir/xxxx/}}xxxx)
0112 log "INFO: Starting partial synchronization of merged files"
0113 if [[ X$latestDir != X ]]
0114 then
0115 for d in $dirs
0116 do
0117 log "INFO: Synchronizing https://cmsweb.cern.ch/dqm/online/data/browse/Merged/$d"
0118 rcurl Merged/$d 1
0119 done
0120 log "INFO: Finished partial synchronization of merged files"
0121 fi
0122 sleep $(( 3600 * 4 ))
0123 done