scripts/harvesting_tools/Upload.sh

0001 #!/bin/zsh
0002
0003 LOCK=upload.lock
0004 if [ -e $LOCK ]; then
0005  echo An update is running with pid $(cat $LOCK)
0006  echo Remove the lock file $LOCK if the job crashed
0007  exit
0008 else
0009  echo $$ > $LOCK
0010 fi
0011
0012 date=`date`
0013
0014
0015 echo
0016 echo "============================"
0017 echo "running Upload at" $date
0018 echo
0019 echo "certificate:"
0020 source /afs/cern.ch/cms/LCG/LCG-2/UI/cms_ui_env.sh
0021 X509_USER_PROXY=$HOME/x509up
0022 #
0023 # note: before installing cronjob, run voms-proxy-init -hours=100000 with X509_USER_PROXY set to sth not on /tmp
0024 #
0025 voms-proxy-info
0026
0027 echo
0028 echo "----------------------------"
0029 for i in "data/dqmoffline" "mc/mc"
0030 do
0031 echo
0032 echo "==== Upload of new files in" $i
0033 Castordir=/castor/cern.ch/cms/store/temp/dqm/offline/harvesting_output/$i
0034 for CMSSW in `nsls $Castordir`;
0035 do
0036 CMSSWdir=$Castordir/$CMSSW
0037 for dataset in `nsls $CMSSWdir`
0038 do
0039 datasetdir=$CMSSWdir/$dataset
0040 echo $CMSSW ":" $dataset
0041 for run in `nsls $datasetdir`;
0042 do
0043 rundir=$datasetdir/$run
0044 for nevents in `nsls $rundir`;
0045 do
0046 neventsdir=$rundir/$nevents
0047 for section in `nsls $neventsdir`;
0048 do
0049 sectiondir=$neventsdir/$section
0050 for file in `nsls $sectiondir`;
0051 do
0052 rootfile=$sectiondir/$file
0053 size=`rfstat $rootfile | grep Size | perl -pe 's/Size \(bytes\)    \: //'`
0054 if [ $size -ne 0 ];
0055 then
0056 ## Definition of ffile changed due to upgrade to crab 2_7_5
0057 ##ffile=DQM_V0$(echo $rootfile | perl -pe 's/.*\/DQM_V0// ; s/_1.root/.root/ ; s/_2.root/.root/; s/_3.root/.root/ ; s/_4.root/.root/ ; s/_5.root/.root/ ; s/_1.root/.root/')
0058 ffile=DQM_V0$(echo $rootfile | perl -pe 's/.*\/DQM_V0// ; s/__DQM.*/__DQM.root/')
0059 file_test=`grep -c "$ffile" upload_bookkeeping.txt`
0060 if [ $file_test -eq 0 ];
0061 then
0062 rfcp $rootfile /tmp/$ffile
0063 if [ `echo $?` != 0 ];
0064 then
0065 break
0066 else
0067 ../VisMonitoring/DQMServer/scripts/visDQMUpload https://cmsweb.cern.ch/dqm/offline /tmp/$ffile
0068 if [ `echo $?` != 0 ];
0069 then
0070 echo "------------------------------------------------------------------------------------------------"
0071 echo /tmp/$ffile could not be uploaded
0072 echo "------------------------------------------------------------------------------------------------"
0073 rm -f /tmp/$ffile
0074 break
0075 else
0076 echo "------------------------------------------------------------------------------------------------"
0077 echo $ffile is uploaded
0078 echo "------------------------------------------------------------------------------------------------"
0079 echo $ffile >> upload_bookkeeping.txt
0080
0081
0082 dataset_test=`grep -c "$dataset" dataset_bookkeeping.txt`
0083 if [ $dataset_test -eq 0 ];
0084 then
0085 echo $dataset >> dataset_bookkeeping.txt
0086 echo $dataset
0087 fi
0088
0089 fi
0090 rm -f $ffile
0091 fi
0092 fi
0093 fi
0094 done
0095 done
0096 done
0097 done
0098 done
0099 done
0100 done
0101
0102 cp upload_bookkeeping.txt /afs/cern.ch/user/n/npietsch/harvesting_backup/upload_bookkeeping_backup.txt
0103
0104 cp dataset_bookkeeping.txt /afs/cern.ch/user/n/npietsch/harvesting_backup/dataset_bookkeeping_backup.txt
0105
0106 echo "done upload"
0107 echo
0108 echo "============================"
0109 echo "... done Upload of" $date
0110 echo "                at" `date`
0111 echo
0112
0113 rm -f $LOCK