|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
|
#!/bin/zsh
LOCK=upload.lock
if [ -e $LOCK ]; then
echo An update is running with pid $(cat $LOCK)
echo Remove the lock file $LOCK if the job crashed
exit
else
echo $$ > $LOCK
fi
date=`date`
echo
echo "============================"
echo "running Upload at" $date
echo
echo "certificate:"
source /afs/cern.ch/cms/LCG/LCG-2/UI/cms_ui_env.sh
X509_USER_PROXY=$HOME/x509up
#
# note: before installing cronjob, run voms-proxy-init -hours=100000 with X509_USER_PROXY set to sth not on /tmp
#
voms-proxy-info
echo
echo "----------------------------"
for i in "data/dqmoffline" "mc/mc"
do
echo
echo "==== Upload of new files in" $i
Castordir=/castor/cern.ch/cms/store/temp/dqm/offline/harvesting_output/$i
for CMSSW in `nsls $Castordir`;
do
CMSSWdir=$Castordir/$CMSSW
for dataset in `nsls $CMSSWdir`
do
datasetdir=$CMSSWdir/$dataset
echo $CMSSW ":" $dataset
for run in `nsls $datasetdir`;
do
rundir=$datasetdir/$run
for nevents in `nsls $rundir`;
do
neventsdir=$rundir/$nevents
for section in `nsls $neventsdir`;
do
sectiondir=$neventsdir/$section
for file in `nsls $sectiondir`;
do
rootfile=$sectiondir/$file
size=`rfstat $rootfile | grep Size | perl -pe 's/Size \(bytes\) \: //'`
if [ $size -ne 0 ];
then
## Definition of ffile changed due to upgrade to crab 2_7_5
##ffile=DQM_V0$(echo $rootfile | perl -pe 's/.*\/DQM_V0// ; s/_1.root/.root/ ; s/_2.root/.root/; s/_3.root/.root/ ; s/_4.root/.root/ ; s/_5.root/.root/ ; s/_1.root/.root/')
ffile=DQM_V0$(echo $rootfile | perl -pe 's/.*\/DQM_V0// ; s/__DQM.*/__DQM.root/')
file_test=`grep -c "$ffile" upload_bookkeeping.txt`
if [ $file_test -eq 0 ];
then
rfcp $rootfile /tmp/$ffile
if [ `echo $?` != 0 ];
then
break
else
../VisMonitoring/DQMServer/scripts/visDQMUpload https://cmsweb.cern.ch/dqm/offline /tmp/$ffile
if [ `echo $?` != 0 ];
then
echo "------------------------------------------------------------------------------------------------"
echo /tmp/$ffile could not be uploaded
echo "------------------------------------------------------------------------------------------------"
rm -f /tmp/$ffile
break
else
echo "------------------------------------------------------------------------------------------------"
echo $ffile is uploaded
echo "------------------------------------------------------------------------------------------------"
echo $ffile >> upload_bookkeeping.txt
dataset_test=`grep -c "$dataset" dataset_bookkeeping.txt`
if [ $dataset_test -eq 0 ];
then
echo $dataset >> dataset_bookkeeping.txt
echo $dataset
fi
fi
rm -f $ffile
fi
fi
fi
done
done
done
done
done
done
done
cp upload_bookkeeping.txt /afs/cern.ch/user/n/npietsch/harvesting_backup/upload_bookkeeping_backup.txt
cp dataset_bookkeeping.txt /afs/cern.ch/user/n/npietsch/harvesting_backup/dataset_bookkeeping_backup.txt
echo "done upload"
echo
echo "============================"
echo "... done Upload of" $date
echo " at" `date`
echo
rm -f $LOCK
|