SiStripMonitorSummary/scripts/Monitor_NoiseRatios.sh

0001 #!/bin/bash
0002
0003 date
0004 # needed to allow the loop on *.png without using "*.png" as value
0005 shopt -s nullglob
0006
0007 if [ $# -ne 4 ]; then
0008     afstokenchecker.sh "You have to provide a <DB>, an <Account>, a <GTAccount> and the <FrontierPath> !!!"
0009     exit
0010 fi
0011
0012 afstokenchecker.sh "Starting execution of Monitor_NoiseRatios $1 $2 $3 $4"
0013
0014 #Example: DB=cms_orcoff_prod
0015 DB=$1
0016 #Example: ACCOUNT=CMS_COND_21X_STRIP
0017 ACCOUNT=$2
0018 #Example: GTACCOUNT=CMS_COND_21X_GLOBALTAG
0019 GTACCOUNT=$3
0020 #Example: FRONTIER=FrontierProd
0021 FRONTIER=$4
0022 DBTAGCOLLECTION=DBTagsIn_${DB}_${ACCOUNT}.txt
0023 GLOBALTAGCOLLECTION=GlobalTagsForDBAccount.txt
0024 DBTAGDIR=DBTagCollection
0025 GLOBALTAGDIR=GlobalTags
0026 STORAGEPATH=/afs/cern.ch/cms/tracker/sistrcalib/WWW/CondDBMonitoring
0027 WORKDIR=$PWD
0028
0029 #Function to publish png pictures on the web. Will be used at the end of the script:
0030 CreateIndex ()
0031 {
0032     cp /afs/cern.ch/cms/tracker/sistrcalib/WWW/index_new.html .
0033
0034     COUNTER=0
0035     LASTUPDATE=`date`
0036
0037     for Plot in *.png; do
0038         if [[ $COUNTER%2 -eq 0 ]]; then
0039             cat >> index_new.html  << EOF
0040 <TR> <TD align=center> <a href="$Plot"><img src="$Plot"hspace=5 vspace=5 border=0 style="width: 90%" ALT="$Plot"></a>
0041   <br> $Plot </TD>
0042 EOF
0043         else
0044             cat >> index_new.html  << EOF
0045   <TD align=center> <a href="$Plot"><img src="$Plot"hspace=5 vspace=5 border=0 style="width: 90%" ALT="$Plot"></a>
0046   <br> $Plot </TD> </TR>
0047 EOF
0048         fi
0049
0050         let COUNTER++
0051     done
0052
0053     cat ${CMSSW_BASE}/src/DQM/SiStripMonitorSummary/data/template_index_foot.html | sed -e "s@insertDate@$LASTUPDATE@g" >> index_new.html
0054
0055     mv -f index_new.html index.html
0056 }
0057
0058 # Creation of all needed directories if not existing yet
0059 if [ ! -d "$STORAGEPATH/$DB" ]; then
0060     afstokenchecker.sh "Creating directory $STORAGEPATH/$DB"
0061     mkdir $STORAGEPATH/$DB;
0062 fi
0063
0064 if [ ! -d "$STORAGEPATH/$DB/$ACCOUNT" ]; then
0065     afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$ACCOUNT"
0066     mkdir $STORAGEPATH/$DB/$ACCOUNT;
0067 fi
0068
0069 if [ ! -d "$STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR" ]; then
0070     afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR"
0071     mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR;
0072 fi
0073
0074 # Array of already analyzed tags
0075
0076 declare -a checkedtags;
0077
0078 # Access of all Global Tags contained in the given DB account
0079 cmscond_tagtree_list -c frontier://cmsfrontier.cern.ch:8000/$FRONTIER/$GTACCOUNT -P /afs/cern.ch/cms/DB/conddb | grep tree | awk '{print $2}' > $GLOBALTAGCOLLECTION
0080
0081 TAGSUBDIR=SiStripNoise
0082
0083 # Loop on all Global Tags
0084 for globaltag in `cat $GLOBALTAGCOLLECTION`; do
0085
0086     afstokenchecker.sh "Processing Global Tag $globaltag";
0087
0088     NEWTAG=False
0089     NEWIOV=False
0090     CFGISSAVED=False
0091     LOGDESTINATION=cout
0092
0093     NOISETAGANDOBJECT=`cmscond_tagtree_list -c frontier://cmsfrontier.cern.ch:8000/$FRONTIER/$GTACCOUNT -P /afs/cern.ch/cms/DB/conddb -T $globaltag | grep SiStripNoise | grep $ACCOUNT | awk '{printf "%s %s",$3,$5}'`
0094
0095     if [ `echo $NOISETAGANDOBJECT | wc -w` -eq 0 ]; then
0096         continue
0097     fi
0098
0099     NOISETAG=`echo $NOISETAGANDOBJECT | awk '{print $1}' | sed -e "s@tag:@@g"`
0100     NOISEOBJECT=`echo $NOISETAGANDOBJECT | awk '{print $2}' | sed -e "s@object:@@g"`
0101
0102     if [ `echo $NOISETAG | grep Ideal | wc -w` -gt 0 ]; then
0103         continue
0104     fi
0105
0106 # grep modified to get ONLY the SiStripApvGainRcd but not the SiStripApvGain2Rcd nor the SiStripApvGainSimRcd
0107     GAINTAGANDOBJECT=`cmscond_tagtree_list -c frontier://cmsfrontier.cern.ch:8000/$FRONTIER/$GTACCOUNT -P /afs/cern.ch/cms/DB/conddb -T $globaltag | grep "record:SiStripApvGainRcd" | grep $ACCOUNT | awk '{printf "%s %s",$3,$5}'`
0108
0109     if [ `echo $GAINTAGANDOBJECT | wc -w` -eq 0 ]; then
0110         continue
0111     fi
0112
0113     GAINTAG=`echo $GAINTAGANDOBJECT | awk '{print $1}' | sed -e "s@tag:@@g"`
0114     GAINOBJECT=`echo $GAINTAGANDOBJECT | awk '{print $2}' | sed -e "s@object:@@g"`
0115
0116     tag=${NOISETAG}_${GAINTAG}
0117     # check if $tag contains blank and if so, issue a warning
0118     if [ `expr index "$tag" " "` -ne 0 ]; then
0119         afstokenchecker.sh "WARNING!! $tag has blank spaces"
0120     fi
0121
0122
0123     # Creation of DB-Tag directory if not existing yet
0124     if [ ! -d "$STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR" ]; then
0125         afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR"
0126         mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR;
0127     fi
0128
0129     if [ ! -d "$STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios" ]; then
0130         afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios"
0131         mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios;
0132     fi
0133
0134     if [ ! -d "$STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag" ]; then
0135         afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag"
0136         mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag;
0137
0138         NEWTAG=True
0139     fi
0140
0141     if [ ! -d "$STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/RelatedGlobalTags" ]; then
0142         afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/RelatedGlobalTags"
0143         mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/RelatedGlobalTags;
0144     fi
0145
0146     # Creation of Global Tag directory if not existing yet
0147     if [ ! -d "$STORAGEPATH/$DB/$GLOBALTAGDIR" ]; then
0148         afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$GLOBALTAGDIR"
0149         mkdir $STORAGEPATH/$DB/$GLOBALTAGDIR;
0150     fi
0151
0152     if [ ! -d "$STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag" ]; then
0153         afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag"
0154         mkdir $STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag;
0155     fi
0156
0157     if [ ! -d "$STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag/NoiseRatios" ]; then
0158         afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag/NoiseRatios"
0159         mkdir $STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag/NoiseRatios;
0160     fi
0161
0162     # Creation of links between the DB-Tag and the respective Global Tags
0163     if [ ! -f $STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag/NoiseRatios/$tag ] || [ ! -f $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/RelatedGlobalTags/$globaltag ]; then
0164         cd $STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag/NoiseRatios;
0165         rm -f $tag;
0166         cat >> $tag << EOF
0167 <html>
0168 <body>
0169 <a href="https://test-stripdbmonitor.web.cern.ch/test-stripdbmonitor/CondDBMonitoring/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag">https://test-stripdbmonitor.web.cern.ch/test-stripdbmonitor/CondDBMonitoring/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag</a>
0170 </body>
0171 </html>
0172 EOF
0173
0174         cd $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/RelatedGlobalTags;
0175         rm -f $globaltag;
0176         cat >> $globaltag << EOF
0177 <html>
0178 <body>
0179 <a href="https://test-stripdbmonitor.web.cern.ch/test-stripdbmonitor/CondDBMonitoring/$DB/$GLOBALTAGDIR/$globaltag">https://test-stripdbmonitor.web.cern.ch/test-stripdbmonitor/CondDBMonitoring/$DB/$GLOBALTAGDIR/$globaltag</a>
0180 </body>
0181 </html>
0182 EOF
0183
0184     fi
0185
0186     cd $WORKDIR;
0187
0188 # check if the tag has been analyzed already
0189
0190     ALREADYCHECKED=0;
0191
0192     for checkedtag in ${checkedtags[*]}; do
0193         if [ $checkedtag == $tag ]; then
0194             ALREADYCHECKED=1
0195         fi
0196     done
0197
0198     if [ $ALREADYCHECKED -eq 1 ]; then
0199         date "+[%c] Tags $tag already checked: skip"
0200         continue
0201     fi
0202
0203     checkedtags[${#checkedtags[*]}]=$tag;
0204
0205     # Get the list of IoVs for the given DB-Tag
0206     iov_list_tag.py -c frontier://cmsfrontier.cern.ch:8000/$FRONTIER/$ACCOUNT -P /afs/cern.ch/cms/DB/conddb -t $NOISETAG > list_Iov.txt # Access via Frontier
0207
0208     # Access DB for the given DB-Tag and dump histograms in .png if not existing yet
0209     afstokenchecker.sh "Now the values of $tag are retrieved from the DB..."
0210
0211     if [ ! -d "$STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/rootfiles" ]; then
0212         mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/rootfiles;
0213         mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/cfg;
0214         mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/plots;
0215     fi
0216
0217 # replaced by the loop below
0218 #    if [ `ls *.png | wc -w` -gt 0 ]; then
0219 #       rm *.png;
0220 #    fi
0221
0222     for OldPlot in *.png; do
0223         rm $OldPlot;
0224     done;
0225
0226     # Process each IOV of the given DB-Tag seperately
0227     for IOV_number in `cat list_Iov.txt`; do
0228
0229         if [ $IOV_number -eq 1 ]; then
0230             FirstRun=$IOV_number
0231             continue
0232         fi
0233
0234         SecondRun=$IOV_number
0235
0236         ROOTFILE="${tag}_Run_${IOV_number}.root"
0237
0238         if [ -f $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/rootfiles/$ROOTFILE ]; then # Skip IOVs already processed. Take only new ones.
0239             FirstRun=$IOV_number
0240             continue
0241         fi
0242
0243         afstokenchecker.sh "New IOV $IOV_number found. Being processed..."
0244
0245         NEWIOV=True
0246
0247         afstokenchecker.sh "Executing cmsRun. Stay tuned ..."
0248
0249         CMSRUNCOMMAND="cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorSummary/test/SiStripCorrelateNoise_conddbmonitoring_cfg.py print connectionString=frontier://$FRONTIER/$ACCOUNT noiseTagName=$NOISETAG gainTagName=$GAINTAG firstRunNumber=$FirstRun secondRunNumber=$SecondRun"
0250         $CMSRUNCOMMAND
0251
0252         FirstRun=$IOV_number
0253
0254         afstokenchecker.sh "cmsRun finished. Now moving the files to the corresponding directories ..."
0255
0256 #       if [ "$NEWTAG" = "True" ] && [ "$CFGISSAVED" = "False" ]; then
0257         cp ${CMSSW_BASE}/src/DQM/SiStripMonitorSummary/test/SiStripCorrelateNoise_conddbmonitoring_cfg.py SiStripCorrelateNoise_cfg.py
0258         cat >> SiStripCorrelateNoise_cfg.py <<EOF
0259 #
0260 # $CMSRUNCOMMAND
0261 #
0262 EOF
0263         mv SiStripCorrelateNoise_cfg.py $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/cfg/${tag}_cfg.py
0264         CFGISSAVED=True
0265 #       fi
0266
0267         mv correlTest.root $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/rootfiles/$ROOTFILE;
0268
0269         rm out.log
0270
0271         for Plot in *.png; do
0272             mv $Plot $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/plots;
0273         done;
0274
0275
0276         #cd $WORKDIR;
0277
0278     done;
0279
0280     # Run the Trends and Publish all histograms on a web page
0281     if [ "$NEWTAG" = "True" ] || [ "$NEWIOV" = "True" ]; then
0282
0283         afstokenchecker.sh "Publishing the new tag $tag (or the new IOV) on the web ..."
0284
0285         cd /afs/cern.ch/cms/tracker/sistrcalib/WWW;
0286         cat ${CMSSW_BASE}/src/DQM/SiStripMonitorSummary/data/template_index_header.html | sed -e "s@insertPageName@Noise Ratios for $NOISETAG and $GAINTAG@g" > index_new.html
0287
0288         cd $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/plots;
0289         CreateIndex
0290
0291     fi
0292
0293     cd $WORKDIR;
0294
0295 done;