SiStripMonitorSummary/scripts/Monitor_NoiseRatios_V2.sh

0001 #!/bin/bash
0002
0003 date
0004 # needed to allow the loop on *.png without using "*.png" as value
0005 shopt -s nullglob
0006
0007 if [ $# -ne 2 ]; then
0008     afstokenchecker.sh "You have to provide a <DB> and a <FrontierPath>!!!"
0009     exit
0010 fi
0011
0012 afstokenchecker.sh "Starting execution of Monitor_NoiseRatios_V2 $1 $2"
0013
0014 #Example: DB=cms_orcoff_prod
0015 DB=$1
0016 #Example: ACCOUNT=CMS_COND_21X_STRIP
0017 ACCOUNT=CMS_CONDITIONS
0018 #Example: FRONTIER=FrontierProd
0019 FRONTIER=$2
0020 DBTAGCOLLECTION=DBTagsIn_${DB}_${ACCOUNT}.txt
0021 GLOBALTAGCOLLECTION=GlobalTagsForDBAccount.txt
0022 DBTAGDIR=DBTagCollection
0023 GLOBALTAGDIR=GlobalTags
0024 STORAGEDIR=CondDBMonitoring
0025 STORAGEPATH=/afs/cern.ch/cms/tracker/sistrcalib/WWW/$STORAGEDIR
0026 WORKDIR=$PWD
0027
0028 #Function to publish png pictures on the web. Will be used at the end of the script:
0029 CreateIndex ()
0030 {
0031     cp /afs/cern.ch/cms/tracker/sistrcalib/WWW/index_new.html .
0032
0033     COUNTER=0
0034     LASTUPDATE=`date`
0035
0036     for Plot in *.png; do
0037         if [[ $COUNTER%2 -eq 0 ]]; then
0038             cat >> index_new.html  << EOF
0039 <TR> <TD align=center> <a href="$Plot"><img src="$Plot"hspace=5 vspace=5 border=0 style="width: 90%" ALT="$Plot"></a>
0040   <br> $Plot </TD>
0041 EOF
0042         else
0043             cat >> index_new.html  << EOF
0044   <TD align=center> <a href="$Plot"><img src="$Plot"hspace=5 vspace=5 border=0 style="width: 90%" ALT="$Plot"></a>
0045   <br> $Plot </TD> </TR>
0046 EOF
0047         fi
0048
0049         let COUNTER++
0050     done
0051
0052     cat ${CMSSW_BASE}/src/DQM/SiStripMonitorSummary/data/template_index_foot.html | sed -e "s@insertDate@$LASTUPDATE@g" >> index_new.html
0053
0054     mv -f index_new.html index.html
0055 }
0056
0057 # Creation of all needed directories if not existing yet
0058 if [ ! -d "$STORAGEPATH/$DB" ]; then
0059     afstokenchecker.sh "Creating directory $STORAGEPATH/$DB"
0060     mkdir $STORAGEPATH/$DB;
0061 fi
0062
0063 if [ ! -d "$STORAGEPATH/$DB/$ACCOUNT" ]; then
0064     afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$ACCOUNT"
0065     mkdir $STORAGEPATH/$DB/$ACCOUNT;
0066 fi
0067
0068 if [ ! -d "$STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR" ]; then
0069     afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR"
0070     mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR;
0071 fi
0072
0073 # Array of already analyzed tags
0074
0075 declare -a checkedtags;
0076
0077 # Access of all Global Tags contained in the given DB account
0078 afstokenchecker.sh "Preparing list of Global Tags"
0079 rm -rf $GLOBALTAGCOLLECTION
0080 conddb --db $DB --nocolors listGTs | grep " GT " | awk '{print $1}' > $GLOBALTAGCOLLECTION
0081
0082 TAGSUBDIR=SiStripNoise
0083
0084 # Loop on all Global Tags
0085 for globaltag in `cat $GLOBALTAGCOLLECTION`; do
0086
0087     afstokenchecker.sh "Processing Global Tag $globaltag";
0088
0089     NEWTAG=False
0090     NEWIOV=False
0091     CFGISSAVED=False
0092     LOGDESTINATION=cout
0093
0094     NOISETAGANDOBJECT=`conddb --db $DB --nocolors list $globaltag | grep SiStripNoisesRcd`
0095
0096     if [ `echo $NOISETAGANDOBJECT | wc -w` -eq 0 ]; then
0097         continue
0098     fi
0099
0100     NOISETAG=`echo $NOISETAGANDOBJECT | awk '{print $3}'`
0101 #    NOISEOBJECT=`echo $NOISETAGANDOBJECT | awk '{print $2}' | sed -e "s@object:@@g"`
0102
0103     if [ `echo $NOISETAG | grep Ideal | wc -w` -gt 0 ]; then
0104         continue
0105     fi
0106
0107 # grep modified to get ONLY the SiStripApvGainRcd but not the SiStripApvGain2Rcd nor the SiStripApvGainSimRcd
0108     GAINTAGANDOBJECT=`conddb --db $DB --nocolors list $globaltag | grep SiStripApvGainRcd`
0109
0110     if [ `echo $GAINTAGANDOBJECT | wc -w` -eq 0 ]; then
0111         continue
0112     fi
0113
0114     GAINTAG=`echo $GAINTAGANDOBJECT | awk '{print $3}'`
0115 #    GAINOBJECT=`echo $GAINTAGANDOBJECT | awk '{print $2}' | sed -e "s@object:@@g"`
0116
0117     tag=${NOISETAG}_${GAINTAG}
0118     # check if $tag contains blank and if so, issue a warning
0119     if [ `expr index "$tag" " "` -ne 0 ]; then
0120         afstokenchecker.sh "WARNING!! $tag has blank spaces"
0121     fi
0122
0123
0124     # Creation of DB-Tag directory if not existing yet
0125     if [ ! -d "$STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR" ]; then
0126         afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR"
0127         mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR;
0128     fi
0129
0130     if [ ! -d "$STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios" ]; then
0131         afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios"
0132         mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios;
0133     fi
0134
0135     if [ ! -d "$STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag" ]; then
0136         afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag"
0137         mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag;
0138
0139         NEWTAG=True
0140     fi
0141
0142     if [ ! -d "$STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/RelatedGlobalTags" ]; then
0143         afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/RelatedGlobalTags"
0144         mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/RelatedGlobalTags;
0145     fi
0146
0147     # Creation of Global Tag directory if not existing yet
0148     if [ ! -d "$STORAGEPATH/$DB/$GLOBALTAGDIR" ]; then
0149         afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$GLOBALTAGDIR"
0150         mkdir $STORAGEPATH/$DB/$GLOBALTAGDIR;
0151     fi
0152
0153     if [ ! -d "$STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag" ]; then
0154         afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag"
0155         mkdir $STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag;
0156     fi
0157
0158     if [ ! -d "$STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag/NoiseRatios" ]; then
0159         afstokenchecker.sh "Creating directory $STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag/NoiseRatios"
0160         mkdir $STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag/NoiseRatios;
0161     fi
0162
0163     # Creation of links between the DB-Tag and the respective Global Tags
0164     if [ ! -f $STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag/NoiseRatios/$tag ] || [ ! -f $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/RelatedGlobalTags/$globaltag ]; then
0165         cd $STORAGEPATH/$DB/$GLOBALTAGDIR/$globaltag/NoiseRatios;
0166         rm -f $tag;
0167         cat >> $tag << EOF
0168 <html>
0169 <body>
0170 <a href="https://test-stripdbmonitor.web.cern.ch/test-stripdbmonitor/$STORAGEDIR/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag">https://test-stripdbmonitor.web.cern.ch/test-stripdbmonitor/$STORAGEDIR/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag</a>
0171 </body>
0172 </html>
0173 EOF
0174
0175         cd $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/RelatedGlobalTags;
0176         rm -f $globaltag;
0177         cat >> $globaltag << EOF
0178 <html>
0179 <body>
0180 <a href="https://test-stripdbmonitor.web.cern.ch/test-stripdbmonitor/$STORAGEDIR/$DB/$GLOBALTAGDIR/$globaltag">https://test-stripdbmonitor.web.cern.ch/test-stripdbmonitor/$STORAGEDIR/$DB/$GLOBALTAGDIR/$globaltag</a>
0181 </body>
0182 </html>
0183 EOF
0184
0185     fi
0186
0187     cd $WORKDIR;
0188
0189 # check if the tag has been analyzed already
0190
0191     ALREADYCHECKED=0;
0192
0193     for checkedtag in ${checkedtags[*]}; do
0194         if [ $checkedtag == $tag ]; then
0195             ALREADYCHECKED=1
0196         fi
0197     done
0198
0199     if [ $ALREADYCHECKED -eq 1 ]; then
0200         date "+[%c] Tags $tag already checked: skip"
0201         continue
0202     fi
0203
0204     checkedtags[${#checkedtags[*]}]=$tag;
0205
0206     # Get the list of IoVs for the given DB-Tag
0207     conddb --db $DB --nocolors list -L 5000 $NOISETAG | awk '{if(match($1,"[a-z]")!=0 || match($1,"-")!=0) {} else {print $1}}' > list_Iov.txt
0208
0209     # Access DB for the given DB-Tag and dump histograms in .png if not existing yet
0210     afstokenchecker.sh "Now the values of $tag are retrieved from the DB..."
0211
0212     if [ ! -d "$STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/rootfiles" ]; then
0213         mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/rootfiles;
0214         mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/cfg;
0215         mkdir $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/plots;
0216     fi
0217
0218 # replaced by the loop below
0219 #    if [ `ls *.png | wc -w` -gt 0 ]; then
0220 #       rm *.png;
0221 #    fi
0222
0223     for OldPlot in *.png; do
0224         rm $OldPlot;
0225     done;
0226
0227     # Process each IOV of the given DB-Tag seperately
0228     for IOV_number in `cat list_Iov.txt`; do
0229
0230         if [ $IOV_number -eq 1 ]; then
0231             FirstRun=$IOV_number
0232             continue
0233         fi
0234
0235         SecondRun=$IOV_number
0236
0237         ROOTFILE="${tag}_Run_${IOV_number}.root"
0238
0239         if [ -f $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/rootfiles/$ROOTFILE ]; then # Skip IOVs already processed. Take only new ones.
0240             FirstRun=$IOV_number
0241             continue
0242         fi
0243
0244         afstokenchecker.sh "New IOV $IOV_number found. Being processed..."
0245
0246         NEWIOV=True
0247
0248         afstokenchecker.sh "Executing cmsRun. Stay tuned ..."
0249
0250         CMSRUNCOMMAND="cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorSummary/test/SiStripCorrelateNoise_conddbmonitoring_cfg.py print connectionString=frontier://$FRONTIER/$ACCOUNT noiseTagName=$NOISETAG gainTagName=$GAINTAG firstRunNumber=$FirstRun secondRunNumber=$SecondRun"
0251         $CMSRUNCOMMAND
0252
0253         FirstRun=$IOV_number
0254
0255         afstokenchecker.sh "cmsRun finished. Now moving the files to the corresponding directories ..."
0256
0257 #       if [ "$NEWTAG" = "True" ] && [ "$CFGISSAVED" = "False" ]; then
0258         cp ${CMSSW_BASE}/src/DQM/SiStripMonitorSummary/test/SiStripCorrelateNoise_conddbmonitoring_cfg.py SiStripCorrelateNoise_cfg.py
0259         cat >> SiStripCorrelateNoise_cfg.py <<EOF
0260 #
0261 # $CMSRUNCOMMAND
0262 #
0263 EOF
0264         mv SiStripCorrelateNoise_cfg.py $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/cfg/${tag}_cfg.py
0265         CFGISSAVED=True
0266 #       fi
0267
0268         mv correlTest.root $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/rootfiles/$ROOTFILE;
0269
0270         rm out.log
0271
0272         for Plot in *.png; do
0273             mv $Plot $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/plots;
0274         done;
0275
0276
0277         #cd $WORKDIR;
0278
0279     done;
0280
0281     # Run the Trends and Publish all histograms on a web page
0282     if [ "$NEWTAG" = "True" ] || [ "$NEWIOV" = "True" ]; then
0283
0284         afstokenchecker.sh "Publishing the new tag $tag (or the new IOV) on the web ..."
0285
0286         cd /afs/cern.ch/cms/tracker/sistrcalib/WWW;
0287         cat ${CMSSW_BASE}/src/DQM/SiStripMonitorSummary/data/template_index_header.html | sed -e "s@insertPageName@Noise Ratios for $NOISETAG and $GAINTAG@g" > index_new.html
0288
0289         cd $STORAGEPATH/$DB/$ACCOUNT/$DBTAGDIR/$TAGSUBDIR/NoiseRatios/$tag/plots;
0290         CreateIndex
0291
0292     fi
0293
0294     cd $WORKDIR;
0295
0296 done;