scripts/cmt/GLOBAL_EOS.sh

0001 #!/bin/bash
0002
0003 WebDir='/store/group/dpg_hcal/comm_hcal/www/HcalRemoteMonitoring'
0004 WebSite='https://cms-conddb.cern.ch/eosweb/hcal/HcalRemoteMonitoring'
0005 HistoDir='/store/group/dpg_hcal/comm_hcal/www/HcalRemoteMonitoring/CMT/histos'
0006 eos='/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select'
0007
0008 # print usage info
0009 if [[ "$1" == "" ]]; then
0010   echo "Usage:"
0011   echo "  $0 file [comment] [-ignore-file] [-das-cache]"
0012   echo "    file  -- a file with run numbers"
0013   echo "    comment  -- add a comment line (instead of spaces use '_')"
0014   echo "    -ignore-file   -- skips production of run html pages. Produces"
0015   echo "                      only the global page. File name is not needed."
0016   echo "    -das-cache   -- whether to save DAS information locally for a reuse"
0017   echo
0018   echo "example: ./GLOBAL.sh Run_List.txt"
0019   exit 1
0020 fi
0021
0022 cmsenv 2>/dev/null
0023 if [ $? == 0 ] ; then
0024     eval `scramv1 runtime -sh`
0025 fi
0026 temp_var=`ls ${eos}`
0027 status="$?"
0028 echo "using eos command <${temp_var}>"
0029 if [ ! ${status} -eq 0 ] ; then
0030     echo "failed to find eos command"
0031     # exit 1
0032 fi
0033
0034
0035 # create log directory
0036 LOG_DIR="dir-Logs"
0037 if [ ! -d ${LOG_DIR} ] ; then mkdir ${LOG_DIR}; fi
0038 rm -f ${LOG_DIR}/*
0039
0040
0041 # Process arguments and set the flags
0042 fileName=$1
0043 comment=$2
0044 if [ ${#comment} -gt 0 ] && [ "${comment:0:1}" == "-" ] ; then comment=""; fi
0045 ignoreFile=0
0046 debug=0
0047 dasCache=0
0048 DAS_DIR="d-DAS-info"
0049
0050 for a in $@ ; do
0051     if [ "$a" == "-ignore-file" ] ; then
0052         echo " ** file will be ignored"
0053         fileName=""
0054         ignoreFile=1
0055     elif [ "$a" == "-das-cache" ] ; then
0056         echo " ** DAS cache ${DAS_DIR} enabled"
0057         dasCache=1
0058         if [ ! -d ${DAS_DIR} ] ; then mkdir ${DAS_DIR}; fi
0059     else
0060         temp_var=${a/-debug/}
0061         if [ ${#a} -gt ${#temp_var} ] ; then
0062             debug=${temp_var}
0063             echo " ** debug detected (debug=${debug})"
0064         fi
0065     fi
0066 done
0067
0068 # Obtain the runList from a file, if needed
0069 runList=""
0070 if [ ${#fileName} -gt 0 ] ; then
0071   if [ -s ${fileName} ] ; then
0072       runList=`cat ${fileName}`
0073   else
0074       echo "<${fileName}> does not seem to be a valid file"
0075       exit 2
0076   fi
0077 else
0078     if [ ${ignoreFile} -eq 0 ] ; then
0079         echo " ! no file provided"
0080     fi
0081     echo " ! will produce only the global html page"
0082 fi
0083
0084
0085 # Check the runList and correct the correctables
0086 # Replace ',' and ';' by empty spaces
0087 runList=`echo "${runList}" | sed 'sk,k\ kg' | sed 'sk;k\ kg'`
0088 ok=1
0089 for r in ${runList} ; do
0090     if [ ! ${#r} -eq 6 ] ; then
0091         echo "run numbers are expected to be of length 6. Check <$r>"
0092         ok=0
0093     fi
0094     debug_loc=0
0095     if [ "$r" -eq "$r" ] 2>/dev/null ; then
0096         if [ ${debug_loc} -eq 1 ] ; then echo "run variable <$r> is a number (ok)"; fi
0097     else
0098         echo "error: run variable <$r> is not an integer number"
0099         ok=0
0100     fi
0101 done
0102
0103 echo "Tested `wc -w <<< "${runList}"` runs from file ${fileName}"
0104 if [ ${ok} -eq 0 ] ; then
0105     echo "errors in the file ${fileName} with run numbers"
0106     exit 3
0107 else
0108     if [ ${#fileName} -gt 0 ] ; then
0109         echo "run numbers in ${fileName} verified ok"
0110     fi
0111 fi
0112
0113 comment=`echo ${comment} | sed sk\_k\ kg`
0114 if [ ${#comment} -gt 0 ] ; then
0115     echo "comment \"${comment}\" will be added to the pages"
0116 fi
0117
0118 if [ ${debug} -eq 3 ] ; then exit; fi
0119
0120
0121 echo
0122 echo
0123 echo
0124 echo 'Run numbers for processing'
0125 echo "${runList}"
0126 echo -e "list complete\n"
0127
0128 #processing
0129
0130 for i in ${runList} ; do
0131     runnumber=$i
0132
0133     logFile="${LOG_DIR}/log_${runnumber}.out"
0134     rm -f ${logFile}
0135
0136 # if [[ "$runnumber" > 233890 ]] ; then
0137     echo
0138     echo
0139     echo
0140     echo  "Run for processing $runnumber"
0141     echo  "always copy root file from /eos !!!"
0142     echo  "file=root://eoscms//cms/$HistoDir/Global_$runnumber.root"
0143 # always copy root file from /eos !!!
0144 ##    if [ ! -s Global_${runnumber}.root ] ; then
0145 ##      xrdcp root://eoscms//eos/cms/$HistoDir/Global_$runnumber.root Global_$runnumber.root
0146         xrdcp -f root://eoscms//eos/cms/$HistoDir/Global_$runnumber.root Global_$runnumber.root
0147         status="$?"
0148         if [ ! ${status} -eq 0 ] ; then
0149             echo "failed to get file Global_${runnumber}.root"
0150             exit 2
0151         fi
0152 ##    fi
0153
0154     #CMT processing
0155     echo -e "\nRemoteMonitoringGLOBAL\n" >> ${logFile}
0156     ./RemoteMonitoringGLOBAL.cc.exe Global_$runnumber.root 2>&1 | tee -a ${logFile}
0157     if [ ! $? -eq 0 ] ; then
0158         echo "GLOBAL processing failed"
0159         exit 2
0160     fi
0161
0162 #    if [ ! -s HELP.html ] ; then
0163 #       echo "GLOBAL failure was not detected. HELP.html is missing"
0164 #       exit 2
0165 #    fi
0166
0167
0168     local_WebDir=dir-CMT-GLOBAL_${runnumber}
0169     rm -rf ${local_WebDir}
0170     if [ ! -d ${local_WebDir} ] ; then mkdir ${local_WebDir}; fi
0171     for j in $(ls -r *.html); do
0172         cat $j | sed 's#cms-cpt-software.web.cern.ch\/cms-cpt-software\/General\/Validation\/SVSuite#cms-conddb.cern.ch\/eosweb\/hcal#g' \
0173             > ${local_WebDir}/$j
0174     done
0175     cp *.png ${local_WebDir}
0176     cp HELP.html ${local_WebDir}
0177     files=`cd ${local_WebDir}; ls`
0178     #echo "CMT files=${files}"
0179
0180     if [ ${debug} -eq 0 ] ; then
0181         cmsMkdir $WebDir/CMT/GLOBAL_$runnumber
0182         if [ ! $? -eq 0 ] ; then
0183             echo "CMT cmsMkdir failed"
0184             exit 2
0185         fi
0186         for f in ${files} ; do
0187             echo "cmsStage -f ${local_WebDir}/${f} $WebDir/CMT/GLOBAL_$runnumber/${f}"
0188             cmsStage -f ${local_WebDir}/${f} $WebDir/CMT/GLOBAL_$runnumber/${f}
0189             if [ ! $? -eq 0 ] ; then
0190                 echo "CMT cmsStage failed for ${f}"
0191                 exit 2
0192             fi
0193         done
0194     else
0195         # debuging
0196         echo "debugging: files are not copied to EOS"
0197     fi
0198
0199     rm *.html
0200     rm *.png
0201
0202
0203
0204     #GlobalRMT processing
0205     echo -e "\nRemoteMonitoringMAP_Global\n" >> ${logFile}
0206     ./RemoteMonitoringMAP_Global.cc.exe Global_$runnumber.root Global_$runnumber.root 2>&1 | tee -a ${logFile}
0207     if [ ! $? -eq 0 ] ; then
0208         echo "MAP_Global processing failed"
0209         exit 2
0210     fi
0211
0212 #    if [ ! -s HELP.html ] ; then
0213 #       echo "MAP_Global failure was not detected. HELP.html is missing"
0214 #       exit 2
0215 #    fi
0216
0217     local_WebDir=dir-GlobalRMT-GLOBAL_${runnumber}
0218     rm -rf ${local_WebDir}
0219     if [ ! -d ${local_WebDir} ] ; then mkdir ${local_WebDir}; fi
0220     for j in $(ls -r *.html); do
0221         cat $j | sed 's#cms-cpt-software.web.cern.ch\/cms-cpt-software\/General\/Validation\/SVSuite#cms-conddb.cern.ch\/eosweb\/hcal#g' \
0222                 > ${local_WebDir}/$j
0223     done
0224     cp *.png ${local_WebDir}
0225     cp HELP.html ${local_WebDir}
0226     files=`cd ${local_WebDir}; ls`
0227     #echo "GlobalRMT files=${files}"
0228
0229     if [ ${debug} -eq 0 ] ; then
0230         cmsMkdir $WebDir/GlobalRMT/GLOBAL_$runnumber
0231         if [ ! $? -eq 0 ] ; then
0232             echo "GlobalRMT cmsMkdir failed"
0233             exit 2
0234         fi
0235         for f in ${files} ; do
0236             echo "cmsStage -f ${local_WebDir}/${f} $WebDir/GlobalRMT/GLOBAL_$runnumber/${f}"
0237             cmsStage -f ${local_WebDir}/${f} $WebDir/GlobalRMT/GLOBAL_$runnumber/${f}
0238             if [ ! $? -eq 0 ] ; then
0239                 echo "GlobalRMT cmsStage failed for ${f}"
0240                 exit 2
0241             fi
0242         done
0243     else
0244         # debuging
0245         echo "debugging: files are not copied to EOS"
0246     fi
0247
0248     rm *.html
0249     rm *.png
0250
0251 #fi
0252
0253 done
0254
0255 if [ ${debug} -eq 2 ] ; then
0256     echo "debug=2 skipping web page creation"
0257     exit 2
0258 fi
0259
0260
0261 # #  #  # # # # # # # # # # #####
0262 # Create global web page
0263 #
0264
0265 echo "Get list of files in ${HistoDir}"
0266 #cmsLs $HistoDir | grep root | awk  '{print $5}' | awk -F / '{print $10}' > rtmp
0267 #cat rtmp | awk -F _ '{print $2}' | awk -F . '{print $1}' > _runlist_
0268 # Warning obtained on Oct 19, 2015:
0269 #   WARNING: this script (cmsLs) is deprecated and will be removed in Jan 2016
0270 #           Please use the 'eos ls' command directly.
0271
0272 histoFiles=`${eos} ls $HistoDir | grep root | awk -F '_' '{print $2}' | awk -F '.' '{print $1}'`
0273 echo -e '\n\nRun numbers on EOS:'
0274 runListEOS=`echo $histoFiles | tee _runlist_`
0275 echo "${runListEOS}"
0276 echo -e "list complete\n"
0277
0278 #making table
0279
0280 # print header to index.html
0281 if [ ${#comment} -eq 0 ] ; then
0282     echo `cat header_GLOBAL_EOS.txt` > index_draft.html
0283 else
0284     echo `head -n -1 header_GLOBAL_EOS.txt` > index_draft.html
0285     echo -e "<td class=\"s1\" align=\"center\">Comment</td>\n</tr>\n" \
0286         >> index_draft.html
0287 fi
0288
0289 #extract run numbers
0290 k=0
0291 for i in ${runListEOS} ; do
0292
0293 #runnumber=$(echo $i | sed -e 's/[^0-9]*//g')
0294 #runnumber=$(echo $i | awk -F 'run' '{print $2}'| awk -F '.' '{print $1}')
0295 runnumber=${i}
0296 #if [[ "$runnumber" > 243400 ]] ; then
0297 let "k = k + 1"
0298 echo
0299 echo
0300 echo
0301 echo 'RUN number = '$runnumber
0302
0303 # extract the date of file
0304 dasInfo=${DAS_DIR}/das_${runnumber}.txt
0305 got=0
0306 if [ ${dasCache} -eq 1 ] ; then
0307     rm -f tmp
0308     if [ -s ${dasInfo} ] ; then
0309         cp ${dasInfo} tmp
0310         got=1
0311     else
0312         echo "no ${dasInfo} found. Will use das_client.py"
0313     fi
0314 fi
0315 if [ ${got} -eq 0 ] ; then
0316     ./das_client.py --query="run=${i} | grep run.beam_e,run.bfield,run.nlumis,run.lhcFill,run.delivered_lumi,run.duration,run.start_time,run.end_time" --limit=0 > tmp
0317     if [ ${dasCache} -eq 1 ] ; then cp tmp ${dasInfo}; fi
0318 fi
0319
0320 #cat tmp
0321 date=`cat tmp | awk '{print $7" "$8}'`
0322 date_end=`cat tmp | awk '{print $9" "$10}'`
0323 E=`cat tmp | awk '{print $1}'`
0324 B=`cat tmp | awk '{print $2}'`
0325 nL=`cat tmp | awk '{print $3}'`
0326 Fill=`cat tmp | awk '{print $4}'`
0327 dLumi=`cat tmp | awk '{print $5}'`
0328 D=`cat tmp | awk '{print $6}'`
0329 rm tmp
0330
0331 #echo 'ver 1'
0332 #${eos} ls $HistoDir/Global_$i.root
0333 #echo 'ver 2'
0334 #${eos} ls -l $HistoDir/Global_$i.root
0335 #old Date_obr=`${eos} ls -l $HistoDir/Global_$i.root | awk '{print $3" "$4}'`
0336
0337 fileinfo=`${eos} ls -l $HistoDir/Global_$i.root`
0338 Date_obr=`echo ${fileinfo} | awk '{print $6" "$7" "$8}'`
0339 echo "Date_obr=$Date_obr"
0340
0341 # extract run type, data, time and number of events
0342 type='Cosmic'
0343 commentariy=''
0344 #cat runs_info
0345
0346 #  for j in $(cat runs_info); do
0347 #    echo $j
0348 #    k= `echo $j | awk  '{print $1}'`
0349 #    if [[ "$runnumber" == "$k" ]] ; then
0350 #      type= `echo $i | awk  '{print $2}'`
0351 #      commentariy=`echo $i | awk  '{print $3}'`
0352 #    fi
0353 #  done
0354
0355 #echo 'RUN Type = '$type
0356 echo 'RUN Start Date = '$date
0357 echo 'RUN Duration = '$D
0358 echo 'RUN End Date = '$date_end
0359 echo 'RUN Energy = '$E
0360 echo 'RUN Magnet field = '$B
0361 echo 'RUN LS number = '$nL
0362 echo 'RUN LHC Fill = '$Fill
0363 echo 'RUN Delivered Luminosity = '$dLumi
0364 echo 'RUN Date processing = '$Date_obr
0365 #echo 'RUN Comment = '$commentariy
0366
0367 #adding entry to list of file index_draft.html
0368 let "raw = (k % 2) + 2"
0369 echo '<tr>'>> index_draft.html
0370 echo '<td class="s1" align="center">'$k'</td>'>> index_draft.html
0371 echo '<td class="s'$raw'" align="center">'$runnumber'</td>'>> index_draft.html
0372 #echo '<td class="s'$raw'" align="center">'$type'</td>'>> index_draft.html
0373 echo '<td class="s'$raw'" align="center">'$nL'</td>'>> index_draft.html
0374 echo '<td class="s'$raw'" align="center">'$Fill'</td>'>> index_draft.html
0375 echo '<td class="s'$raw'" align="center">'$date'</td>'>> index_draft.html
0376 echo '<td class="s'$raw'" align="center">'$D'</td>'>> index_draft.html
0377 echo '<td class="s'$raw'" align="center">'$date_end'</td>'>> index_draft.html
0378 echo '<td class="s'$raw'" align="center"><a href="'$WebSite'/CMT/GLOBAL_'$runnumber'/LumiList.html">CMT_'$runnumber'</a></td>'>> index_draft.html
0379 echo '<td class="s'$raw'" align="center"><a href="'$WebSite'/GlobalRMT/GLOBAL_'$runnumber'/MAP.html">RMT_'$runnumber'</a></td>'>> index_draft.html
0380 echo '<td class="s'$raw'" align="center">'$B' T</td>'>> index_draft.html
0381 echo '<td class="s'$raw'" align="center">'$E' GeV</td>'>> index_draft.html
0382 #echo '<td class="s'$raw'" align="center">'$dLumi' /nb</td>'>> index_draft.html
0383 echo '<td class="s'$raw'" align="center">'$Date_obr' /nb</td>'>> index_draft.html
0384 if [ ${#comment} -gt 0 ] ; then
0385     #echo "runList=${runList}, check ${runnumber}"
0386     temp_var=${runList/${runnumber}/}
0387     if [ ${#temp_var} -lt ${#runList} ] ; then
0388         echo "adding a commentary for this run"
0389         echo "<td class=\"s${raw}\" align=\"center\">${comment}</td>" >> index_draft.html
0390     fi
0391 fi
0392 echo '</tr>'>> index_draft.html
0393 prev=$i
0394
0395 #fi
0396 done
0397
0398
0399 # print footer to index.html
0400 echo `cat footer.txt`>> index_draft.html
0401
0402
0403 status=0
0404 if [ ${debug} -gt 0 ] ; then
0405     echo "debug=${debug}. No upload to eos"
0406     status=-1
0407 else
0408     cmsStage -f index_draft.html $WebDir/CMT/index.html
0409     status="$?"
0410 #rm index_draft.html
0411 fi
0412
0413 # delete temp files
0414
0415 if [ ${debug} -eq 0 ] ; then
0416     rm -f *.root
0417     rm -f _runlist_
0418 fi
0419
0420 # check eos-upload exit code
0421 if [[ "${status}" == "0" ]]; then
0422   echo "Successfully uploaded!"
0423 else
0424   echo "ERROR: Uploading failed"
0425   exit 1
0426 fi
0427
0428 echo "script done"