scripts/cmt/index_EOS.sh

0001 #!/bin/bash
0002 WebDir='/eos/cms/store/group/dpg_hcal/comm_hcal/www/HcalRemoteMonitoring'
0003 WebSite='https://cms-conddb-dev.cern.ch/eosweb/hcal/HcalRemoteMonitoring'
0004 HistoDir='/eos/cms/store/group/dpg_hcal/comm_hcal/www/HcalRemoteMonitoring/CMT/histos'
0005 eos='/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select'
0006
0007 # print usage info
0008 if [[ "$1" == "" ]]; then
0009   echo "Usage:"
0010   echo "  $0 file [comment] [-ignore-file] [-das-cache]"
0011   echo "    file  -- a file with run numbers"
0012   echo "    comment  -- add a comment line (instead of spaces use '_')"
0013   echo "    -ignore-file   -- skips production of run html pages. Produces"
0014   echo "                      only the global page. File name is not needed."
0015   echo "    -das-cache   -- whether to save DAS information locally for a reuse"
0016   echo
0017   echo "example: ./GLOBAL.sh Run_List.txt"
0018   exit 1
0019 fi
0020
0021 cmsenv 2>/dev/null
0022 if [ $? == 0 ] ; then
0023     eval `scramv1 runtime -sh`
0024 fi
0025 temp_var=`ls ${eos}`
0026 status="$?"
0027 echo "using eos command <${temp_var}>"
0028 if [ ! ${status} -eq 0 ] ; then
0029     echo "failed to find eos command"
0030     # exit 1
0031 fi
0032
0033
0034 # Process arguments and set the flags
0035 fileName=$1
0036 comment=$2
0037 if [ ${#comment} -gt 0 ] && [ "${comment:0:1}" == "-" ] ; then comment=""; fi
0038 ignoreFile=0
0039 debug=0
0040 dasCache=0
0041 DAS_DIR="d-DAS-info"
0042
0043 for a in $@ ; do
0044     if [ "$a" == "-ignore-file" ] ; then
0045         echo " ** file will be ignored"
0046         fileName=""
0047         ignoreFile=1
0048     elif [ "$a" == "-das-cache" ] ; then
0049         echo " ** DAS cache ${DAS_DIR} enabled"
0050         dasCache=1
0051         if [ ! -d ${DAS_DIR} ] ; then mkdir ${DAS_DIR}; fi
0052     else
0053         temp_var=${a/-debug/}
0054         if [ ${#a} -gt ${#temp_var} ] ; then
0055             debug=${temp_var}
0056             echo " ** debug detected (debug=${debug})"
0057         fi
0058     fi
0059 done
0060
0061 # Obtain the runList from a file, if needed
0062 runList=""
0063 if [ ${#fileName} -gt 0 ] ; then
0064   if [ -s ${fileName} ] ; then
0065       runList=`cat ${fileName}`
0066   else
0067       echo "<${fileName}> does not seem to be a valid file"
0068       exit 2
0069   fi
0070 else
0071     if [ ${ignoreFile} -eq 0 ] ; then
0072         echo " ! no file provided"
0073     fi
0074     echo " ! will produce only the global html page"
0075 fi
0076
0077
0078 # Check the runList and correct the correctables
0079 # Replace ',' and ';' by empty spaces
0080 runList=`echo "${runList}" | sed 'sk,k\ kg' | sed 'sk;k\ kg'`
0081 ok=1
0082 for r in ${runList} ; do
0083     if [ ! ${#r} -eq 6 ] ; then
0084         echo "run numbers are expected to be of length 6. Check <$r>"
0085         ok=0
0086     fi
0087     debug_loc=0
0088     if [ "$r" -eq "$r" ] 2>/dev/null ; then
0089         if [ ${debug_loc} -eq 1 ] ; then echo "run variable <$r> is a number (ok)"; fi
0090     else
0091         echo "error: run variable <$r> is not an integer number"
0092         ok=0
0093     fi
0094 done
0095
0096 echo "Tested `wc -w <<< "${runList}"` runs from file ${fileName}"
0097 if [ ${ok} -eq 0 ] ; then
0098     echo "errors in the file ${fileName} with run numbers"
0099     exit 3
0100 else
0101     if [ ${#fileName} -gt 0 ] ; then
0102         echo "run numbers in ${fileName} verified ok"
0103     fi
0104 fi
0105
0106 comment=`echo ${comment} | sed sk\_k\ kg`
0107 if [ ${#comment} -gt 0 ] ; then
0108     echo "comment \"${comment}\" will be added to the pages"
0109 fi
0110
0111 if [ ${debug} -eq 3 ] ; then exit; fi
0112
0113
0114 echo
0115 echo
0116 echo
0117 echo 'Run numbers for processing'
0118 echo "${runList}"
0119 echo -e "list complete\n"
0120
0121 #processing skipped
0122
0123
0124 # #  #  # # # # # # # # # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # #####
0125 # Create global web page
0126 #
0127
0128 #echo "Get list of files in ${HistoDir}"
0129 #eos ls $HistoDir | grep root | awk  '{print $5}' | awk -F / '{print $10}' > rtmp
0130 #cat rtmp | awk -F _ '{print $2}' | awk -F . '{print $1}' > _runlist_
0131 #cat $1 > _runlist_
0132
0133 histoFiles=`${eos} ls $HistoDir | grep root | awk -F '_' '{print $2}' | awk -F '.' '{print $1}'`
0134 #echo -e '\n\nRun numbers on EOS now:'
0135 #echo "${histoFiles}"
0136
0137
0138 echo -e '\n\nRun numbers to be on EOS:'
0139 ##runListEOS=`echo $histoFiles | tee _runlist_`
0140 ##runListEOS=`echo $histoFiles | tee _runlist_`
0141 runListEOS=`echo $histoFiles `
0142
0143
0144 echo "${runListEOS}"
0145 echo -e "list complete\n"
0146
0147 #making table
0148 #############                 making table
0149
0150 # skip/comment header:
0151 ## print header to index.html
0152 #if [ ${#comment} -eq 0 ] ; then
0153 #    echo `cat header_GLOBAL_EOS.txt` > index_draft.html
0154 #else
0155 #    echo `head -n -1 header_GLOBAL_EOS.txt` > index_draft.html
0156 #    echo -e "<td class=\"s1\" align=\"center\">Comment</td>\n</tr>\n" \
0157 #       >> index_draft.html
0158 #fi
0159
0160 echo 'next message is Fine because of command rm index.html: '
0161 rm index.html
0162
0163 echo 'copy index.html and  OLDindex.html '
0164 eos cp $WebDir/CMT/index.html index.html
0165 cp index.html OLDindex.html
0166
0167 echo ' delete last line of copied index_draft.html which close the table'
0168 cat index.html | head -n -1 > index_draft.html
0169
0170 echo 'extract run numbers:'
0171 k=0
0172 for i in ${runListEOS} ; do
0173 let "k = k + 1"
0174 done
0175 echo ' ================>>>    k in old list = '$k
0176
0177
0178 #k=1294
0179 k=0
0180 echo ' ================>>> starting hardcoded k = '$k
0181 #===================
0182 for i in ${runListEOS} ; do
0183 #echo 'extract run numbers'
0184
0185 ##runnumber=$(echo $i | sed -e 's/[^0-9]*//g')
0186 ##runnumber=$(echo $i | awk -F 'run' '{print $2}'| awk -F '.' '{print $1}')
0187 runnumber=${i}
0188 let "k = k + 1"
0189 #=======
0190 if [[ "$runnumber" > 309999 ]] ; then
0191 #if [[ "$runnumber" > 306051 ]] ; then
0192 #if [[ "$runnumber" > 305341 ]] ; then
0193 #if [[ "$runnumber" > 304562 ]] ; then
0194 ##if [[ "$runnumber" > 305405 ]] ; then
0195 ##if [[ "$runnumber" > 303825 ]] ; then
0196 ##if [[ "$runnumber" > 290742 ]] ; then
0197 echo
0198 echo
0199 echo 'k = '$k
0200 echo 'RUN number = '$runnumber
0201
0202 # extract the date of file
0203 dasInfo=${DAS_DIR}/das_${runnumber}.txt
0204 got=0
0205 if [ ${dasCache} -eq 1 ] ; then
0206     rm -f tmp
0207     if [ -s ${dasInfo} ] ; then
0208         cp ${dasInfo} tmp
0209         got=1
0210     else
0211         echo "no ${dasInfo} found. Will use das_client.py"
0212     fi
0213 fi
0214 if [ ${got} -eq 0 ] ; then
0215     ./das_client.py --query="run=${i} | grep run.beam_e,run.bfield,run.nlumis,run.lhcFill,run.delivered_lumi,run.duration,run.start_time,run.end_time" --limit=0 > tmp
0216     if [ ${dasCache} -eq 1 ] ; then cp tmp ${dasInfo}; fi
0217 fi
0218
0219 #cat tmp
0220 date=`cat tmp | awk '{print $7" "$8}'`
0221 date_end=`cat tmp | awk '{print $9" "$10}'`
0222 E=`cat tmp | awk '{print $1}'`
0223 B=`cat tmp | awk '{print $2}'`
0224 nL=`cat tmp | awk '{print $3}'`
0225 Fill=`cat tmp | awk '{print $4}'`
0226 dLumi=`cat tmp | awk '{print $5}'`
0227 D=`cat tmp | awk '{print $6}'`
0228 rm tmp
0229
0230 #echo 'ver 1'
0231 #${eos} ls $HistoDir/Global_$i.root
0232 #echo 'ver 2'
0233 #${eos} ls -l $HistoDir/Global_$i.root
0234 #old Date_obr=`${eos} ls -l $HistoDir/Global_$i.root | awk '{print $3" "$4}'`
0235
0236 fileinfo=`${eos} ls -l $HistoDir/Global_$i.root`
0237 Date_obr=`echo ${fileinfo} | awk '{print $6" "$7" "$8}'`
0238 echo "Date_obr=$Date_obr"
0239
0240 # extract run type, data, time and number of events
0241 type='Cosmic'
0242 commentariy=''
0243 #cat runs_info
0244
0245 #  for j in $(cat runs_info); do
0246 #    echo $j
0247 #    k= `echo $j | awk  '{print $1}'`
0248 #    if [[ "$runnumber" == "$k" ]] ; then
0249 #      type= `echo $i | awk  '{print $2}'`
0250 #      commentariy=`echo $i | awk  '{print $3}'`
0251 #    fi
0252 #  done
0253
0254 #echo 'RUN Type = '$type
0255 echo 'RUN Start Date = '$date
0256 echo 'RUN Duration = '$D
0257 echo 'RUN End Date = '$date_end
0258 echo 'RUN Energy = '$E
0259 echo 'RUN Magnet field = '$B
0260 echo 'RUN LS number = '$nL
0261 echo 'RUN LHC Fill = '$Fill
0262 echo 'RUN Delivered Luminosity = '$dLumi
0263 echo 'RUN Date processing = '$Date_obr
0264 #echo 'RUN Comment = '$commentariy
0265
0266 #adding entry to list of file index_draft.html
0267 let "raw = (k % 2) + 2"
0268 echo '<tr>'>> index_draft.html
0269 echo '<td class="s1" align="center">'$k'</td>'>> index_draft.html
0270 echo '<td class="s'$raw'" align="center">'$runnumber'</td>'>> index_draft.html
0271 #echo '<td class="s'$raw'" align="center">'$type'</td>'>> index_draft.html
0272 echo '<td class="s'$raw'" align="center">'$nL'</td>'>> index_draft.html
0273 echo '<td class="s'$raw'" align="center">'$Fill'</td>'>> index_draft.html
0274 echo '<td class="s'$raw'" align="center">'$date'</td>'>> index_draft.html
0275 echo '<td class="s'$raw'" align="center">'$D'</td>'>> index_draft.html
0276 echo '<td class="s'$raw'" align="center">'$date_end'</td>'>> index_draft.html
0277 echo '<td class="s'$raw'" align="center"><a href="'$WebSite'/CMT/GLOBAL_'$runnumber'/LumiList.html">CMT_'$runnumber'</a></td>'>> index_draft.html
0278 echo '<td class="s'$raw'" align="center"><a href="'$WebSite'/GlobalRMT/GLOBAL_'$runnumber'/MAP.html">RMT_'$runnumber'</a></td>'>> index_draft.html
0279 echo '<td class="s'$raw'" align="center">'$B' T</td>'>> index_draft.html
0280 echo '<td class="s'$raw'" align="center">'$E' GeV</td>'>> index_draft.html
0281 #echo '<td class="s'$raw'" align="center">'$dLumi' /nb</td>'>> index_draft.html
0282 echo '<td class="s'$raw'" align="center">'$Date_obr' /nb</td>'>> index_draft.html
0283 if [ ${#comment} -gt 0 ] ; then
0284     #echo "runList=${runList}, check ${runnumber}"
0285     temp_var=${runList/${runnumber}/}
0286     if [ ${#temp_var} -lt ${#runList} ] ; then
0287         echo "adding a commentary for this run"
0288         echo "<td class=\"s${raw}\" align=\"center\">${comment}</td>" >> index_draft.html
0289     fi
0290 fi
0291 echo '</tr>'>> index_draft.html
0292 prev=$i
0293
0294 fi
0295 #=======
0296 done
0297 #===================
0298
0299 # print footer to index.html
0300 echo `cat footer.txt`>> index_draft.html
0301
0302
0303 status=0
0304 if [ ${debug} -gt 0 ] ; then
0305     echo "debug=${debug}. No upload to eos"
0306     status=-1
0307 else
0308 ##    cmsStage -f index_draft.html $WebDir/CMT/index.html
0309 ###    echo "Commented by me:  eos cp index_draft.html $WebDir/CMT/index.html No upload to eos"
0310 #  eos cp OLDindex.html $WebDir/CMT/OLDindex.html
0311 #  eos cp index_draft.html $WebDir/CMT/index.html
0312     status="$?"
0313 #rm index_draft.html
0314 fi
0315
0316 # delete temp files
0317
0318 if [ ${debug} -eq 0 ] ; then
0319     rm -f *.root
0320     rm -f _runlist_
0321 fi
0322
0323 # check eos-upload exit code
0324 if [[ "${status}" == "0" ]]; then
0325   echo "Successfully uploaded!"
0326 else
0327   echo "ERROR: Uploading failed"
0328   exit 1
0329 fi
0330
0331 echo "index script done"