scripts/psm/index_toAddOnlyNewRuns_EOS.shOLD

0001 #!/bin/bash
0002
0003 WebDir='/eos/cms/store/group/dpg_hcal/comm_hcal/www/HcalRemoteMonitoring'
0004 WebSite='https://cms-conddb-dev.cern.ch/eosweb/hcal/HcalRemoteMonitoring'
0005 HistoDir='/eos/cms/store/group/dpg_hcal/comm_hcal/www/HcalRemoteMonitoring/CMT/histos'
0006 eos='/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select'
0007
0008 # print usage info
0009 if [[ "$1" == "" ]]; then
0010   echo "Usage:"
0011   echo "  $0 file [comment] [-ignore-file] [-das-cache]"
0012   echo "    file  -- a file with run numbers"
0013   echo "    comment  -- add a comment line (instead of spaces use '_')"
0014   echo "    -ignore-file   -- skips production of run html pages. Produces"
0015   echo "                      only the global page. File name is not needed."
0016   echo "    -das-cache   -- whether to save DAS information locally for a reuse"
0017   echo
0018   echo "example: ./GLOBAL.sh Run_List.txt"
0019   exit 1
0020 fi
0021
0022 cmsenv 2>/dev/null
0023 if [ $? == 0 ] ; then
0024     eval `scramv1 runtime -sh`
0025 fi
0026 temp_var=`ls ${eos}`
0027 status="$?"
0028 echo "using eos command <${temp_var}>"
0029 if [ ! ${status} -eq 0 ] ; then
0030     echo "failed to find eos command"
0031     # exit 1
0032 fi
0033
0034
0035 # Process arguments and set the flags
0036 fileName=$1
0037 comment=$2
0038 if [ ${#comment} -gt 0 ] && [ "${comment:0:1}" == "-" ] ; then comment=""; fi
0039 ignoreFile=0
0040 debug=0
0041 dasCache=0
0042 DAS_DIR="d-DAS-info"
0043
0044 for a in $@ ; do
0045     if [ "$a" == "-ignore-file" ] ; then
0046         echo " ** file will be ignored"
0047         fileName=""
0048         ignoreFile=1
0049     elif [ "$a" == "-das-cache" ] ; then
0050         echo " ** DAS cache ${DAS_DIR} enabled"
0051         dasCache=1
0052         if [ ! -d ${DAS_DIR} ] ; then mkdir ${DAS_DIR}; fi
0053     else
0054         temp_var=${a/-debug/}
0055         if [ ${#a} -gt ${#temp_var} ] ; then
0056             debug=${temp_var}
0057             echo " ** debug detected (debug=${debug})"
0058         fi
0059     fi
0060 done
0061
0062 # Obtain the runList from a file, if needed
0063 runList=""
0064 if [ ${#fileName} -gt 0 ] ; then
0065   if [ -s ${fileName} ] ; then
0066       runList=`cat ${fileName}`
0067   else
0068       echo "<${fileName}> does not seem to be a valid file"
0069       exit 2
0070   fi
0071 else
0072     if [ ${ignoreFile} -eq 0 ] ; then
0073         echo " ! no file provided"
0074     fi
0075     echo " ! will produce only the global html page"
0076 fi
0077
0078
0079 # Check the runList and correct the correctables
0080 # Replace ',' and ';' by empty spaces
0081 runList=`echo "${runList}" | sed 'sk,k\ kg' | sed 'sk;k\ kg'`
0082 ok=1
0083 for r in ${runList} ; do
0084     if [ ! ${#r} -eq 6 ] ; then
0085         echo "run numbers are expected to be of length 6. Check <$r>"
0086         ok=0
0087     fi
0088     debug_loc=0
0089     if [ "$r" -eq "$r" ] 2>/dev/null ; then
0090         if [ ${debug_loc} -eq 1 ] ; then echo "run variable <$r> is a number (ok)"; fi
0091     else
0092         echo "error: run variable <$r> is not an integer number"
0093         ok=0
0094     fi
0095 done
0096
0097 echo "Tested `wc -w <<< "${runList}"` runs from file ${fileName}"
0098 if [ ${ok} -eq 0 ] ; then
0099     echo "errors in the file ${fileName} with run numbers"
0100     exit 3
0101 else
0102     if [ ${#fileName} -gt 0 ] ; then
0103         echo "run numbers in ${fileName} verified ok"
0104     fi
0105 fi
0106
0107 comment=`echo ${comment} | sed sk\_k\ kg`
0108 if [ ${#comment} -gt 0 ] ; then
0109     echo "comment \"${comment}\" will be added to the pages"
0110 fi
0111
0112 if [ ${debug} -eq 3 ] ; then exit; fi
0113
0114
0115 echo
0116 echo
0117 echo
0118 echo 'Numbers of NEW runs for processing'
0119 echo "${runList}"
0120 echo -e "runList complete\n"
0121
0122 #processing skipped
0123
0124
0125 # #  #  # # # # # # # # # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # ### # # #####
0126 # Create global web page
0127 #
0128
0129 echo "Get list of runss in ${HistoDir}"
0130 #eos ls $HistoDir | grep root | awk  '{print $5}' | awk -F / '{print $10}' > rtmp
0131 #cat rtmp | awk -F _ '{print $2}' | awk -F . '{print $1}' > _runlist_
0132
0133 histoFiles=`${eos} ls $HistoDir | grep root | awk -F '_' '{print $2}' | awk -F '.' '{print $1}'`
0134 echo -e '\n\nRun numbers on EOS:'
0135 runListEOSall=`echo $histoFiles | tee _runlistEOSall_`
0136 echo "Got `wc -w <<< "${runListEOSall}"` runs from HistDir ${HistoDir}"
0137
0138 echo -e '\n\nRun numbers:'
0139 runListEOS=`echo $runList | tee _runlist_`
0140 echo "${runListEOS}"
0141 echo -e "Full runList for EOS complete\n"
0142
0143
0144
0145
0146
0147 ### use to clarify full list of runs:
0148 #echo "Get OLD list of runs in ${HistoDir}"
0149 #echo -e '\n\nRun numbers to be on EOS:'
0150 #runListEOS=`echo $histoFiles `
0151 #echo "${runListEOS}"
0152 #echo -e "OLD list complete\n"
0153
0154
0155 #############                 making table
0156
0157 # skip/comment header:
0158 ## print header to index.html
0159 #if [ ${#comment} -eq 0 ] ; then
0160 #    echo `cat header_GLOBAL_EOS.txt` > index_draft.html
0161 #else
0162 #    echo `head -n -1 header_GLOBAL_EOS.txt` > index_draft.html
0163 #    echo -e "<td class=\"s1\" align=\"center\">Comment</td>\n</tr>\n" \
0164 #       >> index_draft.html
0165 #fi
0166
0167 # copy index.html from EOS:
0168 echo 'next message is Fine: '
0169 rm index.html
0170 #cmsStage $WebDir/CMT/index.html index.html
0171 eoscp $WebDir/CMT/index.html index.html
0172 cp index.html OLDindex.html
0173
0174 # delete last line of copied index_draft.html which close the table
0175 cat index.html | head -n -1 > index_draft.html
0176
0177 #extract run numbers for correct continuation
0178 #k=0
0179 #for i in ${runListEOSall} ; do
0180 #let "k = k + 1"
0181 #done
0182 k=2014
0183 ########################################## type by hands number of new runs k=k-number:
0184 #let "k = k - 1"
0185 echo ' ================>>>    k in old list = '$k
0186
0187 for i in ${runListEOS} ; do
0188
0189 #runnumber=$(echo $i | sed -e 's/[^0-9]*//g')
0190 #runnumber=$(echo $i | awk -F 'run' '{print $2}'| awk -F '.' '{print $1}')
0191 runnumber=${i}
0192 #if [[ "$runnumber" > 243400 ]] ; then
0193 let "k = k + 1"
0194 echo
0195 echo ' ================>>> new k in loop = '$k
0196 echo
0197 echo
0198 echo 'RUN number = '$runnumber
0199
0200 # extract the date of file
0201 dasInfo=${DAS_DIR}/das_${runnumber}.txt
0202 got=0
0203 if [ ${dasCache} -eq 1 ] ; then
0204     rm -f tmp
0205     if [ -s ${dasInfo} ] ; then
0206         cp ${dasInfo} tmp
0207         got=1
0208     else
0209         echo "no ${dasInfo} found. Will use dasgoclient"
0210     fi
0211 fi
0212
0213 if [ ${got} -eq 0 ] ; then
0214 #####  old:                                       1            2          3           4            5               6                 7 8
0215 ##   ./das_client.py --query="run=${i} | grep run.beam_e,run.bfield,run.nlumis,run.lhcFill,run.delivered_lumi,run.duration,run.start_time,run.end_time" --limit=0 > tmp
0216 ###### AZ  used now:                                   1            2          3           4            5               6                 7 8 9      10 11 12
0217 dasgoclient  --query="run=${i} | grep run.lhcEnergy ,run.bfield ,run.lsCount ,run.lhcFill ,  run.lsRanges ,    run.runClassName ,run.startTime ,run.stopTime "  > tmp
0218 # for test:
0219 #dasgoclient  --query="run=${i} | grep run.aaa ,run.lhcEnergy ,run.bfield ,run.lsCount,run.lhcFill ,  run.lsRanges ,    run.runClassName ,run.startTime ,run.stopTime , run.runCreated ,run.modified " > tmp
0220
0221 #    if [ ${dasCache} -eq 1 ] ; then cp tmp ${dasInfo}; fi
0222 fi
0223
0224
0225 ##### AZ used now:
0226 # delete last lines of copied tmp file
0227 cat tmp | head -n -1 > ztmp
0228 rm tmp
0229 #cat ztmp
0230
0231 ###### AZ  used now:
0232 date=`cat ztmp | awk '{print $7" "$8" "$9}'`
0233 date_end=`cat ztmp | awk '{print $10" "$11" "$12}'`
0234 #old:
0235 #date=`cat ztmp | awk '{print $7" "$8}'`
0236 #date_end=`cat ztmp | awk '{print $9" "$10}'`
0237 E=`cat ztmp | awk '{print $1}'`
0238 B=`cat ztmp | awk '{print $2}'`
0239 nL=`cat ztmp | awk '{print $3}'`
0240 Fill=`cat ztmp | awk '{print $4}'`
0241 dLumi=`cat ztmp | awk '{print $5}'`
0242 D=`cat ztmp | awk '{print $6}'`
0243
0244 ###### AZ  used now:
0245 rm ztmp
0246
0247 #echo 'ver 1'
0248 #${eos} ls $HistoDir/Global_$i.root
0249 #echo 'ver 2'
0250 #${eos} ls -l $HistoDir/Global_$i.root
0251 #old Date_obr=`${eos} ls -l $HistoDir/Global_$i.root | awk '{print $3" "$4}'`
0252
0253 fileinfo=`${eos} ls -l $HistoDir/Global_$i.root`
0254 Date_obr=`echo ${fileinfo} | awk '{print $6" "$7" "$8}'`
0255 #echo "Date_obr=$Date_obr"
0256
0257 # extract run type, data, time and number of events
0258 type='Cosmic'
0259 commentariy=''
0260 #cat runs_info
0261
0262 #  for j in $(cat runs_info); do
0263 #    echo $j
0264 #    k= `echo $j | awk  '{print $1}'`
0265 #    if [[ "$runnumber" == "$k" ]] ; then
0266 #      type= `echo $i | awk  '{print $2}'`
0267 #      commentariy=`echo $i | awk  '{print $3}'`
0268 #    fi
0269 #  done
0270
0271 #echo 'RUN Type = '$type
0272 echo 'RUN Start Date = '$date
0273 echo 'RUN Duration = '$D
0274 echo 'RUN End Date = '$date_end
0275 echo 'RUN Energy = '$E
0276 echo 'RUN Magnet field = '$B
0277 echo 'RUN LS number = '$nL
0278 echo 'RUN LHC Fill = '$Fill
0279 echo 'RUN Delivered Luminosity = '$dLumi
0280 echo 'RUN Date processing = '$Date_obr
0281 #echo 'RUN Comment = '$commentariy
0282
0283 #adding entry to list of file index_draft.html
0284 let "raw = (k % 2) + 2"
0285 echo '<tr>'>> index_draft.html
0286 echo '<td class="s1" align="center">'$k'</td>'>> index_draft.html
0287 echo '<td class="s'$raw'" align="center">'$runnumber'</td>'>> index_draft.html
0288 #echo '<td class="s'$raw'" align="center">'$type'</td>'>> index_draft.html
0289 echo '<td class="s'$raw'" align="center">'$nL'</td>'>> index_draft.html
0290 echo '<td class="s'$raw'" align="center">'$Fill'</td>'>> index_draft.html
0291 echo '<td class="s'$raw'" align="center">'$date'</td>'>> index_draft.html
0292 echo '<td class="s'$raw'" align="center">'$D'</td>'>> index_draft.html
0293 echo '<td class="s'$raw'" align="center">'$date_end'</td>'>> index_draft.html
0294 echo '<td class="s'$raw'" align="center"><a href="'$WebSite'/CMT/GLOBAL_'$runnumber'/LumiList.html">CMT_'$runnumber'</a></td>'>> index_draft.html
0295 echo '<td class="s'$raw'" align="center"><a href="'$WebSite'/GlobalRMT/GLOBAL_'$runnumber'/MAP.html">RMT_'$runnumber'</a></td>'>> index_draft.html
0296 echo '<td class="s'$raw'" align="center">'$B' T</td>'>> index_draft.html
0297 echo '<td class="s'$raw'" align="center">'$E' GeV</td>'>> index_draft.html
0298 #echo '<td class="s'$raw'" align="center">'$dLumi' /nb</td>'>> index_draft.html
0299 echo '<td class="s'$raw'" align="center">'$Date_obr'</td>'>> index_draft.html
0300 if [ ${#comment} -gt 0 ] ; then
0301     #echo "runList=${runList}, check ${runnumber}"
0302     temp_var=${runList/${runnumber}/}
0303     if [ ${#temp_var} -lt ${#runList} ] ; then
0304         echo "adding a commentary for this run"
0305         echo "<td class=\"s${raw}\" align=\"center\">${comment}</td>" >> index_draft.html
0306     fi
0307 fi
0308 echo '</tr>'>> index_draft.html
0309 prev=$i
0310
0311 #fi
0312 done
0313
0314
0315 # print footer to index.html
0316 echo `cat footer.txt`>> index_draft.html
0317
0318
0319 status=0
0320 if [ ${debug} -gt 0 ] ; then
0321     echo "debug=${debug}. No upload to eos"
0322     status=-1
0323 else
0324 ###    echo "Commented by me:  eoscp index_draft.html $WebDir/CMT/index.html No upload to eos"
0325 #  eoscp OLDindex.html $WebDir/CMT/OLDindex.html
0326 #  eoscp index_draft.html $WebDir/CMT/index.html
0327
0328
0329     status="$?"
0330 # rm index_draft.html
0331 fi
0332
0333 # delete temp files
0334
0335 if [ ${debug} -eq 0 ] ; then
0336 #    rm -f *.root
0337     rm -f _runlist_
0338     rm -f _runlistEOSall_
0339 fi
0340
0341 # check eos-upload exit code
0342 if [[ "${status}" == "0" ]]; then
0343   echo "Successfully uploaded!"
0344 else
0345   echo "ERROR: Uploading failed"
0346   exit 1
0347 fi
0348
0349 echo "index script done"