SiStripMonitorClient/scripts/TkMap_script_automatic_DB.sh

0001 #!/bin/bash
0002
0003 curdir="$(pwd)"
0004 echo ${curdir}
0005
0006 export PATH=/afs/cern.ch/cms/common:${PATH}
0007 if [[ "$#" == "0" ]]; then
0008     echo "usage: 'TkMap_script_automatic.sh Cosmics|MinimumBias|StreamExpress|StreamExpressCosmics runNumber1 runNumber2...'";
0009     exit 1;
0010 fi
0011
0012 FORCE=0
0013 echo $2
0014 if [ "${2}" == "0" ]; then
0015     FORCE=0
0016 else
0017     if [ "${2}" == "f" ]; then
0018         FORCE=1
0019     fi
0020 fi
0021
0022 export WORKINGDIR=${CMSSW_BASE}/src
0023
0024 cd ${WORKINGDIR}
0025
0026 DataLocalDir=''
0027 DataOflineDir=''
0028
0029 for Run_numb in $@;
0030 do
0031
0032     if [ $Run_numb -gt 284500 ]; then
0033
0034         DataLocalDir='Data2017'
0035         DataOfflineDir='Run2017'
0036     else
0037
0038
0039     if [ "$Run_numb" == "$1" ]; then continue; fi
0040
0041 ##2016 data taking period run > 271024
0042     if [ $Run_numb -gt 271024 ]; then
0043
0044         DataLocalDir='Data2016'
0045         DataOfflineDir='Run2016'
0046     else
0047
0048 #2016 - Commissioning period
0049     if [ $Run_numb -gt 264200 ]; then
0050
0051         DataLocalDir='Data2016'
0052         DataOfflineDir='Commissioning2016'
0053     else
0054
0055     #Run2015A
0056     if [ $Run_numb -gt 246907 ]; then
0057         DataLocalDir='Data2015'
0058         DataOfflineDir='Run2015'
0059     else
0060
0061     #2015 Commissioning period (since January)
0062     if [ $Run_numb -gt 232881 ]; then
0063         DataLocalDir='Data2015'
0064         DataOfflineDir='Commissioning2015'
0065     else
0066     #2013 pp run (2.76 GeV)
0067         if [ $Run_numb -gt 211658 ]; then
0068             DataLocalDir='Data2013'
0069             DataOfflineDir='Run2013'
0070         else
0071     #2013 HI run
0072             if [ $Run_numb -gt 209634 ]; then
0073                 DataLocalDir='Data2013'
0074                 DataOfflineDir='HIRun2013'
0075             else
0076                 if [ $Run_numb -gt 190450 ]; then
0077                     DataLocalDir='Data2012'
0078                     DataOfflineDir='Run2012'
0079                 fi
0080             fi
0081         fi
0082     fi
0083     fi
0084     fi
0085     fi
0086     fi
0087     #loop over datasets
0088     #if Cosmics, do StreamExpressCosmics as well
0089
0090     datasets=${1}
0091     if [ "${1}" == "Cosmics" ]; then
0092         prefix=`echo $Run_numb | awk '{print substr($0,0,3)}'`
0093         checkdir='/data/users/event_display/'${DataLocalDir}'/Cosmics/'${prefix}'/'${Run_numb}'/StreamExpressCosmics'
0094         if [ ! -d $checkdir ]; then
0095             datasets=$datasets' StreamExpressCosmics'
0096             echo "Running on datasets "$datasets
0097         fi
0098     fi
0099
0100     for thisDataset in $datasets
0101       do
0102       echo "Processing "$thisDataset "in "${DataOfflineDir}"..."
0103
0104       nnn=`echo $Run_numb | awk '{print substr($0,0,4)}'`
0105
0106       echo 'Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'${DataOfflineDir}'/'$thisDataset'/000'${nnn}'xx/'
0107
0108     curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'${DataOfflineDir}'/'$thisDataset'/000'${nnn}'xx/' > index.html
0109     dqmFileNames=`cat index.html | grep ${Run_numb} | egrep "_DQM.root|_DQMIO.root" | egrep "Prompt|Express|22Jan2013" | sed 's/.*>\(.*\)<\/a.*/\1/' `
0110     dqmFileName=`expr "$dqmFileNames" : '\(DQM[A-Za-z0-9_/.\-]*root\)'`
0111     echo ' dqmFileNames = '$dqmFileNames
0112     echo ' dqmFileName = ['$dqmFileName']'
0113     curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/$DataOfflineDir/$thisDataset/000${nnn}xx/${dqmFileName} > /tmp/${dqmFileName}
0114     checkFile=`ls /tmp/${dqmFileName} | grep ${Run_numb}`
0115
0116 ##check if the full run is fully processed in GUI (Info/Run summary/ProvInfo/ runIsComplete flag == 1?
0117 ##if not, throw a warning
0118
0119     file_path="/tmp/"
0120
0121     echo "FORCE is " ${FORCE}
0122     ## check if run is complete - LG
0123     echo "get the run status from DQMFile"
0124     runStatus=-1
0125     runStatus="$(${pathTools}getRunStatusFromDQMFile.py ${file_path}/$dqmFileName $Run_numb runIsComplete | wc -l)"
0126     if [[ ${runStatus} == 0 ]]
0127         then
0128         echo ${Run_numb} >> ${curdir}/runsNotComplete_tmp.txt
0129         if [ ${FORCE} == 0 ]
0130         then
0131             continue;
0132         fi
0133     fi
0134     ## LG end
0135
0136     if [ $FORCE == 0 ]; then
0137         check_runcomplete ${file_path}/$dqmFileName
0138         if [ $? -ne 0 ]; then continue; fi
0139     fi
0140
0141     echo Process ${file_path}/$dqmFileName
0142
0143     cd /tmp
0144
0145     nnn=`echo $Run_numb | awk '{print substr($0,0,4)}'`
0146
0147
0148     echo ' dqmFileName = ['$dqmFileName']'
0149
0150      if [[ "${dqmFileName}" == "" ]]
0151      then
0152         echo "Run ${Run_numb} not yet ready"
0153         continue
0154      fi
0155
0156     cd ${WORKINGDIR}
0157
0158     [ -e $Run_numb ] || mkdir $Run_numb;
0159     [ -e $Run_numb/$thisDataset ] || mkdir $Run_numb/$thisDataset;
0160     echo "Run ${Run_numb}"
0161
0162     cd $Run_numb/$thisDataset
0163     echo `pwd`
0164     rm -f *.png
0165     rm -f *.xml
0166     rm -f *.log
0167     rm -f *.txt
0168     rm -f *.html
0169     rm -f *.root
0170
0171
0172     cp ${WORKINGDIR}/DQM/SiStripMonitorClient/scripts/DeadROCCounter.py .
0173     cp ${WORKINGDIR}/DQM/SiStripMonitorClient/scripts/DeadROCCounter_Phase1.py .
0174
0175 # Determine the GlobalTag name used to process the data and the DQM
0176
0177     GLOBALTAG=`getGTfromDQMFile.py ${file_path}/$dqmFileName $Run_numb globalTag_Step1`
0178
0179     if [[ "${GLOBALTAG}" == "" ]]
0180         then
0181         echo " No GlobalTag found: trying from DAS.... "
0182         GLOBALTAG=`getGTscript.sh $dqmFileName $Run_numb`
0183         fi
0184     if [[ "${GLOBALTAG}" == "" ]]
0185     then
0186         echo " No GlobalTag found: skipping this run.... "
0187         continue
0188     fi
0189
0190 #Temporary fix to remove hidden ASCII characters
0191     GLOBALTAG=`echo $GLOBALTAG | cut -c 9-${#GLOBALTAG}`
0192 #    GLOBALTAG=`sed -i 's/[\d128-\d255]//g' <<< "${GLOBALTAG}"`
0193 #    GLOBALTAG=`echo $GLOBALTAG | sed 's/[\d128-\d255]//'`
0194 #    echo `expr length $GLOBALTAG`
0195
0196     echo " Creating the TrackerMap.... "
0197
0198     detIdInfoFileName=`echo "file://TkDetIdInfo_Run${Run_numb}_${thisDataset}.root"`
0199
0200     #cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/SiStripDQM_OfflineTkMap_Template_cfg_DB.py print globalTag=${GLOBALTAG} runNumber=${Run_numb} dqmFile=${file_path}/$dqmFileName  # update GlobalTag
0201     cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/SiStripDQM_OfflineTkMap_Template_cfg_DB.py print globalTag=${GLOBALTAG} runNumber=${Run_numb} dqmFile=${file_path}/$dqmFileName  detIdInfoFile=${detIdInfoFileName} # update GlobalTag
0202
0203 # rename bad module list file
0204
0205      mv QTBadModules.log QualityTest_run${Run_numb}.txt
0206
0207     if [ $thisDataset == "Cosmics" ]; then  # should I add StreamExpressCosmics too
0208         cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap_cosmics.html | sed -e "s@RunNumber@$Run_numb@g" > index.html
0209     else
0210         cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap.html | sed -e "s@RunNumber@$Run_numb@g" > index.html
0211     fi
0212     cp ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/fedmap.html fedmap.html
0213     cp ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/psumap.html psumap.html
0214
0215     echo " Check TrackerMap on $Run_numb/$thisDataset folder"
0216
0217     nnn=`echo ${Run_numb} | awk '{print substr($0,0,3)}'`
0218
0219 ## Producing the list of bad modules
0220     echo " Creating the list of bad modules "
0221
0222     listbadmodule ${file_path}/$dqmFileName PCLBadComponents.log
0223    if [ "$thisDataset" != "StreamExpress" ] ; then
0224        sefile=QualityTest_run${Run_numb}.txt
0225
0226        if [ "$thisDataset" == "Cosmics" ]; then
0227            python ../../DQM/SiStripMonitorClient/scripts/findBadModT9.py -p $sefile -s /data/users/event_display/${DataLocalDir}/Cosmics/${nnn}/${Run_numb}/StreamExpressCosmics/${sefile}
0228        else
0229
0230            python ../../DQM/SiStripMonitorClient/scripts/findBadModT9.py -p $sefile -s /data/users/event_display/${DataLocalDir}/Beam/${nnn}/${Run_numb}/StreamExpress/${sefile}
0231
0232        fi
0233    fi
0234
0235 #    mv QualityTest*txt $Run_numb/$thisDataset
0236
0237 ## Producing the run certification by lumisection
0238     echo " Creating the lumisection certification:"
0239
0240     if [ $thisDataset == "MinimumBias" -o $thisDataset == "StreamExpress" ]; then
0241         ls_cert 0.95 0.95 ${file_path}/$dqmFileName
0242 #       mv Certification_run_* $Run_numb/$thisDataset
0243     fi
0244
0245 ## Producing the PrimaryVertex/BeamSpot quality test by LS..
0246     if [ "$thisDataset" != "Cosmics" ]  &&  [ "$thisDataset" != "StreamExpress" ]  &&  [ "$thisDataset" != "StreamExpressCosmics" ]; then
0247         echo " Creating the BeamSpot Calibration certification summary:"
0248
0249         lsbs_cert  ${file_path}/$dqmFileName
0250
0251 #       mv Certification_BS_run_* $Run_numb/$thisDataset
0252     fi
0253 ## .. and harvest the bad beamspot LS with automatic emailing (if in period and if bad LS found)
0254 #    bs_bad_ls_harvester $Run_numb/$thisDataset $Run_numb
0255     bs_bad_ls_harvester . $Run_numb
0256
0257 ## Producing the Module difference for ExpressStream
0258     if [ $thisDataset == "StreamExpress" -o $thisDataset == "StreamExpressCosmics" ]; then
0259         echo " Creating the Module Status Difference summary:"
0260
0261 #       modulediff $Run_numb ${1}
0262 #       ./modulediff_summary $Run_numb
0263 #       mv ModuleDifference_${Run_numb}.txt $Run_numb/$thisDataset
0264     fi
0265
0266     dest=Beam
0267     if [ $thisDataset == "Cosmics" -o $thisDataset == "StreamExpressCosmics" ]; then dest="Cosmics"; fi
0268
0269 # overwrite destination for tests
0270 # dest=FinalTest
0271
0272 ## create merged list of BadComponent from (PCL, RunInfo and FED Errors)
0273     cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/mergeBadChannel_Template_cfg.py globalTag=${GLOBALTAG} runNumber=${Run_numb} dqmFile=${file_path}/$dqmFileName
0274     mv MergedBadComponents.log MergedBadComponents_run${Run_numb}.txt
0275
0276     rm -f *.xml
0277     rm -f *svg
0278
0279     ssh cctrack@vocms061 "mkdir -p /data/users/event_display/TkCommissioner_runs/${DataLocalDir}/${dest} 2> /dev/null"
0280     scp *.root cctrack@vocms061:/data/users/event_display/TkCommissioner_runs/${DataLocalDir}/${dest}
0281     rm *.root
0282
0283     echo "counting dead pixel ROCs"
0284     echo "DataLocalDir = ${DataLocalDir}"
0285     if [[ $DataLocalDir == "Data2016" || $DataLocalDir == "Data2015" || $DataLocalDir == "Data2013" || $DataLocalDir == "Data2016" ]]; then
0286        ./DeadROCCounter.py ${file_path}/$dqmFileName
0287     else
0288        ./DeadROCCounter_Phase1.py ${file_path}/$dqmFileName
0289     fi
0290     rm -f DeadROCCounter.py
0291     rm -f DeadROCCounter_Phase1.py
0292
0293 #    mkdir -p /data/users/event_display/${DataLocalDir}/${dest}/${nnn}/${Run_numb}/$thisDataset #2> /dev/null
0294 #    cp -r ${Run_numb}/$thisDataset /data/users/event_display/Data2011/${dest}/${nnn}/${Run_numb}/
0295 #    cp -r ${Run_numb}/$thisDataset /data/users/event_display/${DataLocalDir}/${dest}/${nnn}/${Run_numb}/$thisDataset
0296     ssh cctrack@vocms061 "mkdir -p /data/users/event_display/${DataLocalDir}/${dest}/${nnn}/${Run_numb}/$thisDataset 2> /dev/null"
0297     scp -r * cctrack@vocms061:/data/users/event_display/${DataLocalDir}/${dest}/${nnn}/${Run_numb}/$thisDataset
0298
0299      rm ${file_path}/$dqmFileName
0300
0301      cd ${WORKINGDIR}
0302      rm -rf $Run_numb
0303      rm -rf index.html
0304
0305 #done with loop over thisDataset
0306 done
0307
0308 done