CMSSW/DQM/SiStripMonitorClient/scripts/TkMap_script_automatic_DB.sh
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 
#!/bin/bash

curdir="$(pwd)"
echo ${curdir}

export PATH=/afs/cern.ch/cms/common:${PATH}
if [[ "$#" == "0" ]]; then
    echo "usage: 'TkMap_script_automatic.sh Cosmics|MinimumBias|StreamExpress|StreamExpressCosmics runNumber1 runNumber2...'";
    exit 1;
fi

FORCE=0
echo $2
if [ "${2}" == "0" ]; then
    FORCE=0
else
    if [ "${2}" == "f" ]; then
        FORCE=1
    fi
fi

export WORKINGDIR=${CMSSW_BASE}/src

cd ${WORKINGDIR}

DataLocalDir=''
DataOflineDir=''

for Run_numb in $@;
do

    if [ $Run_numb -gt 284500 ]; then

        DataLocalDir='Data2017'
        DataOfflineDir='Run2017'
    else


    if [ "$Run_numb" == "$1" ]; then continue; fi

##2016 data taking period run > 271024
    if [ $Run_numb -gt 271024 ]; then

        DataLocalDir='Data2016'
        DataOfflineDir='Run2016'
    else

#2016 - Commissioning period                                                                                                                               
    if [ $Run_numb -gt 264200 ]; then

        DataLocalDir='Data2016'
        DataOfflineDir='Commissioning2016'
    else

    #Run2015A
    if [ $Run_numb -gt 246907 ]; then
        DataLocalDir='Data2015'
        DataOfflineDir='Run2015'
    else

    #2015 Commissioning period (since January)
    if [ $Run_numb -gt 232881 ]; then
	DataLocalDir='Data2015'
	DataOfflineDir='Commissioning2015'
    else
    #2013 pp run (2.76 GeV)
	if [ $Run_numb -gt 211658 ]; then
	    DataLocalDir='Data2013'
	    DataOfflineDir='Run2013'
	else
    #2013 HI run
	    if [ $Run_numb -gt 209634 ]; then
		DataLocalDir='Data2013'
		DataOfflineDir='HIRun2013'
	    else
		if [ $Run_numb -gt 190450 ]; then
		    DataLocalDir='Data2012'
		    DataOfflineDir='Run2012'
		fi
	    fi
	fi
    fi
    fi
    fi
    fi
    fi
    #loop over datasets
    #if Cosmics, do StreamExpressCosmics as well

    datasets=${1}
    if [ "${1}" == "Cosmics" ]; then
	prefix=`echo $Run_numb | awk '{print substr($0,0,3)}'` 
	checkdir='/data/users/event_display/'${DataLocalDir}'/Cosmics/'${prefix}'/'${Run_numb}'/StreamExpressCosmics'
	if [ ! -d $checkdir ]; then
	    datasets=$datasets' StreamExpressCosmics'
	    echo "Running on datasets "$datasets
	fi
    fi
    
    for thisDataset in $datasets
      do
      echo "Processing "$thisDataset "in "${DataOfflineDir}"..."

      nnn=`echo $Run_numb | awk '{print substr($0,0,4)}'` 

      echo 'Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'${DataOfflineDir}'/'$thisDataset'/000'${nnn}'xx/'
  
    curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'${DataOfflineDir}'/'$thisDataset'/000'${nnn}'xx/' > index.html
    dqmFileNames=`cat index.html | grep ${Run_numb} | egrep "_DQM.root|_DQMIO.root" | egrep "Prompt|Express|22Jan2013" | sed 's/.*>\(.*\)<\/a.*/\1/' `
    dqmFileName=`expr "$dqmFileNames" : '\(DQM[A-Za-z0-9_/.\-]*root\)'`
    echo ' dqmFileNames = '$dqmFileNames
    echo ' dqmFileName = ['$dqmFileName']'
    curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/$DataOfflineDir/$thisDataset/000${nnn}xx/${dqmFileName} > /tmp/${dqmFileName}
    checkFile=`ls /tmp/${dqmFileName} | grep ${Run_numb}`

##check if the full run is fully processed in GUI (Info/Run summary/ProvInfo/ runIsComplete flag == 1? 
##if not, throw a warning

    file_path="/tmp/"

    echo "FORCE is " ${FORCE}
    ## check if run is complete - LG
    echo "get the run status from DQMFile"
    runStatus=-1
    runStatus="$(${pathTools}getRunStatusFromDQMFile.py ${file_path}/$dqmFileName $Run_numb runIsComplete | wc -l)"
    if [[ ${runStatus} == 0 ]] 
	then 
	echo ${Run_numb} >> ${curdir}/runsNotComplete_tmp.txt
        if [ ${FORCE} == 0 ] 
	then 
	    continue; 
	fi
    fi
    ## LG end

    if [ $FORCE == 0 ]; then
	check_runcomplete ${file_path}/$dqmFileName
	if [ $? -ne 0 ]; then continue; fi
    fi

    echo Process ${file_path}/$dqmFileName

    cd /tmp

    nnn=`echo $Run_numb | awk '{print substr($0,0,4)}'` 
  

    echo ' dqmFileName = ['$dqmFileName']'

     if [[ "${dqmFileName}" == "" ]] 
     then
 	echo "Run ${Run_numb} not yet ready"
 	continue
     fi
   
    cd ${WORKINGDIR}    

    [ -e $Run_numb ] || mkdir $Run_numb;
    [ -e $Run_numb/$thisDataset ] || mkdir $Run_numb/$thisDataset;
    echo "Run ${Run_numb}"

    cd $Run_numb/$thisDataset
    echo `pwd`
    rm -f *.png
    rm -f *.xml
    rm -f *.log
    rm -f *.txt
    rm -f *.html
    rm -f *.root


    cp ${WORKINGDIR}/DQM/SiStripMonitorClient/scripts/DeadROCCounter.py .
    cp ${WORKINGDIR}/DQM/SiStripMonitorClient/scripts/DeadROCCounter_Phase1.py .

# Determine the GlobalTag name used to process the data and the DQM

    GLOBALTAG=`getGTfromDQMFile.py ${file_path}/$dqmFileName $Run_numb globalTag_Step1`
    
    if [[ "${GLOBALTAG}" == "" ]]
        then
        echo " No GlobalTag found: trying from DAS.... "
        GLOBALTAG=`getGTscript.sh $dqmFileName $Run_numb`
        fi
    if [[ "${GLOBALTAG}" == "" ]]
    then
        echo " No GlobalTag found: skipping this run.... "
        continue
    fi

#Temporary fix to remove hidden ASCII characters
    GLOBALTAG=`echo $GLOBALTAG | cut -c 9-${#GLOBALTAG}`
#    GLOBALTAG=`sed -i 's/[\d128-\d255]//g' <<< "${GLOBALTAG}"`
#    GLOBALTAG=`echo $GLOBALTAG | sed 's/[\d128-\d255]//'`
#    echo `expr length $GLOBALTAG`

    echo " Creating the TrackerMap.... "

    detIdInfoFileName=`echo "file://TkDetIdInfo_Run${Run_numb}_${thisDataset}.root"`

    #cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/SiStripDQM_OfflineTkMap_Template_cfg_DB.py print globalTag=${GLOBALTAG} runNumber=${Run_numb} dqmFile=${file_path}/$dqmFileName  # update GlobalTag
    cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/SiStripDQM_OfflineTkMap_Template_cfg_DB.py print globalTag=${GLOBALTAG} runNumber=${Run_numb} dqmFile=${file_path}/$dqmFileName  detIdInfoFile=${detIdInfoFileName} # update GlobalTag

# rename bad module list file

     mv QTBadModules.log QualityTest_run${Run_numb}.txt

    if [ $thisDataset == "Cosmics" ]; then  # should I add StreamExpressCosmics too
	cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap_cosmics.html | sed -e "s@RunNumber@$Run_numb@g" > index.html
    else
	cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap.html | sed -e "s@RunNumber@$Run_numb@g" > index.html
    fi
    cp ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/fedmap.html fedmap.html
    cp ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/psumap.html psumap.html

    echo " Check TrackerMap on $Run_numb/$thisDataset folder"

    nnn=`echo ${Run_numb} | awk '{print substr($0,0,3)}'`

## Producing the list of bad modules
    echo " Creating the list of bad modules "
    
    listbadmodule ${file_path}/$dqmFileName PCLBadComponents.log
   if [ "$thisDataset" != "StreamExpress" ] ; then
       sefile=QualityTest_run${Run_numb}.txt

       if [ "$thisDataset" == "Cosmics" ]; then
           python ../../DQM/SiStripMonitorClient/scripts/findBadModT9.py -p $sefile -s /data/users/event_display/${DataLocalDir}/Cosmics/${nnn}/${Run_numb}/StreamExpressCosmics/${sefile}
       else

           python ../../DQM/SiStripMonitorClient/scripts/findBadModT9.py -p $sefile -s /data/users/event_display/${DataLocalDir}/Beam/${nnn}/${Run_numb}/StreamExpress/${sefile}

       fi
   fi

#    mv QualityTest*txt $Run_numb/$thisDataset

## Producing the run certification by lumisection
    echo " Creating the lumisection certification:"

    if [ $thisDataset == "MinimumBias" -o $thisDataset == "StreamExpress" ]; then	
	ls_cert 0.95 0.95 ${file_path}/$dqmFileName
#	mv Certification_run_* $Run_numb/$thisDataset
    fi

## Producing the PrimaryVertex/BeamSpot quality test by LS..
    if [ "$thisDataset" != "Cosmics" ]  &&  [ "$thisDataset" != "StreamExpress" ]  &&  [ "$thisDataset" != "StreamExpressCosmics" ]; then
	echo " Creating the BeamSpot Calibration certification summary:"

	lsbs_cert  ${file_path}/$dqmFileName

#	mv Certification_BS_run_* $Run_numb/$thisDataset
    fi
## .. and harvest the bad beamspot LS with automatic emailing (if in period and if bad LS found)
#    bs_bad_ls_harvester $Run_numb/$thisDataset $Run_numb
    bs_bad_ls_harvester . $Run_numb

## Producing the Module difference for ExpressStream
    if [ $thisDataset == "StreamExpress" -o $thisDataset == "StreamExpressCosmics" ]; then	
	echo " Creating the Module Status Difference summary:"

#	modulediff $Run_numb ${1}
#	./modulediff_summary $Run_numb
#	mv ModuleDifference_${Run_numb}.txt $Run_numb/$thisDataset
    fi

    dest=Beam
    if [ $thisDataset == "Cosmics" -o $thisDataset == "StreamExpressCosmics" ]; then dest="Cosmics"; fi

# overwrite destination for tests
# dest=FinalTest

## create merged list of BadComponent from (PCL, RunInfo and FED Errors)
    cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/mergeBadChannel_Template_cfg.py globalTag=${GLOBALTAG} runNumber=${Run_numb} dqmFile=${file_path}/$dqmFileName
    mv MergedBadComponents.log MergedBadComponents_run${Run_numb}.txt

    rm -f *.xml
    rm -f *svg

    ssh cctrack@vocms061 "mkdir -p /data/users/event_display/TkCommissioner_runs/${DataLocalDir}/${dest} 2> /dev/null"
    scp *.root cctrack@vocms061:/data/users/event_display/TkCommissioner_runs/${DataLocalDir}/${dest}
    rm *.root

    echo "counting dead pixel ROCs" 
    echo "DataLocalDir = ${DataLocalDir}"
    if [[ $DataLocalDir == "Data2016" || $DataLocalDir == "Data2015" || $DataLocalDir == "Data2013" || $DataLocalDir == "Data2016" ]]; then 
       ./DeadROCCounter.py ${file_path}/$dqmFileName
    else 
       ./DeadROCCounter_Phase1.py ${file_path}/$dqmFileName
    fi
    rm -f DeadROCCounter.py
    rm -f DeadROCCounter_Phase1.py

#    mkdir -p /data/users/event_display/${DataLocalDir}/${dest}/${nnn}/${Run_numb}/$thisDataset #2> /dev/null
#    cp -r ${Run_numb}/$thisDataset /data/users/event_display/Data2011/${dest}/${nnn}/${Run_numb}/
#    cp -r ${Run_numb}/$thisDataset /data/users/event_display/${DataLocalDir}/${dest}/${nnn}/${Run_numb}/$thisDataset 
    ssh cctrack@vocms061 "mkdir -p /data/users/event_display/${DataLocalDir}/${dest}/${nnn}/${Run_numb}/$thisDataset 2> /dev/null"
    scp -r * cctrack@vocms061:/data/users/event_display/${DataLocalDir}/${dest}/${nnn}/${Run_numb}/$thisDataset

     rm ${file_path}/$dqmFileName

     cd ${WORKINGDIR}    
     rm -rf $Run_numb
     rm -rf index.html

#done with loop over thisDataset
done

done