Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2021-02-14 13:11:47

0001 #!/bin/bash
0002 
0003 curdir="$(pwd)"
0004 echo ${curdir}
0005 
0006 export PATH=/afs/cern.ch/cms/common:${PATH}
0007 if [[ "$#" == "0" ]]; then
0008     echo "usage: 'TkMap_script_automatic.sh Cosmics|MinimumBias|StreamExpress|StreamExpressCosmics runNumber1 runNumber2...'";
0009     exit 1;
0010 fi
0011 
0012 FORCE=0
0013 echo $2
0014 if [ "${2}" == "0" ]; then
0015     FORCE=0
0016 else
0017     if [ "${2}" == "f" ]; then
0018         FORCE=1
0019     fi
0020 fi
0021 
0022 export WORKINGDIR=${CMSSW_BASE}/src
0023 
0024 cd ${WORKINGDIR}
0025 
0026 DataLocalDir=''
0027 DataOflineDir=''
0028 
0029 for Run_numb in $@;
0030 do
0031 
0032     if [ $Run_numb -gt 284500 ]; then
0033 
0034         DataLocalDir='Data2017'
0035         DataOfflineDir='Run2017'
0036     else
0037 
0038 
0039     if [ "$Run_numb" == "$1" ]; then continue; fi
0040 
0041 ##2016 data taking period run > 271024
0042     if [ $Run_numb -gt 271024 ]; then
0043 
0044         DataLocalDir='Data2016'
0045         DataOfflineDir='Run2016'
0046     else
0047 
0048 #2016 - Commissioning period                                                                                                                               
0049     if [ $Run_numb -gt 264200 ]; then
0050 
0051         DataLocalDir='Data2016'
0052         DataOfflineDir='Commissioning2016'
0053     else
0054 
0055     #Run2015A
0056     if [ $Run_numb -gt 246907 ]; then
0057         DataLocalDir='Data2015'
0058         DataOfflineDir='Run2015'
0059     else
0060 
0061     #2015 Commissioning period (since January)
0062     if [ $Run_numb -gt 232881 ]; then
0063         DataLocalDir='Data2015'
0064         DataOfflineDir='Commissioning2015'
0065     else
0066     #2013 pp run (2.76 GeV)
0067         if [ $Run_numb -gt 211658 ]; then
0068             DataLocalDir='Data2013'
0069             DataOfflineDir='Run2013'
0070         else
0071     #2013 HI run
0072             if [ $Run_numb -gt 209634 ]; then
0073                 DataLocalDir='Data2013'
0074                 DataOfflineDir='HIRun2013'
0075             else
0076                 if [ $Run_numb -gt 190450 ]; then
0077                     DataLocalDir='Data2012'
0078                     DataOfflineDir='Run2012'
0079                 fi
0080             fi
0081         fi
0082     fi
0083     fi
0084     fi
0085     fi
0086     fi
0087     #loop over datasets
0088     #if Cosmics, do StreamExpressCosmics as well
0089 
0090     datasets=${1}
0091     if [ "${1}" == "Cosmics" ]; then
0092         prefix=`echo $Run_numb | awk '{print substr($0,0,3)}'` 
0093         checkdir='/data/users/event_display/'${DataLocalDir}'/Cosmics/'${prefix}'/'${Run_numb}'/StreamExpressCosmics'
0094         if [ ! -d $checkdir ]; then
0095             datasets=$datasets' StreamExpressCosmics'
0096             echo "Running on datasets "$datasets
0097         fi
0098     fi
0099     
0100     for thisDataset in $datasets
0101       do
0102       echo "Processing "$thisDataset "in "${DataOfflineDir}"..."
0103 
0104       nnn=`echo $Run_numb | awk '{print substr($0,0,4)}'` 
0105 
0106       echo 'Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'${DataOfflineDir}'/'$thisDataset'/000'${nnn}'xx/'
0107   
0108     curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'${DataOfflineDir}'/'$thisDataset'/000'${nnn}'xx/' > index.html
0109     dqmFileNames=`cat index.html | grep ${Run_numb} | egrep "_DQM.root|_DQMIO.root" | egrep "Prompt|Express|22Jan2013" | sed 's/.*>\(.*\)<\/a.*/\1/' `
0110     dqmFileName=`expr "$dqmFileNames" : '\(DQM[A-Za-z0-9_/.\-]*root\)'`
0111     echo ' dqmFileNames = '$dqmFileNames
0112     echo ' dqmFileName = ['$dqmFileName']'
0113     curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/$DataOfflineDir/$thisDataset/000${nnn}xx/${dqmFileName} > /tmp/${dqmFileName}
0114     checkFile=`ls /tmp/${dqmFileName} | grep ${Run_numb}`
0115 
0116 ##check if the full run is fully processed in GUI (Info/Run summary/ProvInfo/ runIsComplete flag == 1? 
0117 ##if not, throw a warning
0118 
0119     file_path="/tmp/"
0120 
0121     echo "FORCE is " ${FORCE}
0122     ## check if run is complete - LG
0123     echo "get the run status from DQMFile"
0124     runStatus=-1
0125     runStatus="$(${pathTools}getRunStatusFromDQMFile.py ${file_path}/$dqmFileName $Run_numb runIsComplete | wc -l)"
0126     if [[ ${runStatus} == 0 ]] 
0127         then 
0128         echo ${Run_numb} >> ${curdir}/runsNotComplete_tmp.txt
0129         if [ ${FORCE} == 0 ] 
0130         then 
0131             continue; 
0132         fi
0133     fi
0134     ## LG end
0135 
0136     if [ $FORCE == 0 ]; then
0137         check_runcomplete ${file_path}/$dqmFileName
0138         if [ $? -ne 0 ]; then continue; fi
0139     fi
0140 
0141     echo Process ${file_path}/$dqmFileName
0142 
0143     cd /tmp
0144 
0145     nnn=`echo $Run_numb | awk '{print substr($0,0,4)}'` 
0146   
0147 
0148     echo ' dqmFileName = ['$dqmFileName']'
0149 
0150      if [[ "${dqmFileName}" == "" ]] 
0151      then
0152         echo "Run ${Run_numb} not yet ready"
0153         continue
0154      fi
0155    
0156     cd ${WORKINGDIR}    
0157 
0158     [ -e $Run_numb ] || mkdir $Run_numb;
0159     [ -e $Run_numb/$thisDataset ] || mkdir $Run_numb/$thisDataset;
0160     echo "Run ${Run_numb}"
0161 
0162     cd $Run_numb/$thisDataset
0163     echo `pwd`
0164     rm -f *.png
0165     rm -f *.xml
0166     rm -f *.log
0167     rm -f *.txt
0168     rm -f *.html
0169     rm -f *.root
0170 
0171 
0172     cp ${WORKINGDIR}/DQM/SiStripMonitorClient/scripts/DeadROCCounter.py .
0173     cp ${WORKINGDIR}/DQM/SiStripMonitorClient/scripts/DeadROCCounter_Phase1.py .
0174 
0175 # Determine the GlobalTag name used to process the data and the DQM
0176 
0177     GLOBALTAG=`getGTfromDQMFile.py ${file_path}/$dqmFileName $Run_numb globalTag_Step1`
0178     
0179     if [[ "${GLOBALTAG}" == "" ]]
0180         then
0181         echo " No GlobalTag found: trying from DAS.... "
0182         GLOBALTAG=`getGTscript.sh $dqmFileName $Run_numb`
0183         fi
0184     if [[ "${GLOBALTAG}" == "" ]]
0185     then
0186         echo " No GlobalTag found: skipping this run.... "
0187         continue
0188     fi
0189 
0190 #Temporary fix to remove hidden ASCII characters
0191     GLOBALTAG=`echo $GLOBALTAG | cut -c 9-${#GLOBALTAG}`
0192 #    GLOBALTAG=`sed -i 's/[\d128-\d255]//g' <<< "${GLOBALTAG}"`
0193 #    GLOBALTAG=`echo $GLOBALTAG | sed 's/[\d128-\d255]//'`
0194 #    echo `expr length $GLOBALTAG`
0195 
0196     echo " Creating the TrackerMap.... "
0197 
0198     detIdInfoFileName=`echo "file://TkDetIdInfo_Run${Run_numb}_${thisDataset}.root"`
0199 
0200     #cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/SiStripDQM_OfflineTkMap_Template_cfg_DB.py print globalTag=${GLOBALTAG} runNumber=${Run_numb} dqmFile=${file_path}/$dqmFileName  # update GlobalTag
0201     cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/SiStripDQM_OfflineTkMap_Template_cfg_DB.py print globalTag=${GLOBALTAG} runNumber=${Run_numb} dqmFile=${file_path}/$dqmFileName  detIdInfoFile=${detIdInfoFileName} # update GlobalTag
0202 
0203 # rename bad module list file
0204 
0205      mv QTBadModules.log QualityTest_run${Run_numb}.txt
0206 
0207     if [ $thisDataset == "Cosmics" ]; then  # should I add StreamExpressCosmics too
0208         cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap_cosmics.html | sed -e "s@RunNumber@$Run_numb@g" > index.html
0209     else
0210         cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap.html | sed -e "s@RunNumber@$Run_numb@g" > index.html
0211     fi
0212     cp ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/fedmap.html fedmap.html
0213     cp ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/psumap.html psumap.html
0214 
0215     echo " Check TrackerMap on $Run_numb/$thisDataset folder"
0216 
0217     nnn=`echo ${Run_numb} | awk '{print substr($0,0,3)}'`
0218 
0219 ## Producing the list of bad modules
0220     echo " Creating the list of bad modules "
0221     
0222     listbadmodule ${file_path}/$dqmFileName PCLBadComponents.log
0223    if [ "$thisDataset" != "StreamExpress" ] ; then
0224        sefile=QualityTest_run${Run_numb}.txt
0225 
0226        if [ "$thisDataset" == "Cosmics" ]; then
0227            python ../../DQM/SiStripMonitorClient/scripts/findBadModT9.py -p $sefile -s /data/users/event_display/${DataLocalDir}/Cosmics/${nnn}/${Run_numb}/StreamExpressCosmics/${sefile}
0228        else
0229 
0230            python ../../DQM/SiStripMonitorClient/scripts/findBadModT9.py -p $sefile -s /data/users/event_display/${DataLocalDir}/Beam/${nnn}/${Run_numb}/StreamExpress/${sefile}
0231 
0232        fi
0233    fi
0234 
0235 #    mv QualityTest*txt $Run_numb/$thisDataset
0236 
0237 ## Producing the run certification by lumisection
0238     echo " Creating the lumisection certification:"
0239 
0240     if [ $thisDataset == "MinimumBias" -o $thisDataset == "StreamExpress" ]; then       
0241         ls_cert 0.95 0.95 ${file_path}/$dqmFileName
0242 #       mv Certification_run_* $Run_numb/$thisDataset
0243     fi
0244 
0245 ## Producing the PrimaryVertex/BeamSpot quality test by LS..
0246     if [ "$thisDataset" != "Cosmics" ]  &&  [ "$thisDataset" != "StreamExpress" ]  &&  [ "$thisDataset" != "StreamExpressCosmics" ]; then
0247         echo " Creating the BeamSpot Calibration certification summary:"
0248 
0249         lsbs_cert  ${file_path}/$dqmFileName
0250 
0251 #       mv Certification_BS_run_* $Run_numb/$thisDataset
0252     fi
0253 ## .. and harvest the bad beamspot LS with automatic emailing (if in period and if bad LS found)
0254 #    bs_bad_ls_harvester $Run_numb/$thisDataset $Run_numb
0255     bs_bad_ls_harvester . $Run_numb
0256 
0257 ## Producing the Module difference for ExpressStream
0258     if [ $thisDataset == "StreamExpress" -o $thisDataset == "StreamExpressCosmics" ]; then      
0259         echo " Creating the Module Status Difference summary:"
0260 
0261 #       modulediff $Run_numb ${1}
0262 #       ./modulediff_summary $Run_numb
0263 #       mv ModuleDifference_${Run_numb}.txt $Run_numb/$thisDataset
0264     fi
0265 
0266     dest=Beam
0267     if [ $thisDataset == "Cosmics" -o $thisDataset == "StreamExpressCosmics" ]; then dest="Cosmics"; fi
0268 
0269 # overwrite destination for tests
0270 # dest=FinalTest
0271 
0272 ## create merged list of BadComponent from (PCL, RunInfo and FED Errors)
0273     cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/mergeBadChannel_Template_cfg.py globalTag=${GLOBALTAG} runNumber=${Run_numb} dqmFile=${file_path}/$dqmFileName
0274     mv MergedBadComponents.log MergedBadComponents_run${Run_numb}.txt
0275 
0276     rm -f *.xml
0277     rm -f *svg
0278 
0279     ssh cctrack@vocms061 "mkdir -p /data/users/event_display/TkCommissioner_runs/${DataLocalDir}/${dest} 2> /dev/null"
0280     scp *.root cctrack@vocms061:/data/users/event_display/TkCommissioner_runs/${DataLocalDir}/${dest}
0281     rm *.root
0282 
0283     echo "counting dead pixel ROCs" 
0284     echo "DataLocalDir = ${DataLocalDir}"
0285     if [[ $DataLocalDir == "Data2016" || $DataLocalDir == "Data2015" || $DataLocalDir == "Data2013" || $DataLocalDir == "Data2016" ]]; then 
0286        ./DeadROCCounter.py ${file_path}/$dqmFileName
0287     else 
0288        ./DeadROCCounter_Phase1.py ${file_path}/$dqmFileName
0289     fi
0290     rm -f DeadROCCounter.py
0291     rm -f DeadROCCounter_Phase1.py
0292 
0293 #    mkdir -p /data/users/event_display/${DataLocalDir}/${dest}/${nnn}/${Run_numb}/$thisDataset #2> /dev/null
0294 #    cp -r ${Run_numb}/$thisDataset /data/users/event_display/Data2011/${dest}/${nnn}/${Run_numb}/
0295 #    cp -r ${Run_numb}/$thisDataset /data/users/event_display/${DataLocalDir}/${dest}/${nnn}/${Run_numb}/$thisDataset 
0296     ssh cctrack@vocms061 "mkdir -p /data/users/event_display/${DataLocalDir}/${dest}/${nnn}/${Run_numb}/$thisDataset 2> /dev/null"
0297     scp -r * cctrack@vocms061:/data/users/event_display/${DataLocalDir}/${dest}/${nnn}/${Run_numb}/$thisDataset
0298 
0299      rm ${file_path}/$dqmFileName
0300 
0301      cd ${WORKINGDIR}    
0302      rm -rf $Run_numb
0303      rm -rf index.html
0304 
0305 #done with loop over thisDataset
0306 done
0307 
0308 done