MillePedeAlignmentAlgorithm/templates/mps_runPede_rfcp_template.sh

0001 #!/bin/bash
0002 #
0003 # Run script template for Pede job, copying binary files from mass storage to local disk.
0004 #
0005 # Adjustments might be needed for CMSSW environment.
0006
0007 # The batch job directory (will vanish after job end):
0008 BATCH_DIR=$(pwd)
0009 echo -e "Running at $(date) \n        on ${HOSTNAME} \n        in directory ${BATCH_DIR}."
0010
0011 # set up the CMS environment
0012 cd CMSSW_RELEASE_AREA
0013 eval `scram runtime -sh`
0014 hash -r
0015
0016 cd ${BATCH_DIR}
0017 echo Running directory changed to $(pwd).
0018
0019 # these defaults will be overwritten by MPS
0020 RUNDIR=${HOME}/scratch0/some/path
0021 MSSDIR=/castor/cern.ch/user/u/username/another/path
0022 MSSDIRPOOL=
0023 CONFIG_FILE=
0024
0025 export X509_USER_PROXY=${RUNDIR}/.user_proxy
0026
0027 #get list of treefiles
0028 TREEFILELIST=
0029 if [ "${MSSDIRPOOL}" != "cmscafuser" ]
0030 then
0031     :                           # do nothing
0032 else
0033     TREEFILELIST=$(ls -l ${MSSDIR}/tree_files)
0034 fi
0035 if [[ -z "${TREEFILELIST}" ]]
0036 then
0037     echo -e "\nThe list of treefiles seems to be empty.\n"
0038 fi
0039
0040 clean_up () {
0041 #try to recover log files and root files
0042     echo try to recover log files and root files ...
0043     cp -p pede.dump* ${RUNDIR}
0044     cp -p *.txt.* ${RUNDIR}
0045     cp -p *.log ${RUNDIR}
0046     cp -p *.log.gz ${RUNDIR}
0047     cp -p millePedeMonitor*root ${RUNDIR}
0048     cp -p millepede.res* ${RUNDIR}
0049     cp -p millepede.end ${RUNDIR}
0050     cp -p millepede.his* ${RUNDIR}
0051     cp -p *.db ${RUNDIR}
0052     exit
0053 }
0054 #LSF signals according to http://batch.web.cern.ch/batch/lsf-return-codes.html
0055 trap clean_up HUP INT TERM SEGV USR2 XCPU XFSZ IO
0056
0057
0058 # a helper function to repeatedly try failing copy commands
0059 untilSuccess () {
0060 # trying "${1} ${2} ${3} > /dev/null" until success, if ${4} is a
0061 # positive number run {1} with -f flag and using --cksum md5,
0062 # break after ${5} tries (with four arguments do up to 5 tries).
0063     if  [[ ${#} -lt 4 || ${#} -gt 5 ]]
0064     then
0065         echo ${0} needs 4 or 5 arguments
0066         return 1
0067     fi
0068
0069     TRIES=0
0070     MAX_TRIES=5
0071     if [[ ${#} -eq 5 ]]
0072     then
0073         MAX_TRIES=${5}
0074     fi
0075
0076
0077     if [[ ${4} -gt 0 ]]
0078     then
0079         ${1} -f --cksum md5 ${2} ${3} > /dev/null
0080     else
0081         ${1} ${2} ${3} > /dev/null
0082     fi
0083     while [[ ${?} -ne 0 ]]
0084     do # if not successfull, retry...
0085         if [[ ${TRIES} -ge ${MAX_TRIES} ]]
0086         then # ... but not until infinity!
0087             if [[ ${4} -gt 0 ]]
0088             then
0089                 echo ${0}: Give up doing \"${1} -f --cksum md5 ${2} ${3} \> /dev/null\".
0090                 return 1
0091             else
0092                 echo ${0}: Give up doing \"${1} ${2} ${3} \> /dev/null\".
0093                 return 1
0094             fi
0095         fi
0096         TRIES=$((${TRIES}+1))
0097         if [[ ${4} -gt 0 ]]
0098         then
0099             echo ${0}: WARNING, problems with \"${1} -f --cksum md5 ${2} ${3} \> /dev/null\", try again.
0100             sleep $((${TRIES}*5)) # for before each wait a litte longer...
0101             ${1} -f --cksum md5 ${2} ${3} > /dev/null
0102         else
0103             echo ${0}: WARNING, problems with \"${1} ${2} ${3} \> /dev/null\", try again.
0104             sleep $((${TRIES}*5)) # for before each wait a litte longer...
0105             ${1} ${2} ${3} > /dev/null
0106         fi
0107     done
0108
0109     if [[ ${4} -gt 0 ]]
0110     then
0111         echo successfully executed \"${1} -f --cksum md5 ${2} ${3} \> /dev/null\"
0112     else
0113         echo successfully executed \"${1} ${2} ${3} \> /dev/null\"
0114     fi
0115     return 0
0116 }
0117
0118 copytreefile () {
0119     CHECKFILE=`echo ${TREEFILELIST} | grep -i ${2}`
0120     if [[ -z "${TREEFILELIST}" ]]
0121     then
0122         untilSuccess ${1} ${2} ${3} ${4}
0123     else
0124         if [[ -n "${CHECKFILE}" ]]
0125         then
0126             untilSuccess ${1} ${2} ${3} ${4}
0127         fi
0128     fi
0129 }
0130
0131 # stage and copy the binary file(s), first set castor pool for binary files in ${MSSDIR} area
0132 export -f untilSuccess
0133 export -f copytreefile
0134 rm -rf stager_get-commands.txt; touch stager_get-commands.txt
0135 rm -rf parallel-copy-commands.txt; touch parallel-copy-commands.txt
0136 if [ "${MSSDIRPOOL}" != "cmscafuser" ]; then
0137 # Not using cmscafuser pool => rfcp command must be used
0138   export STAGE_SVCCLASS=${MSSDIRPOOL}
0139   export STAGER_TRACE=
0140   echo stager_get -M ${MSSDIR}/milleBinaryISN.dat.gz >> stager_get-commands.txt
0141   echo untilSuccess rfcp ${MSSDIR}/milleBinaryISN.dat.gz ${BATCH_DIR} 0 >> parallel-copy-commands.txt
0142   echo stager_get -M ${MSSDIR}/treeFileISN.root >> stager_get-commands.txt
0143   echo copytreefile rfcp ${MSSDIR}/treeFileISN.root ${BATCH_DIR} 0 >> parallel-copy-commands.txt
0144 else
0145   MSSCAFDIR=`echo ${MSSDIR} | perl -pe 's/\/castor\/cern.ch\/cms//gi'`
0146   echo untilSuccess xrdcp ${MSSCAFDIR}/binaries/milleBinaryISN.dat.gz milleBinaryISN.dat.gz 1 >> parallel-copy-commands.txt
0147   echo copytreefile xrdcp ${MSSCAFDIR}/tree_files/treeFileISN.root treeFileISN.root 1 >> parallel-copy-commands.txt
0148 fi
0149 xargs -a stager_get-commands.txt -n 1 -P 10 -I {} bash -c '$@' _ {}
0150 xargs -a parallel-copy-commands.txt -n 1 -P 10 -I {} bash -c '$@' _ {}
0151 rm stager_get-commands.txt
0152 rm parallel-copy-commands.txt
0153
0154
0155 # We have gzipped binaries, but the python config looks for .dat
0156 # (could also try to substitute in config ".dat" with ".dat.gz"
0157 #  ONLY for lines which contain "milleBinary" using "sed '/milleBinary/s/.dat/.dat.gz/g'"):
0158 ln -s milleBinaryISN.dat.gz milleBinaryISN.dat
0159
0160 cd ${BATCH_DIR}
0161 echo Running directory changed to $(pwd).
0162
0163 echo -e "\nDirectory content before running cmsRun:"
0164 ls -lh
0165 # Execute. The cfg file name will be overwritten by MPS
0166 time cmsRun ${CONFIG_FILE}
0167
0168 # clean up what has been staged in (to avoid copy mistakes...)
0169 rm treeFileISN.root
0170 rm milleBinaryISN.dat.gz milleBinaryISN.dat
0171
0172 # Gzip one by one in case one argument cannot be expanded:
0173 gzip -f *.log
0174 gzip -f *.txt
0175 gzip -f *.dump
0176
0177 #Try to merge millepede monitor files. This only works successfully if names were assigned to jobs.
0178 mps_merge_millepedemonitor.pl ${RUNDIR}/../../mps.db ${RUNDIR}/../../
0179
0180 # Merge possible alignment monitor and millepede monitor hists...
0181 # ...and remove individual histogram files after merging to save space (if success):
0182 # NOTE: the names "histograms.root" and "millePedeMonitor.root" must match what is in
0183 #      your  alignment_cfg.py!
0184 #hadd histograms_merge.root ${RUNDIR}/../job???/histograms.root
0185 #if [ $? -eq 0 ]; then
0186 #    rm ${RUNDIR}/../job???/histograms.root
0187 #fi
0188 hadd millePedeMonitor_merge.root ${RUNDIR}/../job???/millePedeMonitor*.root
0189 if [[ ${?} -eq 0 ]]
0190 then
0191     rm ${RUNDIR}/../job???/millePedeMonitor*.root
0192 else
0193     rm millePedeMonitor_merge.root
0194 fi
0195
0196 # Macro creating chi2ndfperbinary.pdf with pede chi2/ndf information hists:
0197 if [[ -e ${CMSSW_BASE}/src/Alignment/MillePedeAlignmentAlgorithm/macros/createChi2ndfplot.C ]]
0198 then
0199     # Checked out version if existing:
0200     cp ${CMSSW_BASE}/src/Alignment/MillePedeAlignmentAlgorithm/macros/createChi2ndfplot.C .
0201 else
0202     # If nothing checked out, take from release:
0203     cp ${CMSSW_RELEASE_BASE}/src/Alignment/MillePedeAlignmentAlgorithm/macros/createChi2ndfplot.C .
0204 fi
0205 mps_parse_pedechi2hist.py -d ${RUNDIR}/../../mps.db --his millepede.his -c ${CONFIG_FILE}
0206 if [[ -f chi2pedehis.txt ]]
0207 then
0208     root -l -x -b -q 'createChi2ndfplot.C+("chi2pedehis.txt")'
0209 fi
0210
0211 # Macro creating millepede.his.pdf with pede information hists:
0212 if [[ -e ${CMSSW_BASE}/src/Alignment/MillePedeAlignmentAlgorithm/macros/readPedeHists.C ]]
0213 then
0214     # Checked out version if existing:
0215     cp ${CMSSW_BASE}/src/Alignment/MillePedeAlignmentAlgorithm/macros/readPedeHists.C .
0216 else
0217     # If nothing checked out, take from release:
0218     cp ${CMSSW_RELEASE_BASE}/src/Alignment/MillePedeAlignmentAlgorithm/macros/readPedeHists.C .
0219 fi
0220 root -b -q "readPedeHists.C+(\"print nodraw\")"
0221
0222 # zip plot files:
0223 gzip -f *.pdf
0224 # now zip .his and .res:
0225 gzip -f millepede.*s
0226 # in case of diagonalisation zip this:
0227 gzip -f millepede.eve
0228 # zip monitoring file:
0229 gzip -f millepede.mon
0230
0231 #list IOVs
0232 for tag in $(sqlite3 alignments_MP.db  "SELECT NAME FROM TAG;")
0233 do
0234     conddb --db alignments_MP.db list ${tag}
0235 done
0236
0237 #split the IOVs
0238 aligncond_split_iov.sh alignments_MP.db alignments_split_MP.db
0239
0240 echo -e "\nDirectory content after running cmsRun, zipping log file and merging histogram files:"
0241 ls -lh
0242 # Copy everything you need to MPS directory of your job
0243 # (separate cp's for each item, otherwise you loose all if one file is missing):
0244 cp -p *.root ${RUNDIR}
0245 cp -p *.gz ${RUNDIR}
0246 cp -p *.db ${RUNDIR}
0247 cp -p *.end ${RUNDIR}
0248
0249 # create symlinks of the monitoring files in other (possibly non-existing) jobm folders
0250 nTry=0
0251 while true
0252 do
0253     if [[ ${nTry} -ge 10 ]]     # wait up to 10 times for monitoring files
0254     then
0255         break
0256     fi
0257
0258     monFiles=$(ls --color=never ${RUNDIR}/../jobm*/*.root | egrep -i 'millepedemonitor_.+\.root$')
0259     if [[ ${?} -eq 0 ]]
0260     then
0261         monFiles=$(echo ${monFiles} | xargs -n 1 readlink -e)
0262         break
0263     else
0264         sleep 60                # wait a minute and retry
0265         nTry=$((${nTry} + 1))
0266     fi
0267 done
0268 jobmFolders=$(ls --color=never -d ${RUNDIR}/../jobm* | xargs -n 1 readlink -e)
0269 for folder in ${jobmFolders}
0270 do
0271     for mon in ${monFiles}
0272     do
0273         ln -s ${mon} ${folder}/ > /dev/null 2>&1
0274         ln -s ${mon} > /dev/null 2>&1 # make them also available here
0275     done
0276 done
0277
0278 # copy aligment_merge.py for mps_validate.py
0279 ln -s ${CONFIG_FILE} alignment_merge.py
0280 ln -s ${RUNDIR}/.TrackerTree.root
0281 ln -s ${RUNDIR}/.weights.pkl
0282 # run mps_validate.py
0283 campaign=`basename ${MSSDIR}`
0284 mps_validate.py -m ${campaign} -p ./
0285
0286 cp -pr validation_output ${RUNDIR}