TrackerAlignment/scripts/SkimLooper.tpl

0001 #! /bin/bash
0002
0003 ### $1: file with list of ALCA types you want to process; remember to put for each of them
0004 ###     an ALCARECO style cfg in Alignment/CommonAlignmentProducer/python/
0005 #
0006 ### $2: number, if greater than 2 it cleans the CASTOR area before starting (optional)
0007
0008 source /afs/cern.ch/cms/caf/setup.sh
0009 curdir=$(pwd)
0010
0011 #CASTOR_OUT="/castor/cern.ch/cms/store/user/bonato/CRAFTReproSkims/Craft09/4T/"
0012 ALCAFILELIST=$1
0013 CASTOR_OUT="<CASTOROUT>"
0014 DQM_OUT="${curdir}/MONITORING/DQM/"
0015 MAXEVENTS=10000
0016
0017 #curdir=$( pwd )
0018
0019 #check if output directory exists
0020 nsls /castor/cern.ch/cms/$CASTOR_OUT
0021 if [ $? -ne 0 ]
0022 then
0023 echo "Output directory: "
0024 echo /castor/cern.ch/cms/$CASTOR_OUT
0025 echo "does not exist. Please check the scripts. Exiting."
0026 exit 1
0027 fi
0028
0029
0030 #check if DQM output directory exists
0031 ls $DQM_OUT
0032 if [ $? -ne 0 ]
0033 then
0034 echo "DQM directory: "
0035 echo $DQM_OUT
0036 echo "does not exist. Please check the scripts. Exiting."
0037 exit 1
0038 fi
0039
0040
0041 ## Clean output directory
0042
0043 if [ $# -gt 1 ]
0044 then
0045
0046     if [ $2 -gt 2 ]
0047         then
0048
0049         if [ $(nsls -l /castor/cern.ch/cms/$CASTOR_OUT | wc -l ) -gt 1 ] #there is always the dir 'logfiles'
0050             then
0051             echo "Cleaning output directory: $CASTOR_OUT"
0052
0053             for file in $(nsls /castor/cern.ch/cms/$CASTOR_OUT/ | grep "Skimmed" )
0054               do
0055 #echo
0056               rfrm  /castor/cern.ch/cms/$CASTOR_OUT/$file
0057             done
0058
0059             for file in $(nsls /castor/cern.ch/cms/$CASTOR_OUT/logfiles/)
0060               do
0061 #echo
0062               rfrm  /castor/cern.ch/cms/$CASTOR_OUT/logfiles/$file
0063             done
0064         fi
0065
0066     fi #end if $2 > 2
0067
0068 fi #end if $# > 2
0069
0070
0071 #really needed ?
0072 #export STAGE_SVCCLASS=cmscaf
0073
0074 for ALCATAG in $( cat $ALCAFILELIST  )
0075 do
0076
0077   echo
0078   echo "*****************************************"
0079   echo "*** Starting the ALCATAG: ${ALCATAG}"
0080   echo "*****************************************"
0081   echo
0082 #  DAT_FILE="/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/HIP/bonato/DEVEL/HIPWorkflow/ALCARECOskim/v1.4/data/${ALCATAG}.dat"
0083   DAT_FILE="${curdir}/../data/${ALCATAG}.dat"
0084   TPL_FILE="TkAlCaRecoSkimming.${ALCATAG}.tpl"
0085   TAG=$ALCATAG #"CRAFT"
0086   JOBTAG="ALCASkim_"$TAG
0087   BASE_TPL=$(basename "$TPL_FILE" .tpl)
0088   echo ""
0089   echo "I am in $curdir"
0090   INDEX=1
0091
0092
0093
0094 # echo "*******************************"
0095 # echo "Starting the show"
0096 # echo
0097
0098   TOTFILES=0
0099   for i in $( cat $DAT_FILE )
0100     do
0101
0102 #pick the total nr events in this file from the previously produced nevents.out
0103     let TOTFILES=TOTFILES+1
0104     TOTEVTS=$(sed -n $TOTFILES'p' ../data/nevents${ALCATAG}.out)
0105 #echo "The file #$TOTFILES has $TOTEVTS events"
0106     if [ $TOTEVTS == 0 ]
0107         then
0108         continue
0109     fi
0110     TOTSPLITS=$(( ( $TOTEVTS / $MAXEVENTS ) +1 ))
0111     firstev=0
0112     lastev=-1
0113
0114 #echo "I will split it into $TOTSPLITS"
0115     if [ $TOTSPLITS > 1 ]
0116         then
0117         nsplits=1
0118
0119
0120         while [ $nsplits -le $TOTSPLITS  ]
0121           do
0122 #echo "Splitting the file $TOTFILE : $nsplits"
0123           firstev=$(( $MAXEVENTS*$(( $nsplits-1 ))+1 ))
0124           lastev=$MAXEVENTS    #$(( ($MAXEVENTS*$nsplits) ))
0125           JOB=$JOBTAG"_file"$INDEX
0126           CFG_FILE=$BASE_TPL"."$TAG"_cfg."$INDEX".py"
0127           sed -e "s|<JOB>|${JOB}|g" -e "s|<INPATH>|${i}|g" -e "s|<INIEVT>|${firstev}|g" -e "s|<FINEVT>|${lastev}|g"  -e "s|<ALCATAG>|${TAG}|g"  < $TPL_FILE > $CFG_FILE
0128           let INDEX=INDEX+1
0129           let nsplits=nsplits+1
0130 #     if [ $INDEX -ge 3 ]
0131 #       then
0132 #       echo "Reached a maximum number of files: $INDEX. Stopping the submission"
0133 #       break
0134 #     fi
0135         done
0136
0137     else #file is small and does not contain too many events
0138         firstev=0
0139         lastev=-1
0140         JOB=$JOBTAG"_file"$INDEX
0141         CFG_FILE=$BASE_TPL"."$TAG"_cfg."$INDEX".py"
0142         sed -e "s|<JOB>|${JOB}|g" -e "s|<INPATH>|${i}|g" -e "s|<INIEVT>|${firstev}|g" -e "s|<FINEVT>|${lastev}|g"  -e "s|<ALCATAG>|${TAG}|g"  < $TPL_FILE > $CFG_FILE
0143         let INDEX=INDEX+1
0144
0145 # if [ $INDEX -ge 3 ]
0146 #       then
0147 #           echo "Reached a maximum number of files: $INDEX. Stopping the submission"
0148 #       break
0149 #       fi
0150
0151     fi
0152
0153  #   if [ $INDEX -ge 3 ]
0154 #       then
0155 #       echo "Reached a maximum number of files: $INDEX. Stopping the submission"
0156 #       break
0157 #    fi
0158
0159 #echo "--- moving to next file. At the moment INDEX=$INDEX"
0160
0161
0162   done
0163
0164   TOTCFGFILES=$(( $INDEX -1 ))
0165 #echo "Tot cfg files: $TOTCFGFILES"
0166
0167 ##second loop submitting what we prepared before
0168   INDEX=1
0169   while [ $INDEX -le $TOTCFGFILES ]
0170     do
0171     JOBNAME="ALCASkim"$TAG"_"$INDEX
0172     LOGFILE="${JOBNAME}.log"
0173     CFG_FILE=$BASE_TPL"."$TAG"_cfg."$INDEX".py"
0174
0175
0176     echo "Submitting $JOBNAME with config file $CFG_FILE"
0177
0178     REM=0
0179     let "REM=$INDEX % 300"
0180
0181     if [ $REM -lt 100 ]
0182         then
0183         #echo "dummy A"
0184         bsub -q cmscaf1nd -J $JOBNAME -oo $LOGFILE skim_exec.sh "$curdir/$CFG_FILE" "$CASTOR_OUT" "$DQM_OUT"
0185     elif [ $REM -lt 200 ]
0186         then
0187         #echo "dummy B"
0188         bsub -q cmsexpress -J $JOBNAME -oo $LOGFILE skim_exec.sh "$curdir/$CFG_FILE" "$CASTOR_OUT" "$DQM_OUT"
0189     else
0190         #echo "dummy C"
0191         bsub -q cmscaf1nd    -J $JOBNAME -oo $LOGFILE skim_exec.sh "$curdir/$CFG_FILE" "$CASTOR_OUT" "$DQM_OUT"
0192     fi
0193
0194
0195 ##fi #dummy
0196
0197
0198     let INDEX=INDEX+1
0199   done
0200
0201
0202
0203
0204 done #end 'for loop' on ALCA categories
0205
0206
0207 #cd $curdir
0208 #mv "$BASE_TPL"*".py" $DQM_OUT/../logfiles/