LHEInterface/data/run_generic_tarball_cvmfs_madgraphLO_multithread.sh

0001 #!/bin/bash
0002
0003 #script to run generic lhe generation tarballs
0004 #kept as simply as possible to minimize need
0005 #to update the cmssw release
0006 #(all the logic goes in the run script inside the tarball
0007 # on frontier)
0008 #J.Bendavid
0009
0010 #exit on first error
0011 set -e
0012
0013 echo "[MT] NOTE: The script provides a cure for earlier MadGraph LO gridpacks to enable multi-threading (MT). It is also flexible enough to handle all versions of input gridpack, by applying necessary patches and/or fix bugs depending on the gridpack. In this sense, it disobeys the original goal to make this code simplest and keep ALL the logic in 'runcmsgrid.sh' inside the tarball. It is hence expected to switch back to the original 'run_generic_tarball_cvmfs.sh' at a proper time when the new gridpacks are starting to equip with the MT feature."
0014
0015 echo "   ______________________________________     "
0016 echo "         Running Generic Tarball/Gridpack     "
0017 echo "   ______________________________________     "
0018
0019 path=${1}
0020 echo "gridpack tarball path = $path"
0021
0022 nevt=${2}
0023 echo "%MSG-MG5 number of events requested = $nevt"
0024
0025 rnum=${3}
0026 echo "%MSG-MG5 random seed used for the run = $rnum"
0027
0028 ncpu=${4}
0029 echo "%MSG-MG5 thread count requested = $ncpu"
0030
0031 echo "%MSG-MG5 residual/optional arguments = ${@:5}"
0032
0033 if [ -n "${5}" ]; then
0034   use_gridpack_env=${5}
0035   echo "%MSG-MG5 use_gridpack_env = $use_gridpack_env"
0036 fi
0037
0038 if [ -n "${6}" ]; then
0039   scram_arch_version=${6}
0040   echo "%MSG-MG5 override scram_arch_version = $scram_arch_version"
0041 fi
0042
0043 if [ -n "${7}" ]; then
0044   cmssw_version=${7}
0045   echo "%MSG-MG5 override cmssw_version = $cmssw_version"
0046 fi
0047
0048 LHEWORKDIR=`pwd`
0049
0050 if [ "$use_gridpack_env" = false -a -n "$scram_arch_version" -a -n  "$cmssw_version" ]; then
0051   echo "%MSG-MG5 CMSSW version = $cmssw_version"
0052   export SCRAM_ARCH=${scram_arch_version}
0053   scramv1 project CMSSW ${cmssw_version}
0054   cd ${cmssw_version}/src
0055   eval `scramv1 runtime -sh`
0056   cd $LHEWORKDIR
0057 fi
0058
0059 if [[ -d lheevent ]]
0060     then
0061     echo 'lheevent directory found'
0062     echo 'Setting up the environment'
0063     rm -rf lheevent
0064 fi
0065 mkdir lheevent; cd lheevent
0066
0067 #untar the tarball directly from cvmfs
0068 tar -xaf ${path}
0069
0070 #########################################
0071 # Here starts the new implementation:
0072 # fix the code depending on the gridpack version to enable multi-thread
0073 #########################################
0074
0075 # exit if the gridpack is not a MG LO one
0076 if [[ ! -e process/madevent/SubProcesses/MGVersion.txt ]]; then
0077     echo "[MT] Error: this script only works for the MG LO gridpack, while this gridpack might be a MG NLO or non-MG one. Please set 'scriptName' as 'GeneratorInterface/LHEInterface/data/run_generic_tarball_cvmfs.sh' instead."
0078     exit 1
0079 fi
0080
0081 MGVersion=$(cat process/madevent/SubProcesses/MGVersion.txt)
0082 echo "[MT] Detected MG verion: ${MGVersion}"
0083
0084 MGVersion=(${MGVersion//./ })
0085
0086 if [[ ${MGVersion[1]} -lt 6 ]] || [[ ${MGVersion[1]} -eq 6 && ${MGVersion[2]} -eq 0 ]]; then
0087     echo "[MT] Warning: multi-threading is not supported in MG version < 2.6.1. Will not activate the multi-thread feature."
0088 elif ([[ ${MGVersion[1]} -eq 6 ]] || [[ ${MGVersion[1]} -eq 7 && ${MGVersion[2]} -le 2 ]]) && [[ -e process/madevent/Cards/MadLoopParams.dat ]]; then
0089     echo "[MT] Warning: multi-threading is not supported for loop-induced processes in MG version <= 2.7.2. Will not activate the multi-thread feature."
0090 else
0091     # will activate multi-thread feature
0092     # first fix a multi-thread bug for MG<=2.7.2
0093     if [[ ${MGVersion[1]} -eq 6 ]] || [[ ${MGVersion[1]} -eq 7 && ${MGVersion[2]} -le 2 ]]; then
0094         echo "[MT] Apply a patch to fix multithread bug in 2.6.1<=MG=2.7.2"
0095         patch process/madevent/bin/internal/madevent_interface.py << EOF
0096 === modified file 'madgraph/interface/madevent_interface.py'
0097 --- madgraph/interface/madevent_interface.py    2020-04-23 12:03:18 +0000
0098 +++ madgraph/interface/madevent_interface.py    2020-04-23 15:49:28 +0000
0099 @@ -6667,11 +6667,11 @@
0100                  sum_axsec += result.get('axsec')*gscalefact[Gdir]
0101
0102                  if len(AllEvent) >= 80: #perform a partial unweighting
0103 -                    AllEvent.unweight(pjoin(self.me_dir, "Events", self.run_name, "partials%s.lhe.gz" % partials),
0104 +                    AllEvent.unweight(pjoin(outdir, self.run_name, "partials%s.lhe.gz" % partials),
0105                            get_wgt, log_level=5,  trunc_error=1e-2, event_target=self.nb_event)
0106                      AllEvent = lhe_parser.MultiEventFile()
0107                      AllEvent.banner = self.banner
0108 -                    AllEvent.add(pjoin(self.me_dir, "Events", self.run_name, "partials%s.lhe.gz" % partials),
0109 +                    AllEvent.add(pjoin(outdir, self.run_name, "partials%s.lhe.gz" % partials),
0110                                   sum_xsec,
0111                                   math.sqrt(sum(x**2 for x in sum_xerru)),
0112                                   sum_axsec)
0113 EOF
0114
0115         # fix another bug related to cross-section computing in multiththread mode, as identified in: https://bugs.launchpad.net/mg5amcnlo/+bug/1884085
0116         patch process/madevent/bin/internal/madevent_interface.py << EOF
0117 === modified file 'madgraph/interface/madevent_interface.py'
0118 --- madgraph/interface/madevent_interface.py    2020-06-08 15:08:17 +0000
0119 +++ madgraph/interface/madevent_interface.py    2020-06-18 20:18:17 +0000
0120 @@ -6490,9 +6490,12 @@
0121              os.chdir(self.me_dir)
0122          else:
0123              for line in open(pjoin(self.me_dir,'SubProcesses','subproc.mg')):
0124 -                os.mkdir(line.strip())
0125 +                p = line.strip()
0126 +                os.mkdir(p)
0127 +                files.cp(pjoin(self.me_dir,'SubProcesses',p,'symfact.dat'),
0128 +                         pjoin(p, 'symfact.dat'))
0129
0130
0131      def launch(self, nb_event, seed):
0132 EOF
0133     fi
0134
0135     # fix another "readonly" mode issue related to proper handle of the integration grid,
0136     # as identified in: https://answers.launchpad.net/mg5amcnlo/+question/696856
0137     # this is fixed since 2.9.4 and 3.1.1, so we add a patch if MG is lower then these versions
0138     if [[ ( ${MGVersion[0]} -eq 2 ) && ( ${MGVersion[1]} -lt 9 || ( ${MGVersion[1]} -eq 9 && ${MGVersion[2]} -le 3 ) ) ]] || \
0139        [[ ( ${MGVersion[0]} -eq 3 ) && ( ${MGVersion[1]} -lt 1 || ( ${MGVersion[1]} -eq 1 && ${MGVersion[2]} -le 0 ) ) ]]; then
0140         patch process/madevent/bin/internal/gen_ximprove.py << EOF
0141 === modified file 'madgraph/interface/gen_ximprove.py'
0142 --- madgraph/interface/gen_ximprove.py
0143 +++ madgraph/interface/gen_ximprove.py
0144 @@ -1903,5 +1903,8 @@ class gen_ximprove_gridpack(gen_ximprove_v4):
0145                      'packet': None,
0146                      }
0147 +            if self.readonly:
0148 +                basedir = pjoin(os.path.dirname(__file__), '..','..','SubProcesses', info['P_dir'], info['directory'])
0149 +                info['base_directory'] = basedir
0150
0151
0152              jobs.append(info)
0153 EOF
0154     fi
0155
0156     # fix another multi-thread related bug for MG 2.6.1 only
0157     if [[ ${MGVersion[1]} -eq 6 && ${MGVersion[2]} -eq 1 ]]; then
0158         echo "[MT] Apply another patch to fix multithread bug in MG 2.6.1"
0159         sed -i "/def collect\_result/a\    main_dir = '$(pwd)/process/madevent/SubProcesses'" process/madevent/bin/internal/sum_html.py
0160     fi
0161
0162     # patch on runcmsgrid.sh if old version is detected
0163     if grep -q "succ_setreadonly" runcmsgrid.sh; then
0164         echo "[MT] Congratulations. You are using the new runcmsgrid.sh script with the MG LO multi-thread feature already implemented. Will use this script for event generation without any patch."
0165     else
0166         echo "[MT] Old runcmsgrid.sh script detected. This means you are working on an earlier gridpack where MG LO multi-thread feature is not implemented. Will patch on the runcmsgrid.sh code to enable multi-thread feature."
0167         PATCHDIR=${0%/*}
0168         cp runcmsgrid.sh runcmsgrid.sh.bak
0169         cp ${PATCHDIR}/runcmsgrid_LO_support_multithread.patch .
0170         patch runcmsgrid.sh runcmsgrid_LO_support_multithread.patch
0171     fi
0172 fi
0173 #########################################
0174
0175
0176 # If TMPDIR is unset, set it to the condor scratch area if present
0177 # and fallback to /tmp
0178 export TMPDIR=${TMPDIR:-${_CONDOR_SCRATCH_DIR:-/tmp}}
0179
0180 #generate events
0181 ./runcmsgrid.sh $nevt $rnum $ncpu ${@:5}
0182
0183 mv cmsgrid_final.lhe $LHEWORKDIR/
0184
0185 cd $LHEWORKDIR
0186
0187 #cleanup working directory (save space on worker node for edm output)
0188 rm -rf lheevent
0189
0190 exit 0
0191