MPIServices/src/MPIService.cc

0001 // -*- C++ -*-
0002 #include <cstdlib>
0003 #include <string>
0004
0005 #include <mpi.h>
0006
0007 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0008 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0009 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0010 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0011 #include "FWCore/ServiceRegistry/interface/Service.h"
0012 #include "FWCore/Utilities/interface/Exception.h"
0013 #include "HeterogeneousCore/MPIServices/interface/MPIService.h"
0014
0015 namespace {
0016
0017   // list the MPI thread support levels
0018   const char* const mpi_thread_support_level[] = {
0019       "MPI_THREAD_SINGLE",      // only one thread will execute (the process is single-threaded)
0020       "MPI_THREAD_FUNNELED",    // only the thread that called MPI_Init_thread will make MPI calls
0021       "MPI_THREAD_SERIALIZED",  // only one thread will make MPI library calls at one time
0022       "MPI_THREAD_MULTIPLE"     // multiple threads may call MPI at once with no restrictions
0023   };
0024
0025 }  // namespace
0026
0027 MPIService::MPIService(edm::ParameterSet const& config) {
0028   /* As of Open MPI 4.1.0, `MPI_THREAD_MULTIPLE` is supported by the following transports:
0029    *   - the `ob1` PML, with the following BTLs:
0030    *       - `self`
0031    *       - `sm`
0032    *       - `smcuda`
0033    *       - `tcp`
0034    *       - `ugni`
0035    *       - `usnic`
0036    *   - the `cm` PML, with the following MTLs:
0037    *       - `ofi` (Libfabric)
0038    *       - `portals4`
0039    *   - the `ucx` PML
0040    *
0041    * MPI File operations are not thread safe even if MPI is initialized for `MPI_THREAD_MULTIPLE` support.
0042    *
0043    * See https://github.com/open-mpi/ompi/blob/v4.1.0/README .
0044    */
0045
0046   // set the pmix_server_uri MCA parameter if specified in the configuration and not already set in the environment
0047   if (config.existsAs<std::string>("pmix_server_uri", false)) {
0048     std::string uri = config.getUntrackedParameter<std::string>("pmix_server_uri");
0049     // do not overwrite the environment variable if it is already set
0050     setenv("OMPI_MCA_pmix_server_uri", uri.c_str(), false);
0051   }
0052
0053   // initializes the MPI execution environment, requesting multi-threading support
0054   int provided;
0055   MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided);
0056   if (provided < MPI_THREAD_MULTIPLE) {
0057     throw cms::Exception("UnsupportedFeature")
0058         << "CMSSW requires the " << mpi_thread_support_level[MPI_THREAD_MULTIPLE]
0059         << " multithreading support level, but the MPI library provides only the " << mpi_thread_support_level[provided]
0060         << " level.";
0061   } else {
0062     edm::LogInfo log("MPIService");
0063     log << "The MPI library provides the " << mpi_thread_support_level[provided] << " multithreading support level\n";
0064
0065     // get the number of processes
0066     int world_size;
0067     MPI_Comm_size(MPI_COMM_WORLD, &world_size);
0068     log << "MPI_COMM_WORLD size: " << world_size << '\n';
0069
0070     // get the rank of the process
0071     int world_rank;
0072     MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
0073     log << "MPI_COMM_WORLD rank: " << world_rank << '\n';
0074
0075     // get the name of the processor
0076     char processor_name[MPI_MAX_PROCESSOR_NAME];
0077     int name_len;
0078     MPI_Get_processor_name(processor_name, &name_len);
0079     log << "MPI processor name:  " << processor_name << '\n';
0080
0081     // initialisation done
0082     log << '\n';
0083     log << "MPI successfully initialised";
0084   }
0085 }
0086
0087 MPIService::~MPIService() {
0088   // terminate the MPI execution environment
0089   MPI_Finalize();
0090 }
0091
0092 void MPIService::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0093   edm::ParameterSetDescription desc;
0094   desc.addOptionalUntracked<std::string>("pmix_server_uri")
0095       ->setComment("Set the OpenMPI MCA pmix_server_uri parameter if not already set in the environment");
0096   descriptions.add("MPIService", desc);
0097   descriptions.setComment(R"(This Service provides a common interface to MPI configuration for the CMSSW job.)");
0098 }
0099
0100 void MPIService::required() {
0101   edm::Service<MPIService> s;
0102   if (not s.isAvailable()) {
0103     throw cms::Exception("Configuration") << R"(The MPIService is required by this module.
0104 Please add it to the configuration, for example via
0105
0106 process.load("HeterogeneousCore.MPIServices.MPIService_cfi")
0107 )";
0108   }
0109 }