Issue - 1711 - Subset MPI communcator

.jenkins/mpi_compiler_checks.gvy

@@ -97,6 +97,20 @@ pipeline {
                                     echo ' Build completed '
                                 }
                             }
+                            stage (' Test '){
+                                steps {
+                                    echo ' Running MPI tests with ctest '
+                                    sh """
+                                        module load cmake/3.30.8
+                                        module load boost/1.85.0
+                                        module load ${GNU_COMPILER_MODULE}
+                                        module load ${MPI_MODULE}
+                                        cd build_mpi_${GNU_COMPILER_NAME}_${MPI_NAME}_${CMAKE_HOST_NAME}_${SIMD_NAME}_${env.BUILD_TAG}
+                                        ctest -VV
+                                    """
+                                    echo ' Testing completed '
+                                }
+                            }
                         }
                         post {
                             always {

CMakePresets.json

@@ -170,10 +170,40 @@
         "SPECFEM_BENCHMARKS_BUILD_DIR": "${sourceDir}/benchmarks/build/debug-nosimd"
       }
     },
+    {
+      "name": "debug-mpi",
+      "displayName": "MPI Debug -- EXPERIMENTAL",
+      "binaryDir": "build/debug-mpi",
+      "environment": {
+        "CC": "mpicc",
+        "CXX": "mpicxx",
+        "FC": "mpifort"
+      },
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/bin/debug-mpi",
+        "Kokkos_ARCH_NATIVE": "ON",
+        "Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION": "OFF",
+        "Kokkos_ENABLE_ATOMICS_BYPASS": "OFF",
+        "Kokkos_ENABLE_DEBUG": "ON",
+        "Kokkos_ENABLE_DEBUG_BOUNDS_CHECK": "ON",
+        "SPECFEM_INSTALL": "ON",
+        "SPECFEM_ENABLE_MPI": "ON",
+        "SPECFEM_BUILD_TESTS": "ON",
+        "SPECFEM_ENABLE_SIMD": "ON",
+        "SPECFEM_BUILD_BENCHMARKS": "ON",
+        "SPECFEM_BENCHMARKS_BUILD_DIR": "${sourceDir}/benchmarks/build/debug-mpi"
+      }
+    },
     {
       "name": "release-mpi",
       "displayName": "MPI Release -- EXPERIMENTAL",
       "binaryDir": "build/release-mpi",
+      "environment": {
+        "CC": "mpicc",
+        "CXX": "mpicxx",
+        "FC": "mpifort"
+      },
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Release",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/bin/release-mpi",
@@ -272,6 +302,13 @@
         "all"
       ]
     },
+    {
+      "name": "debug-mpi",
+      "configurePreset": "debug-mpi",
+      "targets": [
+        "all"
+      ]
+    },
     {
       "name": "release-mpi",
       "configurePreset": "release-mpi",

core/specfem.cpp

@@ -61,41 +61,58 @@ int run_simulation(const std::string &dimension, int argc, char **argv,
   try {
     specfem::program::Context context(argc, argv);
 
-    const YAML::Node parameter_dict = YAML::LoadFile(opts.parameters_file);
-
-    // Build LoggerOptions from CLI values
-    std::optional<std::string> log_file_opt;
-    std::optional<bool> per_rank_opt;
-    std::optional<bool> auto_flush_opt;
-    std::optional<std::string> log_level_opt;
-
-    if (flags.log_file_set)
-      log_file_opt = opts.log_file;
-    if (flags.per_rank_set)
-      per_rank_opt = opts.log_per_rank;
-    if (flags.auto_flush_set)
-      auto_flush_opt = opts.log_auto_flush;
-    if (flags.log_level_set)
-      log_level_opt = opts.log_level;
-
-    auto logger_options = specfem::logger::LoggerOptions::from_values(
-        std::move(log_file_opt), per_rank_opt, auto_flush_opt,
-        std::move(log_level_opt));
-    specfem::Logger::apply_options(logger_options);
-
-    // Set log file if specified in parameters and not already set by CLI
-    if (parameter_dict["parameters"]["log-file"]) {
-      const std::string log_file =
-          parameter_dict["parameters"]["log-file"].as<std::string>();
-      specfem::Logger::set_log_file(log_file);
+    // Only run simulation if current rank is part of the subset communicator
+    // (i.e. not excluded by user-defined nprocs)
+    // For simulations that use the entire world communicator, this check will
+    // pass for all ranks.
+    if (specfem::MPI::communicator() != MPI_COMM_NULL) {
+
+      const YAML::Node parameter_dict = YAML::LoadFile(opts.parameters_file);
+
+      // Build LoggerOptions from CLI values
+      std::optional<std::string> log_file_opt;
+      std::optional<bool> per_rank_opt;
+      std::optional<bool> auto_flush_opt;
+      std::optional<std::string> log_level_opt;
+
+      if (flags.log_file_set)
+        log_file_opt = opts.log_file;
+      if (flags.per_rank_set)
+        per_rank_opt = opts.log_per_rank;
+      if (flags.auto_flush_set)
+        auto_flush_opt = opts.log_auto_flush;
+      if (flags.log_level_set)
+        log_level_opt = opts.log_level;
+
+      auto logger_options = specfem::logger::LoggerOptions::from_values(
+          std::move(log_file_opt), per_rank_opt, auto_flush_opt,
+          std::move(log_level_opt));
+      specfem::Logger::apply_options(logger_options);
+
+      // Set log file if specified in parameters and not already set by CLI
+      if (parameter_dict["parameters"]["log-file"]) {
+        const std::string log_file =
+            parameter_dict["parameters"]["log-file"].as<std::string>();
+        specfem::Logger::set_log_file(log_file);
+      }
+
+      const auto success = specfem::program::execute(dimension, parameter_dict);
+
+      if (!success) {
+        std::cerr << "Execution failed" << std::endl;
+        result = 1;
+      }
+    } else {
+      result = 0;
     }
 
-    const auto success = specfem::program::execute(dimension, parameter_dict);
+    SPECFEM_MPI_SAFECALL(MPI_Barrier(MPI_COMM_WORLD));
 
-    if (!success) {
-      std::cerr << "Execution failed" << std::endl;
-      result = 1;
-    }
+    // Ensure all ranks have reached this point, reduce the result across ranks
+    int result_reduced = result;
+    SPECFEM_MPI_SAFECALL(MPI_Allreduce(&result, &result_reduced, 1, MPI_INT,
+                                       MPI_MAX, MPI_COMM_WORLD));
+    result = result_reduced;
 
   } catch (const std::exception &e) {
     std::cerr << "Error during execution: " << e.what() << std::endl;
@@ -137,15 +154,32 @@ int run_qplots(int argc, char **argv, const Qoptions &opts) {
   try {
     specfem::program::Context context(argc, argv);
 
-    const auto success = specfem::program::qplots(
-        opts.Q, opts.minfreq, opts.maxfreq, opts.min_plot_freq,
-        opts.max_plot_freq, opts.output_dir);
-
-    if (!success) {
-      std::cerr << "Q plot generation failed" << std::endl;
-      result = 1;
+    // Only run Q plot generation if current rank is part of the subset
+    // communicator (i.e. not excluded by user-defined nprocs). For simulations
+    // that use the entire world communicator, this check will pass for all
+    // ranks.
+    if (specfem::MPI::communicator() != MPI_COMM_NULL) {
+
+      const auto success = specfem::program::qplots(
+          opts.Q, opts.minfreq, opts.maxfreq, opts.min_plot_freq,
+          opts.max_plot_freq, opts.output_dir);
+
+      if (!success) {
+        std::cerr << "Q plot generation failed" << std::endl;
+        result = 1;
+      }
+    } else {
+      result = 0;
     }
 
+    SPECFEM_MPI_SAFECALL(MPI_Barrier(MPI_COMM_WORLD));
+
+    // Ensure all ranks have reached this point, reduce the result across ranks
+    int result_reduced = result;
+    SPECFEM_MPI_SAFECALL(MPI_Allreduce(&result, &result_reduced, 1, MPI_INT,
+                                       MPI_MAX, MPI_COMM_WORLD));
+    result = result_reduced;
+
   } catch (const std::exception &e) {
     std::cerr << "Error during Q plot generation: " << e.what() << std::endl;
     result = 1;

core/specfem/io/mesh/impl/fortran/dim2/mesh.cpp

@@ -77,15 +77,16 @@ specfem::io::read_2d_mesh(
 
   int nspec_all = mesh.parameters.nspec;
   SPECFEM_MPI_SAFECALL(MPI_Reduce(&mesh.parameters.nspec, &nspec_all, 1,
-                                  MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD));
+                                  MPI_INT, MPI_SUM, 0,
+                                  specfem::MPI::communicator()));
   int nelem_acforcing_all = mesh.parameters.nelem_acforcing;
   SPECFEM_MPI_SAFECALL(MPI_Reduce(&mesh.parameters.nelem_acforcing,
                                   &nelem_acforcing_all, 1, MPI_INT, MPI_SUM, 0,
-                                  MPI_COMM_WORLD));
+                                  specfem::MPI::communicator()));
   int nelem_acoustic_surface_all = mesh.parameters.nelem_acoustic_surface;
   SPECFEM_MPI_SAFECALL(MPI_Reduce(&mesh.parameters.nelem_acoustic_surface,
                                   &nelem_acoustic_surface_all, 1, MPI_INT,
-                                  MPI_SUM, 0, MPI_COMM_WORLD));
+                                  MPI_SUM, 0, specfem::MPI::communicator()));
 
   try {
     auto [n_sls, attenuation_f0_reference, read_velocities_at_f0] =

core/specfem/io/mesh/impl/fortran/dim3/mesh.cpp

@@ -5,7 +5,6 @@
 #include "specfem/io/mesh/impl/fortran/dim3/read_boundaries.hpp"
 #include "specfem/io/mesh/impl/fortran/dim3/read_control_nodes.hpp"
 #include "specfem/io/mesh/impl/fortran/dim3/read_materials.hpp"
-#include "specfem/io/mesh/impl/fortran/dim3/read_mpi_interfaces.hpp"
 #include "specfem/io/mesh/impl/fortran/dim3/read_pml_boundaries.hpp"
 #include <fstream>
 #include <stdexcept>
@@ -53,10 +52,6 @@ specfem::io::read_3d_mesh(const std::string &database_file) {
   // TODO (Rohit: PML_BOUNDARIES): Add support for PML boundaries
   specfem::io::mesh::impl::fortran::dim3::read_pml_boundaries(param_stream);
 
-  // MPI interfaces are not supported yet
-  // TODO (Rohit: MPI_INTERFACES): Add support for MPI interfaces
-  specfem::io::mesh::impl::fortran::dim3::read_mpi_interfaces(param_stream);
-
   // Read adjacency information
   mesh.adjacency_graph =
       specfem::io::mesh::impl::fortran::dim3::read_adjacency_graph(param_stream,

core/specfem/mpi/mpi.cpp

@@ -5,6 +5,7 @@ namespace specfem {
 // Static storage for MPI rank and size (-1 indicates uninitialized)
 int MPI::rank_ = -1;
 int MPI::size_ = -1;
+MPI_Comm MPI::comm_ = MPI_COMM_WORLD;
 
 void MPI::initialize(int *argc, char ***argv) {
 #ifdef SPECFEM_ENABLE_MPI
@@ -15,12 +16,59 @@ void MPI::initialize(int *argc, char ***argv) {
     MPI_Init(argc, argv);
   }
 
-  MPI_Comm_size(MPI_COMM_WORLD, &size_);
-  MPI_Comm_rank(MPI_COMM_WORLD, &rank_);
+  comm_ = MPI_COMM_WORLD;
+
+  MPI_Comm_size(comm_, &size_);
+  MPI_Comm_rank(comm_, &rank_);
+#else
+  // Non-MPI build: single process
+  rank_ = 0;
+  size_ = 1;
+  comm_ = MPI_COMM_WORLD; // Dummy value for non-MPI build
+#endif
+}
+
+void MPI::initialize(int *argc, char ***argv, int nprocs) {
+#ifdef SPECFEM_ENABLE_MPI
+  int initialized;
+  MPI_Initialized(&initialized);
+
+  if (!initialized) {
+    MPI_Init(argc, argv);
+    // Check that requested nprocs does not exceed world size
+    int world_size;
+    MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+    if (nprocs > world_size) {
+      std::cerr << "Error: Requested nprocs (" << nprocs
+                << ") exceeds available world size (" << world_size << ")."
+                << std::endl;
+      MPI_Abort(MPI_COMM_WORLD, 1);
+    }
+
+    // Get rank from world communicator before using in split
+    int world_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
+
+    // Only use the first nprocs ranks in the new communicator
+    int color = (world_rank < nprocs) ? 1 : MPI_UNDEFINED;
+    MPI_Comm_split(MPI_COMM_WORLD, color, world_rank, &comm_);
+  }
+
+  // Handle ranks excluded by subset communicator
+  if (comm_ == MPI_COMM_NULL) {
+    // Rank is not part of the subset communicator
+    rank_ = -1;
+    size_ = -1;
+    return;
+  }
+
+  MPI_Comm_size(comm_, &size_);
+  MPI_Comm_rank(comm_, &rank_);
 #else
   // Non-MPI build: single process
   rank_ = 0;
   size_ = 1;
+  comm_ = MPI_COMM_WORLD; // Dummy value for non-MPI build
 #endif
 }
 
@@ -31,6 +79,11 @@ void MPI::finalize() {
 
   // Only finalize if not already finalized externally
   if (!finalized) {
+    // Free custom communicator if it was created and is valid
+    if (comm_ != MPI_COMM_WORLD && comm_ != MPI_COMM_NULL) {
+      MPI_Comm_free(&comm_);
+      comm_ = MPI_COMM_NULL;
+    }
     MPI_Finalize();
   }
 #endif