Program Listing for File MPIUtils.hpp

↰ Return to documentation for file (src/utils/MPIUtils.hpp)

#ifndef MPIUTILS_HPP
#define MPIUTILS_HPP

#include <algorithm>
#include <iostream> // NOLINT(misc-include-cleaner) -- only used inside initMPI()'s own #ifdef SLUG_MPI block (std::cerr, for the MPI_THREAD_FUNNELED warning), so a build without SLUG_MPI never actually uses it, the same reason <mpi.h> below needs its own identical NOLINT
#include <utility>

#ifdef SLUG_MPI
#   include <mpi.h> // NOLINT(misc-include-cleaner)
#endif
#ifdef _OPENMP
#   include <omp.h> // NOLINT(misc-include-cleaner) -- only used inside initMPI()'s own "provided < MPI_THREAD_FUNNELED" branch, under #ifdef SLUG_MPI as well, so a build with _OPENMP but not SLUG_MPI never actually uses it
#endif

namespace utils
{

    inline constexpr unsigned long mpiUidStride = 1UL << 40U;

    namespace detail
    {
        inline auto cachedMPIRank() -> int&
        {
            static int rank = 0;
            return rank;
        }

        inline auto cachedMPISize() -> int&
        {
            static int size = 1;
            return size;
        }
    } // namespace detail

    inline void initMPI([[maybe_unused]] int* argc, [[maybe_unused]] char*** argv)
    {
#ifdef SLUG_MPI
        int provided = 0;
        MPI_Init_thread(argc, argv, MPI_THREAD_FUNNELED, &provided);
        MPI_Comm_rank(MPI_COMM_WORLD, &detail::cachedMPIRank());
        MPI_Comm_size(MPI_COMM_WORLD, &detail::cachedMPISize());
        if (provided < MPI_THREAD_FUNNELED)
        {
            std::cerr << "slug: warning: MPI implementation provided a lesser "
                "thread support level than MPI_THREAD_FUNNELED; forcing "
                "single-threaded execution for this run, since the MPI "
                "standard does not guarantee correct behavior for a "
                "multithreaded process otherwise\n";
#ifdef _OPENMP
            omp_set_num_threads(1);
#endif
        }
#endif
    }

    inline void finalizeMPI()
    {
#ifdef SLUG_MPI
        MPI_Finalize();
#endif
    }

    class MPIGuard
    {
    public:
        MPIGuard(int* argc, char*** argv) { initMPI(argc, argv); }

        ~MPIGuard() { finalizeMPI(); }
        MPIGuard(const MPIGuard&) = delete;
        MPIGuard(MPIGuard&&) = delete;
        auto operator=(const MPIGuard&) -> MPIGuard& = delete;
        auto operator=(MPIGuard&&) -> MPIGuard& = delete;
    };

    [[nodiscard]] inline auto mpiRank() -> int { return detail::cachedMPIRank(); }

    [[nodiscard]] inline auto mpiSize() -> int { return detail::cachedMPISize(); }

    [[nodiscard]] inline auto isIORank() -> bool { return mpiRank() == 0; }

    inline void mpiBarrier()
    {
#ifdef SLUG_MPI
        MPI_Barrier(MPI_COMM_WORLD);
#endif
    }

    [[nodiscard]] inline auto mpiAllReceivedSigterm(const bool local) -> bool
    {
#ifdef SLUG_MPI
        int localFlag = local ? 1 : 0;
        int anyFlag = 0;
        MPI_Allreduce(&localFlag, &anyFlag, 1, MPI_INT, MPI_LOR, MPI_COMM_WORLD);
        return anyFlag != 0;
#else
        return local;
#endif
    }

    [[nodiscard]] inline auto mpiPartitionRange(
        const unsigned long rangeStart, const unsigned long rangeEnd)
        -> std::pair<unsigned long, unsigned long>
    {
        const auto size = static_cast<unsigned long>(mpiSize());
        const auto rank = static_cast<unsigned long>(mpiRank());
        const unsigned long total = rangeEnd - rangeStart;
        const unsigned long base = total / size;
        const unsigned long remainder = total % size;
        const unsigned long myCount = base + ((rank < remainder) ? 1 : 0);
        const unsigned long precedingCount = rank * base + std::min(rank, remainder);
        const unsigned long start = rangeStart + precedingCount;
        return {start, start + myCount};
    }

} // namespace utils

#endif // MPIUTILS_HPP