From 16179f1f4807a53c6aa3f816fc246165871b03b5 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 00:44:45 -0400 Subject: [PATCH 01/24] Make env.isX bool to match env.userOwnsMpi --- quest/include/environment.h | 12 ++++++------ quest/src/api/environment.cpp | 7 ++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/quest/include/environment.h b/quest/include/environment.h index 608829912..440305b75 100644 --- a/quest/include/environment.h +++ b/quest/include/environment.h @@ -35,20 +35,20 @@ extern "C" { typedef struct { // deployment modes which can be runtime disabled - int isMultithreaded; - int isGpuAccelerated; - int isDistributed; - bool userOwnsMpi; + bool isMultithreaded; + bool isGpuAccelerated; + bool isDistributed; // deployment modes which cannot be directly changed after compilation - int isCuQuantumEnabled; + bool isCuQuantumEnabled; // deployment configurations which can be changed via environment variables - int isGpuSharingEnabled; + bool isGpuSharingEnabled; // distributed configuration int rank; int numNodes; + bool userOwnsMpi; } QuESTEnv; diff --git a/quest/src/api/environment.cpp b/quest/src/api/environment.cpp index 1cc2f6862..82540fdcb 100644 --- a/quest/src/api/environment.cpp +++ b/quest/src/api/environment.cpp @@ -86,7 +86,8 @@ void validateAndInitCustomQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuA // by mpirun believe they are each the main rank. This seems unavoidable. validate_newEnvDeploymentMode(useDistrib, useGpuAccel, useMultithread, caller); - // overwrite deployments left as modeflag::USE_AUTO + // overwrite deployments (left as modeflag::USE_AUTO=-1) with 0,1 (a bool), + // which crucially, resolves useDistrib, permitting its consultation below autodep_chooseQuESTEnvDeployment(useDistrib, useGpuAccel, useMultithread); // optionally initialise MPI; necessary before completing validation, @@ -140,17 +141,17 @@ void validateAndInitCustomQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuA if (globalEnvPtr == nullptr) error_allocOfQuESTEnvFailed(); - // bind deployment info to global instance + // bind deployment info to global instance (autocasting int to bool) globalEnvPtr->isMultithreaded = useMultithread; globalEnvPtr->isGpuAccelerated = useGpuAccel; globalEnvPtr->isDistributed = useDistrib; - globalEnvPtr->userOwnsMpi = userOwnsMpi; globalEnvPtr->isCuQuantumEnabled = useCuQuantum; globalEnvPtr->isGpuSharingEnabled = permitGpuSharing; // bind distributed info globalEnvPtr->rank = (useDistrib)? comm_getRank() : 0; globalEnvPtr->numNodes = (useDistrib)? comm_getNumNodes() : 1; + globalEnvPtr->userOwnsMpi = userOwnsMpi; } void updateQuESTEnvDistInfo() { From 1ddfb6bce559a3953a13e5d2e27b3d0770ea9e84 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 00:45:46 -0400 Subject: [PATCH 02/24] Add MPI status validation --- quest/src/api/environment.cpp | 8 +++++- quest/src/core/validation.cpp | 47 +++++++++++++++++++++++++++++++++++ quest/src/core/validation.hpp | 2 ++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/quest/src/api/environment.cpp b/quest/src/api/environment.cpp index 82540fdcb..9f77421d8 100644 --- a/quest/src/api/environment.cpp +++ b/quest/src/api/environment.cpp @@ -74,9 +74,12 @@ static bool hasEnvBeenFinalized = false; void validateAndInitCustomQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuAccel, int useMultithread, const char* caller) { // ensure that we are never re-initialising QuEST (even after finalize) because - // this leads to undefined behaviour in distributed mode, as per the MPI + // this leads to undefined behaviour in distributed mode, as per the MPI std, + // regardless of whether the user owns MPI validate_envNeverInit(globalEnvPtr != nullptr, hasEnvBeenFinalized, caller); + // load env-vars before validating deployment mode, because some env vars can + // affect validation (such as QUEST_PERMIT_NODES_TO_SHARE_GPU) envvars_validateAndLoadEnvVars(caller); validateconfig_setEpsilonToDefault(); @@ -90,6 +93,9 @@ void validateAndInitCustomQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuA // which crucially, resolves useDistrib, permitting its consultation below autodep_chooseQuESTEnvDeployment(useDistrib, useGpuAccel, useMultithread); + // ensure that current state of MPI is valid + validate_mpiInitStatus(useDistrib, userOwnsMpi, caller); + // optionally initialise MPI; necessary before completing validation, // and before any GPU initialisation and validation, since we will // perform that specifically upon the MPI-process-bound GPU(s). Further, diff --git a/quest/src/core/validation.cpp b/quest/src/core/validation.cpp index 959acb61e..871d94199 100644 --- a/quest/src/core/validation.cpp +++ b/quest/src/core/validation.cpp @@ -107,6 +107,15 @@ namespace report { string CUQUANTUM_DEPLOYED_ON_GPU_WITHOUT_MEM_POOLS = "Cannot use cuQuantum since your GPU does not support memory pools. Recompile with cuQuantum disabled to fall-back to using Thrust and custom kernels."; + string USER_OWNED_MPI_WAS_NOT_INIT = + "User owns MPI but did not prior initialise MPI before initialising QuEST."; + + string QUEST_OWNED_MPI_WAS_PRE_INIT = + "MPI was already initialised prior to QuESTEnv initialisation, but the user did not declare MPI ownership."; + + string QUEST_IS_NON_DISTRIBUTED_BUT_MPI_WAS_INIT = + "QuESTEnv was initialised to be non-distributed but MPI was externally initialised - this is presently unsupported due to a (very minor) technical limitation. If you need this facility, please raise a Github issue!"; + /* * EXISTING QUESTENV @@ -1482,6 +1491,44 @@ void validate_gpuIsCuQuantumCompatible(const char* caller) { assertAllNodesAgreeThat(hasMemPools, report::CUQUANTUM_DEPLOYED_ON_GPU_WITHOUT_MEM_POOLS, caller); } +void validate_mpiInitStatus(bool useDistrib, bool userOwnsMpi, const char* caller) { + + if (!global_isValidationEnabled) + return; + + // Validation prior to this function confirms init(Custom*)QuESTEnv is only ever called + // once, but we must additionally confirm the user has interacted with MPI legally + + bool isMpiInit = comm_isInit(); + + // (A) If the user does not declare ownership of MPI, they are forbidden to initialise it + if (!userOwnsMpi) + assertThat(!isMpiInit, report::QUEST_OWNED_MPI_WAS_PRE_INIT, caller); + + // (B) If QuEST is instructed not to use distribution, we must demand the user is not + // using MPI, because we internally consult comm_isInit() to detect QuEST distribution + // in many functions, and that will give a false positive when the user inits MPI directly. + if (!useDistrib) + assertThat(!isMpiInit, report::QUEST_IS_NON_DISTRIBUTED_BUT_MPI_WAS_INIT, caller); + + // TODO: we can relax above, permitting the user to play with MPI directly while + // disabling it for QuEST, by replacing internal comm_isInit() with e.g. env_isDistributed() + + // (C) If QuEST will use MPI owned by the user, the user must have pre-initialised it + if (useDistrib && userOwnsMpi) + assertThat(isMpiInit, report::USER_OWNED_MPI_WAS_NOT_INIT, caller); + + // Confirmation that all 8 scenarios are handled: + // useDistrib=0, userOwnsMpi=0, isMpiInit=0 (legal: nobody wants MPI) + // (A) useDistrib=0, userOwnsMpi=0, isMpiInit=1 (illegal: user lied about ownership) + // useDistrib=0, userOwnsMpi=1, isMpiInit=0 (legal: user owns MPI but does nothing!) + // (B) useDistrib=0, userOwnsMpi=1, isMpiInit=1 (illegal: comm_isInit() limitation as above) + // useDistrib=1, userOwnsMpi=0, isMpiInit=0 (legal: QuEST will init MPI) + // (A) useDistrib=1, userOwnsMpi=0, isMpiInit=1 (illegal: user lied about ownership) + // (C) useDistrib=1, userOwnsMpi=1, isMpiInit=0 (illegal: user has reponsibility to pre-init) + // useDistrib=1, userOwnsMpi=1, isMpiInit=1 (legal: user fulfilled responsibility to pre-init) +} + /* diff --git a/quest/src/core/validation.hpp b/quest/src/core/validation.hpp index 66fb8f546..345931946 100644 --- a/quest/src/core/validation.hpp +++ b/quest/src/core/validation.hpp @@ -77,6 +77,8 @@ void validate_newEnvNodesEachHaveUniqueGpu(const char* caller); void validate_gpuIsCuQuantumCompatible(const char* caller); +void validate_mpiInitStatus(bool useDistrib, bool userOwnsMpi, const char* caller); + /* From 507c2e46b2f8b8f7a704cac1e28273de08ccabc5 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 01:33:34 -0400 Subject: [PATCH 03/24] Simplify comm_init() --- quest/src/api/environment.cpp | 3 +- quest/src/comm/comm_config.cpp | 53 +++++++--------------------------- quest/src/comm/comm_config.hpp | 2 +- 3 files changed, 14 insertions(+), 44 deletions(-) diff --git a/quest/src/api/environment.cpp b/quest/src/api/environment.cpp index 9f77421d8..3c0b90999 100644 --- a/quest/src/api/environment.cpp +++ b/quest/src/api/environment.cpp @@ -100,7 +100,8 @@ void validateAndInitCustomQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuA // and before any GPU initialisation and validation, since we will // perform that specifically upon the MPI-process-bound GPU(s). Further, // we can make sure validation errors are reported only by the root node. - comm_init(useDistrib, userOwnsMpi); + if (useDistrib) + comm_init(userOwnsMpi); validate_newEnvDistributedBetweenPower2Nodes(caller); diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index 9da8f34e1..f141d1f85 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -103,55 +103,24 @@ bool comm_isInit() { } -void comm_init(int useDistrib, bool userOwnsMpi) { +void comm_init(bool userOwnsMpi) { #if QUEST_COMPILE_MPI - // error if user owns MPI but has not initialised - if (userOwnsMpi && !comm_isInit()) { + // re-assert prior user-validations for robustness + if (userOwnsMpi && !comm_isInit()) error_commNotInit(); - } + if (!userOwnsMpi && comm_isInit()) + error_commAlreadyInit(); - // Overall mpiCommQuest should be set in the following ways - // however only useDistrib = 1 and userOwnsMpi = false - // and useDistrib = 0 and userOwnsMpi = true - // require action here - // - // | useDistrib | userOwnsMpi | mpiCommQuest | - // | ---------- | ----------- | -------------- | - // | 0 | false | MPI_COMM_NULL | - // | ---------- | ----------- | -------------- | - // | 1 | false | MPI_COMM_WORLD | - // | ---------- | ----------- | -------------- | - // | 0 | true | MPI_COMM_SELF | - // | ---------- | ----------- | -------------- | - // | | | MPI_COMM_WORLD | - // | 1 | true | or | - // | | | userQuestComm | - // | ---------- | ----------- | -------------- | - + // init MPI only when it's not the user's responsibility + if (!userOwnsMpi) + MPI_Init(NULL, NULL); - if (useDistrib && !userOwnsMpi) { - // error if attempting re-initialisation - if (comm_isInit()) { - error_commAlreadyInit(); - } else { - MPI_Init(NULL, NULL); - // The user wants MPI and is leaving it to QuEST - MPI_Comm_dup(MPI_COMM_WORLD, &mpiCommQuest); - } - } else if (!useDistrib && userOwnsMpi) { - // The user has initialised MPI but wants QuEST to ignore it - MPI_Comm_dup(MPI_COMM_SELF, &mpiCommQuest); - } else if (useDistrib && userOwnsMpi) { - // if mpiCommQuEST is still MPI_COMM_NULL the user is not - // providing their own MPI_Comm and we should set mpiCommQuest - // to MPI_COMM_WORLD - if (mpiCommQuest == MPI_COMM_NULL) - MPI_Comm_dup(MPI_COMM_WORLD, &mpiCommQuest); - } + // choose communicator only when the user hasn't + if (mpiCommQuest == MPI_COMM_NULL) + MPI_Comm_dup(MPI_COMM_WORLD, &mpiCommQuest); #endif - return; } diff --git a/quest/src/comm/comm_config.hpp b/quest/src/comm/comm_config.hpp index b2d038cd5..b061dd3e2 100644 --- a/quest/src/comm/comm_config.hpp +++ b/quest/src/comm/comm_config.hpp @@ -22,7 +22,7 @@ bool comm_isMpiCompiled(); bool comm_isMpiSubCommunicatorCompiled(); bool comm_isMpiGpuAware(); -void comm_init(int useDistrib, bool userOwnsMpi); +void comm_init(bool userOwnsMpi); void comm_end(bool userOwnsMpi); void comm_sync(); From e80f768a1a5a8343b7131736c51a79c63cfdc4bd Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 01:37:47 -0400 Subject: [PATCH 04/24] Enable error msgs even when MPI config is invalid These exemptions for communicator NULL-ness enable an error message to reach the user even then the user has called MPI_Init themselves but then triggered a validation error before the communicator could be set --- quest/src/comm/comm_config.cpp | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index f141d1f85..7bc9c2f92 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -132,6 +132,13 @@ void comm_end(bool userOwnsMpi) { if (!comm_isInit()) return; + // ungracefully handle when the communicator is still NULL, because comm_end() may be + // triggered by "bad MPI init" validation, during which, the communicator may not yet + // have been set. We choose NOT to divert to MPI_COMM_WORLD, which is likely just to + // stall at MPI_Barrier, and instead let the user's communicator live on; then crash! + if (mpiCommQuest == MPI_COMM_NULL) + return; + MPI_Barrier(mpiCommQuest); MPI_Comm_free(&mpiCommQuest); @@ -152,8 +159,16 @@ int comm_getRank() { if (!comm_isInit()) return ROOT_RANK; + // consult the (potentially sub-) communicator for rank; if it is still + // NULL, as can only validly happen during failed MPI status validation (the + // error msg is attemptedly printed on only the root process), fallback to + // using WORLD (and pray the user hasn't silenced world-root std-out!). We + // COULD safely return ROOT_RANK instead, letting all processes believe they + // are root, but this grossly duplicates the output across ALL processes + MPI_Comm comm = (mpiCommQuest == MPI_COMM_NULL)? MPI_COMM_WORLD : mpiCommQuest; + int rank; - MPI_Comm_rank(mpiCommQuest, &rank); + MPI_Comm_rank(comm, &rank); return rank; #else @@ -200,6 +215,12 @@ void comm_sync() { if (!comm_isInit()) return; + // gracefully handle when the communicator is still NULL, because comm_sync() is + // triggered by "bad MPI init" validation (during the error message printing) + // during which, the communicator may not yet have been overriden + if (mpiCommQuest == MPI_COMM_NULL) + return; + MPI_Barrier(mpiCommQuest); #endif } From 8b73cd3b7ea5f42724e73865233f598d786e4f97 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 01:42:47 -0400 Subject: [PATCH 05/24] Add Oliver's custom MPI examples taken from #712 --- examples/extended/user_owned_mpi.c | 31 ++++++++++++ examples/extended/user_owned_submpi.cpp | 66 +++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 examples/extended/user_owned_mpi.c create mode 100644 examples/extended/user_owned_submpi.cpp diff --git a/examples/extended/user_owned_mpi.c b/examples/extended/user_owned_mpi.c new file mode 100644 index 000000000..55967a4ef --- /dev/null +++ b/examples/extended/user_owned_mpi.c @@ -0,0 +1,31 @@ +/** @file + * + * TODO + * + * @author Oliver Brown + */ + +#include +#include "quest.h" + + + // TODO: + // this file will only receive mpi.h from CMakeLists.txt if + // we are also compiling with QUEST_ENABLE_SUBCOMM. Fix this! + + +int main (void) +{ + const int USE_DISTRIB = 1; + const bool USER_MPI = 1; + const int USE_OPENMP = 1; + const int USE_GPU = 0; + + MPI_Init(NULL, NULL); + initCustomMpiQuESTEnv(USE_DISTRIB, USER_MPI, USE_GPU, USE_OPENMP); + reportQuESTEnv(); + finalizeQuESTEnv(); + MPI_Finalize(); + + return 0; +} diff --git a/examples/extended/user_owned_submpi.cpp b/examples/extended/user_owned_submpi.cpp new file mode 100644 index 000000000..d1d336637 --- /dev/null +++ b/examples/extended/user_owned_submpi.cpp @@ -0,0 +1,66 @@ +/** @file + * + * TODO + * + * @author Oliver Brown + */ + +#include +#include +#include + + + // TODO: + // this file will only receive mpi.h from CMakeLists.txt if + // we are also compiling with QUEST_ENABLE_SUBCOMM. Fix this! + + +int main (void) +{ + int nprocs, quest_nprocs, world_rank, quest_rank; + MPI_Comm comm_split, comm_quantum, comm_classical; + + MPI_Init(NULL, NULL); + + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); + + const int I_AM_QUANTUM = world_rank % 2; + + std::printf("[%d] Hello from rank %d of %d in MPI_COMM_WORLD.\n", world_rank, world_rank, nprocs); + + MPI_Comm_split(MPI_COMM_WORLD, I_AM_QUANTUM, world_rank, &comm_split); + + if (I_AM_QUANTUM) { + MPI_Comm_dup(comm_split, &comm_quantum); + MPI_Comm_size(comm_quantum, &quest_nprocs); + MPI_Comm_rank(comm_quantum, &quest_rank); + std::printf("[%d] Hello from rank %d of %d in comm_quantum.\n", world_rank, quest_rank, quest_nprocs); + } else { + MPI_Comm_dup(comm_split, &comm_classical); + quest_rank = -1; + quest_nprocs = -1; + } + + // only procs in quantum comm initialise QuEST + if (I_AM_QUANTUM) { + std::printf("[%d] Initialising QuEST.\n", world_rank); + initCustomMpiCommQuESTEnv(comm_quantum, modeflag::USE_AUTO, modeflag::USE_AUTO); + + reportQuESTEnv(); + + std::printf("[%d] Finalising QuEST.\n", world_rank); + finalizeQuESTEnv(); + } + + MPI_Comm_free(&comm_split); + if (I_AM_QUANTUM) { + MPI_Comm_free(&comm_quantum); + } else { + MPI_Comm_free(&comm_classical); + } + + MPI_Finalize(); + + return 0; +} From e91f54f9126cd2777093d2920ebce89e823c2b69 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 01:52:27 -0400 Subject: [PATCH 06/24] renamed env.userOwnsMpi to env.isMpiUserOwned for consistency with e.g. env.isMultithreaded. The user arg to e.g. initQuESTEnv() is kept as "userOwnsMpi" for a very superficial consistency with e.g. "useMultithreaded" :^) --- quest/include/environment.h | 2 +- quest/src/api/environment.cpp | 22 ++++++++++------------ tests/unit/environment.cpp | 6 +++--- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/quest/include/environment.h b/quest/include/environment.h index 440305b75..a584192d7 100644 --- a/quest/include/environment.h +++ b/quest/include/environment.h @@ -38,6 +38,7 @@ typedef struct { bool isMultithreaded; bool isGpuAccelerated; bool isDistributed; + bool isMpiUserOwned; // deployment modes which cannot be directly changed after compilation bool isCuQuantumEnabled; @@ -48,7 +49,6 @@ typedef struct { // distributed configuration int rank; int numNodes; - bool userOwnsMpi; } QuESTEnv; diff --git a/quest/src/api/environment.cpp b/quest/src/api/environment.cpp index 3c0b90999..1e740e427 100644 --- a/quest/src/api/environment.cpp +++ b/quest/src/api/environment.cpp @@ -152,13 +152,13 @@ void validateAndInitCustomQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuA globalEnvPtr->isMultithreaded = useMultithread; globalEnvPtr->isGpuAccelerated = useGpuAccel; globalEnvPtr->isDistributed = useDistrib; + globalEnvPtr->isMpiUserOwned = userOwnsMpi; globalEnvPtr->isCuQuantumEnabled = useCuQuantum; globalEnvPtr->isGpuSharingEnabled = permitGpuSharing; // bind distributed info globalEnvPtr->rank = (useDistrib)? comm_getRank() : 0; globalEnvPtr->numNodes = (useDistrib)? comm_getNumNodes() : 1; - globalEnvPtr->userOwnsMpi = userOwnsMpi; } void updateQuESTEnvDistInfo() { @@ -214,7 +214,7 @@ void printDeploymentInfo() { print_table( "deployment", { {"isMpiEnabled", globalEnvPtr->isDistributed}, - {"doesUserOwnMpi", globalEnvPtr->userOwnsMpi}, + {"isMpiUserOwned", globalEnvPtr->isMpiUserOwned}, {"isGpuEnabled", globalEnvPtr->isGpuAccelerated}, {"isOmpEnabled", globalEnvPtr->isMultithreaded}, {"isCuQuantumEnabled", globalEnvPtr->isCuQuantumEnabled}, @@ -457,7 +457,7 @@ void finalizeQuESTEnv() { if (globalEnvPtr->isDistributed) { comm_sync(); - comm_end(globalEnvPtr->userOwnsMpi); + comm_end(globalEnvPtr->isMpiUserOwned); } // free global env's heap memory and flag it as unallocated @@ -517,19 +517,17 @@ void reportQuESTEnv() { void getQuESTEnvironmentString(char str[200]) { validate_envIsInit(__func__); - QuESTEnv env = getQuESTEnv(); - int numThreads = cpu_isOpenmpCompiled()? cpu_getAvailableNumThreads() : 1; - int cuQuantum = env.isGpuAccelerated && gpu_isCuQuantumCompiled(); - int gpuDirect = env.isGpuAccelerated && gpu_isDirectGpuCommPossible(); + int cuQuantum = globalEnvPtr->isGpuAccelerated && gpu_isCuQuantumCompiled(); + int gpuDirect = globalEnvPtr->isGpuAccelerated && gpu_isDirectGpuCommPossible(); snprintf(str, 200, "CUDA=%d OpenMP=%d MPI=%d userOwnsMPI=%d threads=%d ranks=%d cuQuantum=%d gpuDirect=%d", - env.isGpuAccelerated, - env.isMultithreaded, - env.isDistributed, - env.userOwnsMpi, + globalEnvPtr->isGpuAccelerated, + globalEnvPtr->isMultithreaded, + globalEnvPtr->isDistributed, + globalEnvPtr->isMpiUserOwned, numThreads, - env.numNodes, + globalEnvPtr->numNodes, cuQuantum, gpuDirect); } diff --git a/tests/unit/environment.cpp b/tests/unit/environment.cpp index 344ac5864..85d96cf8e 100644 --- a/tests/unit/environment.cpp +++ b/tests/unit/environment.cpp @@ -161,9 +161,9 @@ TEST_CASE( "getQuESTEnv", TEST_CATEGORY ) { REQUIRE( (env.isMultithreaded == 0 || env.isMultithreaded == 1) ); REQUIRE( (env.isGpuAccelerated == 0 || env.isGpuAccelerated == 1) ); REQUIRE( (env.isDistributed == 0 || env.isDistributed == 1) ); - REQUIRE( (env.userOwnsMpi == 0 || env.userOwnsMpi == 1) ); - REQUIRE( (env.isCuQuantumEnabled == 0 || env.isCuQuantumEnabled == 1) ); - REQUIRE( (env.isGpuSharingEnabled == 0 || env.isGpuSharingEnabled == 1) ); + REQUIRE( (env.isMpiUserOwned == 0 || env.isMpiUserOwned == 1) ); // <- pointless since bool + REQUIRE( (env.isCuQuantumEnabled == 0 || env.isCuQuantumEnabled == 1) ); // but you can't be too + REQUIRE( (env.isGpuSharingEnabled == 0 || env.isGpuSharingEnabled == 1) ); // careful ;^) REQUIRE( env.rank >= 0 ); REQUIRE( env.numNodes >= 0 ); From fe1020cf43a96fad2eb7b646cdc3d79f3b5cad76 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 01:53:08 -0400 Subject: [PATCH 07/24] Remove redundant stdbool include --- quest/src/api/subcommunicator.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/quest/src/api/subcommunicator.cpp b/quest/src/api/subcommunicator.cpp index e248f0dba..e2fd00129 100644 --- a/quest/src/api/subcommunicator.cpp +++ b/quest/src/api/subcommunicator.cpp @@ -7,7 +7,6 @@ #if QUEST_COMPILE_MPI && QUEST_COMPILE_SUBCOMM -#include #include void initCustomMpiCommQuESTEnv(MPI_Comm userQuestComm, int useGpuAccel, int useMultithread) { From d363d092c692cab0625043b49e39f760db1ee899 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 02:07:27 -0400 Subject: [PATCH 08/24] Add validation to initCustomMpiCommQuESTEnv --- quest/src/api/subcommunicator.cpp | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/quest/src/api/subcommunicator.cpp b/quest/src/api/subcommunicator.cpp index e2fd00129..dcef1c161 100644 --- a/quest/src/api/subcommunicator.cpp +++ b/quest/src/api/subcommunicator.cpp @@ -2,29 +2,36 @@ #include "quest/include/environment.h" #include "quest/include/subcommunicator.h" +#include "quest/src/core/validation.hpp" #include "quest/src/comm/comm_config.hpp" -#include "quest/src/core/errors.hpp" #if QUEST_COMPILE_MPI && QUEST_COMPILE_SUBCOMM #include + + +// TODO: +// We must resolve this inner function of QuEST initialisation, but which is +// private to api/environment.cpp, and so cannot be exposed in the user-facing +// include/environment.hpp. Grr! For now, we here just cheekily extern it c: +extern void validateAndInitCustomQuESTEnv( + int useDistrib, bool userOwnsMpi, int useGpuAccel, int useMultithread, const char* caller); + + + void initCustomMpiCommQuESTEnv(MPI_Comm userQuestComm, int useGpuAccel, int useMultithread) { + // useDistrib and userOwnsMpi are implied by the user of this initialiser const int useDistrib = 1; const bool userOwnsMpi = true; - // set mpiCommQuest to user provided communicator - if (comm_isInit()) { - comm_setMpiComm(userQuestComm); - } else { - error_commNotInit(); - } - - // initialise QuEST around that communicator - initCustomMpiQuESTEnv(useDistrib, userOwnsMpi, useGpuAccel, useMultithread); + // pre-validate that we are able to set the MPI communicator + validate_mpiInitStatus(useDistrib, userOwnsMpi, __func__); + comm_setMpiComm(userQuestComm); - return; + // perform remaining validation and init QuEST env + validateAndInitCustomQuESTEnv(useDistrib, userOwnsMpi, useGpuAccel, useMultithread, __func__); } #endif From 70ac5693a37b820ad05913a43c57a616aefd37bd Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 02:09:45 -0400 Subject: [PATCH 09/24] Rename mpiCommQuest to global_mpiComm --- quest/src/comm/comm_config.cpp | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index 7bc9c2f92..73f8ffa3b 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -21,7 +21,7 @@ #if QUEST_COMPILE_MPI #include - static MPI_Comm mpiCommQuest = MPI_COMM_NULL; + static MPI_Comm global_mpiComm = MPI_COMM_NULL; #endif @@ -117,8 +117,8 @@ void comm_init(bool userOwnsMpi) { MPI_Init(NULL, NULL); // choose communicator only when the user hasn't - if (mpiCommQuest == MPI_COMM_NULL) - MPI_Comm_dup(MPI_COMM_WORLD, &mpiCommQuest); + if (global_mpiComm == MPI_COMM_NULL) + MPI_Comm_dup(MPI_COMM_WORLD, &global_mpiComm); #endif } @@ -136,11 +136,11 @@ void comm_end(bool userOwnsMpi) { // triggered by "bad MPI init" validation, during which, the communicator may not yet // have been set. We choose NOT to divert to MPI_COMM_WORLD, which is likely just to // stall at MPI_Barrier, and instead let the user's communicator live on; then crash! - if (mpiCommQuest == MPI_COMM_NULL) + if (global_mpiComm == MPI_COMM_NULL) return; - MPI_Barrier(mpiCommQuest); - MPI_Comm_free(&mpiCommQuest); + MPI_Barrier(global_mpiComm); + MPI_Comm_free(&global_mpiComm); // QuEST must finalise MPI if the user does not own it if (!userOwnsMpi) @@ -165,7 +165,7 @@ int comm_getRank() { // using WORLD (and pray the user hasn't silenced world-root std-out!). We // COULD safely return ROOT_RANK instead, letting all processes believe they // are root, but this grossly duplicates the output across ALL processes - MPI_Comm comm = (mpiCommQuest == MPI_COMM_NULL)? MPI_COMM_WORLD : mpiCommQuest; + MPI_Comm comm = (global_mpiComm == MPI_COMM_NULL)? MPI_COMM_WORLD : global_mpiComm; int rank; MPI_Comm_rank(comm, &rank); @@ -197,7 +197,7 @@ int comm_getNumNodes() { return 1; int numNodes; - MPI_Comm_size(mpiCommQuest, &numNodes); + MPI_Comm_size(global_mpiComm, &numNodes); return numNodes; #else @@ -218,29 +218,29 @@ void comm_sync() { // gracefully handle when the communicator is still NULL, because comm_sync() is // triggered by "bad MPI init" validation (during the error message printing) // during which, the communicator may not yet have been overriden - if (mpiCommQuest == MPI_COMM_NULL) + if (global_mpiComm == MPI_COMM_NULL) return; - MPI_Barrier(mpiCommQuest); + MPI_Barrier(global_mpiComm); #endif } #if QUEST_COMPILE_MPI MPI_Comm comm_getMpiComm() { - return mpiCommQuest; + return global_mpiComm; } #if QUEST_COMPILE_SUBCOMM void comm_setMpiComm(MPI_Comm newComm) { - // error if mpiCommQuEST is already set! - if (mpiCommQuest != MPI_COMM_NULL) { - MPI_Barrier(mpiCommQuest); - MPI_Comm_free(&mpiCommQuest); + // error if global_mpiComm is already set! + if (global_mpiComm != MPI_COMM_NULL) { + MPI_Barrier(global_mpiComm); + MPI_Comm_free(&global_mpiComm); error_commDoubleSetMpiComm(); } - int mpi_err = MPI_Comm_dup(newComm, &mpiCommQuest); + int mpi_err = MPI_Comm_dup(newComm, &global_mpiComm); if (mpi_err != MPI_SUCCESS) { error_commInvalidMpiComm(); } From ef6860b543145918fd6f01d776c76a30bd984937 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 02:17:43 -0400 Subject: [PATCH 10/24] Rename mpiCommQuest (local var) to mpiComm and inline where trivial --- quest/src/comm/comm_routines.cpp | 63 +++++++++++--------------------- quest/src/core/errors.cpp | 2 +- 2 files changed, 23 insertions(+), 42 deletions(-) diff --git a/quest/src/comm/comm_routines.cpp b/quest/src/comm/comm_routines.cpp index 0bc90563b..166586606 100644 --- a/quest/src/comm/comm_routines.cpp +++ b/quest/src/comm/comm_routines.cpp @@ -149,8 +149,7 @@ int getMaxNumMessages() { // messages. Beware the max is obtained via a void pointer and might be unset... void* tagUpperBoundPtr; int isAttribSet; - MPI_Comm mpiCommQuest = comm_getMpiComm(); - MPI_Comm_get_attr(mpiCommQuest, MPI_TAG_UB, &tagUpperBoundPtr, &isAttribSet); + MPI_Comm_get_attr(comm_getMpiComm(), MPI_TAG_UB, &tagUpperBoundPtr, &isAttribSet); // if something went wrong with obtaining the tag bound, return the safe minimum if (!isAttribSet) @@ -217,7 +216,7 @@ std::array dividePayloadIntoMessages(qindex numAmps) { void exchangeArrays(qcomp* send, qcomp* recv, qindex numElems, int pairRank) { #if QUEST_COMPILE_MPI - MPI_Comm mpiCommQuest = comm_getMpiComm(); + MPI_Comm mpiComm = comm_getMpiComm(); // each message is asynchronously dispatched with a final wait, as per arxiv.org/abs/2308.07402 @@ -229,8 +228,8 @@ void exchangeArrays(qcomp* send, qcomp* recv, qindex numElems, int pairRank) { // so that messages are permitted to arrive out-of-order (supporting UCX adaptive-routing) for (qindex m=0; m(m); // gauranteed int, but m*messageSize needs qindex - MPI_Irecv(&recv[m*messageSize], messageSize, MPI_QCOMP, pairRank, tag, mpiCommQuest, &requests[2*m]); - MPI_Isend(&send[m*messageSize], messageSize, MPI_QCOMP, pairRank, tag, mpiCommQuest, &requests[2*m+1]); + MPI_Irecv(&recv[m*messageSize], messageSize, MPI_QCOMP, pairRank, tag, mpiComm, &requests[2*m]); + MPI_Isend(&send[m*messageSize], messageSize, MPI_QCOMP, pairRank, tag, mpiComm, &requests[2*m+1]); } // wait for all exchanges to complete (MPI will automatically free the request memory) @@ -251,7 +250,7 @@ void exchangeArrays(qcomp* send, qcomp* recv, qindex numElems, int pairRank) { void asynchSendArray(qcomp* send, qindex numElems, int pairRank) { #if QUEST_COMPILE_MPI - MPI_Comm mpiCommQuest = comm_getMpiComm(); + MPI_Comm mpiComm = comm_getMpiComm(); // we will not track nor wait for the asynch send; instead, the caller will later comm_sync() MPI_Request nullReq = MPI_REQUEST_NULL; @@ -262,7 +261,7 @@ void asynchSendArray(qcomp* send, qindex numElems, int pairRank) { // asynchronously send the uniquely-tagged messages for (qindex m=0; m(m); // gauranteed int, but m*messageSize needs qindex - MPI_Isend(&send[m*messageSize], messageSize, MPI_QCOMP, pairRank, tag, mpiCommQuest, &nullReq); + MPI_Isend(&send[m*messageSize], messageSize, MPI_QCOMP, pairRank, tag, mpiComm, &nullReq); } #else @@ -274,7 +273,7 @@ void asynchSendArray(qcomp* send, qindex numElems, int pairRank) { void receiveArray(qcomp* dest, qindex numElems, int pairRank) { #if QUEST_COMPILE_MPI - MPI_Comm mpiCommQuest = comm_getMpiComm(); + MPI_Comm mpiComm = comm_getMpiComm(); // expect the data in multiple messages auto [messageSize, numMessages] = dividePow2PayloadIntoMessages(numElems); @@ -285,7 +284,7 @@ void receiveArray(qcomp* dest, qindex numElems, int pairRank) { // listen to receive each uniquely-tagged message asynchronously (as per arxiv.org/abs/2308.07402) for (qindex m=0; m(m); // gauranteed int, but m*messageSize needs qindex - MPI_Irecv(&dest[m*messageSize], messageSize, MPI_QCOMP, pairRank, tag, mpiCommQuest, &requests[m]); + MPI_Irecv(&dest[m*messageSize], messageSize, MPI_QCOMP, pairRank, tag, mpiComm, &requests[m]); } // receivers wait for all messages to be received (while sender asynch proceeds) @@ -310,8 +309,7 @@ void globallyCombineNonUniformSubArrays( ) { #if QUEST_COMPILE_MPI - MPI_Comm mpiCommQuest = comm_getMpiComm(); - + auto mpiComm = comm_getMpiComm(); int myRank = comm_getRank(); int numNodes = comm_getNumNodes(); @@ -345,14 +343,14 @@ void globallyCombineNonUniformSubArrays( for (int m=0; m 0) { qindex recvInd = globalRecvIndPerRank[sendRank] + (numBigMsgs * bigMsgSize); requests.push_back(MPI_REQUEST_NULL); - MPI_Ibcast(&recv[recvInd], remMsgSize, MPI_QCOMP, sendRank, mpiCommQuest, &requests.back()); + MPI_Ibcast(&recv[recvInd], remMsgSize, MPI_QCOMP, sendRank, mpiComm, &requests.back()); } } @@ -648,9 +646,7 @@ void comm_exchangeAmpsToBuffers(Qureg qureg, int pairRank) { void comm_broadcastAmp(int sendRank, qcomp* sendAmp) { #if QUEST_COMPILE_MPI - MPI_Comm mpiCommQuest = comm_getMpiComm(); - - MPI_Bcast(sendAmp, 1, MPI_QCOMP, sendRank, mpiCommQuest); + MPI_Bcast(sendAmp, 1, MPI_QCOMP, sendRank, comm_getMpiComm()); #else error_commButEnvNotDistributed(); @@ -661,7 +657,7 @@ void comm_broadcastAmp(int sendRank, qcomp* sendAmp) { void comm_sendAmpsToRoot(int sendRank, qcomp* send, qcomp* recv, qindex numAmps) { #if QUEST_COMPILE_MPI - MPI_Comm mpiCommQuest = comm_getMpiComm(); + MPI_Comm mpiComm = comm_getMpiComm(); // only the sender and root nodes need to continue int recvRank = ROOT_RANK; @@ -678,8 +674,8 @@ void comm_sendAmpsToRoot(int sendRank, qcomp* send, qcomp* recv, qindex numAmps) for (qindex m=0; m(m); (myRank == sendRank)? - MPI_Isend(&send[m*messageSize], messageSize, MPI_QCOMP, recvRank, tag, mpiCommQuest, &requests[m]): // sender - MPI_Irecv(&recv[m*messageSize], messageSize, MPI_QCOMP, sendRank, tag, mpiCommQuest, &requests[m]); // root + MPI_Isend(&send[m*messageSize], messageSize, MPI_QCOMP, recvRank, tag, mpiComm, &requests[m]): // sender + MPI_Irecv(&recv[m*messageSize], messageSize, MPI_QCOMP, sendRank, tag, mpiComm, &requests[m]); // root } // wait for all exchanges to complete (MPI will automatically free the request memory) @@ -692,13 +688,10 @@ void comm_sendAmpsToRoot(int sendRank, qcomp* send, qcomp* recv, qindex numAmps) void comm_broadcastIntsFromRoot(int* arr, qindex length) { - #if QUEST_COMPILE_MPI - MPI_Comm mpiCommQuest = comm_getMpiComm(); - int sendRank = ROOT_RANK; - MPI_Bcast(arr, length, MPI_INT, sendRank, mpiCommQuest); + MPI_Bcast(arr, length, MPI_INT, sendRank, comm_getMpiComm()); #else error_commButEnvNotDistributed(); @@ -709,10 +702,8 @@ void comm_broadcastIntsFromRoot(int* arr, qindex length) { void comm_broadcastUnsignedsFromRoot(unsigned* arr, qindex length) { #if QUEST_COMPILE_MPI - MPI_Comm mpiCommQuest = comm_getMpiComm(); - int sendRank = ROOT_RANK; - MPI_Bcast(arr, length, MPI_UNSIGNED, sendRank, mpiCommQuest); + MPI_Bcast(arr, length, MPI_UNSIGNED, sendRank, comm_getMpiComm()); #else error_commButEnvNotDistributed(); @@ -739,9 +730,7 @@ void comm_combineSubArrays(qcomp* recv, vector recvInds, vector void comm_reduceAmp(qcomp* localAmp) { #if QUEST_COMPILE_MPI - MPI_Comm mpiCommQuest = comm_getMpiComm(); - - MPI_Allreduce(MPI_IN_PLACE, localAmp, 1, MPI_QCOMP, MPI_SUM, mpiCommQuest); + MPI_Allreduce(MPI_IN_PLACE, localAmp, 1, MPI_QCOMP, MPI_SUM, comm_getMpiComm()); #else error_commButEnvNotDistributed(); @@ -752,9 +741,7 @@ void comm_reduceAmp(qcomp* localAmp) { void comm_reduceReal(qreal* localReal) { #if QUEST_COMPILE_MPI - MPI_Comm mpiCommQuest = comm_getMpiComm(); - - MPI_Allreduce(MPI_IN_PLACE, localReal, 1, MPI_QREAL, MPI_SUM, mpiCommQuest); + MPI_Allreduce(MPI_IN_PLACE, localReal, 1, MPI_QREAL, MPI_SUM, comm_getMpiComm()); #else error_commButEnvNotDistributed(); @@ -765,9 +752,7 @@ void comm_reduceReal(qreal* localReal) { void comm_reduceReals(qreal* localReals, qindex numLocalReals) { #if QUEST_COMPILE_MPI - MPI_Comm mpiCommQuest = comm_getMpiComm(); - - MPI_Allreduce(MPI_IN_PLACE, localReals, numLocalReals, MPI_QREAL, MPI_SUM, mpiCommQuest); + MPI_Allreduce(MPI_IN_PLACE, localReals, numLocalReals, MPI_QREAL, MPI_SUM, comm_getMpiComm()); #else error_commButEnvNotDistributed(); @@ -778,12 +763,10 @@ void comm_reduceReals(qreal* localReals, qindex numLocalReals) { bool comm_isTrueOnAllNodes(bool val) { #if QUEST_COMPILE_MPI - MPI_Comm mpiCommQuest = comm_getMpiComm(); - // perform global AND and broadcast result back to all nodes int local = (int) val; int global; - MPI_Allreduce(&local, &global, 1, MPI_INT, MPI_LAND, mpiCommQuest); + MPI_Allreduce(&local, &global, 1, MPI_INT, MPI_LAND, comm_getMpiComm()); return (bool) global; #else @@ -819,8 +802,6 @@ bool comm_isTrueOnRootNode(bool val) { vector comm_gatherStringsToRoot(char* localChars, int maxNumLocalChars) { #if QUEST_COMPILE_MPI - MPI_Comm mpiCommQuest = comm_getMpiComm(); - // no need to validate array sizes and memory alloc successes; // these are trivial O(#nodes)-size arrays containing <20 chars int numNodes = comm_getNumNodes(); @@ -831,7 +812,7 @@ vector comm_gatherStringsToRoot(char* localChars, int maxNumLocalChars) // all nodes send root all their local chars int recvRank = ROOT_RANK; MPI_Gather(localChars, maxNumLocalChars, MPI_CHAR, allChars.data(), - maxNumLocalChars, MPI_CHAR, recvRank, mpiCommQuest); + maxNumLocalChars, MPI_CHAR, recvRank, comm_getMpiComm()); // divide allChars into stings, delimited by each node's terminal char vector out(numNodes); diff --git a/quest/src/core/errors.cpp b/quest/src/core/errors.cpp index 7b624a2f7..2c576439c 100644 --- a/quest/src/core/errors.cpp +++ b/quest/src/core/errors.cpp @@ -188,7 +188,7 @@ void error_commNumMessagesExceedTagMax() { void error_commDoubleSetMpiComm() { - raiseInternalError("An attempt was made to set mpiCommQuest after it had already been set, as indicated by mpiCommQuest != MPI_COMM_NULL."); + raiseInternalError("An attempt was made to set the QuEST MPI communicator after it had already been set (and changed from MPI_COMM_NULL)."); } void assert_commBoundsAreValid(Qureg qureg, qindex sendInd, qindex recvInd, qindex numAmps) { From d85e0641bfebf5f0f0a96dba1deb26e7f9d3c4ba Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 02:21:36 -0400 Subject: [PATCH 11/24] Made environment.cpp adhere to global_ convention shame on 2024 me! --- quest/src/api/environment.cpp | 92 +++++++++++++++++------------------ 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/quest/src/api/environment.cpp b/quest/src/api/environment.cpp index 1e740e427..61e2731ef 100644 --- a/quest/src/api/environment.cpp +++ b/quest/src/api/environment.cpp @@ -48,7 +48,7 @@ using std::string; */ -static QuESTEnv* globalEnvPtr = nullptr; +static QuESTEnv* global_envPtr = nullptr; @@ -62,7 +62,7 @@ static QuESTEnv* globalEnvPtr = nullptr; */ -static bool hasEnvBeenFinalized = false; +static bool global_hasEnvBeenFinalized = false; @@ -76,7 +76,7 @@ void validateAndInitCustomQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuA // ensure that we are never re-initialising QuEST (even after finalize) because // this leads to undefined behaviour in distributed mode, as per the MPI std, // regardless of whether the user owns MPI - validate_envNeverInit(globalEnvPtr != nullptr, hasEnvBeenFinalized, caller); + validate_envNeverInit(global_envPtr != nullptr, global_hasEnvBeenFinalized, caller); // load env-vars before validating deployment mode, because some env vars can // affect validation (such as QUEST_PERMIT_NODES_TO_SHARE_GPU) @@ -142,28 +142,28 @@ void validateAndInitCustomQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuA rand_setSeedsToDefault(); // allocate space for the global QuESTEnv singleton (overwriting nullptr, unless malloc fails) - globalEnvPtr = (QuESTEnv*) malloc(sizeof(QuESTEnv)); + global_envPtr = (QuESTEnv*) malloc(sizeof(QuESTEnv)); // pedantically check that teeny tiny malloc just succeeded - if (globalEnvPtr == nullptr) + if (global_envPtr == nullptr) error_allocOfQuESTEnvFailed(); // bind deployment info to global instance (autocasting int to bool) - globalEnvPtr->isMultithreaded = useMultithread; - globalEnvPtr->isGpuAccelerated = useGpuAccel; - globalEnvPtr->isDistributed = useDistrib; - globalEnvPtr->isMpiUserOwned = userOwnsMpi; - globalEnvPtr->isCuQuantumEnabled = useCuQuantum; - globalEnvPtr->isGpuSharingEnabled = permitGpuSharing; + global_envPtr->isMultithreaded = useMultithread; + global_envPtr->isGpuAccelerated = useGpuAccel; + global_envPtr->isDistributed = useDistrib; + global_envPtr->isMpiUserOwned = userOwnsMpi; + global_envPtr->isCuQuantumEnabled = useCuQuantum; + global_envPtr->isGpuSharingEnabled = permitGpuSharing; // bind distributed info - globalEnvPtr->rank = (useDistrib)? comm_getRank() : 0; - globalEnvPtr->numNodes = (useDistrib)? comm_getNumNodes() : 1; + global_envPtr->rank = (useDistrib)? comm_getRank() : 0; + global_envPtr->numNodes = (useDistrib)? comm_getNumNodes() : 1; } void updateQuESTEnvDistInfo() { - globalEnvPtr->rank = (globalEnvPtr->isDistributed)? comm_getRank() : 0; - globalEnvPtr->numNodes = (globalEnvPtr->isDistributed)? comm_getNumNodes() : 1; + global_envPtr->rank = (global_envPtr->isDistributed)? comm_getRank() : 0; + global_envPtr->numNodes = (global_envPtr->isDistributed)? comm_getNumNodes() : 1; return; } @@ -213,12 +213,12 @@ void printDeploymentInfo() { print_table( "deployment", { - {"isMpiEnabled", globalEnvPtr->isDistributed}, - {"isMpiUserOwned", globalEnvPtr->isMpiUserOwned}, - {"isGpuEnabled", globalEnvPtr->isGpuAccelerated}, - {"isOmpEnabled", globalEnvPtr->isMultithreaded}, - {"isCuQuantumEnabled", globalEnvPtr->isCuQuantumEnabled}, - {"isGpuSharingEnabled", globalEnvPtr->isGpuSharingEnabled}, + {"isMpiEnabled", global_envPtr->isDistributed}, + {"isMpiUserOwned", global_envPtr->isMpiUserOwned}, + {"isGpuEnabled", global_envPtr->isGpuAccelerated}, + {"isOmpEnabled", global_envPtr->isMultithreaded}, + {"isCuQuantumEnabled", global_envPtr->isCuQuantumEnabled}, + {"isGpuSharingEnabled", global_envPtr->isGpuSharingEnabled}, }); } @@ -278,7 +278,7 @@ void printDistributionInfo() { print_table( "distribution", { {"isMpiGpuAware", (comm_isMpiCompiled())? printer_toStr(comm_isMpiGpuAware()) : na}, - {"numMpiNodes", printer_toStr(globalEnvPtr->numNodes)}, + {"numMpiNodes", printer_toStr(global_envPtr->numNodes)}, }); } @@ -288,7 +288,7 @@ void printQuregSizeLimits(bool isDensMatr) { using namespace printer_substrings; // for brevity - int numNodes = globalEnvPtr->numNodes; + int numNodes = global_envPtr->numNodes; // by default, CPU limits are unknown (because memory query might fail) string maxQbForCpu = un; @@ -300,7 +300,7 @@ void printQuregSizeLimits(bool isDensMatr) { maxQbForCpu = printer_toStr(mem_getMaxNumQuregQubitsWhichCanFitInMemory(isDensMatr, 1, cpuMem)); // and the max MPI sizes are only relevant when env is distributed - if (globalEnvPtr->isDistributed) + if (global_envPtr->isDistributed) maxQbForMpiCpu = printer_toStr(mem_getMaxNumQuregQubitsWhichCanFitInMemory(isDensMatr, numNodes, cpuMem)); // when MPI irrelevant, change their status from "unknown" to "N/A" @@ -315,12 +315,12 @@ void printQuregSizeLimits(bool isDensMatr) { string maxQbForMpiGpu = na; // max GPU registers only relevant if env is GPU-accelerated - if (globalEnvPtr->isGpuAccelerated) { + if (global_envPtr->isGpuAccelerated) { qindex gpuMem = gpu_getCurrentAvailableMemoryInBytes(); maxQbForGpu = printer_toStr(mem_getMaxNumQuregQubitsWhichCanFitInMemory(isDensMatr, 1, gpuMem)); // and the max MPI sizes are further only relevant when env is distributed - if (globalEnvPtr->isDistributed) + if (global_envPtr->isDistributed) maxQbForMpiGpu = printer_toStr(mem_getMaxNumQuregQubitsWhichCanFitInMemory(isDensMatr, numNodes, gpuMem)); } @@ -357,7 +357,7 @@ void printQuregAutoDeployments(bool isDensMatr) { // test to theoretically max #qubits, surpassing max that can fit in RAM and GPUs, because // auto-deploy will still try to deploy there to (then subsequent validation will fail) - int maxQubits = mem_getMaxNumQuregQubitsBeforeGlobalMemSizeofOverflow(isDensMatr, globalEnvPtr->numNodes); + int maxQubits = mem_getMaxNumQuregQubitsBeforeGlobalMemSizeofOverflow(isDensMatr, global_envPtr->numNodes); for (int numQubits=1; numQubitsisGpuAccelerated) + if (global_envPtr->isGpuAccelerated) gpu_clearCache(); // syncs first - if (globalEnvPtr->isGpuAccelerated && gpu_isCuQuantumCompiled()) + if (global_envPtr->isGpuAccelerated && gpu_isCuQuantumCompiled()) gpu_finalizeCuQuantum(); - if (globalEnvPtr->isDistributed) { + if (global_envPtr->isDistributed) { comm_sync(); - comm_end(globalEnvPtr->isMpiUserOwned); + comm_end(global_envPtr->isMpiUserOwned); } // free global env's heap memory and flag it as unallocated - free(globalEnvPtr); - globalEnvPtr = nullptr; + free(global_envPtr); + global_envPtr = nullptr; // flag that the environment was finalised, to ensure it is never re-initialised - hasEnvBeenFinalized = true; + global_hasEnvBeenFinalized = true; } void syncQuESTEnv() { validate_envIsInit(__func__); - if (globalEnvPtr->isGpuAccelerated) + if (global_envPtr->isGpuAccelerated) gpu_sync(); - if (globalEnvPtr->isDistributed) { + if (global_envPtr->isDistributed) { comm_sync(); #if QUEST_COMPILE_SUBCOMM updateQuESTEnvDistInfo(); @@ -518,16 +518,16 @@ void getQuESTEnvironmentString(char str[200]) { validate_envIsInit(__func__); int numThreads = cpu_isOpenmpCompiled()? cpu_getAvailableNumThreads() : 1; - int cuQuantum = globalEnvPtr->isGpuAccelerated && gpu_isCuQuantumCompiled(); - int gpuDirect = globalEnvPtr->isGpuAccelerated && gpu_isDirectGpuCommPossible(); + int cuQuantum = global_envPtr->isGpuAccelerated && gpu_isCuQuantumCompiled(); + int gpuDirect = global_envPtr->isGpuAccelerated && gpu_isDirectGpuCommPossible(); snprintf(str, 200, "CUDA=%d OpenMP=%d MPI=%d userOwnsMPI=%d threads=%d ranks=%d cuQuantum=%d gpuDirect=%d", - globalEnvPtr->isGpuAccelerated, - globalEnvPtr->isMultithreaded, - globalEnvPtr->isDistributed, - globalEnvPtr->isMpiUserOwned, + global_envPtr->isGpuAccelerated, + global_envPtr->isMultithreaded, + global_envPtr->isDistributed, + global_envPtr->isMpiUserOwned, numThreads, - globalEnvPtr->numNodes, + global_envPtr->numNodes, cuQuantum, gpuDirect); } From 8fe9bbec3d43a7db13773940cf902b5fa39f9022 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 02:24:22 -0400 Subject: [PATCH 12/24] Remove suspicious updateQuESTEnvDistInfo() --- quest/src/api/environment.cpp | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/quest/src/api/environment.cpp b/quest/src/api/environment.cpp index 61e2731ef..f0d990ad1 100644 --- a/quest/src/api/environment.cpp +++ b/quest/src/api/environment.cpp @@ -161,11 +161,6 @@ void validateAndInitCustomQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuA global_envPtr->numNodes = (useDistrib)? comm_getNumNodes() : 1; } -void updateQuESTEnvDistInfo() { - global_envPtr->rank = (global_envPtr->isDistributed)? comm_getRank() : 0; - global_envPtr->numNodes = (global_envPtr->isDistributed)? comm_getNumNodes() : 1; - return; -} /* @@ -475,12 +470,8 @@ void syncQuESTEnv() { if (global_envPtr->isGpuAccelerated) gpu_sync(); - if (global_envPtr->isDistributed) { + if (global_envPtr->isDistributed) comm_sync(); - #if QUEST_COMPILE_SUBCOMM - updateQuESTEnvDistInfo(); - #endif - } } From 314e72e0e040973bd369876af066f17d696db30b Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 02:31:48 -0400 Subject: [PATCH 13/24] Error in comm_getMpiComm() when comm=NULL --- quest/src/comm/comm_config.cpp | 4 ++++ quest/src/core/errors.cpp | 5 +++++ quest/src/core/errors.hpp | 2 ++ 3 files changed, 11 insertions(+) diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index 73f8ffa3b..aa627a444 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -227,6 +227,10 @@ void comm_sync() { #if QUEST_COMPILE_MPI MPI_Comm comm_getMpiComm() { + + if (global_mpiComm == MPI_COMM_NULL) + error_commMpiCommIsNull(); + return global_mpiComm; } diff --git a/quest/src/core/errors.cpp b/quest/src/core/errors.cpp index 2c576439c..d77bb2d38 100644 --- a/quest/src/core/errors.cpp +++ b/quest/src/core/errors.cpp @@ -191,6 +191,11 @@ void error_commDoubleSetMpiComm() { raiseInternalError("An attempt was made to set the QuEST MPI communicator after it had already been set (and changed from MPI_COMM_NULL)."); } +void error_commMpiCommIsNull() { + + raiseInternalError("The MPI communicator was queried but was unexpectedly still MPI_COMM_NULL."); +} + void assert_commBoundsAreValid(Qureg qureg, qindex sendInd, qindex recvInd, qindex numAmps) { bool valid = ( diff --git a/quest/src/core/errors.hpp b/quest/src/core/errors.hpp index f276c06ad..c41b69851 100644 --- a/quest/src/core/errors.hpp +++ b/quest/src/core/errors.hpp @@ -95,6 +95,8 @@ void error_commNumMessagesExceedTagMax(); void error_commDoubleSetMpiComm(); +void error_commMpiCommIsNull(); + void assert_commBoundsAreValid(Qureg qureg, qindex sendInd, qindex recvInd, qindex numAmps); void assert_commPayloadIsPowerOf2(qindex numAmps); From 51c0731b6fcc2a666f7b454777676cf148de786c Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 02:58:25 -0400 Subject: [PATCH 14/24] Remove MPI leak from comm_config.hpp by just using extern. This is terrible and inadvisable, but offers an easier understanding of the software architecture (and so is easier to fix correctly) than the previous "macros change which signatures this header exposes" design. Also, we removed the unnecessary avoiding of defining comm_setMpiComm when SUBCOMM was not defined, which made the architecture even more confusing. Now, SUBCOMM only influences the contents of subcommunicator.cpp and subcommunicator.hpp. Simple! --- quest/src/api/subcommunicator.cpp | 8 +++-- quest/src/comm/comm_config.cpp | 56 +++++++++++++++++++------------ quest/src/comm/comm_config.hpp | 15 ++------- quest/src/comm/comm_routines.cpp | 3 +- 4 files changed, 45 insertions(+), 37 deletions(-) diff --git a/quest/src/api/subcommunicator.cpp b/quest/src/api/subcommunicator.cpp index dcef1c161..688b2e9eb 100644 --- a/quest/src/api/subcommunicator.cpp +++ b/quest/src/api/subcommunicator.cpp @@ -7,9 +7,14 @@ #if QUEST_COMPILE_MPI && QUEST_COMPILE_SUBCOMM -#include +#include // MPI_Comm +// TODO: +// We must resolve this communicator function which contains an MPI type +// and ergo should not be leaked outside comm_config.cpp. For now, we cheat! +extern void comm_setMpiComm(MPI_Comm newComm); + // TODO: // We must resolve this inner function of QuEST initialisation, but which is @@ -19,7 +24,6 @@ extern void validateAndInitCustomQuESTEnv( int useDistrib, bool userOwnsMpi, int useGpuAccel, int useMultithread, const char* caller); - void initCustomMpiCommQuESTEnv(MPI_Comm userQuestComm, int useGpuAccel, int useMultithread) { // useDistrib and userOwnsMpi are implied by the user of this initialiser diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index aa627a444..5f7a90a24 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -30,6 +30,7 @@ * WARN ABOUT CUDA-AWARENESS */ + #if QUEST_COMPILE_MPI && QUEST_COMPILE_CUDA // this check is OpenMPI specific @@ -54,6 +55,7 @@ /* * MPI ENVIRONMENT MANAGEMENT + * * all of which is safely callable in non-distributed mode */ @@ -124,7 +126,6 @@ void comm_init(bool userOwnsMpi) { } - void comm_end(bool userOwnsMpi) { #if QUEST_COMPILE_MPI @@ -225,31 +226,42 @@ void comm_sync() { #endif } + + +/* + * MPI COMMUNICATOR MANAGEMENT + * + * which requires exposing MPI_Comm in external-facing signatures. + * In lieu of leaking these into comm_config.hpp, callers must + * declare them as extern + */ + + #if QUEST_COMPILE_MPI - MPI_Comm comm_getMpiComm() { - if (global_mpiComm == MPI_COMM_NULL) - error_commMpiCommIsNull(); +MPI_Comm comm_getMpiComm() { - return global_mpiComm; - } + if (global_mpiComm == MPI_COMM_NULL) + error_commMpiCommIsNull(); - #if QUEST_COMPILE_SUBCOMM - void comm_setMpiComm(MPI_Comm newComm) { + return global_mpiComm; +} - // error if global_mpiComm is already set! - if (global_mpiComm != MPI_COMM_NULL) { - MPI_Barrier(global_mpiComm); - MPI_Comm_free(&global_mpiComm); - error_commDoubleSetMpiComm(); - } +void comm_setMpiComm(MPI_Comm newComm) { - int mpi_err = MPI_Comm_dup(newComm, &global_mpiComm); - if (mpi_err != MPI_SUCCESS) { - error_commInvalidMpiComm(); - } + // error if global_mpiComm is already set! + if (global_mpiComm != MPI_COMM_NULL) { + MPI_Barrier(global_mpiComm); + MPI_Comm_free(&global_mpiComm); + error_commDoubleSetMpiComm(); + } - return; - } - #endif -#endif + int mpi_err = MPI_Comm_dup(newComm, &global_mpiComm); + if (mpi_err != MPI_SUCCESS) { + error_commInvalidMpiComm(); + } + + return; +} + +#endif // QUEST_COMPILE_MPI diff --git a/quest/src/comm/comm_config.hpp b/quest/src/comm/comm_config.hpp index b061dd3e2..6b304575c 100644 --- a/quest/src/comm/comm_config.hpp +++ b/quest/src/comm/comm_config.hpp @@ -10,12 +10,6 @@ #ifndef COMM_CONFIG_HPP #define COMM_CONFIG_HPP -#include "quest/include/config.h" - -#if QUEST_COMPILE_MPI - #include -#endif - constexpr int ROOT_RANK = 0; bool comm_isMpiCompiled(); @@ -33,11 +27,8 @@ bool comm_isInit(); bool comm_isRootNode(); bool comm_isRootNode(int rank); -#if QUEST_COMPILE_MPI - MPI_Comm comm_getMpiComm(); - #if QUEST_COMPILE_SUBCOMM - void comm_setMpiComm(MPI_Comm newComm); - #endif -#endif +// Signatures containing MPI types which callers must extern: +// extern MPI_Comm comm_getMpiComm() +// extern void comm_setMpiComm(MPI_Comm newComm) #endif // COMM_CONFIG_HPP diff --git a/quest/src/comm/comm_routines.cpp b/quest/src/comm/comm_routines.cpp index 166586606..cf6956454 100644 --- a/quest/src/comm/comm_routines.cpp +++ b/quest/src/comm/comm_routines.cpp @@ -6,7 +6,7 @@ * * @author Tyson Jones * @author Jakub Adamski (sped-up large comm by asynch messages) - * @author Oliver Brown (patched max-message inference, consulted on AR and MPICH support) + * @author Oliver Brown (added custom communicators, patched max-message inference, consulted on AR and MPICH support) * @author Ania (Anna) Brown (developed QuEST v1 logic) */ @@ -24,6 +24,7 @@ #if QUEST_COMPILE_MPI #include + extern MPI_Comm comm_getMpiComm(); // comm_config.cpp does not leak MPI_Comm #endif #include From 047ede75d6ceee2e121c47c135e5e8eef1bc9daa Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 03:02:19 -0400 Subject: [PATCH 15/24] Rename comm_isMpiSubCommunicatorCompiled to comm_isMpiSubCommCompiled although I suspect this is a poor choice of name. The logic should always be considered "compiled" when MPI is known, and we choose instead whether to "expose" the MPI signature to the users --- quest/src/api/environment.cpp | 2 +- quest/src/comm/comm_config.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/quest/src/api/environment.cpp b/quest/src/api/environment.cpp index f0d990ad1..ab18ced91 100644 --- a/quest/src/api/environment.cpp +++ b/quest/src/api/environment.cpp @@ -196,7 +196,7 @@ void printCompilationInfo() { print_table( "compilation", { {"isMpiCompiled", comm_isMpiCompiled()}, - {"isMpiSubCommunicatorCompiled", comm_isMpiSubCommunicatorCompiled()}, + {"isMpiSubCommCompiled", comm_isMpiSubCommCompiled()}, {"isGpuCompiled", gpu_isGpuCompiled()}, {"isOmpCompiled", cpu_isOpenmpCompiled()}, {"isCuQuantumCompiled", gpu_isCuQuantumCompiled()}, diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index 5f7a90a24..c71cc16b8 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -64,7 +64,7 @@ bool comm_isMpiCompiled() { return (bool) QUEST_COMPILE_MPI; } -bool comm_isMpiSubCommunicatorCompiled() { +bool comm_isMpiSubCommCompiled() { return (bool) QUEST_COMPILE_SUBCOMM; } From 1680a12600429fc94f3e6cc1ec14616644601838 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 03:05:37 -0400 Subject: [PATCH 16/24] Replace magic number --- quest/src/core/validation.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quest/src/core/validation.cpp b/quest/src/core/validation.cpp index 871d94199..7f53e87bc 100644 --- a/quest/src/core/validation.cpp +++ b/quest/src/core/validation.cpp @@ -1164,10 +1164,10 @@ void default_inputErrorHandler(const char* func, const char* msg) { // will then attempt to instantly abort all nodes, losing the error message. comm_sync(); - // finalise MPI before error-exit to avoid scaring user with giant MPI error message + // finalise MPI before error-exit to avoid scaring user with giant MPI error message; // we always "take ownership" of MPI here since we're about to kill the whole program if (comm_isInit()) - comm_end(0); + comm_end(/*userOwnsMpi=*/false); // simply exit, interrupting any other process (potentially leaking) exit(EXIT_FAILURE); From 00332a844ba5edfb12d1cdf2f744aff2dfb1cc0e Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 03:25:30 -0400 Subject: [PATCH 17/24] Make initCustomMpiCommQuESTEnv validate against re-init without triggering an internal error --- quest/src/api/subcommunicator.cpp | 11 ++++++++--- quest/src/comm/comm_config.cpp | 26 ++++++++++++++++---------- quest/src/comm/comm_config.hpp | 4 +++- quest/src/core/errors.cpp | 2 +- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/quest/src/api/subcommunicator.cpp b/quest/src/api/subcommunicator.cpp index 688b2e9eb..47461cbab 100644 --- a/quest/src/api/subcommunicator.cpp +++ b/quest/src/api/subcommunicator.cpp @@ -7,7 +7,7 @@ #if QUEST_COMPILE_MPI && QUEST_COMPILE_SUBCOMM -#include // MPI_Comm +#include // TODO: @@ -32,9 +32,14 @@ void initCustomMpiCommQuESTEnv(MPI_Comm userQuestComm, int useGpuAccel, int useM // pre-validate that we are able to set the MPI communicator validate_mpiInitStatus(useDistrib, userOwnsMpi, __func__); - comm_setMpiComm(userQuestComm); - // perform remaining validation and init QuEST env + // avoid re-setting the MPI comm (to avoid an internal error), which happens + // if a user illegally re-calls this function, which will be subsequently + // caught by the validation in validateAndInitCustomQuESTEnv() below + if (comm_isMpiCommSet()) + comm_setMpiComm(userQuestComm); + + // perform remaining validation (some is harmlessly repeated) and init QuEST env validateAndInitCustomQuESTEnv(useDistrib, userOwnsMpi, useGpuAccel, useMultithread, __func__); } diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index c71cc16b8..b90861cc9 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -231,11 +231,16 @@ void comm_sync() { /* * MPI COMMUNICATOR MANAGEMENT * - * which requires exposing MPI_Comm in external-facing signatures. - * In lieu of leaking these into comm_config.hpp, callers must - * declare them as extern + * some of which requires exposing MPI_Comm in external-facing signatures. + * In lieu of leaking these into comm_config.hpp, callers must extern them. */ +bool comm_isMpiCommSet() { + + // once comm_init() or comm_setMpiComm() overwrite + // the communicator, is can never return to NULL + return (global_mpiComm == MPI_COMM_NULL); +} #if QUEST_COMPILE_MPI @@ -249,17 +254,18 @@ MPI_Comm comm_getMpiComm() { void comm_setMpiComm(MPI_Comm newComm) { - // error if global_mpiComm is already set! - if (global_mpiComm != MPI_COMM_NULL) { - MPI_Barrier(global_mpiComm); - MPI_Comm_free(&global_mpiComm); + // this is called prior to QuEST initialisation, + // and merely seeks to overwrite global_mpiComm + + if (global_mpiComm != MPI_COMM_NULL) error_commDoubleSetMpiComm(); - } + if (newComm == MPI_COMM_NULL) + error_commMpiCommIsNull(); int mpi_err = MPI_Comm_dup(newComm, &global_mpiComm); - if (mpi_err != MPI_SUCCESS) { + + if (mpi_err != MPI_SUCCESS) error_commInvalidMpiComm(); - } return; } diff --git a/quest/src/comm/comm_config.hpp b/quest/src/comm/comm_config.hpp index 6b304575c..ffe49eaed 100644 --- a/quest/src/comm/comm_config.hpp +++ b/quest/src/comm/comm_config.hpp @@ -13,7 +13,7 @@ constexpr int ROOT_RANK = 0; bool comm_isMpiCompiled(); -bool comm_isMpiSubCommunicatorCompiled(); +bool comm_isMpiSubCommCompiled(); bool comm_isMpiGpuAware(); void comm_init(bool userOwnsMpi); @@ -27,6 +27,8 @@ bool comm_isInit(); bool comm_isRootNode(); bool comm_isRootNode(int rank); +bool comm_isMpiCommSet(); + // Signatures containing MPI types which callers must extern: // extern MPI_Comm comm_getMpiComm() // extern void comm_setMpiComm(MPI_Comm newComm) diff --git a/quest/src/core/errors.cpp b/quest/src/core/errors.cpp index d77bb2d38..6e354d267 100644 --- a/quest/src/core/errors.cpp +++ b/quest/src/core/errors.cpp @@ -193,7 +193,7 @@ void error_commDoubleSetMpiComm() { void error_commMpiCommIsNull() { - raiseInternalError("The MPI communicator was queried but was unexpectedly still MPI_COMM_NULL."); + raiseInternalError("The MPI communicator was queried (or set) but was unexpectedly MPI_COMM_NULL (or set to be)."); } void assert_commBoundsAreValid(Qureg qureg, qindex sendInd, qindex recvInd, qindex numAmps) { From 6763af08b3a6e65f15d43f6ca5996d386069a272 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 03:33:22 -0400 Subject: [PATCH 18/24] Make initCustomMpiCommQuESTEnv validate subcomm is non-null --- quest/src/api/subcommunicator.cpp | 1 + quest/src/comm/comm_config.cpp | 20 +++++++++++--------- quest/src/core/validation.cpp | 11 +++++++++++ quest/src/core/validation.hpp | 2 ++ 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/quest/src/api/subcommunicator.cpp b/quest/src/api/subcommunicator.cpp index 47461cbab..e85b939ad 100644 --- a/quest/src/api/subcommunicator.cpp +++ b/quest/src/api/subcommunicator.cpp @@ -32,6 +32,7 @@ void initCustomMpiCommQuESTEnv(MPI_Comm userQuestComm, int useGpuAccel, int useM // pre-validate that we are able to set the MPI communicator validate_mpiInitStatus(useDistrib, userOwnsMpi, __func__); + validate_mpiSubCommIsNonNull(userQuestComm != MPI_COMM_NULL, __func__); // avoid re-setting the MPI comm (to avoid an internal error), which happens // if a user illegally re-calls this function, which will be subsequently diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index b90861cc9..c5a36a3f0 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -133,7 +133,7 @@ void comm_end(bool userOwnsMpi) { if (!comm_isInit()) return; - // ungracefully handle when the communicator is still NULL, because comm_end() may be + // gracefully handle when the communicator is still NULL, because comm_end() may be // triggered by "bad MPI init" validation, during which, the communicator may not yet // have been set. We choose NOT to divert to MPI_COMM_WORLD, which is likely just to // stall at MPI_Barrier, and instead let the user's communicator live on; then crash! @@ -160,16 +160,18 @@ int comm_getRank() { if (!comm_isInit()) return ROOT_RANK; - // consult the (potentially sub-) communicator for rank; if it is still - // NULL, as can only validly happen during failed MPI status validation (the - // error msg is attemptedly printed on only the root process), fallback to - // using WORLD (and pray the user hasn't silenced world-root std-out!). We - // COULD safely return ROOT_RANK instead, letting all processes believe they - // are root, but this grossly duplicates the output across ALL processes - MPI_Comm comm = (global_mpiComm == MPI_COMM_NULL)? MPI_COMM_WORLD : global_mpiComm; + // Consult the (potentially sub-) communicator for rank; if it is still + // NULL, as can only validly happen during failed QuESTEnv init validation + // (which triggers root-only error printing and ergo this function), we + // fall back to every process believing it is root and so attempting to + // print. This safely avoids consulting a potentially bugged MPI communicator + // and losing the message. We once tried to fallback to MPI_COMM_WORLD here, + // to avoid duplicate output, but it is not worth the risk of msg loss! + if (global_mpiComm == MPI_COMM_NULL) + return ROOT_RANK; int rank; - MPI_Comm_rank(comm, &rank); + MPI_Comm_rank(global_mpiComm, &rank); return rank; #else diff --git a/quest/src/core/validation.cpp b/quest/src/core/validation.cpp index 7f53e87bc..0d8a49423 100644 --- a/quest/src/core/validation.cpp +++ b/quest/src/core/validation.cpp @@ -110,6 +110,9 @@ namespace report { string USER_OWNED_MPI_WAS_NOT_INIT = "User owns MPI but did not prior initialise MPI before initialising QuEST."; + string USER_GIVEN_MPI_COMMUNICATOR_IS_NULL = + "The provided MPI communicator was null (MPI_COMM_NULL)."; + string QUEST_OWNED_MPI_WAS_PRE_INIT = "MPI was already initialised prior to QuESTEnv initialisation, but the user did not declare MPI ownership."; @@ -1529,6 +1532,14 @@ void validate_mpiInitStatus(bool useDistrib, bool userOwnsMpi, const char* calle // useDistrib=1, userOwnsMpi=1, isMpiInit=1 (legal: user fulfilled responsibility to pre-init) } +void validate_mpiSubCommIsNonNull(bool isNonNull, const char* caller) { + + if (!global_isValidationEnabled) + return; + + assertThat(isNonNull, report::USER_GIVEN_MPI_COMMUNICATOR_IS_NULL, caller); +} + /* diff --git a/quest/src/core/validation.hpp b/quest/src/core/validation.hpp index 345931946..109728643 100644 --- a/quest/src/core/validation.hpp +++ b/quest/src/core/validation.hpp @@ -79,6 +79,8 @@ void validate_gpuIsCuQuantumCompatible(const char* caller); void validate_mpiInitStatus(bool useDistrib, bool userOwnsMpi, const char* caller); +void validate_mpiSubCommIsNonNull(bool isNonNull, const char* caller); + /* From 1c9072cee12f8158c300f317957b1c835b8fe949 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 03:45:51 -0400 Subject: [PATCH 19/24] Make initCustomMpiCommQuESTEnv validate set-subcomm succeeds replacing the original internal error. Note that all of the other MPI functions between comm_config.cpp and comm_subroutines.cpp are unguarded; we should create a macro around them --- quest/src/api/subcommunicator.cpp | 8 +++++--- quest/src/comm/comm_config.cpp | 12 ++++-------- quest/src/comm/comm_config.hpp | 2 +- quest/src/core/errors.cpp | 5 ----- quest/src/core/errors.hpp | 2 -- quest/src/core/validation.cpp | 11 +++++++++++ quest/src/core/validation.hpp | 2 ++ 7 files changed, 23 insertions(+), 19 deletions(-) diff --git a/quest/src/api/subcommunicator.cpp b/quest/src/api/subcommunicator.cpp index e85b939ad..74c05293a 100644 --- a/quest/src/api/subcommunicator.cpp +++ b/quest/src/api/subcommunicator.cpp @@ -13,7 +13,7 @@ // TODO: // We must resolve this communicator function which contains an MPI type // and ergo should not be leaked outside comm_config.cpp. For now, we cheat! -extern void comm_setMpiComm(MPI_Comm newComm); +extern bool comm_setMpiComm(MPI_Comm newComm); // TODO: @@ -37,8 +37,10 @@ void initCustomMpiCommQuESTEnv(MPI_Comm userQuestComm, int useGpuAccel, int useM // avoid re-setting the MPI comm (to avoid an internal error), which happens // if a user illegally re-calls this function, which will be subsequently // caught by the validation in validateAndInitCustomQuESTEnv() below - if (comm_isMpiCommSet()) - comm_setMpiComm(userQuestComm); + if (!comm_isMpiCommSet()) { + bool success = comm_setMpiComm(userQuestComm); + validate_mpiSubCommSetSucceeded(success, __func__); + } // perform remaining validation (some is harmlessly repeated) and init QuEST env validateAndInitCustomQuESTEnv(useDistrib, userOwnsMpi, useGpuAccel, useMultithread, __func__); diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index c5a36a3f0..20f8ac026 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -241,7 +241,7 @@ bool comm_isMpiCommSet() { // once comm_init() or comm_setMpiComm() overwrite // the communicator, is can never return to NULL - return (global_mpiComm == MPI_COMM_NULL); + return (global_mpiComm != MPI_COMM_NULL); } #if QUEST_COMPILE_MPI @@ -254,7 +254,7 @@ MPI_Comm comm_getMpiComm() { return global_mpiComm; } -void comm_setMpiComm(MPI_Comm newComm) { +bool comm_setMpiComm(MPI_Comm newComm) { // this is called prior to QuEST initialisation, // and merely seeks to overwrite global_mpiComm @@ -264,12 +264,8 @@ void comm_setMpiComm(MPI_Comm newComm) { if (newComm == MPI_COMM_NULL) error_commMpiCommIsNull(); - int mpi_err = MPI_Comm_dup(newComm, &global_mpiComm); - - if (mpi_err != MPI_SUCCESS) - error_commInvalidMpiComm(); - - return; + auto status = MPI_Comm_dup(newComm, &global_mpiComm); + return status == MPI_SUCCESS; } #endif // QUEST_COMPILE_MPI diff --git a/quest/src/comm/comm_config.hpp b/quest/src/comm/comm_config.hpp index ffe49eaed..8441dbc23 100644 --- a/quest/src/comm/comm_config.hpp +++ b/quest/src/comm/comm_config.hpp @@ -31,6 +31,6 @@ bool comm_isMpiCommSet(); // Signatures containing MPI types which callers must extern: // extern MPI_Comm comm_getMpiComm() -// extern void comm_setMpiComm(MPI_Comm newComm) +// extern bool comm_setMpiComm(MPI_Comm newComm) #endif // COMM_CONFIG_HPP diff --git a/quest/src/core/errors.cpp b/quest/src/core/errors.cpp index 6e354d267..7e2c6ab73 100644 --- a/quest/src/core/errors.cpp +++ b/quest/src/core/errors.cpp @@ -156,11 +156,6 @@ void error_commAlreadyInit() { raiseInternalError("The MPI communication environment was attemptedly re-initialised despite the QuEST environment already existing."); } -void error_commInvalidMpiComm() { - - raiseInternalError("The supplied MPI communicator was MPI_COMM_NULL, or duplication failed."); -} - void error_commButEnvNotDistributed() { raiseInternalError("A function attempted to invoke communication despite QuEST being compiled in non-distributed mode."); diff --git a/quest/src/core/errors.hpp b/quest/src/core/errors.hpp index c41b69851..d557d39b2 100644 --- a/quest/src/core/errors.hpp +++ b/quest/src/core/errors.hpp @@ -81,8 +81,6 @@ void error_commNotInit(); void error_commAlreadyInit(); -void error_commInvalidMpiComm(); - void error_commButEnvNotDistributed(); void error_commOutOfBounds(); diff --git a/quest/src/core/validation.cpp b/quest/src/core/validation.cpp index 0d8a49423..e1df0af76 100644 --- a/quest/src/core/validation.cpp +++ b/quest/src/core/validation.cpp @@ -113,6 +113,9 @@ namespace report { string USER_GIVEN_MPI_COMMUNICATOR_IS_NULL = "The provided MPI communicator was null (MPI_COMM_NULL)."; + string USER_GIVEN_MPI_COMMUNICATOR_FAILED_TO_SET = + "The provided MPI communicator could not be used; MPI_Comm_dup() was not successful."; + string QUEST_OWNED_MPI_WAS_PRE_INIT = "MPI was already initialised prior to QuESTEnv initialisation, but the user did not declare MPI ownership."; @@ -1540,6 +1543,14 @@ void validate_mpiSubCommIsNonNull(bool isNonNull, const char* caller) { assertThat(isNonNull, report::USER_GIVEN_MPI_COMMUNICATOR_IS_NULL, caller); } +void validate_mpiSubCommSetSucceeded(bool success, const char* caller) { + + if (!global_isValidationEnabled) + return; + + assertThat(success, report::USER_GIVEN_MPI_COMMUNICATOR_FAILED_TO_SET, caller); +} + /* diff --git a/quest/src/core/validation.hpp b/quest/src/core/validation.hpp index 109728643..787316326 100644 --- a/quest/src/core/validation.hpp +++ b/quest/src/core/validation.hpp @@ -81,6 +81,8 @@ void validate_mpiInitStatus(bool useDistrib, bool userOwnsMpi, const char* calle void validate_mpiSubCommIsNonNull(bool isNonNull, const char* caller); +void validate_mpiSubCommSetSucceeded(bool success, const char* caller); + /* From 7c75e72f42f3950c173c41d12ca057be74e1ebbc Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 03:47:44 -0400 Subject: [PATCH 20/24] Remove redundant env.bool tests --- tests/unit/environment.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/unit/environment.cpp b/tests/unit/environment.cpp index 85d96cf8e..9ecf8e376 100644 --- a/tests/unit/environment.cpp +++ b/tests/unit/environment.cpp @@ -158,13 +158,6 @@ TEST_CASE( "getQuESTEnv", TEST_CATEGORY ) { QuESTEnv env = getQuESTEnv(); - REQUIRE( (env.isMultithreaded == 0 || env.isMultithreaded == 1) ); - REQUIRE( (env.isGpuAccelerated == 0 || env.isGpuAccelerated == 1) ); - REQUIRE( (env.isDistributed == 0 || env.isDistributed == 1) ); - REQUIRE( (env.isMpiUserOwned == 0 || env.isMpiUserOwned == 1) ); // <- pointless since bool - REQUIRE( (env.isCuQuantumEnabled == 0 || env.isCuQuantumEnabled == 1) ); // but you can't be too - REQUIRE( (env.isGpuSharingEnabled == 0 || env.isGpuSharingEnabled == 1) ); // careful ;^) - REQUIRE( env.rank >= 0 ); REQUIRE( env.numNodes >= 0 ); From 93f30f28d51f39ce4edc35a7a9c87457f1f9b949 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 03:48:55 -0400 Subject: [PATCH 21/24] Rename error_commDoubleSetMpiComm --- quest/src/comm/comm_config.cpp | 2 +- quest/src/core/errors.cpp | 2 +- quest/src/core/errors.hpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index 20f8ac026..27926f5d8 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -260,7 +260,7 @@ bool comm_setMpiComm(MPI_Comm newComm) { // and merely seeks to overwrite global_mpiComm if (global_mpiComm != MPI_COMM_NULL) - error_commDoubleSetMpiComm(); + error_commAlreadyHasSetMpiComm(); if (newComm == MPI_COMM_NULL) error_commMpiCommIsNull(); diff --git a/quest/src/core/errors.cpp b/quest/src/core/errors.cpp index 7e2c6ab73..862136a9c 100644 --- a/quest/src/core/errors.cpp +++ b/quest/src/core/errors.cpp @@ -181,7 +181,7 @@ void error_commNumMessagesExceedTagMax() { raiseInternalError("A function attempted to communicate via more messages than permitted (since there would be more uniquely-tagged messages than the tag upperbound)."); } -void error_commDoubleSetMpiComm() { +void error_commAlreadyHasSetMpiComm() { raiseInternalError("An attempt was made to set the QuEST MPI communicator after it had already been set (and changed from MPI_COMM_NULL)."); } diff --git a/quest/src/core/errors.hpp b/quest/src/core/errors.hpp index d557d39b2..33cc0661d 100644 --- a/quest/src/core/errors.hpp +++ b/quest/src/core/errors.hpp @@ -91,7 +91,7 @@ void error_commGivenInconsistentNumSubArraysANodes(); void error_commNumMessagesExceedTagMax(); -void error_commDoubleSetMpiComm(); +void error_commAlreadyHasSetMpiComm(); void error_commMpiCommIsNull(); From 790d11c4041f2c6a94d810f5388bdaac93251c41 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 04:03:49 -0400 Subject: [PATCH 22/24] Skip custom MPI examples when no MPI --- examples/extended/user_owned_mpi.c | 20 ++++++++++++++----- examples/extended/user_owned_submpi.cpp | 26 +++++++++++++++++++++---- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/examples/extended/user_owned_mpi.c b/examples/extended/user_owned_mpi.c index 55967a4ef..5142b9c3c 100644 --- a/examples/extended/user_owned_mpi.c +++ b/examples/extended/user_owned_mpi.c @@ -5,16 +5,24 @@ * @author Oliver Brown */ -#include #include "quest.h" +// This example requires linking with MPI, which the CMake +// build only enables when QUEST_ENABLE_SUBCOMM is ON, which +// results in quest.h defining QUEST_COMPILE_SUBCOMM +#if ! QUEST_COMPILE_SUBCOMM + +int main(void) +{ + std::printf("Example skipped since MPI is not linked.\n"); + return 0; +} - // TODO: - // this file will only receive mpi.h from CMakeLists.txt if - // we are also compiling with QUEST_ENABLE_SUBCOMM. Fix this! +#else +#include -int main (void) +int main(void) { const int USE_DISTRIB = 1; const bool USER_MPI = 1; @@ -29,3 +37,5 @@ int main (void) return 0; } + +#endif // QUEST_COMPILE_SUBCOMM diff --git a/examples/extended/user_owned_submpi.cpp b/examples/extended/user_owned_submpi.cpp index d1d336637..164ecd419 100644 --- a/examples/extended/user_owned_submpi.cpp +++ b/examples/extended/user_owned_submpi.cpp @@ -5,15 +5,31 @@ * @author Oliver Brown */ +#include "quest.h" #include -#include -#include // TODO: - // this file will only receive mpi.h from CMakeLists.txt if - // we are also compiling with QUEST_ENABLE_SUBCOMM. Fix this! + // this example sees some processes print to std-out while + // QuEST is reporting, colliding with output. May be worth + // introducing a sync to force non-QuEST-processes to wait + // during QUEST reporting + + +// This example requires linking with MPI, which the CMake +// build only enables when QUEST_ENABLE_SUBCOMM is ON, which +// results in quest.h defining QUEST_COMPILE_SUBCOMM +#if ! QUEST_COMPILE_SUBCOMM +int main() +{ + std::printf("Example skipped since MPI is not linked.\n"); + return 0; +} + +#else + +#include int main (void) { @@ -64,3 +80,5 @@ int main (void) return 0; } + +#endif // QUEST_COMPILE_SUBCOMM From ac86d12370832ab34af6229dde340f4d6231f18f Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 29 May 2026 04:07:49 -0400 Subject: [PATCH 23/24] Patches --- examples/extended/user_owned_mpi.c | 3 ++- quest/src/comm/comm_config.cpp | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/extended/user_owned_mpi.c b/examples/extended/user_owned_mpi.c index 5142b9c3c..f5117f48f 100644 --- a/examples/extended/user_owned_mpi.c +++ b/examples/extended/user_owned_mpi.c @@ -6,6 +6,7 @@ */ #include "quest.h" +#include // This example requires linking with MPI, which the CMake // build only enables when QUEST_ENABLE_SUBCOMM is ON, which @@ -14,7 +15,7 @@ int main(void) { - std::printf("Example skipped since MPI is not linked.\n"); + printf("Example skipped since MPI is not linked.\n"); return 0; } diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index 27926f5d8..c69e72919 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -238,10 +238,14 @@ void comm_sync() { */ bool comm_isMpiCommSet() { +#if QUEST_COMPILE_MPI // once comm_init() or comm_setMpiComm() overwrite // the communicator, is can never return to NULL return (global_mpiComm != MPI_COMM_NULL); +# else + return false; +#endif } #if QUEST_COMPILE_MPI From 28ddedd47ca9ae1891b7b8284da1d1094c0761b1 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Sun, 31 May 2026 02:01:26 -0400 Subject: [PATCH 24/24] Polished examples (and extended to C and C++) --- examples/extended/user_owned_mpi.c | 15 +++-- examples/extended/user_owned_mpi.cpp | 49 +++++++++++++++ examples/extended/user_owned_submpi.c | 84 +++++++++++++++++++++++++ examples/extended/user_owned_submpi.cpp | 22 +++---- 4 files changed, 155 insertions(+), 15 deletions(-) create mode 100644 examples/extended/user_owned_mpi.cpp create mode 100644 examples/extended/user_owned_submpi.c diff --git a/examples/extended/user_owned_mpi.c b/examples/extended/user_owned_mpi.c index f5117f48f..4e3c766f4 100644 --- a/examples/extended/user_owned_mpi.c +++ b/examples/extended/user_owned_mpi.c @@ -1,26 +1,32 @@ /** @file * - * TODO + * An example of using QuEST's experimental + * initCustomMpiQuESTEnv() function, to + * initialise QuEST in an environment where + * MPI is owned and controlled by the user. * * @author Oliver Brown + * @author Tyson Jones (doc) */ #include "quest.h" #include + // This example requires linking with MPI, which the CMake // build only enables when QUEST_ENABLE_SUBCOMM is ON, which -// results in quest.h defining QUEST_COMPILE_SUBCOMM +// results in quest.h defining QUEST_COMPILE_SUBCOMM. To +// enable this example to always be compilable (like during +// our CI), we guard against when QUEST_ENABLE_SUBCOMM is OFF. #if ! QUEST_COMPILE_SUBCOMM - int main(void) { printf("Example skipped since MPI is not linked.\n"); return 0; } - #else + #include int main(void) @@ -39,4 +45,5 @@ int main(void) return 0; } + #endif // QUEST_COMPILE_SUBCOMM diff --git a/examples/extended/user_owned_mpi.cpp b/examples/extended/user_owned_mpi.cpp new file mode 100644 index 000000000..54345d576 --- /dev/null +++ b/examples/extended/user_owned_mpi.cpp @@ -0,0 +1,49 @@ +/** @file + * + * An example of using QuEST's experimental + * initCustomMpiQuESTEnv() function to + * initialise QuEST in an environment where + * MPI is owned and controlled by the user. + * + * @author Oliver Brown + * @author Tyson Jones (doc) + */ + +#include "quest.h" +#include + + +// This example requires linking with MPI, which the CMake +// build only enables when QUEST_ENABLE_SUBCOMM is ON, which +// results in quest.h defining QUEST_COMPILE_SUBCOMM. To +// enable this example to always be compilable (like during +// our CI), we guard against when QUEST_ENABLE_SUBCOMM is OFF. +#if ! QUEST_COMPILE_SUBCOMM +int main(void) +{ + std::printf("Example skipped since MPI is not linked.\n"); + return 0; +} +#else + + +#include + +int main(void) +{ + const int USE_DISTRIB = 1; + const bool USER_MPI = 1; + const int USE_OPENMP = 1; + const int USE_GPU = 0; + + MPI_Init(NULL, NULL); + initCustomMpiQuESTEnv(USE_DISTRIB, USER_MPI, USE_GPU, USE_OPENMP); + reportQuESTEnv(); + finalizeQuESTEnv(); + MPI_Finalize(); + + return 0; +} + + +#endif // QUEST_COMPILE_SUBCOMM diff --git a/examples/extended/user_owned_submpi.c b/examples/extended/user_owned_submpi.c new file mode 100644 index 000000000..6f2ea6290 --- /dev/null +++ b/examples/extended/user_owned_submpi.c @@ -0,0 +1,84 @@ +/** @file + * + * An example of using QuEST's experimental + * initCustomMpiCommQuESTEnv() function to + * dedicate only some user-owned MPI processes + * to QuEST, and dedicate the remainder to + * other tasks. + * + * @author Oliver Brown + * @author Tyson Jones (doc) + */ + +#include "quest.h" +#include + + +// This example requires linking with MPI, which the CMake +// build only enables when QUEST_ENABLE_SUBCOMM is ON, which +// results in quest.h defining QUEST_COMPILE_SUBCOMM. To +// enable this example to always be compilable (like during +// our CI), we guard against when QUEST_ENABLE_SUBCOMM is OFF. +#if ! QUEST_COMPILE_SUBCOMM +int main() +{ + printf("Example skipped since MPI is not linked.\n"); + return 0; +} +#else + + +#include + +int main (void) +{ + int nprocs, quest_nprocs, world_rank, quest_rank; + MPI_Comm comm_split, comm_quantum, comm_classical; + + MPI_Init(NULL, NULL); + + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); + + const int I_AM_QUANTUM = world_rank % 2; + + printf("[%d] Hello from rank %d of %d in MPI_COMM_WORLD.\n", world_rank, world_rank, nprocs); + + MPI_Comm_split(MPI_COMM_WORLD, I_AM_QUANTUM, world_rank, &comm_split); + + if (I_AM_QUANTUM) { + MPI_Comm_dup(comm_split, &comm_quantum); + MPI_Comm_size(comm_quantum, &quest_nprocs); + MPI_Comm_rank(comm_quantum, &quest_rank); + printf("[%d] Hello from rank %d of %d in comm_quantum.\n", world_rank, quest_rank, quest_nprocs); + } else { + MPI_Comm_dup(comm_split, &comm_classical); + quest_rank = -1; + quest_nprocs = -1; + } + + // only procs in quantum comm initialise QuEST + if (I_AM_QUANTUM) { + printf("[%d] Initialising QuEST.\n", world_rank); + initCustomMpiCommQuESTEnv(comm_quantum, -1, -1); // -1 = auto-deployments + + reportQuESTEnv(); + + printf("[%d] Finalising QuEST.\n", world_rank); + finalizeQuESTEnv(); + } + + MPI_Comm_free(&comm_split); + if (I_AM_QUANTUM) { + MPI_Comm_free(&comm_quantum); + } else { + MPI_Comm_free(&comm_classical); + } + + MPI_Finalize(); + + return 0; +} + + +#endif // QUEST_COMPILE_SUBCOMM diff --git a/examples/extended/user_owned_submpi.cpp b/examples/extended/user_owned_submpi.cpp index 164ecd419..ea82a4f9d 100644 --- a/examples/extended/user_owned_submpi.cpp +++ b/examples/extended/user_owned_submpi.cpp @@ -1,34 +1,33 @@ /** @file * - * TODO + * An example of using QuEST's experimental + * initCustomMpiCommQuESTEnv() function to + * dedicate only some user-owned MPI processes + * to QuEST, and dedicate the remainder to + * other tasks. * * @author Oliver Brown + * @author Tyson Jones (doc) */ #include "quest.h" #include - // TODO: - // this example sees some processes print to std-out while - // QuEST is reporting, colliding with output. May be worth - // introducing a sync to force non-QuEST-processes to wait - // during QUEST reporting - - // This example requires linking with MPI, which the CMake // build only enables when QUEST_ENABLE_SUBCOMM is ON, which -// results in quest.h defining QUEST_COMPILE_SUBCOMM +// results in quest.h defining QUEST_COMPILE_SUBCOMM. To +// enable this example to always be compilable (like during +// our CI), we guard against when QUEST_ENABLE_SUBCOMM is OFF. #if ! QUEST_COMPILE_SUBCOMM - int main() { std::printf("Example skipped since MPI is not linked.\n"); return 0; } - #else + #include int main (void) @@ -81,4 +80,5 @@ int main (void) return 0; } + #endif // QUEST_COMPILE_SUBCOMM