diff --git a/.gitignore b/.gitignore index 111d35e..d23e90c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ build/ build-*/ build_tests/ cmake-build-*/ -cmake/ CMakeFiles/ CMakeCache.txt wget-log* diff --git a/CMakeLists.txt b/CMakeLists.txt index fb0fc1c..cc8226b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,71 @@ option(BUILD_TESTING "Build the testing tree." ON) add_library(gpufl STATIC) add_library(gpufl::gpufl ALIAS gpufl) +# One counter registry per PROCESS, not per module. `gpufl` is static and is +# linked separately into gpufl_inject, the Python extension and any host +# application, so without this each would hold its own registry - a profiled +# target would tick one and the injected evaluator would read another. Only a C +# ABI crosses this boundary; see include/gpufl/abi/gpufl_counter_abi.h. +add_library(gpufl_counter_runtime SHARED + runtime/counter_runtime.cpp + include/gpufl/core/counter_registry.cpp +) +target_include_directories(gpufl_counter_runtime PUBLIC include) + +# Installed unconditionally, NOT only with the Python bindings. A launcher-only +# install that omits it leaves the injected evaluator and the target application +# each falling back to their own in-process registry - the exact split this +# library exists to prevent, and one that shows up as a counter reading Missing +# forever rather than as a missing file. +install(TARGETS gpufl_counter_runtime + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} +) +set_target_properties(gpufl_counter_runtime PROPERTIES + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON +) + +# Counters WITHOUT the profiler. What an application links to tick counters a +# rule can watch (`gpufl::counter("tokens").add(n)`), while the profiling +# itself arrives from outside via `gpufl trace`. Linking gpufl::gpufl for two +# lines of instrumentation reads as "I am embedding the SDK" - and the target +# name is most of the answer to that confusion; nothing here starts a session, +# and none of the SDK's dependencies (CUPTI, NVML, zlib, OpenSSL, the +# uploader) are in it. +# +# `gpufl` layers ON TOP of this rather than compiling the same sources again: +# two archives both defining the registry is a duplicate-symbol error for +# anyone who links both. +add_library(gpufl_counters STATIC + include/gpufl/core/counter_registry.cpp + include/gpufl/core/counter_api.cpp + include/gpufl/core/counter_provider.cpp + include/gpufl/core/debug_logger.cpp +) +add_library(gpufl::counters ALIAS gpufl_counters) +# The ALIAS above only exists in this build tree. The INSTALLED name comes +# from EXPORT_NAME plus the export namespace, and without this the package +# would ship the target as gpufl::gpufl_counters while every document says +# gpufl::counters. +set_target_properties(gpufl_counters PROPERTIES EXPORT_NAME counters) +target_include_directories(gpufl_counters + PUBLIC + $ + $ +) +target_compile_features(gpufl_counters INTERFACE cxx_std_17) +# The provider dlopens the shared runtime and the registry uses std::mutex; +# consumers must not have to know either. Threads::Threads rather than a raw +# `pthread`: the raw name would be exported verbatim into the targets file and +# bypass whatever the consumer's toolchain says threading means there. +find_package(Threads REQUIRED) +target_link_libraries(gpufl_counters PUBLIC Threads::Threads) +if(UNIX) + target_link_libraries(gpufl_counters PUBLIC ${CMAKE_DL_LIBS}) +endif() +set_target_properties(gpufl_counters PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_include_directories(gpufl PUBLIC $ @@ -79,6 +144,11 @@ target_compile_definitions(gpufl PUBLIC # Enable PIC for static library (required when linking into shared libraries like Python modules) set_target_properties(gpufl PROPERTIES POSITION_INDEPENDENT_CODE ON) +# PUBLIC: consumers of the full SDK get the counter layer through this link, +# and the counter sources live in gpufl_counters ONLY - see that target for +# why they must not be compiled into both archives. +target_link_libraries(gpufl PUBLIC gpufl_counters) + target_sources(gpufl PRIVATE include/gpufl/core/dictionary_manager.cpp include/gpufl/core/sass_compressor.cpp @@ -107,12 +177,16 @@ target_sources(gpufl PRIVATE include/gpufl/core/runtime.cpp include/gpufl/core/backend_factory.cpp include/gpufl/core/monitor_adapter.cpp + include/gpufl/core/nvtx_counters.cpp + include/gpufl/core/metric_id.cpp + include/gpufl/core/deep_window_rule.cpp + include/gpufl/core/deep_window_rules.cpp + include/gpufl/core/metric_registry.cpp include/gpufl/core/monitor_batch_manager.cpp include/gpufl/core/monitor_record_builders.cpp include/gpufl/core/monitor.cpp include/gpufl/core/gpufl.cpp include/gpufl/core/common.cpp - include/gpufl/core/debug_logger.cpp include/gpufl/core/stack_trace.cpp include/gpufl/core/itanium_demangle.cpp include/gpufl/core/scope_registry.cpp @@ -147,7 +221,15 @@ else() GIT_REPOSITORY https://github.com/madler/zlib.git GIT_TAG v1.3.1 ) + # zlib's own install rules cache ABSOLUTE destinations at configure time + # (INSTALL_LIB_DIR = ${CMAKE_INSTALL_PREFIX}/lib), so they ignore + # `cmake --install --prefix` and try to write into Program Files - which is + # how the install-tree consumer test failed before this. Skip them all; the + # archive is installed below with a relative destination that the prefix + # override can move. + set(SKIP_INSTALL_ALL ON) FetchContent_MakeAvailable(zlib) + unset(SKIP_INSTALL_ALL) # zlib.h lives in the source dir; zconf.h is generated in the binary dir. # Add both privately to gpufl - consumers never include zlib headers directly. target_link_libraries(gpufl PRIVATE zlibstatic) @@ -188,6 +270,12 @@ FetchContent_Declare( # cpp-httplib's CMakeLists defines build flags for its own tests / examples. # Turn them off so we only build the header-only interface target. set(HTTPLIB_COMPILE OFF CACHE BOOL "" FORCE) +# HTTPLIB_INSTALL stays ON, deliberately: gpufl's link interface records +# httplib::httplib, and install(EXPORT gpufl_clientTargets) refuses to +# generate unless that target is in SOME export set - httplib's own is what +# satisfies the check. Turning it off fails the whole configure. The price is +# cpp-httplib's headers and package files landing in the install prefix; the +# fix that removes them is reworking the SDK export, not this switch. FetchContent_MakeAvailable(httplib) find_package(OpenSSL QUIET) @@ -635,7 +723,28 @@ if(BUILD_GPUFL_INJECT AND ((UNIX AND NOT APPLE) OR WIN32)) add_library(gpufl_inject SHARED include/gpufl/inject/inject_entry.cpp ) + if(UNIX AND NOT APPLE AND GPUFL_HAS_CUDA) + # Compile CUDA boundary wrappers against the toolkit's official ABI. + # inject_entry.cpp keeps its header-free fallback for no-CUDA builds. + target_sources(gpufl_inject PRIVATE + include/gpufl/inject/cuda_interpose_linux.cpp + ) + target_compile_definitions(gpufl_inject PRIVATE + GPUFL_TYPED_CUDA_INTERPOSE=1 + ) + endif() target_link_libraries(gpufl_inject PRIVATE gpufl::gpufl) + + # The injected DLL is loaded by the CUDA driver into the profiled target, + # whose DLL search path has no reason to include our bin directory. The + # provider resolves the runtime from the injection DLL's own location, so + # it has to be there. + add_custom_command(TARGET gpufl_inject POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMENT "Colocating gpufl_counter_runtime with gpufl_inject" + ) if(CUDAToolkit_INCLUDE_DIRS) target_include_directories(gpufl_inject PRIVATE ${CUDAToolkit_INCLUDE_DIRS}) endif() @@ -731,6 +840,10 @@ endfunction() if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR AND BUILD_TESTING) enable_testing() + # Keep googletest's install rules out of the package: the install-tree + # consumer test installs this build into a scratch prefix, and gtest + # headers and archives are not something gpufl_client ships. + set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) add_subdirectory(tests) endif() @@ -758,6 +871,15 @@ if(BUILD_PYTHON) target_link_libraries(_gpufl_client PRIVATE gpufl::gpufl) + # Same for the Python extension: it may be the first module to bind, in + # which case it is the one that loads the runtime for the whole process. + add_custom_command(TARGET _gpufl_client POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMENT "Colocating gpufl_counter_runtime with _gpufl_client" + ) + # If CUDA is available, link it to the Python module if(GPUFL_HAS_CUDA) target_link_libraries(_gpufl_client PRIVATE CUDA::cudart) @@ -776,6 +898,28 @@ include(GNUInstallDirs) # Install header files install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT counters +) + +# Two export sets, deliberately. The generated targets file REFUSES to load +# when any imported target it references is absent, and the SDK records its +# private dependencies (httplib, zlib, OpenSSL, CUDA) in its link interface - +# so shipping counters in the SDK's set made `find_package(gpufl_client)` fail +# for a counters-only consumer with "httplib::httplib is missing". Measured, +# not theoretical. The counters set references nothing but system dl/pthread +# and loads anywhere; the SDK set loads only for consumers that opt in AND +# have its dependencies. +install(TARGETS gpufl_counters + EXPORT gpufl_countersTargets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT counters + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) +install(EXPORT gpufl_countersTargets + FILE gpufl_countersTargets.cmake + NAMESPACE gpufl:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/gpufl_client + COMPONENT counters ) install(TARGETS gpufl @@ -791,3 +935,95 @@ install(EXPORT gpufl_clientTargets NAMESPACE gpufl:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/gpufl_client ) + +# What makes `find_package(gpufl_client)` actually resolve: CMake looks for +# gpufl_clientConfig.cmake, and a bare Targets file is not one. Version file +# alongside so `find_package(gpufl_client 1.2)` can hold a floor. +include(CMakePackageConfigHelpers) +configure_package_config_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/gpufl_clientConfig.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/gpufl_clientConfig.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/gpufl_client +) +write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/gpufl_clientConfigVersion.cmake + VERSION ${PROJECT_VERSION} + COMPATIBILITY SameMajorVersion +) +install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/gpufl_clientConfig.cmake + ${CMAKE_CURRENT_BINARY_DIR}/gpufl_clientConfigVersion.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/gpufl_client + COMPONENT counters +) + +# Cross-module counter sharing. Cannot be proven from inside one executable - it +# needs the Python extension AND the shared runtime as separate modules - so it +# runs as its own process rather than as a gtest case. +# +# Lives here, not in tests/, because tests/ is processed before _gpufl_client is +# defined; gating on that target there is always false and the check silently +# never registers - which is how it went unrun in the first place. +# +# Staged into the build tree rather than run against the source tree: the +# extension has to sit INSIDE the package to be importable, and copying a build +# artifact into source would leave it there for every later run. +if(BUILD_TESTING AND BUILD_PYTHON AND TARGET _gpufl_client + AND TARGET gpufl_counter_runtime + AND CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) + # The interpreter the EXTENSION was built for, not whichever one a fresh + # find_package picks. A different minor version cannot import a cp3XX + # module at all, so the check would silently run against the stub and + # report that the registries are split when nothing of the sort happened. + if(DEFINED Python_EXECUTABLE) + set(GPUFL_XMOD_PYTHON "${Python_EXECUTABLE}") + elseif(DEFINED PYTHON_EXECUTABLE) + set(GPUFL_XMOD_PYTHON "${PYTHON_EXECUTABLE}") + endif() + if(GPUFL_XMOD_PYTHON) + set(GPUFL_XMOD_STAGE ${CMAKE_BINARY_DIR}/xmod_stage) + add_custom_target(counter_xmod_stage ALL + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${CMAKE_SOURCE_DIR}/python/gpufl + ${GPUFL_XMOD_STAGE}/python/gpufl + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + ${GPUFL_XMOD_STAGE}/python/gpufl/ + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + ${GPUFL_XMOD_STAGE}/python/gpufl/ + COMMENT "Staging the cross-module counter check") + add_dependencies(counter_xmod_stage _gpufl_client gpufl_counter_runtime) + + add_test(NAME counter_cross_module + COMMAND ${GPUFL_XMOD_PYTHON} + ${CMAKE_SOURCE_DIR}/scripts/counter_cross_module_check.py) + set_tests_properties(counter_cross_module PROPERTIES + ENVIRONMENT "GPUFL_REPO=${GPUFL_XMOD_STAGE}") + endif() +endif() + +# The install-tree consumer, as CTest rather than a manual run: install into a +# scratch prefix, configure the fixture consumer against it with find_package, +# build it, run it. This is the only test that can see EXPORT_NAME, the Config +# file install, and the counters export set's isolation from the SDK's +# dependencies - the unit suite links the build tree and passes no matter what +# the package ships. +if(BUILD_TESTING) + enable_testing() + add_test(NAME counters_package_consumer + COMMAND ${CMAKE_COMMAND} + -DGPUFL_BINARY_DIR=${CMAKE_BINARY_DIR} + -DGPUFL_SOURCE_DIR=${CMAKE_SOURCE_DIR} + -DGPUFL_CONFIG=$ + -DGPUFL_GENERATOR=${CMAKE_GENERATOR} + -DGPUFL_PLATFORM=${CMAKE_GENERATOR_PLATFORM} + -DGPUFL_TOOLSET=${CMAKE_GENERATOR_TOOLSET} + -P ${CMAKE_SOURCE_DIR}/tests/package/counters_consumer_check.cmake) + # Self-contained: the script builds gpufl_counters itself and installs + # ONLY the counters component, so this test does not depend on - and + # cannot be broken by - the rest of the project's install rules. A Debug + # run without gpufl_counter_runtime built proved why that isolation + # matters. + set_tests_properties(counters_package_consumer PROPERTIES TIMEOUT 300) +endif() diff --git a/cmake/gpufl_clientConfig.cmake.in b/cmake/gpufl_clientConfig.cmake.in new file mode 100644 index 0000000..7e57836 --- /dev/null +++ b/cmake/gpufl_clientConfig.cmake.in @@ -0,0 +1,24 @@ +@PACKAGE_INIT@ + +# find_package(gpufl_client) entry point. What it guarantees is the counters +# layer: +# +# find_package(gpufl_client REQUIRED) +# target_link_libraries(app PRIVATE gpufl::counters) +# +# Threads is the one real dependency in its link interface, resolved here so +# the consumer does not have to know. +# +# The full SDK target is NOT loaded by this file. Its targets file records +# private dependencies (httplib, zlib, OpenSSL, CUDA) and refuses to load +# unless the consumer already provides those imported targets - publishing it +# as a component would be documenting a flow that only works by accident. +# Consuming the SDK from an install tree needs find_dependency() wiring that +# does not exist yet. + +include(CMakeFindDependencyMacro) +find_dependency(Threads) + +include("${CMAKE_CURRENT_LIST_DIR}/gpufl_countersTargets.cmake") + +check_required_components(gpufl_client) diff --git a/daemon/launcher/CMakeLists.txt b/daemon/launcher/CMakeLists.txt index e95b5cd..ace2f50 100644 --- a/daemon/launcher/CMakeLists.txt +++ b/daemon/launcher/CMakeLists.txt @@ -24,6 +24,7 @@ add_executable(gpufl_launcher cli_parse.cpp info_command.cpp trace_command_common.cpp + deep_window_env.cpp ${GPUFL_LAUNCHER_TRACE_IMPL} monitor_command.cpp ../monitor/monitor_runner.cpp @@ -74,6 +75,18 @@ if(WIN32 AND TARGET gpufl_inject) COMMENT "Colocating gpufl_inject.dll next to gpufl.exe") endif() +# The counter runtime travels with the inject DLL. Once injected, gpufl_inject +# resolves it from its OWN directory - the target process's search path is not +# ours to rely on - so it has to be wherever the inject DLL ended up. +if(WIN32 AND TARGET gpufl_counter_runtime) + add_dependencies(gpufl_launcher gpufl_counter_runtime) + add_custom_command(TARGET gpufl_launcher POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMENT "Colocating gpufl_counter_runtime.dll next to gpufl.exe") +endif() + # On Windows, copy CUPTI / NVPERF DLLs next to gpufl.exe (= beside the # colocated gpufl_inject.dll) so the inject DLL's dependencies resolve when # the driver loads it into the target. trace_command_win.cpp also prepends diff --git a/daemon/launcher/agent_launcher.cpp b/daemon/launcher/agent_launcher.cpp index d9198c7..42cc67e 100644 --- a/daemon/launcher/agent_launcher.cpp +++ b/daemon/launcher/agent_launcher.cpp @@ -1,5 +1,6 @@ #include "agent_launcher.hpp" +#include #include #include #include @@ -170,36 +171,46 @@ bool AgentProcess::start(const std::vector& command, #endif } -bool AgentProcess::waitForExit(int timeoutMs) { - if (!running_) return true; +AgentWaitResult AgentProcess::waitForExit(int timeoutMs) { + if (!running_) return {true, 0}; #ifdef _WIN32 auto process = process_; const DWORD r = WaitForSingleObject( process, timeoutMs < 0 ? INFINITE : static_cast(timeoutMs)); if (r == WAIT_OBJECT_0) { + DWORD exit_code = 0; + const bool have_exit_code = GetExitCodeProcess(process, &exit_code) != 0; CloseHandle(thread_); CloseHandle(process); process_ = nullptr; thread_ = nullptr; running_ = false; - return true; + return {true, have_exit_code ? static_cast(exit_code) : -1}; } - return false; // timeout/failure - caller falls back to stop() + return {}; // timeout/failure - caller falls back to stop() #else constexpr int step_ms = 100; int waited = 0; int status = 0; while (timeoutMs < 0 || waited < timeoutMs) { const pid_t rc = ::waitpid(pid_, &status, WNOHANG); - if (rc == pid_ || rc < 0) { // exited, or already reaped / gone + if (rc == pid_) { + running_ = false; + pid_ = -1; + if (WIFEXITED(status)) return {true, WEXITSTATUS(status)}; + if (WIFSIGNALED(status)) return {true, 128 + WTERMSIG(status)}; + return {true, -1}; + } + if (rc < 0) { + if (errno == EINTR) continue; running_ = false; pid_ = -1; - return true; + return {true, -1}; } usleep(step_ms * 1000); waited += step_ms; } - return false; + return {}; #endif } diff --git a/daemon/launcher/agent_launcher.hpp b/daemon/launcher/agent_launcher.hpp index 5406444..cbdc968 100644 --- a/daemon/launcher/agent_launcher.hpp +++ b/daemon/launcher/agent_launcher.hpp @@ -26,6 +26,15 @@ struct AgentLaunchPlan { std::string description; }; +struct AgentWaitResult { + bool exited = false; + // Valid when exited=true. A negative value means the platform could not + // recover a trustworthy child status and must never be treated as success. + int exit_code = -1; + + bool succeeded() const { return exited && exit_code == 0; } +}; + class AgentProcess { public: AgentProcess() = default; @@ -35,7 +44,7 @@ class AgentProcess { bool start(const std::vector& command, std::string& error); void stop(); - bool waitForExit(int timeoutMs); + AgentWaitResult waitForExit(int timeoutMs); bool isRunning() const { return running_; } private: diff --git a/daemon/launcher/cli_parse.cpp b/daemon/launcher/cli_parse.cpp index e2d7537..4b353c3 100644 --- a/daemon/launcher/cli_parse.cpp +++ b/daemon/launcher/cli_parse.cpp @@ -168,22 +168,28 @@ const char* traceHelp() { " Hard cap on total target runtime (safety).\n" " --after-window=\n" " What to do at window end. Only 'stop' today.\n" - " --deep-after= Arm the DEEP engines this long into the run,\n" + " A deep window is ONE adaptive run: gpufl picks the deep engine, so\n" + " --deep-* cannot be combined with --passes. Which engine was selected\n" + " is printed at startup rather than promised here - it depends on the\n" + " GPU, and today only PM sampling is selected.\n" + "\n" + " --deep-after= Arm the deep engine this long into the run,\n" " then disarm. Unlike --window the target keeps\n" " running. Needs a bound below. Default: 0 (arm\n" " at the first kernel launch).\n" " --deep-for= How long the deep window stays armed. Note this\n" " bounds TIME, which does not bound how much the\n" " engines actually collect - see --deep-launches.\n" + " --deep-when= Open the window when a metric crosses a threshold,\n" + " e.g. \"custom.token_rate<1000 for 2s\". This and\n" + " --deep-after are two answers to the same\n" + " question, so pass one or the other.\n" " --deep-launches= Kernel-launch bound on the deep window; ends it\n" - " at whichever bound is hit first. PREFER THIS.\n" - " For SASS / Range replay re-runs every kernel, so a\n" - " second of wall time covers ~25x less work there\n" - " than under PM sampling. For PcSampling it is what\n" - " decides whether you get data at all: samples only\n" - " become readable after a few thousand launches, so\n" - " a short --deep-for window (or any window over\n" - " slow kernels) can collect nothing.\n" + " at whichever bound is hit first. PREFER THIS:\n" + " wall time does not bound how much an engine\n" + " collects, and how far one second of it goes\n" + " differs by more than an order of magnitude\n" + " between engines.\n" " --deep-cooldown=\n" " Quiet time before another window may open.\n" " --pc-sample-period=\n" @@ -417,10 +423,28 @@ TraceParseResult parseTraceArgs(const std::vector& argv) { "--deep-launches " + v + " to bound it by kernel " "launches instead"}; } - if (key == "--deep-after") out.deep_after_ms = ms; + if (key == "--deep-after") { + out.deep_after_ms = ms; + out.deep_after_set = true; + } else if (key == "--deep-for") out.deep_for_ms = ms; else out.deep_cooldown_ms = ms; out.deep_requested = true; + } else if (key == "--deep-when") { + std::string v; + auto err = take_value(v); + if (!err.empty()) return {std::nullopt, err}; + if (v.empty()) { + return {std::nullopt, + "--deep-when needs an expression, e.g. " + "--deep-when=\"custom.token_rate<1000 for 2s\""}; + } + // Parsed by the client, not here: telling a misspelled built-in + // from a custom counter that has simply not registered yet needs + // the metric registry, and duplicating that check would give two + // places to disagree about what a metric name means. + out.deep_when = v; + out.deep_requested = true; } else if (key == "--deep-launches") { std::string v; auto err = take_value(v); @@ -463,6 +487,30 @@ TraceParseResult parseTraceArgs(const std::vector& argv) { if (out.command.empty()) { return {std::nullopt, "no command specified after `--`"}; } + // Explicit passes and deep windows are different execution models, and + // mixing them cannot mean anything coherent: the deep engines have to be + // fixed before the first CUDA call, so a --passes list either already + // contains what the window would arm (making the flag redundant) or does + // not (making the window arm nothing - which is what + // `--passes=Trace --deep-after=30s` silently did). Rejected before the + // target is launched, whichever order the flags came in. + if (const std::string mode_error = validateTraceExecutionMode(out); + !mode_error.empty()) { + return {std::nullopt, mode_error}; + } + + // Two answers to "when does the window open" is one too many. Measured on + // the 3090: with both set, the scheduled window opens at t=0, the rule's + // request is refused as busy, and the rule then waits for a rearm that a + // still-busy workload never gives - it reported `never_true` for a + // condition that held the whole run. + if (!out.deep_when.empty() && out.deep_after_set) { + return {std::nullopt, + "--deep-when and --deep-after are two different triggers for " + "the same window. Pass --deep-when to open it on a metric, or " + "--deep-after to open it at a fixed time"}; + } + // A deep window with neither bound would arm and never disarm, which is // just "profile deeply for the whole run" with extra steps. if (out.deep_requested && out.deep_for_ms == 0 && out.deep_launches == 0) { @@ -766,9 +814,63 @@ InfoParseResult parseInfoArgs(const std::vector& argv) { return {out, ""}; } +std::string validateTraceExecutionMode(const TraceArgs& args) { + if (!args.deep_requested || args.passes.empty()) return {}; + return "--passes cannot be combined with --deep-* flags.\n" + "\n" + " --passes runs the engines you name, relaunching the target " + "once per pass.\n" + " --deep-* runs ONE adaptive pass: gpufl selects a compatible " + "deep engine\n" + " and arms it only inside the window.\n" + "\n" + "Drop --passes to use a deep window."; +} + +CaptureMode resolveCaptureMode(const TraceArgs& args) { + return args.deep_requested ? CaptureMode::AdaptiveDeepWindow + : CaptureMode::ExplicitPasses; +} + +AdaptiveCapturePlan resolveAdaptivePlan(const TraceArgs& args) { + AdaptiveCapturePlan plan; + if (!args.deep_requested) return plan; // selected_deep stays empty + + // Trace is pinned as the base rather than left to the deep engine's own + // policy: kernel_launch_rate and recent_kernel_ms are computed from its + // completed-kernel records, and a rule that silently loses its metric + // reads as "condition never held". + plan.base = "Trace"; + + // PM only, for now. It is the one deep engine that works on a 3090 at all: + // PC sampling fails configuration under injection there and SASS emits no + // records, so neither has a dormant cost anyone has measured. + // + // Earlier overhead measurements covered PM selected-but-not-initialized, + // because initialization still lived in the arm path at the time. They are + // intentionally not used as a policy budget here; the prepared-and-idle + // configuration must be measured again after the lifecycle split. + plan.selected_deep = {"PmSampling"}; + plan.arm_window_only = true; + return plan; +} + std::vector resolvePassPlan(const TraceArgs& args) { - if (args.passes.empty()) return {"Trace"}; - return args.passes; + if (!args.passes.empty()) return args.passes; + + // Exactly one pass for an adaptive run. Relaunching the target per pass is + // what --passes is for; a window that triggers on a live condition cannot + // be reproduced across relaunches, so splitting it would change what is + // being measured. + if (resolveCaptureMode(args) == CaptureMode::AdaptiveDeepWindow) { + const AdaptiveCapturePlan plan = resolveAdaptivePlan(args); + std::string composite = plan.base; + for (const std::string& engine : plan.selected_deep) { + composite += "+" + engine; + } + return {composite}; + } + return {"Trace"}; } } // namespace gpufl::launcher diff --git a/daemon/launcher/cli_parse.hpp b/daemon/launcher/cli_parse.hpp index f8a35d1..f13206b 100644 --- a/daemon/launcher/cli_parse.hpp +++ b/daemon/launcher/cli_parse.hpp @@ -44,8 +44,12 @@ struct TraceArgs { // gpufl::deepWindow(), so time is the trigger the launcher can offer. // Any of them turns on window-only arming (GPUFL_DEEP_ARM=window). int64_t deep_after_ms = 0; // --deep-after; 0 = arm at the first launch + bool deep_after_set = false; // distinguishes "--deep-after=0" from absent int64_t deep_for_ms = 0; // --deep-for; duration bound, 0 = none uint64_t deep_launches = 0; // --deep-launches; launch bound, 0 = none + // --deep-when: open the window when a metric crosses a threshold, e.g. + // "custom.token_rate<1000 for 2s". Empty = time/launch triggered only. + std::string deep_when; int64_t deep_cooldown_ms = 0; // --deep-cooldown; quiet time between windows bool deep_requested = false; // any --deep-* flag was given // PC sampling period as a log2 exponent (2^N GPU cycles/sample, valid 5..31; @@ -124,11 +128,65 @@ struct TraceParseResult { TraceParseResult parseTraceArgs(const std::vector& argv); +/** + * Validate execution-mode invariants independently of argument parsing. + * + * TraceArgs is intentionally a simple value type and is also constructed by + * tests and shared launcher code. Keep this check at both the parser boundary + * and the execution boundary so those callers cannot create a mixed mode. + * Returns an empty string when valid, otherwise a user-facing error. + */ +std::string validateTraceExecutionMode(const TraceArgs& args); + +/** + * How a run decides which engines to select - two modes, never mixed. + * + * ExplicitPasses is the caller saying "run exactly these engines, relaunching + * the target once per pass". AdaptiveDeepWindow is the caller saying "watch for + * a condition and profile deeply when it happens" and leaving the engines to + * gpufl. + * + * They cannot be combined because the engine set is fixed before the trigger: + * an adaptive window may arm only engines selected when the target starts. + * Accepting a --passes list alongside a deep flag would let a user ask for a + * base with nothing a window could arm - `--passes=Trace --deep-after=30s` + * silently produced a window that armed nothing. + */ +enum class CaptureMode { + ExplicitPasses, + AdaptiveDeepWindow, +}; + +CaptureMode resolveCaptureMode(const TraceArgs& args); + +/** + * The engines an adaptive run SELECTS, and when it arms them. + * + * Deliberately NOT `ProfilingEngine::Deep`. That enum means "the deepest + * analysis this GPU supports" and picks SASS-or-PC plus PM; it does not + * guarantee the base Trace activity that `recent_kernel_ms` and + * `kernel_launch_rate` are computed from, and its base policy varies with the + * path chosen. An adaptive run needs the base pinned. + */ +struct AdaptiveCapturePlan { + // Always on for the whole run. Kernel-timing conditions need it. + std::string base = "Trace"; + // Selected by the launcher. The engine prepares after the first valid CUDA + // context exists and remains idle until a window opens. + std::vector selected_deep; + bool arm_window_only = true; +}; + +/** The plan for an adaptive run. Empty selected_deep if mode is explicit. */ +AdaptiveCapturePlan resolveAdaptivePlan(const TraceArgs& args); + // Resolves the ordered capture plan (one isolated CUPTI engine per pass) from // parsed trace args. Precedence: // 1. explicit --passes -> the listed engines, one pass each (Deep is the // Deep engine, not an expansion); -// 2. otherwise -> a single Trace pass. +// 2. any --deep-* flag -> a single adaptive pass, engines chosen by +// resolveAdaptivePlan; +// 3. otherwise -> a single Trace pass. // A returned size() > 1 is a multi-pass run (the launcher assigns one // analysis_id and labels each pass), shared by the Linux and Windows launchers. std::vector resolvePassPlan(const TraceArgs& args); diff --git a/daemon/launcher/deep_window_env.cpp b/daemon/launcher/deep_window_env.cpp new file mode 100644 index 0000000..c5bdea2 --- /dev/null +++ b/daemon/launcher/deep_window_env.cpp @@ -0,0 +1,84 @@ +// Publishing the deep-window options into the target's environment. +// +// Its own translation unit so a test can link it without the rest of the trace +// command, which pulls in zlib, the agent launcher and the uploader. What is +// being tested here is a decision, not a launch. + +#include +#include + +#include "cli_parse.hpp" +#include "gpufl/core/env_vars.hpp" +#include "trace_command_common.hpp" + +namespace gpufl::launcher { + +bool setEnvOrPrint(const TracePlatform& platform, const char* key, + const std::string& value) { + std::string error; + if (platform.setEnv(key, value, error)) return true; + std::fprintf(stderr, "gpufl: %s\n", error.c_str()); + return false; +} + +bool unsetEnvOrPrint(const TracePlatform& platform, const char* key) { + std::string error; + if (platform.unsetEnv(key, error)) return true; + std::fprintf(stderr, "gpufl: %s\n", error.c_str()); + return false; +} + +// --deep-*: bound how long the DEEP engines stay armed inside a target that +// keeps running. Distinct from --window, which bounds the target's lifetime. +// Asking for a deep window implies window-only arming, or the engines would be +// armed from the first kernel and the window would bound nothing. +bool applyDeepWindowEnv(const TraceArgs& args, const TracePlatform& platform) { + if (!args.deep_requested) { + // Not this run's business. A `--passes` run sets neither trigger and + // scrubs neither: configuring a window purely through the environment + // is the supported way to reach an engine the adaptive plan does not + // select yet. + return true; + } + if (!setEnvOrPrint(platform, env::kDeepArm, "window")) return false; + + // Both trigger variables install by EXISTING, whatever their value, so the + // run has to say which one it owns and REMOVE the other. Merely not + // setting one leaves whatever the parent shell had: a --deep-when run + // under an exported GPUFL_DEEP_AFTER_MS opened a scheduled window at t=0 + // and the rule spent the run refused behind it, and a --deep-after run + // under an exported GPUFL_DEEP_WHEN installed a rule nobody asked for. The + // CLI rejects the two flags together; this is the same rule applied to the + // environment the target inherits. + if (!args.deep_when.empty()) { + if (!unsetEnvOrPrint(platform, env::kDeepAfterMs)) return false; + if (!setEnvOrPrint(platform, env::kDeepWhen, args.deep_when)) { + return false; + } + } else { + if (!unsetEnvOrPrint(platform, env::kDeepWhen)) return false; + if (!setEnvOrPrint(platform, env::kDeepAfterMs, + std::to_string(args.deep_after_ms))) { + return false; + } + } + + if (args.deep_for_ms > 0 && + !setEnvOrPrint(platform, env::kDeepWindowMs, + std::to_string(args.deep_for_ms))) { + return false; + } + if (args.deep_launches > 0 && + !setEnvOrPrint(platform, env::kDeepWindowMaxLaunches, + std::to_string(args.deep_launches))) { + return false; + } + if (args.deep_cooldown_ms > 0 && + !setEnvOrPrint(platform, env::kDeepWindowCooldownMs, + std::to_string(args.deep_cooldown_ms))) { + return false; + } + return true; +} + +} // namespace gpufl::launcher diff --git a/daemon/launcher/trace_command.cpp b/daemon/launcher/trace_command.cpp index fc9260f..11243f2 100644 --- a/daemon/launcher/trace_command.cpp +++ b/daemon/launcher/trace_command.cpp @@ -78,6 +78,15 @@ class PosixTracePlatform final : public TracePlatform { return false; } + bool unsetEnv(const char* key, std::string& error) const override { + // unsetenv succeeds when the name is not set, which is the outcome + // being asked for either way. + if (::unsetenv(key) == 0) return true; + error = "unsetenv " + std::string(key) + " failed: " + + std::strerror(errno); + return false; + } + bool prepareInjectionEnv(const fs::path& inject_lib, std::string& error) const override { std::string ld_preload = inject_lib.string(); diff --git a/daemon/launcher/trace_command_common.cpp b/daemon/launcher/trace_command_common.cpp index 9cb9b88..8f05dff 100644 --- a/daemon/launcher/trace_command_common.cpp +++ b/daemon/launcher/trace_command_common.cpp @@ -39,15 +39,6 @@ fs::path findInjectLib(const TracePlatform& platform, const fs::path& exe) { return {}; } -bool setEnvOrPrint(const TracePlatform& platform, - const char* key, - const std::string& value) { - std::string error; - if (platform.setEnv(key, value, error)) return true; - std::fprintf(stderr, "gpufl: %s\n", error.c_str()); - return false; -} - // A '+'-joined pass token ("Trace+PcSampling") runs those engines together in // one process via GPUFL_ENGINE_COMBO. Returns the comma-joined combo for a // composite token, or "" for a single-engine token. @@ -444,7 +435,10 @@ void signalSessionsComplete(const fs::path& output_dir, if (res.second) { GFL_LOG_DEBUG("signalled upload-complete for session ", res.first); } else if (!quiet) { - GFL_LOG_ERROR("failed to signal upload-complete for session ", res.first); + std::fprintf(stderr, + "gpufl trace --upload: failed to signal " + "upload-complete for session %s\n", + res.first.c_str()); } } } @@ -522,6 +516,19 @@ int repairUncompressedLogs(const fs::path& root) { } // namespace int runTraceCommon(const TraceArgs& args, const TracePlatform& platform) { + // Re-checked here, not only in the parser. The two modes are enforced by + // the CLI, but TraceArgs is a plain struct: anything constructing one + // directly - a test, a future caller, a refactor that reorders parsing - + // can set both fields and reach this function in a state the parser would + // have refused. resolvePassPlan() would then honour `passes` while the + // block below still exported the deep environment, producing a run that is + // neither mode. + if (const std::string mode_error = validateTraceExecutionMode(args); + !mode_error.empty()) { + std::fprintf(stderr, "gpufl: %s\n", mode_error.c_str()); + return 2; + } + const fs::path exe = platform.selfExe(); if (exe.empty()) { std::fprintf(stderr, "gpufl: cannot resolve launcher path (%s)\n", @@ -616,33 +623,7 @@ int runTraceCommon(const TraceArgs& args, const TracePlatform& platform) { return 2; } - // --deep-*: bound how long the DEEP engines stay armed inside a target - // that keeps running. Distinct from --window above, which bounds the - // target's lifetime. Asking for a deep window implies window-only - // arming, or the engines would be armed from the first kernel and the - // window would bound nothing. - if (args.deep_requested) { - if (!setEnvOrPrint(platform, env::kDeepArm, "window") || - !setEnvOrPrint(platform, env::kDeepAfterMs, - std::to_string(args.deep_after_ms))) { - return 2; - } - if (args.deep_for_ms > 0 && - !setEnvOrPrint(platform, env::kDeepWindowMs, - std::to_string(args.deep_for_ms))) { - return 2; - } - if (args.deep_launches > 0 && - !setEnvOrPrint(platform, env::kDeepWindowMaxLaunches, - std::to_string(args.deep_launches))) { - return 2; - } - if (args.deep_cooldown_ms > 0 && - !setEnvOrPrint(platform, env::kDeepWindowCooldownMs, - std::to_string(args.deep_cooldown_ms))) { - return 2; - } - } + if (!applyDeepWindowEnv(args, platform)) return 2; // A bounded window stops the target after warmup+window wall-clock; // run_ms == 0 keeps the historical "run until the target exits" behavior. @@ -701,6 +682,16 @@ int runTraceCommon(const TraceArgs& args, const TracePlatform& platform) { if (!args.quiet) { std::fprintf(stderr, "[gpufl] capturing -> %s\n", output_dir.string().c_str()); + if (args.deep_requested) { + const AdaptiveCapturePlan adaptive = resolveAdaptivePlan(args); + std::fprintf(stderr, "[gpufl] adaptive capture: base=%s; selected deep=", + adaptive.base.c_str()); + for (size_t i = 0; i < adaptive.selected_deep.size(); ++i) { + std::fprintf(stderr, "%s%s", i == 0 ? "" : ",", + adaptive.selected_deep[i].c_str()); + } + std::fprintf(stderr, "; arm=window-only\n"); + } if (multipass) { std::fprintf(stderr, "[gpufl] multi-pass analysis %s - %zu passes:", analysis_id.c_str(), plan.size()); @@ -816,7 +807,8 @@ int runTraceCommon(const TraceArgs& args, const TracePlatform& platform) { // hard-kill the agent mid-upload and drop its late windows. const int cap = args.agent_drain_ms; GFL_LOG_DEBUG("waiting up to ", cap / 1000.0, "s for agent to finish uploading"); - if (agent.waitForExit(cap)) { + const AgentWaitResult wait = agent.waitForExit(cap); + if (wait.succeeded()) { GFL_LOG_DEBUG("agent finished uploading"); // Clean drain: every window was 202-accepted, so no more chunks are // coming. Signal upload-complete from here (outside the agent's racing @@ -826,12 +818,23 @@ int runTraceCommon(const TraceArgs& args, const TracePlatform& platform) { config.api_key = resolveOption(args.api_key, env::kApiKey); config.api_path = args.api_version.empty() ? "" : "/api/" + args.api_version; signalSessionsComplete(output_dir, config, args.quiet); + } else if (wait.exited) { + std::fprintf( + stderr, + "gpufl trace --upload: agent exited before completing the upload " + "(exit code %d); session-complete was not sent\n", + wait.exit_code); + if (overall_rc == 0) overall_rc = 4; } else { if (!args.quiet) { - GFL_LOG_ERROR("agent still running after ", cap / 1000.0, - "s cap - stopping (late windows may need a post-hoc `gpufl upload`) "); + std::fprintf( + stderr, + "gpufl trace --upload: agent still running after %.1fs - " + "stopping; late windows may require a post-hoc `gpufl upload`\n", + cap / 1000.0); } agent.stop(); + if (overall_rc == 0) overall_rc = 4; } } diff --git a/daemon/launcher/trace_command_common.hpp b/daemon/launcher/trace_command_common.hpp index 51c1c5f..9654dcc 100644 --- a/daemon/launcher/trace_command_common.hpp +++ b/daemon/launcher/trace_command_common.hpp @@ -45,6 +45,17 @@ class TracePlatform { virtual bool setEnv(const char* key, const std::string& value, std::string& error) const = 0; + /** + * Remove a variable from the environment the target will inherit. + * + * Needed because some GPUFL variables are triggers that install by + * EXISTING, not by their value: not setting one is not the same as the + * target not seeing one. Not-present is the only way to say "off", so a + * mode that owns a trigger has to be able to take the other one away. + * + * Removing a variable that is not set is success, not an error. + */ + virtual bool unsetEnv(const char* key, std::string& error) const = 0; virtual bool prepareInjectionEnv(const fs::path& inject_lib, std::string& error) const = 0; virtual TraceProcessResult runProcess( @@ -52,6 +63,25 @@ class TracePlatform { const RunOptions& opts) const = 0; }; +/** @brief platform.setEnv, printing the platform's reason on failure. */ +bool setEnvOrPrint(const TracePlatform& platform, const char* key, + const std::string& value); +/** @brief platform.unsetEnv, printing the platform's reason on failure. */ +bool unsetEnvOrPrint(const TracePlatform& platform, const char* key); + +/** + * @brief Publish the deep-window options into the target's environment. + * + * Separate from runTraceCommon so the trigger-ownership rule can be tested + * without launching a process: which variable a mode sets is only half of it, + * and the half that bit - which one it REMOVES - is invisible in any test that + * only inspects what was set. + * + * @return false when the platform refused an environment change; the reason is + * already on stderr. + */ +bool applyDeepWindowEnv(const TraceArgs& args, const TracePlatform& platform); + int runTraceCommon(const TraceArgs& args, const TracePlatform& platform); } // namespace gpufl::launcher diff --git a/daemon/launcher/trace_command_win.cpp b/daemon/launcher/trace_command_win.cpp index 6514156..720f2c9 100644 --- a/daemon/launcher/trace_command_win.cpp +++ b/daemon/launcher/trace_command_win.cpp @@ -134,6 +134,18 @@ class WindowsTracePlatform final : public TracePlatform { return false; } + bool unsetEnv(const char* key, std::string& error) const override { + // A null value deletes the variable. Deleting one that was never set + // reports ERROR_ENVVAR_NOT_FOUND, which is the state being asked for. + if (SetEnvironmentVariableA(key, nullptr)) return true; + const DWORD err = GetLastError(); + if (err == ERROR_ENVVAR_NOT_FOUND) return true; + error = "SetEnvironmentVariable(" + std::string(key) + + ", null) failed (err=" + + std::to_string(static_cast(err)) + ")"; + return false; + } + bool prepareInjectionEnv(const fs::path& inject_lib, std::string& error) const override { std::string path = inject_lib.parent_path().string(); diff --git a/daemon/launcher/upload_command.cpp b/daemon/launcher/upload_command.cpp index 9efb75a..c507571 100644 --- a/daemon/launcher/upload_command.cpp +++ b/daemon/launcher/upload_command.cpp @@ -121,9 +121,17 @@ int runUpload(const UploadArgs& args) { return 3; } const int cap_ms = args.timeout_s > 0 ? args.timeout_s * 1000 : 300000; - if (agent.waitForExit(cap_ms)) { - if (!args.quiet) std::printf("Upload complete.\n"); - return 0; + const AgentWaitResult wait = agent.waitForExit(cap_ms); + if (wait.exited) { + if (wait.succeeded()) { + if (!args.quiet) std::printf("Upload complete.\n"); + return 0; + } + std::fprintf(stderr, + "gpufl upload: agent exited before completing the upload " + "(exit code %d)\n", + wait.exit_code); + return 4; } agent.stop(); std::fprintf(stderr, diff --git a/example/cuda/deep_window_demo.cu b/example/cuda/deep_window_demo.cu index 48db5a0..a311564 100644 --- a/example/cuda/deep_window_demo.cu +++ b/example/cuda/deep_window_demo.cu @@ -9,7 +9,7 @@ // whole run. // // ── WHAT TO CHECK ──────────────────────────────────────────────────────────── -// 1. Before the trigger: the engine is armed but IDLE, so the session +// 1. Before the trigger: the engine is prepared but IDLE, so the session // carries no PC / PM / SASS samples from those iterations. // 2. During the window: samples appear, bounded to the window. // 3. After the window: the process keeps running to completion. A window diff --git a/include/gpufl/abi/gpufl_counter_abi.h b/include/gpufl/abi/gpufl_counter_abi.h new file mode 100644 index 0000000..649cd67 --- /dev/null +++ b/include/gpufl/abi/gpufl_counter_abi.h @@ -0,0 +1,98 @@ +#ifndef GPUFL_COUNTER_ABI_H +#define GPUFL_COUNTER_ABI_H + +/* + * Counter registry ABI. + * + * Exists because `gpufl` is a STATIC library, linked separately into + * gpufl_inject.dll, the Python extension, and any host application. Each copy + * would hold its own registry, so a Python server under `gpufl trace` would + * tick one and the injected evaluator would read another - the counter would + * read as Missing forever, which is exactly the case counters were added for. + * + * A small shared runtime owns the registry and everyone binds to it. Only C + * types cross the boundary: no std:: types, no atomics passed by address, no + * layout that depends on the compiler that built either side. + * + * Versioning: `abi_version` gates compatibility and `struct_size` allows + * appending members. A consumer must check both before calling anything, and + * must not assume a member exists because its own header declares it. + */ + +#include +#include + +#if defined(_WIN32) +# define GPUFL_COUNTER_EXPORT __declspec(dllexport) +#else +# define GPUFL_COUNTER_EXPORT __attribute__((visibility("default"))) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define GPUFL_COUNTER_ABI_VERSION 1u + +/* Opaque. Stable for the life of the process once returned. */ +typedef void* gpufl_counter_handle; + +typedef struct gpufl_counter_provider_v1 { + uint32_t abi_version; + uint32_t struct_size; + + /* + * Find or create the counter named [name, name + name_length). + * Returns NULL when the name is invalid or the counter limit is reached. + * The same name always returns the same handle, including across modules + * and from several threads at once. + */ + gpufl_counter_handle (*register_counter)(const char* name, size_t name_length); + + /* + * Add to a counter. The caller validates its own input; this ignores a + * NULL handle. Values wrap, and readers take unsigned deltas, which stay + * correct across a wrap. + */ + void (*add)(gpufl_counter_handle handle, uint64_t value); + + /* Raw value, including anything added before the current session. */ + uint64_t (*load)(gpufl_counter_handle handle); + + /* Value accrued since the current session's baseline. */ + uint64_t (*load_since_baseline)(gpufl_counter_handle handle); + + /* + * Baseline every counter at its current value. Called when a runtime + * initialises, so ticks from a previous session - or from while no runtime + * was active - are not counted as this one's. + */ + void (*begin_session)(void); + void (*end_session)(void); + int (*session_active)(void); + + /* + * Find WITHOUT creating. Returns NULL when the application has not + * registered this counter. + * + * A rule naming a counter must not bring it into existence: doing so makes + * "the application never registered it" indistinguishable from "registered + * but never ticked", and those need different answers - the first is a + * config mistake worth reporting, the second is a workload that is simply + * idle. + */ + gpufl_counter_handle (*lookup)(const char* name, size_t name_length); +} gpufl_counter_provider_v1; + +/* + * Entry point the shared runtime exports. Never returns NULL from a runtime + * that loaded successfully. + */ +GPUFL_COUNTER_EXPORT const gpufl_counter_provider_v1* +gpufl_get_counter_provider_v1(void); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* GPUFL_COUNTER_ABI_H */ diff --git a/include/gpufl/backends/nvidia/capture_capability_resolver.cpp b/include/gpufl/backends/nvidia/capture_capability_resolver.cpp index 7aad0a8..1db8849 100644 --- a/include/gpufl/backends/nvidia/capture_capability_resolver.cpp +++ b/include/gpufl/backends/nvidia/capture_capability_resolver.cpp @@ -36,7 +36,7 @@ std::vector BuildCaptureCapabilityWarnings( // for a deep window those differ by a lot, and quoting the wrong one // contradicts the advice. warnings.push_back( - "[gpufl] PC sampling collected 0 stall samples - too few kernel " + "PC sampling collected 0 stall samples - too few kernel " "launches were sampled. PC sampling accumulates per kernel and " "needs a few thousand launches before any samples are readable. " "Sample more launches - note that a longer wall-clock window is " @@ -47,14 +47,14 @@ std::vector BuildCaptureCapabilityWarnings( if (input.requests.sass && input.engine_state.sass.active && !input.engine_state.sass.has_data) { warnings.push_back( - "[gpufl] SASS metrics collected 0 instruction samples - the " + "SASS metrics collected 0 instruction samples - the " "profiled kernels were too short. Run more iterations / a " "longer-running kernel to collect instruction-level data."); } if (input.requests.pm && input.engine_state.pm.active && !input.engine_state.pm.has_data) { warnings.push_back( - "[gpufl] PM sampling collected 0 hardware samples - the profiled " + "PM sampling collected 0 hardware samples - the profiled " "workload was too short for the sampling interval."); } return warnings; diff --git a/include/gpufl/backends/nvidia/cupti_backend.cpp b/include/gpufl/backends/nvidia/cupti_backend.cpp index 07a3212..86ca9a2 100644 --- a/include/gpufl/backends/nvidia/cupti_backend.cpp +++ b/include/gpufl/backends/nvidia/cupti_backend.cpp @@ -23,6 +23,7 @@ #include "gpufl/core/common.hpp" #include "gpufl/core/debug_logger.hpp" #include "gpufl/core/deep_window.hpp" +#include "gpufl/core/deep_window_rules.hpp" #include "gpufl/core/logger/logger.hpp" #include "gpufl/core/monitor.hpp" // Monitor::RequestSyntheticDrainAndWait #include "gpufl/core/model/perf_metric_model.hpp" @@ -220,14 +221,14 @@ CUptiResult (*CuptiBackend::get_value())(CUpti_ActivityKind) { void CuptiBackend::start() { if (!initialized_) return; - // SASS safe mode keeps kernel-activity OFF (it deadlocks), so orphan launch - // metas would become synthetic kernels with host-dispatch timing only. Keep - // suppressing that path for SASS. PC sampling intentionally leaves the - // synthetic drain enabled so callback-derived kernel rows can be emitted - // when kernel activity is unavailable; drainSyntheticKernels filters - // non-kernel memcpy/memset API metas before emitting rows. + // Establish the timing-provenance contract before any launch callback can + // arrive. Real-record modes never convert an unmatched launch into a kernel + // row: the launch-to-launch host gap is not GPU execution time. Synthetic + // rows remain available only in modes that explicitly use callback-derived + // kernels (for example PC sampling). SetSuppressOrphanSyntheticKernels( - opts_.profiling_engine == ProfilingEngine::SassMetrics); + ShouldSuppressOrphanKernelSynthesis(collectsKernelEvents(), + WillEmitSyntheticKernels())); // Windows-injection PC sampling AND Deep both lose their final flush to the // process-exit teardown race (gpufl::shutdown runs during DLL detach, after // the OS starts tearing the process down, so the shutdown drain can be cut @@ -746,6 +747,11 @@ void CuptiBackend::EngineLaunchTick() { // stop/collect. Closing before the engine tick also spares the engine a // beat it would only spend on a window that is already over. DeepWindow::OnLaunch(); + // The HOST launch rate a rule may watch. Gated on a rule wanting it, so a + // run without one pays a single relaxed atomic load per launch. + if (detail::DeepWindowRules::WantsLaunchFeed()) { + detail::DeepWindowRules::NoteKernelLaunch(detail::GetTimestampNs()); + } if (engine_) engine_->onLaunchTick(); } @@ -762,6 +768,10 @@ void CuptiBackend::DrainProfilingData() { if (rebound) cuCtxSetCurrent(prev); } +bool CuptiBackend::DeepEnginesPrepared() const { + return engine_ != nullptr && engine_->isPrepared(); +} + std::vector CuptiBackend::ArmedEngineWireNames_() const { // Reuses the inspector the capability rows are built from, so a combo, a // Deep run and a single engine all narrow the same way: each path reports diff --git a/include/gpufl/backends/nvidia/cupti_backend.hpp b/include/gpufl/backends/nvidia/cupti_backend.hpp index 8441451..adf5b17 100644 --- a/include/gpufl/backends/nvidia/cupti_backend.hpp +++ b/include/gpufl/backends/nvidia/cupti_backend.hpp @@ -27,6 +27,30 @@ namespace gpufl { class ICuptiHandler; +// Decision for Monitor::Shutdown's orphan-synthesis suppression, kept as a +// pure function so the truth table is unit-testable without a CUPTI session. +// True = this session expected real kernel activity records, launches +// happened, and none arrived - so orphan launch metas must NOT be resurrected +// as synthetic kernel rows (their "durations" would be host dispatch gaps). +// Modes that synthesize by design (PC sampling / SASS safe mode) pass +// will_emit_synthetic=true and are never suppressed here. +inline bool KernelActivityExpectedButMissing(bool collects_kernel_events, + bool will_emit_synthetic, + uint64_t launch_callbacks, + uint64_t valid_activity_rows) { + return collects_kernel_events && !will_emit_synthetic && + launch_callbacks > 0 && valid_activity_rows == 0; +} + +// Host launch-to-launch gaps are not kernel durations. A mode that requested +// real CUPTI kernel activity must therefore drop every unmatched launch meta, +// even if some other launches produced valid activity rows. Only modes whose +// explicit product is callback-derived kernel rows may synthesize orphans. +inline bool ShouldSuppressOrphanKernelSynthesis(bool collects_kernel_events, + bool will_emit_synthetic) { + return collects_kernel_events || !will_emit_synthetic; +} + /** * @brief CUPTI-based monitoring backend for NVIDIA GPUs. * @@ -79,16 +103,24 @@ class CuptiBackend : public IMonitorBackend { } return opts_.profiling_engine == ProfilingEngine::RangeProfilerKernelReplay; } - // True when CUPTI kernel ACTIVITY records won't be collected, so every - // launch must be reported from its callback as a synthetic kernel (PC - // Sampling, or SASS profiler safe mode without GPUFL_SASS_ALLOW_KERNEL_ - // ACTIVITY). Mirrors KernelLaunchHandler::requiredActivityKinds() returning - // {}. In these modes the launch callback precomputes the simplified kernel - // occupancy (the activity record that would otherwise carry it never - // arrives); see KernelLaunchHandler::handle + drainSyntheticKernels. + // True only when callback-derived kernel rows are the selected mode's + // deliberate product. If real CUPTI kernel activity is enabled, unmatched + // launches are data loss and must never be turned into host-gap durations. + // SASSMetrics safe mode also suppresses synthetic rows because its + // execution signature is carried separately. bool WillEmitSyntheticKernels() const { - return opts_.profiling_engine == ProfilingEngine::PcSampling || - !AllowSassKernelActivity(); + if (collectsKernelEvents()) return false; + return opts_.profiling_engine != ProfilingEngine::SassMetrics; + } + // See IMonitorBackend. Observed live (Linux 3090, Trace+PM adaptive, + // NVTX-counter targets, ~1/4 runs): every cuptiActivityEnable succeeds, + // SYNCHRONIZATION records flow, but zero KERNEL/CONCURRENT_KERNEL + // records ever arrive. Valid only after stop()'s disable + flush. + bool kernelActivityExpectedButMissing() const override { + return KernelActivityExpectedButMissing( + collectsKernelEvents(), WillEmitSyntheticKernels(), + kernel_launch_callback_count_.load(std::memory_order_acquire), + kernel_activity_emitted_.load(std::memory_order_relaxed)); } bool AllowSassMarkerActivity() const { return resolved_plan_.allow_sass_marker_activity; @@ -247,6 +279,14 @@ class CuptiBackend : public IMonitorBackend { // or disarms it. std::vector ArmedEngineWireNames_() const; + // Capability gate for conditional windows: does this run have an engine a + // window could arm at all? A Trace-only run answers no, which is what stops + // a rule burning its budget on windows that collect nothing. + bool DeepEnginesPrepared() const override; + bool DeepEnginePreparationPending() const override { + return engine_start_pending_.load(std::memory_order_acquire); + } + bool ShouldEnableNvtxMarkerActivityBeforeEngine_() const; bool ShouldEnableNvtxMarkerActivityForSelectedEngine_() const; static void EnableNvtxMarkerActivity_(const char* phase); diff --git a/include/gpufl/backends/nvidia/cupti_capture_capabilities.cpp b/include/gpufl/backends/nvidia/cupti_capture_capabilities.cpp index 322bc02..a3361e8 100644 --- a/include/gpufl/backends/nvidia/cupti_capture_capabilities.cpp +++ b/include/gpufl/backends/nvidia/cupti_capture_capabilities.cpp @@ -1,7 +1,6 @@ #include "gpufl/backends/nvidia/cupti_backend.hpp" #include -#include #include #include "gpufl/backends/nvidia/capture_capability_resolver.hpp" @@ -83,10 +82,30 @@ void CuptiBackend::EmitCaptureCapabilities_() const { // run otherwise gives no local hint that a too-short workload starved the // sampler. Point at the remedies. for (const std::string& warning : BuildCaptureCapabilityWarnings(input)) { - std::fprintf(stderr, "%s\n", warning.c_str()); + GFL_LOG_WARN(warning); + } + + CaptureCapabilitiesEvent evt = BuildCaptureCapabilitiesEvent(input); + + // Scope attribution gave up on some samples. Reported through the + // capability matrix rather than the local log alone: a log line is invisible + // to whoever reads the session later, and the samples still upload - they + // just carry no scope. Without this the dashboard presents partial + // attribution as though it were complete. + const uint64_t truncated = Monitor::ScopeAttributionTruncated(); + if (truncated > 0 && Monitor::PmSampleRowsSeen() > 0) { + CaptureCapability cap; + cap.feature = "scope_attribution"; + cap.requested = true; + cap.status = "partial"; + cap.reason_code = "scope_attribution_truncated"; + cap.message = "Evicted " + std::to_string(truncated) + + " completed scope records after the retention cap was reached; " + "PM scope attribution may be incomplete. Raise pm_sampling_max_samples " + "or lengthen the sampling interval so decodes keep up."; + evt.capabilities.push_back(std::move(cap)); } - const CaptureCapabilitiesEvent evt = BuildCaptureCapabilitiesEvent(input); rt->logger->write(model::CaptureCapabilitiesModel(evt)); } diff --git a/include/gpufl/backends/nvidia/cupti_engine_selection.cpp b/include/gpufl/backends/nvidia/cupti_engine_selection.cpp index 1fc1a22..5acbc19 100644 --- a/include/gpufl/backends/nvidia/cupti_engine_selection.cpp +++ b/include/gpufl/backends/nvidia/cupti_engine_selection.cpp @@ -215,7 +215,7 @@ EngineRuntimeState InspectEngineRuntimeState(const IProfilingEngine* engine, if (comboActive) { // Keep the matrix log close to the single place that inspects // runtime engine state. - GFL_LOG_ERROR("[Composite][matrix] ", sub->name(), " armed=", + GFL_LOG_DEBUG("[Composite][matrix] ", sub->name(), " armed=", armed ? "yes" : "no", " produced=", produced ? "yes" : "no"); } diff --git a/include/gpufl/backends/nvidia/engine/composite_engine.hpp b/include/gpufl/backends/nvidia/engine/composite_engine.hpp index 97b468b..ed6ed92 100644 --- a/include/gpufl/backends/nvidia/engine/composite_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/composite_engine.hpp @@ -109,6 +109,13 @@ class CompositeEngine final : public IProfilingEngine { for (auto& e : engines_) if (e && e->stallReasonsUnavailable()) return true; return false; } + /// Any, not all: a combo whose PM prepared and whose SASS did not still + /// has something for a window to arm, and refusing there would give up a + /// window that would have collected data. + bool isPrepared() const override { + for (auto& e : engines_) if (e && e->isPrepared()) return true; + return false; + } bool isOperational() const override { for (auto& e : engines_) if (e && e->isOperational()) return true; return false; diff --git a/include/gpufl/backends/nvidia/engine/pc_sampling_engine.hpp b/include/gpufl/backends/nvidia/engine/pc_sampling_engine.hpp index c3eb214..e595354 100644 --- a/include/gpufl/backends/nvidia/engine/pc_sampling_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/pc_sampling_engine.hpp @@ -70,6 +70,21 @@ class PcSamplingEngine final : public IProfilingEngine { && !sampling_api_blocked_.load(std::memory_order_relaxed); } + /** + * Ready to arm inside a window. + * + * Stricter than isOperational() on the SamplingAPI path: picking that + * method only records which API is in play, while the enable + configure + * that a later cuptiPCSamplingStart needs can still fail - and on this + * 3090 it does, with INVALID_OPERATION. ActivityAPI has nothing deferred: + * cuptiActivityEnable already succeeded, so selecting it IS being ready. + */ + bool isPrepared() const override { + if (sampling_api_blocked_.load(std::memory_order_relaxed)) return false; + return pc_sampling_method_ == Method::ActivityAPI || + sampling_api_ready_.load(std::memory_order_acquire); + } + /** True once at least one PC sample was emitted this session. */ bool producedData() const override { return produced_data_.load(std::memory_order_relaxed); diff --git a/include/gpufl/backends/nvidia/engine/pc_sampling_with_sass_engine.hpp b/include/gpufl/backends/nvidia/engine/pc_sampling_with_sass_engine.hpp index df278e2..4f03ab1 100644 --- a/include/gpufl/backends/nvidia/engine/pc_sampling_with_sass_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/pc_sampling_with_sass_engine.hpp @@ -85,6 +85,18 @@ class PcSamplingWithSassEngine final : public IProfilingEngine { return (pc_ && pc_->isOperational()) || sass_ok_; } + /** + * Ready to arm inside a window if ANY sub-engine is. + * + * Spelled out rather than derived from isOperational(): this engine keeps + * whichever of PC / SASS / PM survived start(), and a window that arms one + * of the three is still a useful window. + */ + bool isPrepared() const override { + return (pc_ && pc_->isPrepared()) || (sass_ && sass_->isPrepared()) || + (pm_ && pm_->isPrepared()); + } + bool sassActive() const { return sass_ok_; } bool pcSamplingActive() const { return pc_ && pc_->isOperational(); } diff --git a/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp b/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp index e412da8..0737115 100644 --- a/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp +++ b/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp @@ -6,6 +6,7 @@ #include #endif +#include #include #include #include @@ -36,6 +37,11 @@ bool PmSamplingEngine::initialize(const MonitorOptions& opts, opts_ = opts; ctx_ = ctx; metrics_ = ResolveMetrics_(); + prepared_.store(false, std::memory_order_relaxed); + operational_.store(false, std::memory_order_relaxed); + attempted_.store(false, std::memory_order_relaxed); + produced_data_.store(false, std::memory_order_relaxed); + insufficient_privileges_.store(false, std::memory_order_relaxed); GFL_LOG_DEBUG("[PmSamplingEngine] initialized preset=", opts_.pm_sampling_preset, " metrics=", metrics_.size(), " interval_us=", opts_.pm_sampling_interval_us, @@ -49,12 +55,18 @@ void PmSamplingEngine::start() { if (!ScopeGated_()) { StartPmSampling_(); } else { - attempted_.store(true, std::memory_order_relaxed); - if (!config_emitted_) { - EmitConfig_(); - config_emitted_ = true; + // Prepare while the first valid CUDA context is current, but do not + // start hardware sampling. CuptiBackend invokes this either directly + // from start() or from its CONTEXT_CREATED deferred-start path, then + // restores activity subscriptions that Profiler initialization may + // reset. A deep-window open therefore performs only the bounded arm. + if (PreparePmSampling_()) { + if (opts_.deep_arm_mode == DeepArmMode::WindowOnly) { + GFL_LOG_INFO("deep engine prepared: nvidia.pm_sampling"); + } + GFL_LOG_DEBUG("[PmSamplingEngine] prepared; sampling remains " + "idle until a deep window opens"); } - GFL_LOG_DEBUG("[PmSamplingEngine] scope-only mode: PM sampling arms on scope start"); } #else GFL_LOG_ERROR("[PmSamplingEngine] Not built with GPUFL_HAS_PERFWORKS"); @@ -81,6 +93,7 @@ void PmSamplingEngine::shutdown() { } #endif operational_.store(false, std::memory_order_relaxed); + prepared_.store(false, std::memory_order_release); } void PmSamplingEngine::onScopeStart(const char*) { @@ -109,6 +122,34 @@ void PmSamplingEngine::EmitConfig_() const { } #if GPUFL_HAS_PERFWORKS +bool PmSamplingEngine::PreparePmSampling_() { + std::lock_guard lk(pm_mu_); + return PreparePmSamplingLocked_(); +} + +bool PmSamplingEngine::PreparePmSamplingLocked_() { + attempted_.store(true, std::memory_order_relaxed); + if (!config_emitted_) { + EmitConfig_(); + config_emitted_ = true; + } + if (prepared_.load(std::memory_order_relaxed)) return true; + if (!InitializePmSampling_()) { + prepared_.store(false, std::memory_order_release); + return false; + } + // Allocate the full destination image during preparation. Allocating it + // when the window opens would consume the front of the bounded interval + // and would make dormant-memory measurements exclude the largest PM-owned + // allocation. + if (!CreateCounterDataImage_()) { + prepared_.store(false, std::memory_order_release); + return false; + } + prepared_.store(true, std::memory_order_release); + return true; +} + bool PmSamplingEngine::InitializePmSampling_() { if (pm_initialized_) return true; if (!ctx_.cuda_ctx) { @@ -407,9 +448,17 @@ void PmSamplingEngine::DecodeAndEmit_() { LogCuptiErrorIfFailed(this->name(), "cuptiPmSamplingDecodeData", res); return; } - if (decode.overflow || res == CUPTI_ERROR_OUT_OF_MEMORY) { + const bool overflowed = decode.overflow || res == CUPTI_ERROR_OUT_OF_MEMORY; + if (overflowed) { GFL_LOG_ERROR("[PmSamplingEngine] PM sampling hardware buffer overflow; increase pm_sampling_max_samples or interval_us"); } + // CUPTI states outright whether the hardware buffer was exhausted. + // END_OF_RECORDS is the only reason that proves nothing older is left to + // come; COUNTER_DATA_FULL means records remain because the destination + // image filled first. + const bool buffer_exhausted = + res == CUPTI_SUCCESS && !overflowed && + decode.decodeStopReason == CUPTI_PM_SAMPLING_DECODE_STOP_REASON_END_OF_RECORDS; CUpti_PmSampling_GetCounterDataInfo_Params info = { CUpti_PmSampling_GetCounterDataInfo_Params_STRUCT_SIZE}; @@ -447,7 +496,7 @@ void PmSamplingEngine::DecodeAndEmit_() { if (LogCuptiErrorIfFailed(this->name(), "cuptiProfilerHostEvaluateToGpuValues", res)) continue; const uint64_t mid = sampleInfo.startTimestamp + - ((sampleInfo.endTimestamp - sampleInfo.startTimestamp) / 2ull); + (sampleInfo.endTimestamp - sampleInfo.startTimestamp) / 2ull; // CUPTI sample ts -> wall-clock via the kernel anchor, so PM lines up // with the kernel timeline. const int64_t mid_wall_ns = ctx_.base_cpu_ns + @@ -469,7 +518,26 @@ void PmSamplingEngine::DecodeAndEmit_() { if (!rows.empty()) { Monitor::PushPmSamples(rows); - produced_data_.store(true, std::memory_order_relaxed); + if (!produced_data_.exchange(true, std::memory_order_acq_rel) && + opts_.deep_arm_mode == DeepArmMode::WindowOnly) { + GFL_LOG_INFO("deep engine produced data: nvidia.pm_sampling"); + } + + // Release the scopes no later sample can reach - but only when CUPTI + // has told us the hardware buffer is actually empty. Sampling is time + // ordered, so once it is, every future record is newer than everything + // just decoded and the newest timestamp here is a real boundary. + // + // Deriving one from the buffer span instead would be an inference about + // capacity, not a guarantee, and it would be wrong precisely when + // records were left behind. On overflow or COUNTER_DATA_FULL the + // watermark simply stays where it is, so a truncated decode cannot + // strand the scopes its leftovers still need. + if (buffer_exhausted) { + int64_t newest_ns = std::numeric_limits::min(); + for (const auto& row : rows) newest_ns = std::max(newest_ns, row.ts_ns); + Monitor::PublishScopeRetentionWatermark(newest_ns); + } } // Hand the image back empty so the next decode starts from a clean slate @@ -509,25 +577,27 @@ void PmSamplingEngine::DisablePmSampling_() { } profiler_initialized_ = false; profiler_init_owned_ = false; + prepared_.store(false, std::memory_order_release); } void PmSamplingEngine::StartPmSampling_() { std::lock_guard lk(pm_mu_); if (running_) return; - attempted_.store(true, std::memory_order_relaxed); - if (!config_emitted_) { - EmitConfig_(); - config_emitted_ = true; - } - if (!InitializePmSampling_()) { + if (!PreparePmSamplingLocked_()) { operational_.store(false, std::memory_order_relaxed); return; } - if (!CreateCounterDataImage_()) { + // The image was allocated during prepare and is reset after every decode. + // Reset again at the arm boundary so a failed/partial prior window cannot + // leak samples into this one. + if (!ResetCounterDataImage_()) { operational_.store(false, std::memory_order_relaxed); return; } + // Capture before the CUPTI call so the boundary is conservative: even a + // sample produced during a slow successful start is covered. + const int64_t attribution_start_ns = detail::GetTimestampNs(); CUpti_PmSampling_Start_Params start = {CUpti_PmSampling_Start_Params_STRUCT_SIZE}; start.pPmSamplingObject = pm_object_; CUptiResult res = cuptiPmSamplingStart(&start); @@ -536,7 +606,11 @@ void PmSamplingEngine::StartPmSampling_() { return; } running_ = true; + Monitor::BeginPmScopeAttribution(attribution_start_ns); operational_.store(true, std::memory_order_relaxed); + if (opts_.deep_arm_mode == DeepArmMode::WindowOnly) { + GFL_LOG_INFO("deep window armed: nvidia.pm_sampling"); + } GFL_LOG_DEBUG("[PmSamplingEngine] >>> STARTED (Scope Begin) <<<"); } @@ -551,6 +625,7 @@ void PmSamplingEngine::StopPmSampling_() { GFL_LOG_DEBUG("[PmSamplingEngine] <<< COLLECTING (Scope End) >>>"); DecodeAndEmit_(); } + Monitor::EndPmScopeAttribution(); running_ = false; } #endif diff --git a/include/gpufl/backends/nvidia/engine/pm_sampling_engine.hpp b/include/gpufl/backends/nvidia/engine/pm_sampling_engine.hpp index 2409fec..fe658e2 100644 --- a/include/gpufl/backends/nvidia/engine/pm_sampling_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/pm_sampling_engine.hpp @@ -36,9 +36,11 @@ class PmSamplingEngine final : public IProfilingEngine { bool hasInsufficientPrivileges() const override { return insufficient_privileges_.load(std::memory_order_relaxed); } + bool isPrepared() const override { + return prepared_.load(std::memory_order_acquire); + } bool isOperational() const override { - return operational_.load(std::memory_order_relaxed) || - attempted_.load(std::memory_order_relaxed); + return operational_.load(std::memory_order_relaxed); } bool producedData() const override { return produced_data_.load(std::memory_order_relaxed); @@ -56,6 +58,8 @@ class PmSamplingEngine final : public IProfilingEngine { void EmitConfig_() const; #if GPUFL_HAS_PERFWORKS + bool PreparePmSampling_(); + bool PreparePmSamplingLocked_(); bool InitializePmSampling_(); bool BuildConfigImage_(); bool ResetCounterDataImage_(); @@ -90,6 +94,7 @@ class PmSamplingEngine final : public IProfilingEngine { std::vector metrics_; bool running_ = false; std::atomic operational_{false}; + std::atomic prepared_{false}; std::atomic attempted_{false}; std::atomic produced_data_{false}; std::atomic insufficient_privileges_{false}; diff --git a/include/gpufl/backends/nvidia/engine/profiling_engine.hpp b/include/gpufl/backends/nvidia/engine/profiling_engine.hpp index cd7ed6c..4ed8a84 100644 --- a/include/gpufl/backends/nvidia/engine/profiling_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/profiling_engine.hpp @@ -127,6 +127,23 @@ class IProfilingEngine { */ virtual bool stallReasonsUnavailable() const { return false; } + /** + * @brief True once all context-bound configuration and storage needed to + * arm this engine have been created successfully. + * + * Window-gated engines use this to distinguish "selected" from "ready to + * arm". A conditional rule asks this before it spends a window, so the + * default is `false` and every engine that can be armed says so itself. + * + * Deferring to isOperational() was the wrong default: that one answers + * "did this pass start", it defaults to true, and a new engine would + * silently inherit a yes it had never earned - the rule would spend its + * budget on windows that arm nothing and report `fired` for a window with + * no data in it. An engine that forgets to override this collects nothing + * from windows, which is visible; the other way round is not. + */ + virtual bool isPrepared() const { return false; } + /** * @brief True if this engine started successfully and is producing * data. False if start() was skipped, failed, or the engine is None. diff --git a/include/gpufl/backends/nvidia/engine/range_profiler_engine.cpp b/include/gpufl/backends/nvidia/engine/range_profiler_engine.cpp index 16eab19..a737b78 100644 --- a/include/gpufl/backends/nvidia/engine/range_profiler_engine.cpp +++ b/include/gpufl/backends/nvidia/engine/range_profiler_engine.cpp @@ -173,6 +173,10 @@ void RangeProfilerEngine::stop() { } void RangeProfilerEngine::shutdown() { + // Cleared first, not with operational_ at the end: the profiler object is + // disabled below, and a window opened in between would arm a session that + // is already being torn down. + session_ready_.store(false, std::memory_order_release); #if GPUFL_HAS_PERFWORKS if (mode_ == Mode::KernelReplay && kernel_replay_running_) { stop(); @@ -606,6 +610,7 @@ bool RangeProfilerEngine::InitPerfworksSession_(bool require_single_pass) { } perf_session_active_ = true; + session_ready_.store(true, std::memory_order_release); operational_.store(true, std::memory_order_relaxed); GFL_LOG_DEBUG("[RangeProfilerEngine] Session initialized for chip: ", ctx_.chip_name); diff --git a/include/gpufl/backends/nvidia/engine/range_profiler_engine.hpp b/include/gpufl/backends/nvidia/engine/range_profiler_engine.hpp index 03a0766..27b7ca9 100644 --- a/include/gpufl/backends/nvidia/engine/range_profiler_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/range_profiler_engine.hpp @@ -44,6 +44,23 @@ class RangeProfilerEngine final : public IProfilingEngine { return operational_.load(std::memory_order_relaxed) || attempted_.load(std::memory_order_relaxed); } + + /** + * Ready to arm inside a window: the Perfworks session exists AND + * cuptiRangeProfilerSetConfig succeeded. + * + * Cannot reuse isOperational(): that one counts `attempted_`, so a session + * whose creation failed still reports true - deliberately, so the pass + * reports itself rather than vanishing. A rule asking the same question + * would spend a window on it. + * + * Mirrored into an atomic rather than read from `perf_session_active_`, + * which is guarded by a mutex a decode can hold for a long time. The rule + * asks this on the collector beat and must not block behind one. + */ + bool isPrepared() const override { + return session_ready_.load(std::memory_order_acquire); + } bool producedData() const override { return produced_data_.load(std::memory_order_relaxed); } @@ -79,6 +96,8 @@ class RangeProfilerEngine final : public IProfilingEngine { EngineContext ctx_; std::atomic operational_{false}; std::atomic attempted_{false}; + /// Lock-free mirror of perf_session_active_; see isPrepared(). + std::atomic session_ready_{false}; std::atomic produced_data_{false}; }; diff --git a/include/gpufl/backends/nvidia/engine/sass_metrics_engine.hpp b/include/gpufl/backends/nvidia/engine/sass_metrics_engine.hpp index 5cab3b8..36bd282 100644 --- a/include/gpufl/backends/nvidia/engine/sass_metrics_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/sass_metrics_engine.hpp @@ -41,6 +41,13 @@ class SassMetricsEngine final : public IProfilingEngine { bool isOperational() const override { return isEnabled(); } + /** + * Ready to arm inside a window: the profiler is initialized and the metric + * config is set. `enabled_` - the actual arm - is deliberately not part of + * this; a WindowOnly run is prepared precisely while it is NOT armed. + */ + bool isPrepared() const override { return isEnabled(); } + /** True once at least one SASS metric sample was pushed this session. */ bool producedData() const override { return produced_data_.load(std::memory_order_relaxed); diff --git a/include/gpufl/backends/nvidia/kernel_launch_handler.cpp b/include/gpufl/backends/nvidia/kernel_launch_handler.cpp index b025d45..845a6d1 100644 --- a/include/gpufl/backends/nvidia/kernel_launch_handler.cpp +++ b/include/gpufl/backends/nvidia/kernel_launch_handler.cpp @@ -1,5 +1,6 @@ #include "gpufl/backends/nvidia/kernel_launch_handler.hpp" +#include "gpufl/core/deep_window_rules.hpp" #include "gpufl/core/env_vars.hpp" #include // std::min(initializer_list) - see occupancy calc below @@ -545,6 +546,21 @@ bool KernelLaunchHandler::handleActivityRecord(const CUpti_Activity* record, kernelName); out.cpu_start_ns = wallStartNs; out.duration_ns = static_cast(k->end - k->start); + // Feed recent_kernel_ms. This is the only place a COMPLETED kernel's + // duration exists - the launch callback sees a launch, not a result - so a + // rule watching kernel duration reads nothing at all without this. + // + // Stamped with when the kernel ENDED, not when it started. Freshness is + // measured against the source event, so a two-second kernel reported with + // its start time arrives already two seconds old and can read as stale the + // moment it lands - the slower the kernel, the more certainly a rule + // watching for slow kernels would discard it. + if (out.duration_ns >= 0 && + wallStartNs <= INT64_MAX - out.duration_ns) { + const int64_t wallEndNs = wallStartNs + out.duration_ns; + detail::DeepWindowRules::NoteKernelDuration( + wallEndNs, static_cast(out.duration_ns) / 1e6); + } out.dyn_shared = k->dynamicSharedMemory; out.static_shared = k->staticSharedMemory; out.num_regs = k->registersPerThread; diff --git a/include/gpufl/core/counter_api.cpp b/include/gpufl/core/counter_api.cpp new file mode 100644 index 0000000..85f030b --- /dev/null +++ b/include/gpufl/core/counter_api.cpp @@ -0,0 +1,91 @@ +#include "gpufl.hpp" + +#include "gpufl/abi/gpufl_counter_abi.h" +#include "gpufl/core/counter_provider.hpp" +#include "gpufl/core/counter_registry.hpp" + +namespace gpufl { +namespace { + +// Fallback provider, used when no shared runtime is present. Correct for an +// embedded host, which holds the only copy of gpufl in the process. Under +// injection it is NOT correct - the target and the evaluator are separate +// modules and would each get their own registry - which is why +// CounterProvider::isShared() exists for the evaluator to gate on. +// +// Implemented as a provider rather than a separate branch in add() so both +// paths go through one code shape and cannot drift. +using detail::CounterRegistry; + +using Slot = CounterRegistry::Slot; + +Slot* AsSlot(gpufl_counter_handle h) { return static_cast(h); } + +gpufl_counter_handle LocalRegister(const char* name, size_t len) { + if (name == nullptr) return nullptr; + CounterRegistry& reg = CounterRegistry::instance(); + return reg.slotFor(reg.registerCounter(std::string(name, len))); +} + +gpufl_counter_handle LocalLookup(const char* name, size_t len) { + if (name == nullptr) return nullptr; + CounterRegistry& reg = CounterRegistry::instance(); + return reg.slotFor(reg.findCounter(std::string(name, len))); +} + +void LocalAdd(gpufl_counter_handle h, uint64_t v) { + CounterRegistry::addRaw(AsSlot(h), v); +} + +uint64_t LocalLoad(gpufl_counter_handle h) { + return CounterRegistry::rawValue(AsSlot(h)); +} + +uint64_t LocalLoadSinceBaseline(gpufl_counter_handle h) { + return CounterRegistry::valueSinceBaseline(AsSlot(h)); +} + +void LocalBeginSession() { CounterRegistry::instance().beginSession(); } +void LocalEndSession() { CounterRegistry::instance().endSession(); } +int LocalSessionActive() { return CounterRegistry::instance().sessionActive() ? 1 : 0; } + +const gpufl_counter_provider_v1 kLocalProvider = { + GPUFL_COUNTER_ABI_VERSION, + sizeof(gpufl_counter_provider_v1), + &LocalRegister, &LocalAdd, &LocalLoad, &LocalLoadSinceBaseline, + &LocalBeginSession, &LocalEndSession, &LocalSessionActive, + &LocalLookup, +}; + +} // namespace + +namespace detail { + +const gpufl_counter_provider_v1* ActiveCounterProvider() { + if (const auto* shared = CounterProvider::get()) return shared; + return &kLocalProvider; +} + +} // namespace detail + +void Counter::add(const int64_t n) const { + if (handle_ == nullptr || n <= 0 || n > kMaxAddPerCall) return; + static_cast(provider_) + ->add(handle_, static_cast(n)); +} + +Counter counter(const std::string& name) { + const gpufl_counter_provider_v1* provider = detail::ActiveCounterProvider(); + gpufl_counter_handle handle = + provider->register_counter(name.c_str(), name.size()); + if (handle == nullptr) return Counter{}; + return Counter{provider, handle}; +} + +void tick(const std::string& name, const int64_t n) { + // Deliberately does the lookup every call. Documented as not for tight + // loops; anyone who cares holds a Counter instead. + counter(name).add(n); +} + +} // namespace gpufl diff --git a/include/gpufl/core/counter_provider.cpp b/include/gpufl/core/counter_provider.cpp new file mode 100644 index 0000000..8d83420 --- /dev/null +++ b/include/gpufl/core/counter_provider.cpp @@ -0,0 +1,267 @@ +#include "gpufl/core/counter_provider.hpp" + +#include +#include +#include + +#if defined(_WIN32) +# include +#else +# include +#endif + +#include + +#include "gpufl/core/counter_registry.hpp" +#include "gpufl/core/env_vars.hpp" +#include "gpufl/core/debug_logger.hpp" + +namespace gpufl::detail { +namespace { + +#if defined(_WIN32) +constexpr const char* kRuntimeName = "gpufl_counter_runtime.dll"; +#elif defined(__APPLE__) +constexpr const char* kRuntimeName = "libgpufl_counter_runtime.dylib"; +#else +constexpr const char* kRuntimeName = "libgpufl_counter_runtime.so"; +#endif + +std::mutex g_mu; +// Only a SUCCESSFUL bind is cached. A failure is not final: under injection the +// runtime appears when the driver loads gpufl_inject, which is after the target +// may already have registered a counter. Caching that failure forever is what +// would leave the application on one registry and the evaluator on another. +bool g_resolved = false; +bool g_failed_once = false; +const gpufl_counter_provider_v1* g_provider = nullptr; +bool g_shared = false; + +/** + * Directory of the module this code is linked into - the injection DLL, the + * Python extension, or the host executable. NOT the process directory: under + * CUDA_INJECTION64_PATH the process is the profiled target, which has no + * reason to sit next to our files. + */ +std::string ThisModuleDir() { +#if defined(_WIN32) + HMODULE module = nullptr; + if (!GetModuleHandleExA( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(&ThisModuleDir), &module)) { + return {}; + } + char path[MAX_PATH] = {}; + const DWORD n = GetModuleFileNameA(module, path, sizeof(path)); + if (n == 0 || n >= sizeof(path)) return {}; + std::string full(path, n); + const auto slash = full.find_last_of("\\/"); + return slash == std::string::npos ? std::string{} : full.substr(0, slash); +#else + Dl_info info{}; + if (dladdr(reinterpret_cast(&ThisModuleDir), &info) == 0 || + info.dli_fname == nullptr) { + return {}; + } + const std::string full(info.dli_fname); + const auto slash = full.find_last_of('/'); + return slash == std::string::npos ? std::string{} : full.substr(0, slash); +#endif +} + +/** + * A copy already mapped into this process, or nullptr. + * + * Tried before any path, and it is what actually delivers the single-instance + * property. Deployment colocates the runtime with each consumer - beside the + * injection DLL, inside the Python wheel - so several copies exist on disk, and + * loading them by full path would map them as separate modules with separate + * registries. That is the very split this ABI exists to prevent. Binding to + * whatever is already loaded makes the number of copies on disk irrelevant. + */ +void* AlreadyLoaded() { +#if defined(_WIN32) + return reinterpret_cast(GetModuleHandleA(kRuntimeName)); +#else + // NOLOAD returns a handle only if the object is already mapped. + return dlopen(kRuntimeName, RTLD_NOW | RTLD_NOLOAD); +#endif +} + +void* OpenLibrary(const std::string& path) { +#if defined(_WIN32) + return LoadLibraryA(path.c_str()); +#else + // RTLD_GLOBAL so a second binder resolves to this same object rather than + // mapping its own copy. + return dlopen(path.c_str(), RTLD_NOW | RTLD_GLOBAL); +#endif +} + +void* FindSymbol(void* handle, const char* name) { +#if defined(_WIN32) + return reinterpret_cast( + GetProcAddress(static_cast(handle), name)); +#else + return dlsym(handle, name); +#endif +} + +/** Directory part of a path, or empty. */ +std::string DirOf(const std::string& path) { + const auto slash = path.find_last_of("\\/"); + return slash == std::string::npos ? std::string{} : path.substr(0, slash); +} + +/** Candidate locations, most specific first. */ +std::vector BuildCandidates(const char* injection_path, + const char* explicit_runtime_path) { + std::vector out; + + // An explicit override wins. Deployment layouts we do not control need a + // way to say where the runtime is without guessing. + if (explicit_runtime_path != nullptr && explicit_runtime_path[0] != '\0') { + out.emplace_back(explicit_runtime_path); + } + + // Beside the injection library. This is what makes an ordinary C++ target + // work: it links gpufl statically, so ThisModuleDir() is the TARGET's + // directory, which has no runtime in it. If that target calls counter() + // before its first CUDA call - entirely normal - it would bind to its own + // local registry and stay there, invisible to the evaluator that loads + // later. The launcher puts this variable in the child's environment before + // exec, so it is readable from the very first instruction. + if (injection_path != nullptr && injection_path[0] != '\0') { + if (const std::string dir = DirOf(injection_path); !dir.empty()) { + out.push_back(dir + "/" + kRuntimeName); + } + } + + const std::string dir = ThisModuleDir(); + if (!dir.empty()) { + out.push_back(dir + "/" + kRuntimeName); + // Build trees put the shared runtime beside the config directory + // rather than next to every consumer. + out.push_back(dir + "/../" + kRuntimeName); + out.push_back(dir + "/../bin/" + kRuntimeName); + } + // Last resort: the platform loader's own search. Also the path that finds + // an already-loaded copy by name, which is the common case for whichever + // module binds second. + out.emplace_back(kRuntimeName); + return out; +} + +bool Resolve() { + // Whatever is already mapped wins, before any path is considered. See + // AlreadyLoaded(): several copies of this library legitimately exist on + // disk, and binding by path would defeat the point. + std::vector candidates = + BuildCandidates(std::getenv(env::kCudaInjection64Path), + std::getenv(env::kCounterRuntimePath)); + candidates.insert(candidates.begin(), std::string{}); // "" = already loaded + + for (const std::string& candidate : candidates) { + void* handle = candidate.empty() ? AlreadyLoaded() : OpenLibrary(candidate); + if (handle == nullptr) continue; + + using GetProviderFn = const gpufl_counter_provider_v1* (*)(); + const auto entry = reinterpret_cast( + FindSymbol(handle, "gpufl_get_counter_provider_v1")); + if (entry == nullptr) continue; + + const gpufl_counter_provider_v1* provider = entry(); + if (provider == nullptr) continue; + // Check both before calling through: a newer runtime may have appended + // members this build does not know about, and an older one may lack + // members this build assumes. + if (provider->abi_version != GPUFL_COUNTER_ABI_VERSION || + provider->struct_size < sizeof(gpufl_counter_provider_v1)) { + GFL_LOG_ERROR("[CounterProvider] ", candidate, + " reports ABI ", provider->abi_version, + " (size ", provider->struct_size, "); this build needs ", + GPUFL_COUNTER_ABI_VERSION, " (size ", + sizeof(gpufl_counter_provider_v1), ")"); + continue; + } + + g_provider = provider; + g_shared = true; + GFL_LOG_DEBUG("[CounterProvider] bound to shared runtime: ", + candidate.empty() ? "(already loaded)" : candidate.c_str()); + return true; + } + + // No shared runtime. Correct for an embedded host, which holds the only + // copy of gpufl in the process; wrong under injection, where the target and + // the evaluator are separate modules and would each get their own registry. + // isShared() is what lets the evaluator refuse a rule it cannot honour, + // rather than reporting a counter that is being ticked as Missing. + g_provider = nullptr; + g_shared = false; + GFL_LOG_DEBUG("[CounterProvider] ", kRuntimeName, + " not found; counters stay local to this module"); + return false; +} + +} // namespace + +std::vector CounterRuntimeCandidatesForTesting( + const char* injection_path, const char* explicit_runtime_path) { + return BuildCandidates(injection_path, explicit_runtime_path); +} + +const gpufl_counter_provider_v1* CounterProvider::get() { + std::lock_guard lk(g_mu); + if (g_resolved) return g_provider; + + if (g_failed_once) { + // A full path scan already came up empty. Repeating it on every + // counter() would mean a filesystem probe per registration for the + // entire life of an ordinary embedded run, which has no shared runtime + // and never will. + // + // What CAN change is that the runtime gets mapped later - the injected + // library loads at the first CUDA call. So retry only that: one cheap + // already-loaded check, no path search. + void* handle = AlreadyLoaded(); + if (handle == nullptr) return g_provider; + using GetProviderFn = const gpufl_counter_provider_v1* (*)(); + const auto entry = reinterpret_cast( + FindSymbol(handle, "gpufl_get_counter_provider_v1")); + if (entry == nullptr) return g_provider; + const gpufl_counter_provider_v1* provider = entry(); + if (provider == nullptr || + provider->abi_version != GPUFL_COUNTER_ABI_VERSION || + provider->struct_size < sizeof(gpufl_counter_provider_v1)) { + return g_provider; + } + g_provider = provider; + g_shared = true; + g_resolved = true; + GFL_LOG_DEBUG("[CounterProvider] bound to shared runtime that appeared " + "after the first attempt"); + return g_provider; + } + + g_resolved = Resolve(); + if (!g_resolved) g_failed_once = true; + return g_provider; +} + +bool CounterProvider::isShared() { + get(); + std::lock_guard lk(g_mu); + return g_shared; +} + +void CounterProvider::resetForTesting() { + std::lock_guard lk(g_mu); + g_resolved = false; + g_failed_once = false; + g_provider = nullptr; + g_shared = false; +} + +} // namespace gpufl::detail diff --git a/include/gpufl/core/counter_provider.hpp b/include/gpufl/core/counter_provider.hpp new file mode 100644 index 0000000..47576d8 --- /dev/null +++ b/include/gpufl/core/counter_provider.hpp @@ -0,0 +1,66 @@ +#pragma once + +#include +#include + +#include "gpufl/abi/gpufl_counter_abi.h" + +namespace gpufl::detail { + +/** + * @brief Binds this module to the process-wide counter runtime. + * + * Every module holding a copy of the static `gpufl` library calls this and + * ends up on the same registry, which is the whole point: without it a Python + * target ticks the extension's registry while the injected evaluator reads the + * injection DLL's, and the counter reads as Missing forever. + * + * Loaded explicitly by absolute path rather than through an import table. + * Injection arrives via CUDA_INJECTION64_PATH, so the driver loads + * gpufl_inject.dll into a process whose DLL search path has no reason to + * include our bin directory - an import entry would fail there. Resolving the + * path from THIS module's own location works wherever the module was loaded + * from. + * + * Whoever binds first loads the library; everyone after gets the same instance, + * which is what makes ordering between Python import and CUDA init irrelevant. + */ +class CounterProvider { +public: + /** @brief The provider, or nullptr if the runtime could not be loaded. */ + static const gpufl_counter_provider_v1* get(); + + /** + * @brief True when counters are shared across modules. + * + * False means the shared runtime was not found and this module fell back to + * its own in-process registry. That is correct for an embedded host, which + * has exactly one copy of gpufl, and wrong under injection, where the + * target and the evaluator are different modules. Callers that care - the + * rule evaluator - must refuse custom counter rules when this is false and + * more than one module is in play. + */ + static bool isShared(); + + /** @brief Test seam: forget the binding so the next get() resolves again. */ + static void resetForTesting(); +}; + +/** + * @brief Candidate runtime paths, most specific first. Test seam. + * + * Takes the two environment inputs explicitly rather than reading them, so a + * test can pin the ORDER and the injection-directory rule without mutating + * process environment that other tests share. + */ +std::vector CounterRuntimeCandidatesForTesting( + const char* injection_path, const char* explicit_runtime_path); + +/** + * @brief The provider in use: the shared runtime, or this module's own. + * + * Never null, so callers do not each need a fallback branch. + */ +const gpufl_counter_provider_v1* ActiveCounterProvider(); + +} // namespace gpufl::detail diff --git a/include/gpufl/core/counter_registry.cpp b/include/gpufl/core/counter_registry.cpp new file mode 100644 index 0000000..3a456d1 --- /dev/null +++ b/include/gpufl/core/counter_registry.cpp @@ -0,0 +1,127 @@ +#include "gpufl/core/counter_registry.hpp" + +#include + +#include "gpufl/core/debug_logger.hpp" + +namespace gpufl::detail { + +CounterRegistry& CounterRegistry::instance() { + // Function-local static: the table has to outlive every runtime, and this + // is the only storage duration that guarantees it without an init order + // dependency on whoever registers first. + static CounterRegistry registry; + return registry; +} + +bool CounterRegistry::nameIsValid(const std::string& name) { + if (name.empty() || name.size() > kMaxNameLength) return false; + // Explicit ASCII, not std::isalnum: that one answers according to the + // current locale, and the accepted set here is part of a wire contract + // rather than something a host program's locale gets to widen. + return std::all_of(name.begin(), name.end(), [](const char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_' || c == '.' || c == '-'; + }); +} + +std::string CounterRegistry::forLog(const std::string& name) { + // A rejected name is attacker- or bug-supplied and can be arbitrarily long. + // Show enough to identify it, never enough to bloat the log. + constexpr size_t kMaxLogged = 32; + if (name.size() <= kMaxLogged) return name; + return name.substr(0, kMaxLogged) + "...(" + std::to_string(name.size()) + " chars)"; +} + +CounterRegistry::SlotId CounterRegistry::findCounter(const std::string& name) const { + std::lock_guard lk(mu_); + const auto it = byName_.find(name); + return it == byName_.end() ? kInvalidSlot : it->second; +} + +CounterRegistry::SlotId CounterRegistry::registerCounter(const std::string& name) { + if (!nameIsValid(name)) { + std::lock_guard lk(mu_); + if (!loggedInvalidName_) { + loggedInvalidName_ = true; + GFL_LOG_ERROR("[CounterRegistry] rejected counter name '", forLog(name), + "': must be 1-", kMaxNameLength, + " characters of [A-Za-z0-9._-] " + "(further name rejections suppressed)"); + } + return kInvalidSlot; + } + + std::lock_guard lk(mu_); + // Same name from several threads resolves to one slot; the map lookup + // under the lock is what makes that true. + if (const auto it = byName_.find(name); it != byName_.end()) return it->second; + + if (slots_.size() >= kMaxCounters) { + if (!loggedLimitReached_) { + loggedLimitReached_ = true; + GFL_LOG_ERROR("[CounterRegistry] counter limit reached (", kMaxCounters, + "); '", forLog(name), + "' not registered (further rejections suppressed)"); + } + return kInvalidSlot; + } + + const auto slot = static_cast(slots_.size()); + slots_.emplace_back(); + slots_.back().name = name; + // Registered mid-generation: baseline at the current value so ticks from a + // previous session, or from before init(), do not land in this one. + slots_.back().baseline.store( + slots_.back().value.load(std::memory_order_relaxed), + std::memory_order_relaxed); + byName_.emplace(name, slot); + return slot; +} + +CounterRegistry::Slot* CounterRegistry::slotFor(const SlotId slot) { + std::lock_guard lk(mu_); + if (slot >= slots_.size()) return nullptr; + // Safe to hand out and keep: a deque never relocates existing elements, and + // slots are never erased outside resetForTesting(). + return &slots_[slot]; +} + +const std::string& CounterRegistry::name(SlotId slot) const { + static const std::string kEmpty; + std::lock_guard lk(mu_); + if (slot >= slots_.size()) return kEmpty; + return slots_[slot].name; +} + +size_t CounterRegistry::counterCount() const { + std::lock_guard lk(mu_); + return slots_.size(); +} + +void CounterRegistry::beginSession() { + std::lock_guard lk(mu_); + for (auto& slot : slots_) { + slot.baseline.store(slot.value.load(std::memory_order_relaxed), + std::memory_order_relaxed); + } + sessionActive_.store(true, std::memory_order_release); +} + +void CounterRegistry::endSession() { + // Values stay. A handle that outlives this keeps pointing at a live slot; + // it is the baseline taken by the next beginSession that stops those adds + // from being counted twice. + sessionActive_.store(false, std::memory_order_release); +} + +void CounterRegistry::resetForTesting() { + std::lock_guard lk(mu_); + slots_.clear(); + byName_.clear(); + sessionActive_.store(false, std::memory_order_release); + loggedInvalidName_ = false; + loggedLimitReached_ = false; +} + +} // namespace gpufl::detail diff --git a/include/gpufl/core/counter_registry.hpp b/include/gpufl/core/counter_registry.hpp new file mode 100644 index 0000000..e5b3b1d --- /dev/null +++ b/include/gpufl/core/counter_registry.hpp @@ -0,0 +1,182 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace gpufl::detail { + +/** + * @brief Named counters the application increments, read as rates. + * + * Exists so a rule can watch something only the application knows - tokens, + * steps, requests - without the application computing a rate or reaching for a + * scope. A scope would cost two locked batch pushes and a wire row per + * iteration, which rules it out of a decode loop; this costs one relaxed atomic + * add, and unlike a scope it can carry a count, so a step that produced eight + * tokens says so. + * + * ## Slots outlive the runtime, deliberately + * + * A handle can be stored in a static, or held across a shutdown()/init() cycle + * by an embedded host. A generation number alone would not make that safe: it + * cannot stop shutdown() freeing state while another thread is inside add(). + * So the slot itself is never freed. init()/shutdown() only change which + * runtime reads it, and nothing add() touches can disappear underneath it. + * + * That also removes any need to rebind a stale handle. There is nothing to + * rebind to - the slot a handle points at is still the right one - so the hot + * path carries no generation check either. + * + * The cost is a value that survives sessions, so each runtime records a + * baseline at init() and reports only what accrued since. Adds made while no + * runtime is active land in the slot and are excluded from the next session + * rather than counted twice. + */ +class CounterRegistry { +public: + /** @brief Index into the permanent slot table. kInvalidSlot on rejection. */ + using SlotId = uint32_t; + static constexpr SlotId kInvalidSlot = 0xFFFFFFFFu; + + /** + * @brief One counter's storage, addressed directly by a handle. + * + * Public because the hot path is a pointer to this, not an index. Going + * through an index means bounds-checking the container, and checking it + * means locking it - which would serialise every ticking thread on one + * mutex and distort the very throughput a rule is trying to measure. + * + * Never freed, so the pointer stays valid across shutdown()/init(); a deque + * also never relocates existing elements, so registration cannot move one + * out from under a caller that already holds it. + */ + struct Slot { + std::atomic value{0}; + /// Atomic so valueSinceBaseline stays lock-free too. Written only by + /// beginSession and by registration, both rare. + std::atomic baseline{0}; + std::string name; ///< guarded by mu_, never changes once set + }; + + /** Bounds. A permanent table is exactly what must not grow without limit. */ + static constexpr size_t kMaxCounters = 256; + static constexpr size_t kMaxNameLength = 96; + /** + * Largest single add accepted. + * + * Not an overflow guard - the counter is 64-bit and rates are unsigned + * deltas, so a wrap is both unreachable in practice and handled if it + * happened. This catches a caller passing something that is not a count at + * all: a pointer, an uninitialised value, a negative cast to unsigned. One + * of those silently makes every rate meaningless, which is worse than being + * told the value was refused. + */ + static constexpr int64_t kMaxAddPerCall = 1LL << 40; // ~1.1e12 + + static CounterRegistry& instance(); + + /** + * @brief Find or create the slot for @p name. + * + * Rejects a name that is empty, over kMaxNameLength, contains anything + * outside [A-Za-z0-9._-], or would exceed kMaxCounters. Registering the + * same name twice returns the same slot, including from several threads at + * once. + */ + SlotId registerCounter(const std::string& name); + + /** + * @brief Stable address of a slot, or nullptr for kInvalidSlot. + * + * Taken once at registration and held. Everything on the hot path goes + * through this rather than through a SlotId, so no lock is needed and the + * container is never touched. + */ + Slot* slotFor(SlotId slot); + + /** + * @brief Find @p name WITHOUT creating it. kInvalidSlot if absent. + * + * For readers that must not bring a counter into existence by asking about + * it - a rule naming a counter is a question, not a registration, and + * conflating the two hides the difference between a config typo and an idle + * workload. + */ + SlotId findCounter(const std::string& name) const; + + /** + * @brief Add to a slot. One relaxed atomic, no lock, no lookup. + * + * The whole point of handing out a Slot*: a decode loop ticking a counter + * must not serialise on a registry mutex, or the profiler changes the + * throughput the rule is watching. + * + * Validation happens at gpufl::Counter::add, before this is reached; + * re-checking here would duplicate a contract in two places that could + * then disagree. + */ + static void addRaw(Slot* slot, uint64_t value) { + if (slot != nullptr) slot->value.fetch_add(value, std::memory_order_relaxed); + } + + /** @brief Raw slot value, including anything added before this runtime. */ + static uint64_t rawValue(const Slot* slot) { + return slot == nullptr ? 0 : slot->value.load(std::memory_order_relaxed); + } + + /** @brief Value accrued since the current runtime took its baseline. */ + static uint64_t valueSinceBaseline(const Slot* slot) { + if (slot == nullptr) return 0; + // Unsigned subtraction, deliberately: correct across a wrap, where a + // signed difference would go negative. + return slot->value.load(std::memory_order_relaxed) - + slot->baseline.load(std::memory_order_relaxed); + } + + const std::string& name(SlotId slot) const; + size_t counterCount() const; + + /** + * @brief Start a session, baselining every slot at its current value. + * + * What keeps a permanent slot from leaking one session's ticks into the + * next. Anything added while no session was active is excluded here. + */ + void beginSession(); + /** @brief End the current session. Slots keep their values on purpose. */ + void endSession(); + /** @brief Whether a session is currently active. */ + bool sessionActive() const { return sessionActive_.load(std::memory_order_acquire); } + + /** @brief Test seam. Drops every slot, which a real process never does. */ + void resetForTesting(); + +private: + CounterRegistry() = default; + + static bool nameIsValid(const std::string& name); + /** @brief Truncate an untrusted name so a rejection cannot bloat the log. */ + static std::string forLog(const std::string& name); + + // Guards registration and the name map. Never taken by addRaw/rawValue. + mutable std::mutex mu_; + // Deque, not vector: a handle holds a Slot* and a vector would invalidate + // it on growth. Slots are never erased. + std::deque slots_; + std::unordered_map byName_; + // A flag, not a counter. Nothing needs to tell one session from another - + // baselines already separate them - and a monotonic epoch that no reader + // consults is a thing that drifts out of date without anyone noticing. + std::atomic sessionActive_{false}; + // Rejections are logged once each. A name that fails validation usually + // fails on every call, and tick() in a decode loop would otherwise write a + // line per iteration. + bool loggedInvalidName_ = false; + bool loggedLimitReached_ = false; +}; + +} // namespace gpufl::detail diff --git a/include/gpufl/core/debug_logger.hpp b/include/gpufl/core/debug_logger.hpp index bfcce74..fc49c71 100644 --- a/include/gpufl/core/debug_logger.hpp +++ b/include/gpufl/core/debug_logger.hpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace gpufl { @@ -21,6 +22,22 @@ class DebugLogger { } } + template + static void info(const char* prefix, Args&&... args) { + std::stringstream ss; + ss << prefix; + (ss << ... << std::forward(args)); + std::cerr << ss.str() << std::endl; + } + + template + static void warn(const char* prefix, Args&&... args) { + std::stringstream ss; + ss << prefix; + (ss << ... << std::forward(args)); + std::cerr << ss.str() << std::endl; + } + template static void error(const char* prefix, Args&&... args) { // Errors ALWAYS print, regardless of the debug-output flag. @@ -39,6 +56,8 @@ class DebugLogger { }; #define GFL_LOG_DEBUG(...) ::gpufl::DebugLogger::log("[GPUFL] ", __VA_ARGS__) +#define GFL_LOG_INFO(...) ::gpufl::DebugLogger::info("[GPUFL] ", __VA_ARGS__) +#define GFL_LOG_WARN(...) ::gpufl::DebugLogger::warn("[GPUFL-WARN] ", __VA_ARGS__) #define GFL_LOG_ERROR(...) \ ::gpufl::DebugLogger::error("[GPUFL-ERROR] " __FILE__ ":", __LINE__, ": ", __VA_ARGS__) diff --git a/include/gpufl/core/deep_window.cpp b/include/gpufl/core/deep_window.cpp index bec910d..cd0ca4c 100644 --- a/include/gpufl/core/deep_window.cpp +++ b/include/gpufl/core/deep_window.cpp @@ -39,15 +39,48 @@ std::atomic g_close_requested{false}; std::atomic g_close_reason{static_cast(DeepWindowClose::Deadline)}; // An open asked for by a thread that can't arm one itself. Checked lock-free -// on the launch beat; g_pending_spec is only read once this is set, so the +// on the launch beat; g_pending is only read once this is set, so the // hot path pays for it exactly when a trigger is waiting. std::atomic g_open_requested{false}; std::atomic g_pending_open_at_ns{0}; // 0 = at the next launch -DeepWindowSpec g_pending_spec; // guarded by g_mu + +// The queued request, as ONE record. Spec and owner token were separate fields +// once, and an untagged request arriving after a tagged one then replaced only +// the spec - so a manual window opened carrying a rule's token and was charged +// to that rule's budget. Replacing the record replaces both or neither. +struct PendingOpen { + DeepWindowSpec spec; + uint64_t owner_token = 0; // 0 = nobody is waiting on this one +}; + +// First request wins; a second is refused while one is still queued. +// +// The alternative - newest wins - silently discards whichever trigger asked +// first, and with both --deep-after and a rule configured that is decided by +// which one happens to run first rather than by anything the user chose. A +// refusal is at least reported: a rule gets token 0 and goes back to armed, +// and the scheduled window logs that it kept its place. +bool PendingIsQueued() { return g_open_requested.load(std::memory_order_acquire); } +PendingOpen g_pending; // guarded by g_mu // When the last window closed, so a cooldown can be enforced. 0 = never. std::atomic g_last_close_ns{0}; +// Attribution for windows opened through a tagged request. +// +// A plain "a window opened" counter is not enough: a manual or scheduled +// window opening while a rule's request is outstanding would be counted +// against that rule's budget, and the budget is meant to bound what the RULE +// costs. Tokens start at 1 so 0 keeps its meaning of "not from a request". +std::atomic g_next_open_token{1}; +std::atomic g_last_opened_token{0}; +std::atomic g_opens_completed{0}; +// Token of the request currently being serviced. Only TakePendingOpen_ sets +// it, and only for the duration of the Open() call it drives, so a direct +// Open() can never inherit someone else's attribution. +thread_local uint64_t g_claimed_token = 0; + +DeepWindowTrigger g_trigger; // guarded by g_mu int64_t g_opened_ns = 0; int64_t g_requested_duration_ms = 0; uint64_t g_requested_max_launches = 0; @@ -165,6 +198,15 @@ bool DeepWindow::Active() { bool DeepWindow::Open(const DeepWindowSpec& spec) { if (const Runtime* rt = runtime(); !rt || !rt->logger) return false; + // Scheduled and manual requests do not pass through RequestOpenTagged. + // Apply the same real-readiness gate here so they cannot publish a window + // that was active in name only and armed no engine. + if (IMonitorBackend* backend = Monitor::GetBackend(); + backend != nullptr && !backend->DeepEnginesPrepared()) { + GFL_LOG_ERROR("[DeepWindow] open refused: selected deep engine is not " + "prepared"); + return false; + } std::string name; { @@ -182,7 +224,14 @@ bool DeepWindow::Open(const DeepWindowSpec& spec) { return false; } + // Publish who this open belongs to, then clear the claim so a later + // direct Open() cannot inherit it. The claim is taken only by + // RequestOpenTagged and consumed exactly once, here. + g_last_opened_token.store(g_claimed_token, std::memory_order_release); + g_opens_completed.fetch_add(1, std::memory_order_acq_rel); + g_opened_ns = detail::GetTimestampNs(); + g_trigger = spec.trigger; g_name = spec.name.empty() ? "deep_window" : spec.name; g_requested_duration_ms = spec.max_duration_ms; g_requested_max_launches = spec.max_launches; @@ -250,6 +299,7 @@ void DeepWindow::Close(const DeepWindowClose reason) { // the engine teardown, so a slow disarm doesn't shorten the quiet time. g_last_close_ns.store(end_ns, std::memory_order_relaxed); + ev.trigger = g_trigger; ev.pid = detail::GetPid(); ev.name = g_name; ev.close_reason = DeepWindowCloseName(reason); @@ -296,6 +346,10 @@ void DeepWindow::Close(const DeepWindowClose reason) { close_row.name_id = Monitor::InternScopeName(name); close_row.event_type = 1; close_row.depth = 0; // ignored on close; the open row carries it + // end_ns intentionally precedes engine disarm so the window event + // measures the requested boundary. Publish it only after the final + // drain above, immediately before the scope-state transition. + Monitor::MarkScopeClosePending(g_scope_instance_id, end_ns); Monitor::PushScopeRow(close_row); g_scope_instance_id = 0; } @@ -317,17 +371,113 @@ void DeepWindow::RequestOpen(const DeepWindowSpec& spec) { ScheduleOpenAfter(0, spec); } +OpenRequestResult DeepWindow::RequestOpenTagged(const DeepWindowSpec& spec) { + // Rule installation may precede CONTEXT_CREATED under Windows injection, + // so it gates on engine selection. By the time a condition fires, require + // the context-bound preparation to have really succeeded. + // + // Pending and failed are reported apart because they call for opposite + // responses: pending means try again shortly, failed means this session + // will never open a useful window and the rule should say so rather than + // retrying until shutdown and reporting `never_true`. + if (IMonitorBackend* backend = Monitor::GetBackend(); + backend != nullptr && !backend->DeepEnginesPrepared()) { + if (backend->DeepEnginePreparationPending()) { + GFL_LOG_DEBUG("[DeepWindow] conditional open deferred: deep engine " + "preparation has not run yet"); + return {0, OpenRequestStatus::PreparationPending}; + } + GFL_LOG_ERROR("[DeepWindow] conditional open refused: selected deep " + "engine is not prepared"); + return {0, OpenRequestStatus::EngineUnavailable}; + } + + uint64_t token = 0; + { + std::lock_guard lk(g_mu); + // Decided here so the caller learns now, rather than discovering later + // that a window it counted on never opened. + if (g_active.load(std::memory_order_relaxed)) { + return {0, OpenRequestStatus::Busy}; + } + if (InCooldown(spec)) return {0, OpenRequestStatus::Cooldown}; + if (PendingIsQueued()) { + // Someone else - typically the launcher's --deep-after window - is + // already waiting. Refusing here means the rule retries later + // instead of cancelling a window the user explicitly scheduled. + GFL_LOG_DEBUG("[DeepWindow] tagged open refused: a request is " + "already queued"); + return {0, OpenRequestStatus::Busy}; + } + + token = g_next_open_token.fetch_add(1, std::memory_order_relaxed); + // Spec and token published together, under one lock. Splitting them is + // how an untagged request could inherit a rule's attribution. + g_pending.spec = spec; + g_pending.owner_token = token; + g_pending_open_at_ns.store(0, std::memory_order_relaxed); + // Published INSIDE the lock. Releasing first left a window where a + // second caller took the lock, saw no request queued, and overwrote + // this one - which is precisely the first-wins rule this is meant to + // enforce, defeated by two threads asking at once. + g_open_requested.store(true, std::memory_order_release); + } + GFL_LOG_DEBUG("[DeepWindow] tagged open requested token=", token, + " duration_ms=", spec.max_duration_ms); + return {token, OpenRequestStatus::Accepted}; +} + +const char* toString(const OpenRequestStatus status) { + switch (status) { + case OpenRequestStatus::Accepted: return "accepted"; + case OpenRequestStatus::PreparationPending: return "preparation_pending"; + case OpenRequestStatus::EngineUnavailable: return "deep_engine_not_prepared"; + case OpenRequestStatus::Busy: return "busy"; + case OpenRequestStatus::Cooldown: return "cooldown"; + case OpenRequestStatus::InvalidResult: return "invalid_open_result"; + } + return "unknown"; +} + +uint64_t DeepWindow::LastOpenedToken() { + return g_last_opened_token.load(std::memory_order_acquire); +} + +uint64_t DeepWindow::PendingOpenToken() { + if (!g_open_requested.load(std::memory_order_acquire)) return 0; + std::lock_guard lk(g_mu); + return g_pending.owner_token; +} + +uint64_t DeepWindow::OpensCompleted() { + return g_opens_completed.load(std::memory_order_acquire); +} + void DeepWindow::ScheduleOpenAfter(const int64_t delay_ms, const DeepWindowSpec& spec) { { std::lock_guard lk(g_mu); - g_pending_spec = spec; + if (PendingIsQueued()) { + // Keeps the queued request rather than replacing it. Overwriting + // would let a scheduled window cancel a rule's window - or the + // reverse - purely on call order. + GFL_LOG_DEBUG("[DeepWindow] open request ignored: one is already " + "queued (owner_token=", g_pending.owner_token, ")"); + return; + } + g_pending.spec = spec; + // Untagged: nobody is counting this window against a budget. Cleared + // explicitly, so replacing a tagged request cannot leave its owner + // attached to somebody else's window. + g_pending.owner_token = 0; g_pending_open_at_ns.store( delay_ms > 0 ? detail::GetTimestampNs() + delay_ms * 1000000 : 0, std::memory_order_relaxed); + // Under the lock, for the same reason as the tagged path: the launch + // beat still reads the spec only once this is set, and a concurrent + // caller can no longer slip in between the two. + g_open_requested.store(true, std::memory_order_release); } - // Published last: the launch beat reads the spec only once this is set. - g_open_requested.store(true, std::memory_order_release); GFL_LOG_DEBUG("[DeepWindow] open requested delay_ms=", delay_ms, " duration_ms=", spec.max_duration_ms, " max_launches=", spec.max_launches); @@ -336,15 +486,27 @@ void DeepWindow::ScheduleOpenAfter(const int64_t delay_ms, // Runs on the collector, off the CUPTI callback path. Claims the request // before opening so nothing can act on it twice. void DeepWindow::TakePendingOpen_() { - if (!g_open_requested.exchange(false, std::memory_order_acq_rel)) return; - DeepWindowSpec spec; { std::lock_guard lk(g_mu); - spec = g_pending_spec; + // The flag and the record are claimed TOGETHER. Clearing the flag + // first and then taking the lock left a gap: a producer could acquire + // the lock, see nothing queued, and store its own request - which this + // collector then consumed believing it was the earlier one, while the + // producer's flag stayed set and let the same request open a second + // window later. + if (!g_open_requested.exchange(false, std::memory_order_acq_rel)) return; + spec = g_pending.spec; + // Claim the token here, not in Open(): Open can still refuse, and a + // token left behind would keep reading as "queued" for the rest of the + // run, so a rule would wait forever for a window that was already + // turned down. + g_claimed_token = g_pending.owner_token; + g_pending.owner_token = 0; } // Outside the lock: Open takes it too. Open(spec); + g_claimed_token = 0; } void DeepWindow::OnLaunch() { @@ -410,6 +572,10 @@ void DeepWindow::ServicePending() { void DeepWindow::ResetForTesting() { std::lock_guard lk(g_mu); g_active.store(false, std::memory_order_release); + g_next_open_token.store(1, std::memory_order_relaxed); + g_last_opened_token.store(0, std::memory_order_relaxed); + g_opens_completed.store(0, std::memory_order_relaxed); + g_pending = PendingOpen{}; g_deadline_ns.store(0, std::memory_order_relaxed); g_launches_remaining.store(0, std::memory_order_relaxed); g_launches_covered.store(0, std::memory_order_relaxed); @@ -418,7 +584,7 @@ void DeepWindow::ResetForTesting() { std::memory_order_relaxed); g_open_requested.store(false, std::memory_order_relaxed); g_pending_open_at_ns.store(0, std::memory_order_relaxed); - g_pending_spec = DeepWindowSpec{}; + g_pending = PendingOpen{}; g_last_close_ns.store(0, std::memory_order_relaxed); g_opened_ns = 0; g_requested_duration_ms = 0; diff --git a/include/gpufl/core/deep_window.hpp b/include/gpufl/core/deep_window.hpp index e6313e4..b452362 100644 --- a/include/gpufl/core/deep_window.hpp +++ b/include/gpufl/core/deep_window.hpp @@ -3,6 +3,8 @@ #include #include +#include "gpufl/core/events.hpp" + namespace gpufl { /** @@ -37,6 +39,52 @@ struct DeepWindowSpec { // here rather than in the caller's trigger. int64_t cooldown_ms = 0; std::string name = "deep_window"; + // What asked for this window, when a rule did. Travels with the spec so the + // window carries its own explanation rather than needing a lookup that + // would have to survive the window closing. + DeepWindowTrigger trigger; +}; + +/** + * @brief Why a tagged open was refused, when it was. + * + * A single "refused" answer is not enough for a rule to act on. Cooldown and a + * window already being open are ordinary and temporary - the rule should wait + * and try again. A deep engine that failed its context-bound preparation is + * permanent, and a rule that keeps retrying it reports `never_true` at + * shutdown, which says the condition never held when in fact it held and could + * not be acted on. + */ +enum class OpenRequestStatus { + Accepted, + /// The engine has not reached its first CUDA context yet. Temporary: + /// under Windows injection a rule is installed before CONTEXT_CREATED. + PreparationPending, + /// Preparation ran and failed. Permanent for this session. + EngineUnavailable, + /// A window is already open, or another request is queued. + Busy, + Cooldown, + /// The coordinator answered with something that cannot be acted on - + /// accepted but no token. Its own value rather than a silent fallback: it + /// means a coordinator is broken, and reporting it as cooldown would hide + /// that behind a state the user would read as normal. + InvalidResult, +}; + +const char* toString(OpenRequestStatus status); + +struct OpenRequestResult { + uint64_t token = 0; ///< non-zero only when Accepted + /// Defaults to a refusal, not to Accepted. A default-constructed result is + /// one nobody filled in, and taking that as "yes, token 0" left the + /// evaluator waiting in Opening for a window that was never asked for. + OpenRequestStatus status = OpenRequestStatus::InvalidResult; + + /// True only when this result can actually be waited on. + bool accepted() const { + return status == OpenRequestStatus::Accepted && token != 0; + } }; /** @@ -75,10 +123,40 @@ class DeepWindow { * records the request here and the next launch performs the open. The * mirror of how a deadline reached off the app thread defers its close. * - * A pending request is replaced, not queued: the newest spec wins. + * A request is REFUSED while another is still queued - first one wins. + * Replacing it would let a scheduled window cancel a rule's window, or + * the reverse, decided by nothing but which ran first. */ static void RequestOpen(const DeepWindowSpec& spec); + /** + * @brief Request an open and get back a token identifying it. + * + * For callers that must know whether the window they asked for actually + * happened - a rule with a window budget cannot count a window that never + * opened, and cannot let a manual one consume its budget either. + * + * Returns 0 when the request is refused outright (a window is already open, + * or the cooldown has not elapsed). A non-zero token matches + * LastOpenedToken() once, and only once, the requested window opens. + */ + static OpenRequestResult RequestOpenTagged(const DeepWindowSpec& spec); + + /** @brief Token of the most recent open; 0 when it was not from a request. */ + static uint64_t LastOpenedToken(); + + /** + * @brief Token of the request still queued, or 0 when none is. + * + * An open is serviced on a later beat, so "has not opened yet" and "will + * never open" look identical without this. A caller that treated the first + * as the second would abandon a window that was about to open. + */ + static uint64_t PendingOpenToken(); + + /** @brief Monotonic count of windows that actually opened. */ + static uint64_t OpensCompleted(); + /** * @brief Same, but not before `delay_ms` have passed. * diff --git a/include/gpufl/core/deep_window_rule.cpp b/include/gpufl/core/deep_window_rule.cpp new file mode 100644 index 0000000..b33147d --- /dev/null +++ b/include/gpufl/core/deep_window_rule.cpp @@ -0,0 +1,606 @@ +#include "gpufl/core/deep_window_rule.hpp" + +#include +#include +#include +#include + +#include "gpufl/core/debug_logger.hpp" + +namespace gpufl::detail { +namespace { + +/** Largest window a rule may ask for. Property 4: nothing is unbounded. */ +constexpr int64_t kMaxWindowDurationMs = 60 * 1000; +constexpr uint64_t kMaxWindowLaunches = 5000000; +constexpr int kMaxWindowsHardLimit = 64; + +std::string trim(const std::string& s) { + const size_t b = s.find_first_not_of(" \t"); + if (b == std::string::npos) return {}; + const size_t e = s.find_last_not_of(" \t"); + return s.substr(b, e - b + 1); +} + +/** Parse "2s" / "500ms" / "2000" (bare = milliseconds). Negative on failure. */ +int64_t parseDurationMs(const std::string& text) { + const std::string s = trim(text); + if (s.empty()) return -1; + + size_t digits = 0; + while (digits < s.size() && s[digits] >= '0' && s[digits] <= '9') ++digits; + if (digits == 0) return -1; + + errno = 0; + const long long value = std::strtoll(s.substr(0, digits).c_str(), nullptr, 10); + if (errno != 0 || value < 0) return -1; + + const std::string unit = trim(s.substr(digits)); + if (unit.empty() || unit == "ms") return value; + if (unit == "s") { + if (value > kMaxWindowDurationMs) return -1; // also guards the *1000 + return value * 1000; + } + return -1; +} + +bool parseDouble(const std::string& text, double* out) { + const std::string s = trim(text); + if (s.empty()) return false; + char* end = nullptr; + errno = 0; + const double v = std::strtod(s.c_str(), &end); + if (errno != 0 || end == s.c_str() || *end != '\0') return false; + if (!std::isfinite(v)) return false; + *out = v; + return true; +} + +} // namespace + +const char* toString(const Comparison op) { + return op == Comparison::LessThan ? "<" : ">"; +} + +const char* toString(const RuleError e) { + switch (e) { + case RuleError::None: return "ok"; + case RuleError::BadMetric: return "bad_metric"; + case RuleError::BadTiming: return "bad_timing"; + case RuleError::ThresholdNotFinite: return "threshold_not_finite"; + case RuleError::RearmWrongSide: return "rearm_wrong_side"; + case RuleError::MaxWindowsOutOfRange: return "max_windows_out_of_range"; + case RuleError::WindowBoundsMissing: return "window_bounds_missing"; + case RuleError::WindowBoundsTooLarge: return "window_bounds_too_large"; + case RuleError::Unparsable: return "unparsable"; + case RuleError::DuplicateRule: return "duplicate_rule"; + } + return "unknown"; +} + +const char* toString(const RuleState s) { + switch (s) { + case RuleState::Inactive: return "inactive"; + case RuleState::WarmingUp: return "warming_up"; + case RuleState::Armed: return "armed"; + case RuleState::Pending: return "pending"; + case RuleState::Opening: return "opening"; + case RuleState::Blackout: return "blackout"; + case RuleState::Recovery: return "recovery"; + case RuleState::WaitingForRearm: return "waiting_for_rearm"; + } + return "unknown"; +} + +const char* toString(const RuleOutcome o) { + switch (o) { + case RuleOutcome::None: return "none"; + case RuleOutcome::NeverTrue: return "never_true"; + case RuleOutcome::Blocked: return "blocked"; + case RuleOutcome::Fired: return "fired"; + case RuleOutcome::Exhausted: return "exhausted"; + case RuleOutcome::Unsupported: return "unsupported"; + case RuleOutcome::InvalidConfig: return "invalid_config"; + } + return "unknown"; +} + +const char* toString(const RuleGate g) { + switch (g) { + case RuleGate::Ok: return "ok"; + case RuleGate::MetricUnavailable: return "metric_unavailable"; + case RuleGate::CountersNotShared: return "counters_not_shared"; + case RuleGate::NoDeepEngine: return "no_deep_engine"; + case RuleGate::WindowsUnsupported: return "windows_unsupported"; + } + return "unknown"; +} + +// ------------------------------------------------------------------- parsing + +RuleParseResult parseRuleExpression(const std::string& text) { + RuleParseResult out; + + const size_t op_at = text.find_first_of("<>"); + if (op_at == std::string::npos) { + out.error = RuleError::Unparsable; + out.detail = "expected '<' or '>' in \"" + text + "\""; + return out; + } + out.rule.op = text[op_at] == '<' ? Comparison::LessThan : Comparison::GreaterThan; + + const MetricParseResult metric = parseMetric(trim(text.substr(0, op_at))); + if (!metric.ok()) { + out.error = RuleError::BadMetric; + out.metric_error = metric.error; + out.detail = toString(metric.error); + return out; + } + out.rule.metric = metric.id; + + std::string rest = text.substr(op_at + 1); + // " for " is optional; without it the rule fires on the first + // fresh true reading. + int64_t sustained_ms = 0; + if (const size_t f = rest.find(" for "); f != std::string::npos) { + sustained_ms = parseDurationMs(rest.substr(f + 5)); + if (sustained_ms < 0) { + out.error = RuleError::Unparsable; + out.detail = "bad duration in \"" + text + "\""; + return out; + } + rest = rest.substr(0, f); + } + + if (!parseDouble(rest, &out.rule.threshold)) { + out.error = RuleError::Unparsable; + out.detail = "bad threshold in \"" + text + "\""; + return out; + } + + out.rule.timing.sustained_ms = sustained_ms; + // Plain "condition false" until an explicit hysteresis value is supplied. + out.rule.rearm_threshold = out.rule.threshold; + return out; +} + +RuleParseResult validateRule(const DeepWindowRule& rule) { + RuleParseResult out; + out.rule = rule; + + if (!std::isfinite(rule.threshold) || !std::isfinite(rule.rearm_threshold)) { + // NaN makes every comparison false, so the rule would sit armed forever + // while looking perfectly healthy. + out.error = RuleError::ThresholdNotFinite; + return out; + } + + // A rearm on the wrong side of the operator can never be reached, so the + // rule fires exactly once and then waits for a condition that cannot occur. + const bool rearm_ok = rule.op == Comparison::LessThan + ? rule.rearm_threshold >= rule.threshold + : rule.rearm_threshold <= rule.threshold; + if (!rearm_ok) { + out.error = RuleError::RearmWrongSide; + out.detail = std::string("rearm ") + toString(rule.op) + + " rule needs rearm on the other side of the threshold"; + return out; + } + + if (rule.max_windows < 1 || rule.max_windows > kMaxWindowsHardLimit) { + out.error = RuleError::MaxWindowsOutOfRange; + return out; + } + + if (rule.window.max_duration_ms <= 0 && rule.window.max_launches == 0) { + // A window with neither bound never closes on its own, which turns a + // bounded-cost feature into an always-on one. + out.error = RuleError::WindowBoundsMissing; + return out; + } + if (rule.window.max_duration_ms > kMaxWindowDurationMs || + rule.window.max_launches > kMaxWindowLaunches) { + out.error = RuleError::WindowBoundsTooLarge; + return out; + } + + if (const ConfigError e = validate(rule.timing); e != ConfigError::None) { + out.error = RuleError::BadTiming; + out.config_error = e; + out.detail = explain(rule.timing, e); + return out; + } + + return out; +} + +// ----------------------------------------------------------------- evaluator + +RuleEvaluator::Hooks RuleEvaluator::liveHooks() { + Hooks h; + h.request_open = [](void*, const DeepWindowSpec& spec) { + return DeepWindow::RequestOpenTagged(spec); + }; + h.window_active = [](void*) { return DeepWindow::Active(); }; + h.opens_completed = [](void*) { return DeepWindow::OpensCompleted(); }; + h.last_opened_token = [](void*) { return DeepWindow::LastOpenedToken(); }; + h.pending_open_token = [](void*) { return DeepWindow::PendingOpenToken(); }; + h.ctx = nullptr; + return h; +} + +RuleEvaluator::RuleEvaluator(DeepWindowRule rule, std::string rule_id, + const RuleCapabilities& caps, MetricSource* source, + Hooks hooks) + : rule_(std::move(rule)), + rule_id_(std::move(rule_id)), + source_(source), + hooks_(hooks) { + // Two independent gates, each with its own reason. A rule can be perfectly + // valid and still have nothing to watch, or nothing to arm. + RuleGate gate = RuleGate::Ok; + if (!caps.windows_supported) { + gate = RuleGate::WindowsUnsupported; + } else if (!caps.deep_engine_prepared) { + // An enum check on the configured engine is not enough: without this a + // rule spends its whole budget opening windows that arm nothing. + gate = RuleGate::NoDeepEngine; + } else if (rule_.metric.resolvesLazily() && caps.multi_module && + !caps.counters_shared) { + // The target ticks one registry and this evaluator reads another, so + // the counter can never be seen. Refusing here is the honest answer; + // the alternative is reporting a counter that is being ticked as + // Missing for the whole run. + gate = RuleGate::CountersNotShared; + } else if (caps.device_count > 0 && !rule_.metric.resolvesLazily() && + rule_.metric.shape() == MetricShape::Gauge && + rule_.metric.device_index >= caps.device_count) { + // Built-in metrics are decided eagerly - unlike a custom counter, the + // answer cannot change later. + gate = RuleGate::MetricUnavailable; + } + + if (gate != RuleGate::Ok) { + state_ = RuleState::Inactive; + terminal_ = RuleOutcome::Unsupported; + reason_ = toString(gate); + } +} + +DeepWindowSpec RuleEvaluator::specWithTrigger(const MetricSample& sample) const { + DeepWindowSpec spec = rule_.window; + spec.trigger.present = true; + spec.trigger.rule_id = rule_id_; + spec.trigger.metric = rule_.metric.canonical; + spec.trigger.op = toString(rule_.op); + spec.trigger.threshold = rule_.threshold; + spec.trigger.rearm_threshold = rule_.rearm_threshold; + spec.trigger.observed = sample.value; + spec.trigger.rate_window_ms = rule_.timing.rate_window_ms; + spec.trigger.sustained_ms = rule_.timing.sustained_ms; + spec.trigger.first_true_ns = first_true_observed_ns_.value_or(sample.observed_ns); + spec.trigger.fired_ns = sample.observed_ns; + return spec; +} + +bool RuleEvaluator::requestWindow(const MetricSample& sample) { + const OpenRequestResult result = + hooks_.request_open(hooks_.ctx, specWithTrigger(sample)); + + // Reaching here at all means the condition held long enough to act on. + // Never cleared: what it answers at shutdown is "did this rule ever get as + // far as asking", which decides whether `never_true` is an honest verdict. + // Set before the answer is known on purpose - an accepted request whose + // window is later dropped also asked and also got nothing. + open_was_attempted_ = true; + + // Accepted with no token is not a yes. Waiting on token 0 leaves the + // evaluator in Opening for the rest of the run, watching a request that + // was never queued - a silent stall rather than a reported fault. + if (result.status == OpenRequestStatus::Accepted && !result.accepted()) { + GFL_LOG_ERROR("[DeepWindowRule] window coordinator accepted a request " + "without a token; treating the rule as unsupported"); + terminal_ = RuleOutcome::Unsupported; + reason_ = toString(OpenRequestStatus::InvalidResult); + state_ = RuleState::Inactive; + ++state_sequence_; + return false; + } + + switch (result.status) { + case OpenRequestStatus::Accepted: + pending_token_ = result.token; + state_ = RuleState::Opening; + ++state_sequence_; + return true; + + case OpenRequestStatus::EngineUnavailable: + case OpenRequestStatus::InvalidResult: + // Permanent for this session. Retrying until shutdown would end in + // `never_true`, which says the condition was never met - the exact + // opposite of what happened. + terminal_ = RuleOutcome::Unsupported; + reason_ = toString(result.status); + state_ = RuleState::Inactive; + ++state_sequence_; + return false; + + case OpenRequestStatus::PreparationPending: + case OpenRequestStatus::Busy: + case OpenRequestStatus::Cooldown: + // Ordinary and temporary. Back to armed rather than holding + // Pending, which would fire the instant the refusal lifted on + // evidence gathered long before. + reason_ = toString(result.status); + toArmed(); + return false; + } + return false; +} + +bool RuleEvaluator::conditionHolds(const double value) const { + return rule_.op == Comparison::LessThan ? value < rule_.threshold + : value > rule_.threshold; +} + +bool RuleEvaluator::rearmHolds(const double value) const { + // One predicate, not "false or threshold". With rearm == threshold this is + // exactly "condition false", which is the no-hysteresis default. + return rule_.op == Comparison::LessThan ? value >= rule_.rearm_threshold + : value <= rule_.rearm_threshold; +} + +void RuleEvaluator::toArmed() { + state_ = RuleState::Armed; + first_true_observed_ns_.reset(); + ++state_sequence_; +} + +void RuleEvaluator::enterBlackout(const int64_t) { + state_ = RuleState::Blackout; + first_true_observed_ns_.reset(); + ++state_sequence_; +} + +void RuleEvaluator::poll(const int64_t now_ns) { + if (state_ == RuleState::Inactive) return; + + const bool active = hooks_.window_active(hooks_.ctx); + + // A window can open and close entirely between two beats - a launch-bounded + // one over a busy loop routinely does. The counter catches that; a boolean + // would not, and the samples taken during it would feed the rule as clean. + const uint64_t opens = hooks_.opens_completed != nullptr + ? hooks_.opens_completed(hooks_.ctx) : 0; + const bool missed_window = have_opens_ && opens != opens_seen_ && !active; + opens_seen_ = opens; + have_opens_ = true; + + // 1. Confirm a requested open before anything else. The window we caused + // also puts us into blackout, so checking blackout first would lose the + // only chance to count it. + if (state_ == RuleState::Opening) { + if (pending_token_ != 0 && + hooks_.last_opened_token(hooks_.ctx) == pending_token_) { + pending_token_ = 0; + ++windows_opened_; + if (windows_opened_ >= static_cast(rule_.max_windows)) { + // Marked at the open that reaches the limit, not later: a run + // that crashes during this window still explains itself. + terminal_ = RuleOutcome::Exhausted; + } + enterBlackout(now_ns); + window_was_active_ = true; + return; + } + else if (hooks_.pending_open_token(hooks_.ctx) != pending_token_) { + // No longer queued and never opened: the coordinator turned it down + // when it got round to it. No budget is consumed - the budget + // bounds what the rule COST, and this cost nothing. + // + // Checked against the queue rather than "is a window open yet", + // because an open is serviced on a later beat and treating "not + // yet" as "never" would abandon a window about to open. + pending_token_ = 0; + reason_ = "open_request_dropped"; + toArmed(); + } + } + + // 2. Any open window contaminates, whoever opened it. Contamination does + // not care who asked. + if (active) { + if (state_ != RuleState::Blackout) enterBlackout(now_ns); + window_was_active_ = true; + return; + } + + // 3. A window just closed - either one we watched, or one that came and + // went between beats. Blackout and recovery are distinct: blackout is + // "discard everything", recovery is "refill the clean epoch". Merging + // them would let contaminated samples prove the workload recovered. + if (window_was_active_ || missed_window) { + window_was_active_ = false; + source_->resetEpoch(now_ns); + if (terminal_ == RuleOutcome::Exhausted) { + state_ = RuleState::Inactive; + ++state_sequence_; + return; + } + state_ = RuleState::Recovery; + ++state_sequence_; + } + + const MetricSample sample = source_->poll(now_ns); + last_metric_state_ = sample.state; + + // Staleness is checked BEFORE the repeat filter, deliberately. A source + // that dies stops advancing its sequence, so a rule that only looked at new + // samples would never notice - it would sit in Pending on a reading nobody + // is taking any more, which is the exact failure the two-timestamp design + // exists to prevent. + if (sample.state != MetricState::Fresh) { + // Not evidence the condition stopped holding, but not evidence it + // continued either, so the run is broken rather than completed. + if (state_ == RuleState::Pending) { + first_true_observed_ns_.reset(); + state_ = RuleState::Armed; + ++state_sequence_; + } + return; + } + + // Only a NEW fresh reading is evidence. The evaluator runs 100-2000x faster + // than a metric publishes, so counting repeats would let a single reading + // satisfy any sustained_ms on its own. + const bool is_new = !have_sequence_ || sample.sequence != last_sequence_; + if (!is_new) return; + last_sequence_ = sample.sequence; + have_sequence_ = true; + + ++samples_seen_; + truncated_samples_ = sample.truncated_samples; + last_value_ = sample.value; + last_observed_ns_ = sample.observed_ns; + + switch (state_) { + case RuleState::WarmingUp: + case RuleState::Recovery: + // A full window of clean data has now arrived, since a Fresh sample + // implies it. + state_ = state_ == RuleState::Recovery ? RuleState::WaitingForRearm + : RuleState::Armed; + ++state_sequence_; + if (state_ == RuleState::Armed) break; + [[fallthrough]]; + + case RuleState::WaitingForRearm: + if (rearmHolds(sample.value)) { + toArmed(); + } + break; + + case RuleState::Armed: + if (conditionHolds(sample.value)) { + first_true_observed_ns_ = sample.observed_ns; + state_ = RuleState::Pending; + ++state_sequence_; + // sustained_ms == 0 fires on this same reading. + if (rule_.timing.sustained_ms == 0) requestWindow(sample); + } + break; + + case RuleState::Pending: { + if (!conditionHolds(sample.value)) { + toArmed(); + break; + } + if (!first_true_observed_ns_.has_value()) { + first_true_observed_ns_ = sample.observed_ns; + break; + } + // A span between two observations, not an accumulation - so one + // stale reading re-read many times can never satisfy it. + const int64_t held_ns = sample.observed_ns - *first_true_observed_ns_; + if (held_ns < rule_.timing.sustained_ms * 1000000) break; + + requestWindow(sample); + break; + } + + case RuleState::Opening: + case RuleState::Blackout: + case RuleState::Inactive: + break; + } +} + +bool RuleEvaluator::takeTerminalToEmit() { + if (terminal_emitted_) return false; + if (terminal_ != RuleOutcome::Exhausted && + terminal_ != RuleOutcome::Unsupported) { + return false; + } + terminal_emitted_ = true; + return true; +} + +RuleSummary RuleEvaluator::snapshot(const int64_t now_ns) const { + RuleSummary s; + s.rule_id = rule_id_; + s.state = state_; + s.outcome = terminal_; + s.samples_seen = samples_seen_; + s.windows_opened = windows_opened_; + s.last_value = last_value_; + s.last_observed_ns = last_observed_ns_; + s.last_metric_state = last_metric_state_; + s.truncated_samples = truncated_samples_; + if (source_ != nullptr) { + s.metric_quality_resets = source_->qualityResets(); + s.last_quality_reason = source_->lastQualityReason(); + } + s.reason = reason_; + s.state_sequence = state_sequence_; + s.emitted_ns = now_ns; + return s; +} + +RuleSummary RuleEvaluator::finish(const int64_t now_ns) { + RuleSummary s = snapshot(now_ns); + + // Precedence, highest first. Several can apply at once and the most + // specific has to win, or a rule that was refused outright would report as + // one that simply never matched. + if (terminal_ == RuleOutcome::InvalidConfig || + terminal_ == RuleOutcome::Unsupported) { + s.outcome = terminal_; + } else if (terminal_ == RuleOutcome::Exhausted) { + s.outcome = RuleOutcome::Exhausted; + } else if (windows_opened_ > 0) { + s.outcome = RuleOutcome::Fired; + } else if (open_was_attempted_) { + // The condition held long enough to ask, and every ask was turned + // down - by a manual window holding the coordinator, by a cooldown + // longer than the run had left, or by preparation that never + // completed. `never_true` would say the condition never occurred, + // which is the opposite, and would send the user to look at their + // threshold instead of at what was blocking the window. + s.outcome = RuleOutcome::Blocked; + if (s.reason.empty()) s.reason = "open_refused"; + } else { + s.outcome = RuleOutcome::NeverTrue; + if (s.reason.empty()) { + // Distinguish "watched it and it never happened" from "never had + // anything to watch"; both leave windows_opened at 0. + if (last_metric_state_ == MetricState::Missing) { + s.reason = "custom_metric_never_registered"; + } else if (samples_seen_ == 0) { + // Watched, but the source never produced a usable reading - + // a device that does not exist, or a sampler that never ran. + s.reason = "metric_source_never_reported"; + } else { + s.reason = "condition_never_held"; + } + } + } + + s.state_sequence = ++state_sequence_; + return s; +} + +RuleSummary RuleEvaluator::refused(std::string rule_id, const RuleOutcome outcome, + std::string reason, const int64_t now_ns) { + RuleSummary s; + s.rule_id = std::move(rule_id); + // The terminal outcomes leave the evaluator standing in none of the running + // states, which is what `inactive` is for. + s.state = RuleState::Inactive; + s.outcome = outcome; + s.reason = std::move(reason); + s.emitted_ns = now_ns; + s.state_sequence = 1; + return s; +} + +} // namespace gpufl::detail diff --git a/include/gpufl/core/deep_window_rule.hpp b/include/gpufl/core/deep_window_rule.hpp new file mode 100644 index 0000000..3332930 --- /dev/null +++ b/include/gpufl/core/deep_window_rule.hpp @@ -0,0 +1,339 @@ +#pragma once + +#include +#include +#include +#include + +#include "gpufl/core/deep_window.hpp" +#include "gpufl/core/metric_id.hpp" +#include "gpufl/core/metric_registry.hpp" + +namespace gpufl::detail { + +enum class Comparison { LessThan, GreaterThan }; + +const char* toString(Comparison op); + +/** + * @brief A condition that opens a deep window. + * + * The predicate form already exists - it is the `if` a caller writes around + * gpufl::deepWindow(). A rule earns its place on two counts only: metrics + * gpufl has and the application does not, and thresholds that live in + * configuration rather than in code. + */ +struct DeepWindowRule { + MetricId metric; + Comparison op = Comparison::LessThan; + double threshold = 0.0; + /** + * Value the metric must recover past before the rule may fire again. + * + * Equal to `threshold` by default, which degenerates to plain "condition + * false" - the no-hysteresis case. A rule whose rearm sits on the wrong + * side of the operator can never rearm, so the direction is validated. + */ + double rearm_threshold = 0.0; + MetricWindowConfig timing; + int max_windows = 3; + DeepWindowSpec window; +}; + +/** + * @brief Why a rule was refused, or could not be honoured. + * + * Property 5 of the design: every reason a rule did not fire is recorded. A + * rejected rule that leaves no trace is indistinguishable from one that was + * simply never true, and the two call for opposite responses from the user. + */ +enum class RuleError { + None, + /// The metric name itself was refused; carries the parse reason. + BadMetric, + /// The timing combination cannot produce evidence; carries the config reason. + BadTiming, + ThresholdNotFinite, ///< NaN makes every comparison false, silently + RearmWrongSide, ///< LessThan needs rearm >= threshold, and vice versa + MaxWindowsOutOfRange, + WindowBoundsMissing, ///< a window with no bound at all never closes + WindowBoundsTooLarge, + Unparsable, ///< the --deep-when expression did not parse + DuplicateRule, ///< MVP allows exactly one +}; + +const char* toString(RuleError e); + +/** @brief Parse ` for `, e.g. "x<1000 for 2s". */ +struct RuleParseResult { + DeepWindowRule rule; + RuleError error = RuleError::None; + /// Set when error is BadMetric / BadTiming, so the reason is not lost. + MetricParseError metric_error = MetricParseError::None; + ConfigError config_error = ConfigError::None; + std::string detail; + + bool ok() const { return error == RuleError::None; } +}; + +/** + * @brief Parse the expression half of the config. + * + * Only the metric, operator, threshold and sustained duration come from the + * expression; everything else is a separate option. Splitting them keeps each + * refusal able to name the field it is about. + */ +RuleParseResult parseRuleExpression(const std::string& text); + +/** @brief Check a fully assembled rule. Returns RuleError::None when usable. */ +RuleParseResult validateRule(const DeepWindowRule& rule); + +// --------------------------------------------------------------------------- + +/** + * @brief Where the evaluator is standing. + * + * Never a verdict on the session - see RuleOutcome for that. Collapsing the + * two would make `armed` look like a conclusion, and force every reader to + * guess which kind of answer it was holding. + */ +enum class RuleState { + Inactive, ///< terminal: invalid, unsupported, or budget spent + WarmingUp, + Armed, + Pending, ///< condition true, waiting out sustained_ms + Opening, ///< an open was requested; waiting to see it happen + Blackout, ///< a window is open; everything observed is contaminated + Recovery, ///< window closed; refilling the clean epoch + WaitingForRearm, +}; + +const char* toString(RuleState s); + +/** @brief What the session concluded about the rule. */ +enum class RuleOutcome { + None, + NeverTrue, + /** + * The condition held and a window was asked for, but none ever opened. + * + * Separate from NeverTrue because the two send the user to opposite + * places: never_true says look at your threshold, blocked says look at + * what is holding the window - a manual window, a cooldown longer than + * the run, an engine still preparing. Both leave windows_opened at 0, so + * one field cannot carry them. + */ + Blocked, + Fired, + Exhausted, + Unsupported, + InvalidConfig, +}; + +const char* toString(RuleOutcome o); + +/** @brief Which gate refused a rule that was otherwise well-formed. */ +enum class RuleGate { + Ok, + /// The metric cannot be produced: no such device, sampler absent, or the + /// base mode does not generate it. + MetricUnavailable, + /// Custom counters are not shared across modules, so a counter the target + /// ticks is invisible to the evaluator that would read it. + CountersNotShared, + /// Windows are meaningless without an engine to arm inside them. An enum + /// check on the configured engine is necessary but not sufficient: without + /// this a rule burns its budget opening windows that arm nothing. + NoDeepEngine, + /// The base mode does not support bounded windows at all. + WindowsUnsupported, +}; + +const char* toString(RuleGate g); + +/** + * @brief Everything the session learned about one rule. + * + * `state` and `outcome` are separate on purpose. A crash mid-run leaves a + * meaningful state and no outcome; a clean run with a rule that never matched + * leaves `armed` and `never_true`. One field cannot carry both. + */ +struct RuleSummary { + std::string rule_id; + RuleState state = RuleState::Inactive; + RuleOutcome outcome = RuleOutcome::None; + uint64_t samples_seen = 0; + uint32_t windows_opened = 0; + /// Absent, not NaN: there is a real "no value yet", and NaN does not + /// survive the JSON and DB boundaries cleanly. + std::optional last_value; + std::optional last_observed_ns; + MetricState last_metric_state = MetricState::Missing; + /** + * Completed kernels discarded across the run. + * + * Reported so a conclusion drawn from a partial percentile is not + * presented as one drawn from all of it. Counted rather than used to + * suppress the metric: truncation starts at launch rates far below what + * the workloads this feature targets actually reach. + */ + uint64_t truncated_samples = 0; + /** + * Rate windows THIS rule's metric discarded for failed reads, and why the + * last one was. Per rule, never session-wide: copying a session's whole + * tally onto every rule would make an unrelated counter's errors look + * like they broke this rule. + */ + uint64_t metric_quality_resets = 0; + std::string last_quality_reason; + std::string reason; + uint64_t state_sequence = 0; + int64_t emitted_ns = 0; +}; + +/** + * @brief The gates a rule must pass before it can open anything. + * + * Supplied by the caller rather than queried here so the evaluator stays + * testable without a GPU, and so the two gates are visibly independent - each + * has its own recorded failure reason. + */ +struct RuleCapabilities { + bool windows_supported = true; + bool deep_engine_prepared = true; + bool counters_shared = true; + /// More than one module holds a copy of gpufl - injection. Only then does + /// an unshared counter registry actually break anything. + bool multi_module = false; + int device_count = 1; +}; + +/** + * @brief Drives one rule from metric samples to a window request. + * + * Runs on the collector beat (~1ms) alongside serviceDeepWindow. + */ +class RuleEvaluator { +public: + /** @brief Injected so tests drive the coordinator without a GPU. */ + struct Hooks { + /// Ask for a window. The REASON matters, not just the refusal: + /// cooldown and busy are temporary and worth retrying, a failed + /// engine preparation is permanent and retrying it until shutdown + /// makes the summary claim the condition never held. + OpenRequestResult (*request_open)(void* ctx, const DeepWindowSpec&) = nullptr; + /// Is a window - any window, whoever opened it - currently open? + bool (*window_active)(void* ctx) = nullptr; + /// Monotonic count of windows that have opened. A short launch-bounded + /// window can open AND close between two beats, so polling a boolean + /// would miss it entirely and its contaminated samples would feed the + /// rule as if profiling had never been on. + uint64_t (*opens_completed)(void* ctx) = nullptr; + /// Token of the most recent open, so a manual window is not mistaken + /// for the one this rule asked for and charged to its budget. + uint64_t (*last_opened_token)(void* ctx) = nullptr; + /// Token of the request still queued, or 0. Without it "has not opened + /// yet" and "will never open" are indistinguishable, and the evaluator + /// abandons a window that was about to open. + uint64_t (*pending_open_token)(void* ctx) = nullptr; + void* ctx = nullptr; + }; + + /** @brief Hooks wired to the real DeepWindow coordinator. */ + static Hooks liveHooks(); + + RuleEvaluator(DeepWindowRule rule, std::string rule_id, + const RuleCapabilities& caps, MetricSource* source, Hooks hooks); + + /** @brief Advance the machine. Call every collector beat. */ + void poll(int64_t now_ns); + + /** @brief Finalise at shutdown and return what the session concluded. */ + RuleSummary finish(int64_t now_ns); + + /** @brief Current summary without concluding, for a mid-run emit. */ + RuleSummary snapshot(int64_t now_ns) const; + + /** + * @brief True once, when a terminal outcome is first reached. + * + * `exhausted` and `unsupported` are conclusions the run can reach long + * before it ends. Holding them until shutdown means a process that crashes + * afterwards explains nothing, and the session looks like one where the + * rule simply never fired. Reported here so the caller can write it at the + * transition; the shutdown summary still follows, with a higher sequence. + */ + bool takeTerminalToEmit(); + + RuleState state() const { return state_; } + uint32_t windowsOpened() const { return windows_opened_; } + + /** + * @brief Mark a rule that was refused before it could run. + * + * Kept separate from the constructor because an invalid rule must never + * fail init(): configuration is parsed during init, and a hard failure + * there would leave no session and no telemetry writer - nowhere to record + * the very outcome that has to be reported. + */ + static RuleSummary refused(std::string rule_id, RuleOutcome outcome, + std::string reason, int64_t now_ns); + +private: + /** + * The window spec plus the comparison that produced it. + * + * Built at the request, not at the close: by the time the window closes the + * reading that caused it is long gone, and a bare observed value with no + * threshold beside it stops being readable the first time the rule changes. + */ + DeepWindowSpec specWithTrigger(const MetricSample& sample) const; + /** + * Ask for a window and act on the answer. + * + * One place, so the two call sites - sustained_ms == 0 and the + * held-long-enough path - cannot reach different conclusions about what a + * refusal means. Returns true only when a window was actually requested. + */ + bool requestWindow(const MetricSample& sample); + bool conditionHolds(double value) const; + bool rearmHolds(double value) const; + void enterBlackout(int64_t now_ns); + void toArmed(); + + DeepWindowRule rule_; + std::string rule_id_; + MetricSource* source_ = nullptr; + Hooks hooks_; + + RuleState state_ = RuleState::WarmingUp; + RuleOutcome terminal_ = RuleOutcome::None; + std::string reason_; + + uint64_t samples_seen_ = 0; + uint64_t last_sequence_ = 0; + bool have_sequence_ = false; + uint32_t windows_opened_ = 0; + uint64_t state_sequence_ = 0; + + /// Start of the current unbroken run of true readings. Compared against a + /// later sample's timestamp, so sustained_ms is a span between two + /// observations rather than an accumulation that one stale reading could + /// satisfy on its own. + std::optional first_true_observed_ns_; + std::optional last_value_; + std::optional last_observed_ns_; + MetricState last_metric_state_ = MetricState::Missing; + uint64_t truncated_samples_ = 0; + + /// Whether this rule ever got as far as asking for a window. Decides + /// `blocked` against `never_true` at shutdown; never cleared. + bool open_was_attempted_ = false; + uint64_t pending_token_ = 0; + bool window_was_active_ = false; + uint64_t opens_seen_ = 0; + bool have_opens_ = false; + bool terminal_emitted_ = false; +}; + +} // namespace gpufl::detail diff --git a/include/gpufl/core/deep_window_rules.cpp b/include/gpufl/core/deep_window_rules.cpp new file mode 100644 index 0000000..0bc8627 --- /dev/null +++ b/include/gpufl/core/deep_window_rules.cpp @@ -0,0 +1,510 @@ +#include "gpufl/core/deep_window_rules.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include "gpufl/core/common.hpp" +#include "gpufl/core/counter_provider.hpp" +#include "gpufl/core/debug_logger.hpp" +#include "gpufl/core/deep_window_rule.hpp" +#include "gpufl/core/env_vars.hpp" +#include "gpufl/core/events.hpp" +#include "gpufl/core/logger/logger.hpp" +#include "gpufl/core/metric_registry.hpp" +#include "gpufl/core/nvtx_counters.hpp" +#include "gpufl/core/model/deep_window_model.hpp" +#include "gpufl.hpp" +#include "gpufl/core/monitor.hpp" +#include "gpufl/core/monitor_backend.hpp" +#include "gpufl/core/runtime.hpp" + +namespace gpufl::detail { +namespace { + +std::mutex g_mu; +bool g_installed = false; +bool g_finished = false; +std::string g_expression; +std::string g_rule_id; + +/** + * Process-lifetime feeds, never destroyed. + * + * The launch callback writes here without taking any lock, so the object must + * not be able to disappear underneath it. Same reasoning as the counter slots: + * a lifetime that ends is a race the hot path would have to pay to defend + * against. Contents are cleared on install instead. + */ +MetricFeeds& Feeds() { + static MetricFeeds feeds; + return feeds; +} + +std::unique_ptr g_source; +std::unique_ptr g_eval; +// Set when the rule was refused before it could run, so Finish() still has +// something to report. Holding the summary rather than an error string keeps +// the refused and the ran paths on one shape. +std::unique_ptr g_refused; +bool g_refused_emitted = false; + +// Read by the launch callback on every launch, so it must not take the lock. +std::atomic g_wants_launch_feed{false}; +std::atomic g_wants_duration_feed{false}; +// Mirrors g_installed without the mutex, for the feed entry points. +std::atomic g_installed_relaxed{false}; + +const char* EnvOrNull(const char* name) { + const char* v = std::getenv(name); + return (v && v[0] != '\0') ? v : nullptr; +} + +/** + * Integer from env, or the fallback when unset. + * + * A malformed value is a HARD failure, reported through @p bad rather than + * quietly replaced by the default. Substituting silently means a typo in a + * threshold or a budget still opens real windows, under settings the user never + * asked for and cannot see. + */ +int64_t EnvIntOr(const char* name, const int64_t fallback, std::string* bad) { + const char* v = EnvOrNull(name); + if (!v) return fallback; + char* end = nullptr; + errno = 0; + const long long n = std::strtoll(v, &end, 10); + // ERANGE too: strtoll saturates at LLONG_MAX, and a saturated value then + // overflows the derived-default arithmetic below rather than being caught. + if (end == v || *end != 0 || errno == ERANGE) { + if (bad->empty()) *bad = std::string(name) + "='" + v + "' is not an integer"; + return fallback; + } + return n; +} + +bool EnvDoubleOr(const char* name, double* out, std::string* bad) { + const char* v = EnvOrNull(name); + if (!v) return false; + char* end = nullptr; + errno = 0; + const double d = std::strtod(v, &end); + if (end == v || *end != 0 || errno == ERANGE) { + if (bad->empty()) *bad = std::string(name) + "='" + v + "' is not a number"; + return false; + } + *out = d; + return true; +} + +/** + * Short, stable id for a rule. + * + * Hashed over the CANONICAL form, so two spellings of one rule do not produce + * two ids. An invalid rule has no canonical form, so it hashes its normalised + * raw input instead - a rejected rule still needs an id to be reported under. + */ +std::string RuleId(const std::string& canonical_input) { + uint64_t h = 1469598103934665603ull; // FNV-1a + for (const unsigned char c : canonical_input) { + h ^= c; + h *= 1099511628211ull; + } + std::ostringstream oss; + oss << std::hex << h; + return oss.str().substr(0, 12); +} + +/** Upper bound accepted from config; the rule validator enforces the same. */ +constexpr int64_t kMaxWindowsConfigurable = 64; + +/** rate_window + sustained + bucket + 1s, or 0 with @p bad set on overflow. */ +int64_t CheckedStaleDefault(const MetricWindowConfig& t, std::string* bad) { + constexpr int64_t kMax = INT64_MAX; // not numeric_limits: windows.h defines max() + constexpr int64_t kSlack = 1000; + int64_t total = t.rate_window_ms; + for (const int64_t term : {t.sustained_ms, t.bucketIntervalMs(), kSlack}) { + if (term < 0 || total > kMax - term) { + if (bad->empty()) { + *bad = "rate window and sustained are too large to derive a " + "stale-after default; set " + + std::string(env::kDeepStaleAfterMs) + " explicitly"; + } + return 0; + } + total += term; + } + return total; +} + +std::string CanonicalConfig(const DeepWindowRule& r) { + std::ostringstream oss; + oss << "v1|" << r.metric.canonical << '|' << toString(r.op) << '|' + << r.threshold << '|' << r.rearm_threshold << '|' + << r.timing.rate_window_ms << '|' << r.timing.sustained_ms << '|' + << r.timing.stale_after_ms << '|' << r.max_windows << '|' + << r.window.max_duration_ms << '|' << r.window.max_launches; + return oss.str(); +} + +RuleCapabilities QueryCapabilities() { + RuleCapabilities caps; + // A Monitor-only run has no engine to arm inside a window at all. + // The RESOLVED engine, not the requested one: a request can be overridden + // at init, and gating on what was asked for would answer about a run that + // is not the one happening. + caps.windows_supported = + Monitor::ResolvedProfilingEngine() != ProfilingEngine::Monitor; + + IMonitorBackend* backend = Monitor::GetBackend(); + // Installation can run before Windows injection receives CONTEXT_CREATED. + // Pending is acceptable here; the queued open checks actual preparation + // after the first CUDA context exists. A completed preparation failure is + // rejected now instead of leaving a rule that can only open empty windows. + caps.deep_engine_prepared = + backend != nullptr && + (backend->DeepEnginesPrepared() || + backend->DeepEnginePreparationPending()); + + caps.counters_shared = CounterProvider::isShared(); + // Under injection the target and this evaluator are separate modules, which + // is the only situation where an unshared registry actually breaks a rule. + caps.multi_module = EnvOrNull(env::kCudaInjection64Path) != nullptr; + + // 0 = unknown. The device list is not enumerated until the sampler takes + // its first measurement, which is after init(), so a rule naming a device + // cannot be refused eagerly without guessing. Guessing 1 would refuse a + // valid gpu[3] rule on a four-GPU host - worse than deciding late. A device + // that never reports is named in the summary instead. + caps.device_count = 0; + return caps; +} + +void RefuseLocked(const RuleOutcome outcome, const std::string& reason) { + g_refused = std::make_unique(RuleEvaluator::refused( + g_rule_id, outcome, reason, detail::GetTimestampNs())); + g_installed = true; + GFL_LOG_ERROR("[DeepWindowRule] ", env::kDeepWhen, "='", g_expression, + "' disabled: ", reason, + ". Profiling continues; only the trigger is off."); +} + +/** + * Write one summary row. + * + * Shared by the terminal-transition emit and the shutdown emit so the two + * cannot describe the same rule differently. The backend upsert accepts only a + * strictly greater state_sequence, so the shutdown row - which always carries a + * higher one - wins, and a redelivery of either is a no-op. + */ +void EmitSummary(const RuleSummary& summary, const std::string& expression) { + const Runtime* rt = runtime(); + if (!rt || !rt->logger) { + GFL_LOG_ERROR("[DeepWindowRule] no logger; summary lost: ", + toString(summary.outcome), " ", summary.reason); + return; + } + + DeepWindowRuleSummaryEvent ev; + ev.pid = detail::GetPid(); + ev.app = rt->app_name; + ev.session_id = rt->session_id; + ev.rule_id = summary.rule_id; + ev.expression = expression; + ev.state = toString(summary.state); + ev.outcome = toString(summary.outcome); + ev.reason = summary.reason; + ev.metric_state = toString(summary.last_metric_state); + ev.samples_seen = summary.samples_seen; + ev.windows_opened = summary.windows_opened; + ev.truncated_samples = summary.truncated_samples; + ev.metric_quality_resets = summary.metric_quality_resets; + ev.last_quality_reason = summary.last_quality_reason; + ev.has_last_value = summary.last_value.has_value(); + if (ev.has_last_value) { + ev.last_value = *summary.last_value; + ev.last_observed_ns = summary.last_observed_ns.value_or(0); + } + ev.state_sequence = summary.state_sequence; + ev.emitted_ns = summary.emitted_ns; + + rt->logger->write(model::DeepWindowRuleSummaryModel(ev)); + GFL_LOG_DEBUG("[DeepWindowRule] summary id=", ev.rule_id, + " outcome=", ev.outcome, " windows=", ev.windows_opened, + " seq=", ev.state_sequence, " reason=", ev.reason); +} + +/** + * Drop the session so a later init() installs cleanly. + * + * An embedded host can shutdown() and init() again in one process, and a + * session left claimed makes the second run's rule look like a duplicate - + * that run then has no trigger at all and nothing says why. + */ +void ReleaseSession() { + std::lock_guard lk(g_mu); + g_installed = false; + g_finished = false; + g_eval.reset(); + g_source.reset(); + g_refused.reset(); + g_refused_emitted = false; + g_expression.clear(); + g_rule_id.clear(); +} + +} // namespace + +void DeepWindowRules::InstallFromEnv() { + const char* expr = EnvOrNull(env::kDeepWhen); + if (expr == nullptr) return; + + std::lock_guard lk(g_mu); + if (g_installed) { + // One rule for the MVP. Rejected loudly rather than silently ignored: + // a second rule that quietly does nothing is worse than an error. + GFL_LOG_ERROR("[DeepWindowRule] a rule is already installed; ignoring '", + expr, "'"); + return; + } + + g_expression = expr; + g_rule_id = RuleId("v1|raw|" + g_expression); + + RuleParseResult parsed = parseRuleExpression(g_expression); + if (!parsed.ok()) { + RefuseLocked(RuleOutcome::InvalidConfig, + parsed.detail.empty() ? toString(parsed.error) : parsed.detail); + return; + } + + std::string bad_env; + DeepWindowRule rule = parsed.rule; + rule.timing.rate_window_ms = EnvIntOr(env::kDeepRateWindowMs, 1000, &bad_env); + // Derived from the other two so the out-of-the-box combination is one that + // CAN fire, rather than one the validator then rejects. Summed with checks: + // the inputs are attacker- or typo-supplied, and an overflow here would + // produce a negative default that then reads as a different error entirely. + const int64_t derived_stale = CheckedStaleDefault(rule.timing, &bad_env); + rule.timing.stale_after_ms = + EnvIntOr(env::kDeepStaleAfterMs, derived_stale, &bad_env); + double rearm = 0.0; + if (EnvDoubleOr(env::kDeepRearmAt, &rearm, &bad_env)) rule.rearm_threshold = rearm; + // Range-checked BEFORE narrowing. 4294967297 fits an int64 and survives + // the ERANGE check, then narrows to 1 - a value the validator happily + // accepts, so the run silently uses a budget nobody asked for. + const int64_t max_windows = EnvIntOr(env::kDeepMaxWindows, 3, &bad_env); + if (max_windows < 1 || max_windows > kMaxWindowsConfigurable) { + if (bad_env.empty()) { + bad_env = std::string(env::kDeepMaxWindows) + "=" + + std::to_string(max_windows) + " is out of range"; + } + } + rule.max_windows = static_cast( + max_windows < 1 || max_windows > kMaxWindowsConfigurable ? 1 : max_windows); + + rule.window.max_duration_ms = EnvIntOr(env::kDeepWindowMs, 0, &bad_env); + rule.window.max_launches = + static_cast(EnvIntOr(env::kDeepWindowMaxLaunches, 0, &bad_env)); + rule.window.cooldown_ms = EnvIntOr(env::kDeepWindowCooldownMs, 0, &bad_env); + rule.window.name = "deep_window"; + + if (!bad_env.empty()) { + // Fail closed. A malformed number silently replaced by a default opens + // real windows under settings nobody chose. + RefuseLocked(RuleOutcome::InvalidConfig, bad_env); + return; + } + + const RuleParseResult checked = validateRule(rule); + if (!checked.ok()) { + RefuseLocked(RuleOutcome::InvalidConfig, + checked.detail.empty() ? toString(checked.error) + : checked.detail); + return; + } + + g_rule_id = RuleId(CanonicalConfig(rule)); + // Cleared rather than reallocated: the feeds outlive every session so the + // lock-free launch path never has to check whether they still exist. + Feeds().resetForTesting(); + Feeds().seedStartup(detail::GetTimestampNs()); + g_source = std::make_unique(rule.metric, rule.timing, + &Feeds(), + ActiveCounterProvider()); + g_eval = std::make_unique(rule, g_rule_id, QueryCapabilities(), + g_source.get(), + RuleEvaluator::liveHooks()); + g_installed = true; + // The launch feed costs an atomic per launch, so only a rule that reads it + // turns it on. + g_wants_launch_feed.store(rule.metric.kind == MetricKind::KernelLaunchRate, + std::memory_order_release); + g_wants_duration_feed.store(rule.metric.kind == MetricKind::RecentKernelMs, + std::memory_order_release); + g_installed_relaxed.store(true, std::memory_order_release); + + GFL_LOG_DEBUG("[DeepWindowRule] installed id=", g_rule_id, " '", g_expression, + "' state=", toString(g_eval->state())); +} + +bool DeepWindowRules::Installed() { + std::lock_guard lk(g_mu); + return g_installed; +} + +bool DeepWindowRules::WantsLaunchFeed() { + return g_wants_launch_feed.load(std::memory_order_acquire); +} + +void DeepWindowRules::NoteKernelLaunch(const int64_t ts_ns) { + // No lock at all: one atomic gate, then three relaxed atomics. This runs on + // the application's launch path, and taking g_mu here would queue it behind + // the collector's evaluation - changing the launch rate being measured. + if (!WantsLaunchFeed()) return; + Feeds().noteKernelLaunch(ts_ns); +} + +void DeepWindowRules::NoteKernelDuration(const int64_t ts_ns, const double ms) { + // Fed from activity processing, not from the per-launch path, so the + // feed's own lock is acceptable here. + if (!g_wants_duration_feed.load(std::memory_order_acquire)) return; + Feeds().noteKernelDuration(ts_ns, ms); +} + +void DeepWindowRules::NoteDeviceSample(const DeviceSample& sample, + const int64_t ts_ns) { + if (!g_installed_relaxed.load(std::memory_order_acquire)) return; + Feeds().noteDeviceSample(sample, ts_ns); +} + +void DeepWindowRules::Service() { + RuleSummary terminal; + std::string expression; + { + std::lock_guard lk(g_mu); + if (g_finished) return; + + // A rule refused before it could run reports as soon as there is a + // logger to report to, rather than waiting for a shutdown that may + // never come. + if (g_refused && !g_refused_emitted) { + g_refused_emitted = true; + terminal = *g_refused; + expression = g_expression; + } else if (g_eval) { + const int64_t now = detail::GetTimestampNs(); + g_eval->poll(now); + // Emitted at the transition, not at shutdown: a run that crashes + // after spending its budget would otherwise look like one whose + // rule simply never fired. + if (!g_eval->takeTerminalToEmit()) return; + terminal = g_eval->snapshot(now); + expression = g_expression; + } else { + return; + } + } + // Outside the lock: the logger write can block, and the launch path must + // never queue behind it. + EmitSummary(terminal, expression); +} + +void DeepWindowRules::EmitCounterQuality() { + // Separate from Finish() because this event exists WITHOUT a rule: NVTX + // counters flow through the bridge whether or not anything watches them, + // and a refused registration with no rule configured still needs a place + // to be reported. + NvtxCounterBridge::QualitySnapshot snap = + NvtxCounterBridge::instance().takeSessionSnapshot(); + + uint64_t discarded = 0; + { + std::lock_guard lk(g_mu); + if (g_source) discarded = g_source->qualityResets(); + } + + // Silent when this SESSION had nothing to say. Gating on trackedCount() + // was wrong: the table is process-lifetime, so one registration in an + // earlier embedded session would emit an all-zero row for every session + // after it - and an all-zero row with no denominator cannot tell "clean" + // from "nothing was watched". samples_observed is session-scoped, so this + // gate is too. + if (!snap.any() && discarded == 0 && snap.samples_observed == 0) { + return; + } + + const Runtime* rt = runtime(); + if (!rt || !rt->logger) return; + + CounterDataQualitySummaryEvent ev; + ev.pid = detail::GetPid(); + ev.app = rt->app_name; + ev.session_id = rt->session_id; + ev.tracked_counters = NvtxCounterBridge::instance().trackedCount(); + ev.samples_observed = snap.samples_observed; + ev.registration_rejected = snap.registration_rejected; + ev.unknown_id_samples = snap.unknown_id_samples; + ev.unavailable_samples = snap.unavailable_samples; + ev.negative_delta_samples = snap.negative_delta_samples; + ev.rate_windows_discarded = discarded; + ev.emitted_ns = detail::GetTimestampNs(); + rt->logger->write(model::CounterDataQualitySummaryModel(ev)); + GFL_LOG_DEBUG("[NvtxCounters] quality summary rejected=", + ev.registration_rejected, " unknown=", ev.unknown_id_samples, + " unavailable=", ev.unavailable_samples, + " negative=", ev.negative_delta_samples, + " discarded=", ev.rate_windows_discarded); +} + +void DeepWindowRules::Finish() { + RuleSummary summary; + std::string expression; + { + std::lock_guard lk(g_mu); + if (!g_installed || g_finished) return; + g_finished = true; + // Set before producing the summary, so a collector beat that is already + // inside Service() cannot advance the evaluator past what is reported. + g_wants_launch_feed.store(false, std::memory_order_release); + g_wants_duration_feed.store(false, std::memory_order_release); + g_installed_relaxed.store(false, std::memory_order_release); + if (g_refused) { + summary = *g_refused; + } else if (g_eval) { + summary = g_eval->finish(detail::GetTimestampNs()); + } else { + return; + } + expression = g_expression; + } + + // Released BEFORE the write is attempted. Tying the release to a successful + // write meant that a shutdown with no logger left the session claimed + // forever, and a host that called init() again got no rule at all - a + // reporting failure silently turning into a functional one. + ReleaseSession(); + EmitSummary(summary, expression); +} + +void DeepWindowRules::ResetForTesting() { + std::lock_guard lk(g_mu); + g_installed = false; + g_finished = false; + g_expression.clear(); + g_rule_id.clear(); + g_eval.reset(); + g_source.reset(); + Feeds().resetForTesting(); + g_refused.reset(); + g_refused_emitted = false; + g_wants_launch_feed.store(false, std::memory_order_release); + g_wants_duration_feed.store(false, std::memory_order_release); + g_installed_relaxed.store(false, std::memory_order_release); +} + +} // namespace gpufl::detail diff --git a/include/gpufl/core/deep_window_rules.hpp b/include/gpufl/core/deep_window_rules.hpp new file mode 100644 index 0000000..ed3b092 --- /dev/null +++ b/include/gpufl/core/deep_window_rules.hpp @@ -0,0 +1,76 @@ +#pragma once + +#include +#include + +namespace gpufl { +struct DeviceSample; +} + +namespace gpufl::detail { + +/** + * @brief Owns the one conditional rule for the session. + * + * A facade rather than an object the caller holds, because the feed points sit + * in the launch callback and the sampler while the evaluation sits on the + * collector - three call sites that have no way to pass an instance between + * them. + * + * One rule for the MVP. Multi-rule arbitration is deferred rather than guessed: + * two rules that both want a window need a policy for who wins, and inventing + * one before anybody has asked is how it ends up wrong. + */ +class DeepWindowRules { +public: + /** + * @brief Read GPUFL_DEEP_WHEN and friends. Called once from init(). + * + * NEVER fails init(), whatever the configuration says. Config is parsed + * during init, so a hard failure here would leave no session and no + * telemetry writer - nowhere to record the very outcome a rejected rule has + * to report. An invalid rule disables the trigger and nothing else; the + * profiling session runs normally. + */ + static void InstallFromEnv(); + + /** @brief Advance the rule. Called on the collector beat. */ + static void Service(); + + /** @brief A kernel launch was observed at the host launch API. */ + static void NoteKernelLaunch(int64_t ts_ns); + /** @brief A completed kernel's duration, for recent_kernel_ms. */ + static void NoteKernelDuration(int64_t ts_ns, double duration_ms); + /** @brief A successful device measurement. Never called on a poll. */ + static void NoteDeviceSample(const DeviceSample& sample, int64_t ts_ns); + + /** + * @brief Write the rule summary. Called once during shutdown. + * + * Emitted even when the rule never fired: "log once" is invisible in the + * UI, and no record at all is indistinguishable from a rule that was simply + * never true. + */ + static void Finish(); + + /** + * @brief Write the session's counter data-quality summary, if any. + * + * Called at shutdown beside Finish(), but independent of it: the event + * reports what the APPLICATION sent (refused registrations, failed reads, + * negative deltas) and exists whether or not a rule was configured. + * Advances the bridge's session baseline, so an embedded re-init reports + * only its own session's problems. + */ + static void EmitCounterQuality(); + + /** @brief True when a rule is installed - valid or refused. */ + static bool Installed(); + + /** @brief Cheap enough for the launch callback to ask every launch. */ + static bool WantsLaunchFeed(); + + static void ResetForTesting(); +}; + +} // namespace gpufl::detail diff --git a/include/gpufl/core/env_vars.hpp b/include/gpufl/core/env_vars.hpp index 9978b96..3a22db3 100644 --- a/include/gpufl/core/env_vars.hpp +++ b/include/gpufl/core/env_vars.hpp @@ -148,6 +148,24 @@ constexpr const char* kDeepWindowCooldownMs = "GPUFL_DEEP_WINDOW_COOLDOWN_MS"; // it from --deep-after. Unset = no scheduled window. constexpr const char* kDeepAfterMs = "GPUFL_DEEP_AFTER_MS"; +// ── Conditional deep window ───────────────────────────────────────────────── +// Open a window when a metric crosses a threshold, e.g. +// "custom.token_rate<1000 for 2s". Separate options rather than one grammar: +// the rule carries more fields than an expression can hold readably, and each +// one needs its own validation message when it is refused. +constexpr const char* kDeepWhen = "GPUFL_DEEP_WHEN"; +// Window the rate is measured over. Also sizes the warm-up. +constexpr const char* kDeepRateWindowMs = "GPUFL_DEEP_RATE_WINDOW_MS"; +// How long the source may go quiet before its readings stop counting as +// evidence. Validated against rate window + sustained, since a value shorter +// than those can never fire. +constexpr const char* kDeepStaleAfterMs = "GPUFL_DEEP_STALE_AFTER_MS"; +// Hysteresis: the value the metric must recover past before the rule may fire +// again. Defaults to the trigger threshold, which is plain "condition false". +constexpr const char* kDeepRearmAt = "GPUFL_DEEP_REARM_AT"; +// How many windows one rule may open in a session. +constexpr const char* kDeepMaxWindows = "GPUFL_DEEP_MAX_WINDOWS"; + // ── SASS metrics knobs ────────────────────────────────────────────────────── constexpr const char* kSassMetricsOnly = "GPUFL_SASS_METRICS_ONLY"; constexpr const char* kSassForceSafeActivity = "GPUFL_SASS_FORCE_SAFE_ACTIVITY"; @@ -176,6 +194,9 @@ constexpr const char* kMonitorIntervalMs = "GPUFL_MONITOR_INTERVAL_MS"; // specific spots - centralized so the exact spelling lives in one place. constexpr const char* kCudaModuleLoading = "CUDA_MODULE_LOADING"; constexpr const char* kCudaInjection64Path = "CUDA_INJECTION64_PATH"; +// Explicit location of gpufl_counter_runtime, for deployment layouts where it +// sits beside neither the calling module nor the injection library. +constexpr const char* kCounterRuntimePath = "GPUFL_COUNTER_RUNTIME_PATH"; constexpr const char* kNvtxInjection64Path = "NVTX_INJECTION64_PATH"; constexpr const char* kCudaPath = "CUDA_PATH"; constexpr const char* kLdPreload = "LD_PRELOAD"; diff --git a/include/gpufl/core/events.hpp b/include/gpufl/core/events.hpp index 922d393..df5667c 100644 --- a/include/gpufl/core/events.hpp +++ b/include/gpufl/core/events.hpp @@ -649,7 +649,32 @@ struct NvtxMarkerEvent { * `close_reason` is what tells the reader that was the deadline expiring * rather than the profiler failing. */ +/** + * What a rule observed at the moment it asked for a window. + * + * Carried on the window rather than looked up later. A bare `trigger_value=842` + * becomes unreadable the first time somebody edits the threshold, so the whole + * comparison travels with the window it caused. + * + * `present` is false for a window nobody triggered - manual, scheduled, or the + * launcher's --deep-after. + */ +struct DeepWindowTrigger { + bool present = false; + std::string rule_id; + std::string metric; // canonical name, device index included + std::string op; // "<" or ">" + double threshold = 0.0; + double rearm_threshold = 0.0; + double observed = 0.0; + int64_t rate_window_ms = 0; + int64_t sustained_ms = 0; + int64_t first_true_ns = 0; // start of the run of true readings + int64_t fired_ns = 0; +}; + struct DeepWindowEvent { + DeepWindowTrigger trigger; int pid = 0; std::string app; std::string session_id; @@ -669,6 +694,93 @@ struct DeepWindowEvent { uint64_t requested_max_launches = 0; }; +/** + * What the session concluded about a conditional rule. + * + * Emitted even when the rule never fired. A rule that leaves no record is + * indistinguishable from one that was never true, and from a run that crashed + * before it could report - three situations calling for different responses. + * + * `state` and `outcome` are separate: state is where the evaluator was + * standing, outcome is the verdict. One field cannot carry both without making + * `armed` look like a conclusion. + */ +struct DeepWindowRuleSummaryEvent { + int pid = 0; + std::string app; + std::string session_id; + std::string rule_id; + std::string expression; // the configured rule, as written + std::string state; + std::string outcome; + std::string reason; + std::string metric_state; + uint64_t samples_seen = 0; + uint32_t windows_opened = 0; + // Absent rather than a sentinel: there is a real "no value yet", and NaN + // does not survive the JSON and DB boundaries cleanly. + bool has_last_value = false; + double last_value = 0.0; + int64_t last_observed_ns = 0; + /** + * Completed kernels discarded before the percentile was computed. + * + * 0 for every metric that is not a percentile, and for a percentile that + * kept everything. Non-zero says the conclusion rests on a subset - which + * a value alone can never show. + */ + uint64_t truncated_samples = 0; + /// Rate windows THIS rule's metric discarded for failed reads (NVTX + /// SAMPLE_UNAVAILABLE), and why the last one was. Per rule, never the + /// session total - an unrelated counter's errors must not look like they + /// broke this rule. + uint64_t metric_quality_resets = 0; + std::string last_quality_reason; + // Monotonic, so a redelivered or late record cannot overwrite a newer one. + uint64_t state_sequence = 0; + int64_t emitted_ns = 0; +}; + +/** + * Session-scoped data quality of application-fed counters. + * + * NOT capture capability: none of these say what the GPU or driver supports. + * They say what the APPLICATION sent - refused registrations, samples for ids + * nobody issued, reads the application itself failed, deltas that went + * backwards - and how often the metric layer had to discard a rate window + * because of it. Each field has ONE meaning; a combined tally is a number + * nobody can act on. + * + * Values are this SESSION's, not the process totals: the tallies live for the + * process (like counter slots) and an embedded host re-initialises, so raw + * totals would re-report session one's problems as session two's. + */ +struct CounterDataQualitySummaryEvent { + int pid = 0; + std::string app; + std::string session_id; + /** + * Which counter path these tallies observed. Only "nvtx" today: the + * gpufl::counter() API's own rejections are not routed through the + * bridge, and a generic-looking row would claim coverage it does not + * have - a gpufl::counter registration failure beside + * registration_rejected: 0 would read as "nothing went wrong". + */ + std::string source = "nvtx"; + int schema_version = 1; + /// Registration-table size at emit (process-lifetime context). + uint64_t tracked_counters = 0; + /// Valid samples THIS session - the denominator that distinguishes + /// "0 failures out of many" from "0 out of 0". + uint64_t samples_observed = 0; + uint64_t registration_rejected = 0; + uint64_t unknown_id_samples = 0; + uint64_t unavailable_samples = 0; + uint64_t negative_delta_samples = 0; + uint64_t rate_windows_discarded = 0; + int64_t emitted_ns = 0; +}; + /** * One CUDA graph launch event captured by CUPTI's * CUPTI_ACTIVITY_KIND_GRAPH_TRACE stream. diff --git a/include/gpufl/core/gpufl.cpp b/include/gpufl/core/gpufl.cpp index f8e64f3..6b60252 100644 --- a/include/gpufl/core/gpufl.cpp +++ b/include/gpufl/core/gpufl.cpp @@ -22,6 +22,7 @@ #include "gpufl/core/config_file_loader.hpp" #include "gpufl/core/debug_logger.hpp" #include "gpufl/core/deep_window.hpp" +#include "gpufl/core/deep_window_rules.hpp" #include "gpufl/core/events.hpp" #include "gpufl/core/logger/logger.hpp" #include "gpufl/core/remote_config.hpp" @@ -612,6 +613,10 @@ bool init(const InitOptions& opts) { // from here: it only records the request; the arm itself happens on the // app thread at the first launch past the delay. scheduleEnvDeepWindow(); + // Same reason as above: the rule needs a live backend to ask about deep + // engine capability, and a metric window that starts from a running + // runtime rather than from a half-built one. + detail::DeepWindowRules::InstallFromEnv(); GFL_LOG_DEBUG("Initialization complete!"); return true; @@ -848,14 +853,14 @@ ScopedMonitor::~ScopedMonitor() { auto& stack = getThreadScopeStack(); if (!stack.empty()) stack.pop_back(); const int depth = static_cast(stack.size()); - const int64_t end_ns = detail::GetTimestampNs(); ScopeBatchRow row; - row.ts_ns = end_ns; row.scope_instance_id = scope_id_; row.name_id = Monitor::InternScopeName(name_); row.event_type = 1; // end row.depth = depth; + const int64_t end_ns = Monitor::CaptureScopeCloseTimestamp(scope_id_); + row.ts_ns = end_ns; Monitor::PushScopeRow(row); // Scopes are recorded via scope_event only - we no longer echo each diff --git a/include/gpufl/core/metric_id.cpp b/include/gpufl/core/metric_id.cpp new file mode 100644 index 0000000..6a3cff2 --- /dev/null +++ b/include/gpufl/core/metric_id.cpp @@ -0,0 +1,271 @@ +#include "gpufl/core/metric_id.hpp" + +#include +#include +#include + +#include "gpufl/core/counter_registry.hpp" + +namespace gpufl::detail { +namespace { + +constexpr const char* kCustomPrefix = "custom."; +constexpr const char* kRateSuffix = "_rate"; + +bool startsWith(const std::string& s, const char* prefix) { + const size_t n = std::char_traits::length(prefix); + return s.size() >= n && s.compare(0, n, prefix) == 0; +} + +bool endsWith(const std::string& s, const char* suffix) { + const size_t n = std::char_traits::length(suffix); + return s.size() >= n && s.compare(s.size() - n, n, suffix) == 0; +} + +/** + * Same charset the counter registry enforces. Shared on purpose: a name the + * registry would refuse must not parse into a rule that then waits forever for + * a counter that can never exist. + */ +bool customNameCharsetOk(const std::string& name) { + for (const char c : name) { + const bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '.' || c == '_' || c == '-'; + if (!ok) return false; + } + return true; +} + +/** Parse `gpu[N].`. Returns false if the shape does not match at all. */ +bool parseDeviceMetric(const std::string& text, MetricParseResult* out) { + if (!startsWith(text, "gpu[")) return false; + + const size_t close = text.find(']'); + if (close == std::string::npos || close == 4 || text.size() <= close + 1 || + text[close + 1] != '.') { + out->error = MetricParseError::MalformedDeviceIndex; + return true; + } + + int64_t index = 0; + for (size_t i = 4; i < close; ++i) { + const char c = text[i]; + if (c < '0' || c > '9') { + out->error = MetricParseError::MalformedDeviceIndex; + return true; + } + index = index * 10 + (c - '0'); + if (index > 4095) { // no plausible host, and keeps the parse bounded + out->error = MetricParseError::MalformedDeviceIndex; + return true; + } + } + + const std::string field = text.substr(close + 2); + if (field == "util_pct") { + out->id.kind = MetricKind::GpuUtilPct; + } else if (field == "power_mw") { + out->id.kind = MetricKind::GpuPowerMw; + } else if (field == "sm_clock_mhz") { + out->id.kind = MetricKind::GpuSmClockMhz; + } else { + out->error = MetricParseError::UnknownBuiltinMetric; + return true; + } + + out->id.device_index = static_cast(index); + // Canonicalised so gpu[00] and gpu[0] cannot hash to two different rule ids. + char buf[64]; + std::snprintf(buf, sizeof(buf), "gpu[%d].%s", out->id.device_index, field.c_str()); + out->id.canonical = buf; + return true; +} + +} // namespace + +MetricShape MetricId::shape() const { + switch (kind) { + case MetricKind::KernelLaunchRate: + case MetricKind::CustomRate: + return MetricShape::Rate; + case MetricKind::RecentKernelMs: + return MetricShape::Percentile; + case MetricKind::GpuUtilPct: + case MetricKind::GpuPowerMw: + case MetricKind::GpuSmClockMhz: + break; + } + return MetricShape::Gauge; +} + +const char* toString(MetricKind k) { + switch (k) { + case MetricKind::GpuUtilPct: return "gpu_util_pct"; + case MetricKind::GpuPowerMw: return "gpu_power_mw"; + case MetricKind::GpuSmClockMhz: return "gpu_sm_clock_mhz"; + case MetricKind::KernelLaunchRate: return "kernel_launch_rate"; + case MetricKind::RecentKernelMs: return "recent_kernel_ms"; + case MetricKind::CustomRate: return "custom_rate"; + } + return "unknown"; +} + +const char* toString(MetricParseError e) { + switch (e) { + case MetricParseError::None: return "ok"; + case MetricParseError::Empty: return "empty_metric_name"; + case MetricParseError::UnknownBuiltinMetric: return "unknown_builtin_metric"; + case MetricParseError::MissingCustomPrefix: return "missing_custom_prefix"; + case MetricParseError::MalformedDeviceIndex: return "malformed_device_index"; + case MetricParseError::MalformedCustomMetric: return "malformed_custom_metric"; + case MetricParseError::CustomNameTooLong: return "custom_name_too_long"; + case MetricParseError::CustomNameCharset: return "custom_name_charset"; + } + return "unknown"; +} + +MetricParseResult parseMetric(const std::string& text) { + MetricParseResult out; + + if (text.empty()) { + out.error = MetricParseError::Empty; + return out; + } + + if (parseDeviceMetric(text, &out)) return out; + + if (text == "kernel_launch_rate") { + out.id.kind = MetricKind::KernelLaunchRate; + out.id.canonical = text; + return out; + } + if (text == "recent_kernel_ms") { + out.id.kind = MetricKind::RecentKernelMs; + out.id.canonical = text; + return out; + } + + if (startsWith(text, kCustomPrefix)) { + const std::string body = + text.substr(std::char_traits::length(kCustomPrefix)); + if (!endsWith(body, kRateSuffix)) { + out.error = MetricParseError::MalformedCustomMetric; + return out; + } + const std::string name = + body.substr(0, body.size() - std::char_traits::length(kRateSuffix)); + if (name.empty()) { + out.error = MetricParseError::MalformedCustomMetric; + return out; + } + if (name.size() > CounterRegistry::kMaxNameLength) { + out.error = MetricParseError::CustomNameTooLong; + return out; + } + if (!customNameCharsetOk(name)) { + out.error = MetricParseError::CustomNameCharset; + return out; + } + out.id.kind = MetricKind::CustomRate; + out.id.custom_name = name; + out.id.canonical = std::string(kCustomPrefix) + name + kRateSuffix; + return out; + } + + // Anything left is either a misspelled built-in or a custom counter written + // without its prefix. Both are config errors, and both are caught here + // rather than at shutdown, which is the entire reason the prefix exists. + out.error = MetricParseError::MissingCustomPrefix; + return out; +} + +int64_t MetricWindowConfig::bucketIntervalMs() const { + if (rate_window_ms <= 0) return 10; + return std::max(10, std::min(100, rate_window_ms / 10)); +} + +int64_t MetricWindowConfig::bucketCount() const { + const int64_t bucket = bucketIntervalMs(); + if (rate_window_ms <= 0) return 1; + return std::max(1, (rate_window_ms + bucket - 1) / bucket); +} + +const char* toString(ConfigError e) { + switch (e) { + case ConfigError::None: return "ok"; + case ConfigError::RateWindowNotPositive: return "rate_window_not_positive"; + case ConfigError::RateWindowTooLarge: return "rate_window_too_large"; + case ConfigError::SustainedNegative: return "sustained_negative"; + case ConfigError::StaleAfterNotPositive: return "stale_after_not_positive"; + case ConfigError::StaleBeforeEvidence: return "stale_before_evidence"; + } + return "unknown"; +} + +ConfigError validate(const MetricWindowConfig& cfg) { + if (cfg.rate_window_ms <= 0) return ConfigError::RateWindowNotPositive; + if (cfg.rate_window_ms > MetricWindowConfig::kMaxRateWindowMs) { + return ConfigError::RateWindowTooLarge; + } + if (cfg.sustained_ms < 0) return ConfigError::SustainedNegative; + if (cfg.stale_after_ms <= 0) return ConfigError::StaleAfterNotPositive; + + // Overflow-safe: the sum is the point of the check, so it must not be the + // thing that breaks it. + constexpr int64_t kMax = std::numeric_limits::max(); + int64_t need = cfg.rate_window_ms; + if (cfg.sustained_ms > kMax - need) return ConfigError::StaleBeforeEvidence; + need += cfg.sustained_ms; + const int64_t bucket = cfg.bucketIntervalMs(); + if (bucket > kMax - need) return ConfigError::StaleBeforeEvidence; + need += bucket; + + if (cfg.stale_after_ms < need) return ConfigError::StaleBeforeEvidence; + return ConfigError::None; +} + +std::string explain(const MetricWindowConfig& cfg, const ConfigError e) { + char buf[320]; + switch (e) { + case ConfigError::None: + return "ok"; + case ConfigError::RateWindowNotPositive: + std::snprintf(buf, sizeof(buf), + "rate window must be > 0 (got %lldms)", + static_cast(cfg.rate_window_ms)); + break; + case ConfigError::RateWindowTooLarge: + std::snprintf(buf, sizeof(buf), + "rate window %lldms exceeds the %lldms limit", + static_cast(cfg.rate_window_ms), + static_cast(MetricWindowConfig::kMaxRateWindowMs)); + break; + case ConfigError::SustainedNegative: + std::snprintf(buf, sizeof(buf), + "sustained must be >= 0 (got %lldms)", + static_cast(cfg.sustained_ms)); + break; + case ConfigError::StaleAfterNotPositive: + std::snprintf(buf, sizeof(buf), + "stale-after must be > 0 (got %lldms)", + static_cast(cfg.stale_after_ms)); + break; + case ConfigError::StaleBeforeEvidence: + // Spell out the arithmetic: the fields are individually reasonable + // and the reader has no way to see why the combination cannot fire. + std::snprintf( + buf, sizeof(buf), + "stale-after %lldms is too short to ever fire: needs >= rate " + "window %lld + sustained %lld + bucket %lld = %lldms", + static_cast(cfg.stale_after_ms), + static_cast(cfg.rate_window_ms), + static_cast(cfg.sustained_ms), + static_cast(cfg.bucketIntervalMs()), + static_cast(cfg.rate_window_ms + cfg.sustained_ms + + cfg.bucketIntervalMs())); + break; + } + return buf; +} + +} // namespace gpufl::detail diff --git a/include/gpufl/core/metric_id.hpp b/include/gpufl/core/metric_id.hpp new file mode 100644 index 0000000..6560d6c --- /dev/null +++ b/include/gpufl/core/metric_id.hpp @@ -0,0 +1,150 @@ +#pragma once + +#include +#include + +namespace gpufl::detail { + +/** + * @brief What a rule is allowed to watch. + * + * Deliberately small. Every entry is something gpufl knows and the application + * does not - otherwise the caller may as well write the `if` themselves, which + * is the alternative this feature has to beat. + */ +enum class MetricKind { + GpuUtilPct, ///< gpu[N].util_pct + GpuPowerMw, ///< gpu[N].power_mw + GpuSmClockMhz, ///< gpu[N].sm_clock_mhz + KernelLaunchRate, ///< kernel_launch_rate, launches/s at the HOST API + RecentKernelMs, ///< recent_kernel_ms, p50 kernel duration over the window + CustomRate, ///< custom._rate, ticks/s +}; + +/** @brief How a metric behaves, which decides warm-up and empty-window rules. */ +enum class MetricShape { + /// Events per second. An empty window is a real 0. + Rate, + /// A percentile over samples. An empty window has no value at all. + Percentile, + /// Last successful measurement. Advances only on a real measurement. + Gauge, +}; + +/** + * @brief A parsed, canonicalised metric name. + * + * `canonical` is what the rule id hashes over, so `gpu[00].util_pct` and + * `gpu[0].util_pct` cannot produce two different ids for one rule. + */ +struct MetricId { + MetricKind kind = MetricKind::KernelLaunchRate; + int device_index = 0; ///< only meaningful for gpu[N].* metrics + std::string custom_name; ///< the in custom._rate + std::string canonical; + + MetricShape shape() const; + /** @brief True when the metric cannot be resolved until the app registers it. */ + bool resolvesLazily() const { return kind == MetricKind::CustomRate; } +}; + +/** + * @brief Why a metric name or a rule config was refused. + * + * Every value here surfaces as `outcome=invalid_config` with this string as the + * reason. A rejected rule that leaves no trace is indistinguishable from one + * that was simply never true, so nothing may fail silently. + */ +enum class MetricParseError { + None, + Empty, + UnknownBuiltinMetric, ///< looks built-in, is not: gpu[0].temperature_pct + MissingCustomPrefix, ///< bare name that is not built-in: tokne_rate + MalformedDeviceIndex, ///< gpu[].util_pct, gpu[-1].util_pct, gpu[x].util_pct + MalformedCustomMetric, ///< custom._rate with a bad or missing name + CustomNameTooLong, + CustomNameCharset, +}; + +const char* toString(MetricParseError e); +const char* toString(MetricKind k); + +struct MetricParseResult { + MetricId id; + MetricParseError error = MetricParseError::None; + bool ok() const { return error == MetricParseError::None; } +}; + +/** + * @brief Parse a metric name from config. + * + * The `custom.` prefix is what makes a typo detectable here rather than at + * shutdown. Without it `tokne_rate` is syntactically indistinguishable from a + * counter that has simply not registered yet, and the mistake only surfaces + * once the run is over. + */ +MetricParseResult parseMetric(const std::string& text); + +/** + * @brief Timing configuration shared by the metric source and the rule. + * + * Held together because the three fields are not independently valid: a + * combination that can never produce evidence is a config error even though + * each field on its own looks reasonable. + */ +struct MetricWindowConfig { + /** + * Upper bound on the rate window. + * + * Property 4 of the design is that no setting is unbounded. The window also + * sizes the bucket ring, so an unbounded value is an unbounded allocation + * driven by a config string. + */ + static constexpr int64_t kMaxRateWindowMs = 5 * 60 * 1000; // 5 minutes + + int64_t rate_window_ms = 1000; + int64_t sustained_ms = 2000; + int64_t stale_after_ms = 5000; + + /** + * @brief Bucket width, derived so the validator and the producer agree. + * + * Internal and not configurable. Defined once here precisely because two + * independent definitions would eventually disagree and the validator would + * start accepting configurations the producer cannot satisfy. + */ + int64_t bucketIntervalMs() const; + + /** @brief Number of buckets covering the rate window (>= 1). */ + int64_t bucketCount() const; +}; + +enum class ConfigError { + None, + RateWindowNotPositive, + RateWindowTooLarge, + SustainedNegative, + StaleAfterNotPositive, + /// stale_after_ms < rate_window + sustained + bucket: the rule goes stale + /// before it can ever accumulate the evidence needed to fire. + StaleBeforeEvidence, +}; + +const char* toString(ConfigError e); + +/** + * @brief Reject a configuration that cannot work, with the arithmetic named. + * + * The interesting case is StaleBeforeEvidence. A total stall only fires if the + * metric stays Fresh long enough to accumulate `sustained_ms` of zero readings, + * and the first zero does not appear until a full rate window has elapsed. The + * weaker pairwise checks (`stale > window`, `stale >= sustained`) accept + * `window=4s, sustained=4s, stale=5s`, which produces its first zero at t=4 and + * goes stale at t=5 - four seconds short of ever firing. + */ +ConfigError validate(const MetricWindowConfig& cfg); + +/** @brief Human-readable explanation including the numbers, for the summary. */ +std::string explain(const MetricWindowConfig& cfg, ConfigError e); + +} // namespace gpufl::detail diff --git a/include/gpufl/core/metric_registry.cpp b/include/gpufl/core/metric_registry.cpp new file mode 100644 index 0000000..b0f3384 --- /dev/null +++ b/include/gpufl/core/metric_registry.cpp @@ -0,0 +1,451 @@ +#include "gpufl/core/metric_registry.hpp" + +#include +#include + +#include "gpufl/core/events.hpp" +#include "gpufl/core/nvtx_counters.hpp" + +namespace gpufl::detail { +namespace { + +constexpr int64_t kNsPerMs = 1000000; + +} // namespace + +const char* toString(const MetricState s) { + switch (s) { + case MetricState::Missing: return "missing"; + case MetricState::WarmingUp: return "warming_up"; + case MetricState::Fresh: return "fresh"; + case MetricState::Stale: return "stale"; + } + return "unknown"; +} + +// ---------------------------------------------------------------- MetricFeeds + +void MetricFeeds::noteKernelLaunch(const int64_t ts_ns) { + // Lock-free: this is the application's launch path. + launch_count_.fetch_add(1, std::memory_order_relaxed); + launch_last_ns_.store(ts_ns, std::memory_order_relaxed); + launch_seeded_.store(true, std::memory_order_release); +} + +void MetricFeeds::noteKernelDuration(const int64_t ts_ns, const double duration_ms) { + std::lock_guard lk(mu_); + // Bounded here, at the push. Trimming after a drain left the buffer free to + // grow without limit until that drain came - and the case where it grows + // fastest is a stalled collector, which is also the case where the drain is + // late. + if (durations_.samples.size() >= kMaxPendingDurations) { + ++durations_.dropped; + } else { + durations_.samples.push_back(DurationSample{ts_ns, duration_ms}); + } + // Advances even when the sample is dropped: the SOURCE is alive, and + // freezing this would report a busy workload as a dead one. + durations_.last_event_ns = ts_ns; +} + +void MetricFeeds::noteDeviceSample(const DeviceSample& sample, const int64_t ts_ns) { + if (sample.device_id < 0) return; + std::lock_guard lk(mu_); + const auto index = static_cast(sample.device_id); + if (index >= devices_.size()) devices_.resize(index + 1); + DeviceGauges& g = devices_[index]; + + // The measurement timestamp advances only here, never on a poll. A metric + // whose sequence advanced because someone asked about it could never be + // detected as dead. + g.util.value = static_cast(sample.gpu_util); + g.util.last_event_ns = ts_ns; + ++g.util.measurements; + + g.power.value = static_cast(sample.power_mw); + g.power.last_event_ns = ts_ns; + ++g.power.measurements; + + g.sm_clock.value = static_cast(sample.clock_sm); + g.sm_clock.last_event_ns = ts_ns; + ++g.sm_clock.measurements; +} + +void MetricFeeds::seedStartup(const int64_t ts_ns) { + { + std::lock_guard lk(mu_); + durations_.last_event_ns = ts_ns; + } + if (launch_seeded_.load(std::memory_order_acquire)) return; + launch_last_ns_.store(ts_ns, std::memory_order_relaxed); + launch_seeded_.store(true, std::memory_order_release); +} + +int MetricFeeds::deviceCount() const { + std::lock_guard lk(mu_); + return static_cast(devices_.size()); +} + +MetricFeeds::LaunchFeed MetricFeeds::launchFeed() const { + LaunchFeed out; + // Seeded first: it is released last on the write side, so observing it + // true means the count and timestamp beside it are already visible. + out.seeded = launch_seeded_.load(std::memory_order_acquire); + out.count = launch_count_.load(std::memory_order_relaxed); + out.last_event_ns = launch_last_ns_.load(std::memory_order_relaxed); + return out; +} + +MetricFeeds::DurationFeed MetricFeeds::drainDurationsUpTo(const int64_t boundary_ns) { + std::lock_guard lk(mu_); + DurationFeed out; + out.last_event_ns = durations_.last_event_ns; + out.dropped = durations_.dropped; + durations_.dropped = 0; + + // Kernels complete roughly in order, so a linear partition is enough; the + // point is that a sample from after the boundary stays for the bucket it + // belongs to rather than being counted in an earlier one. + std::vector keep; + keep.reserve(durations_.samples.size()); + for (const DurationSample& s : durations_.samples) { + if (s.ts_ns <= boundary_ns) { + out.samples.push_back(s); + } else { + keep.push_back(s); + } + } + durations_.samples.swap(keep); + return out; +} + +MetricFeeds::DurationFeed MetricFeeds::drainDurations() { + std::lock_guard lk(mu_); + DurationFeed out; + out.samples = std::move(durations_.samples); + out.last_event_ns = durations_.last_event_ns; + out.dropped = durations_.dropped; + durations_.samples.clear(); + durations_.dropped = 0; + return out; +} + +int64_t MetricFeeds::durationsLastEventNs() const { + std::lock_guard lk(mu_); + return durations_.last_event_ns; +} + +MetricFeeds::GaugeFeed MetricFeeds::gaugeFeed(const MetricKind kind, + const int device_index) const { + std::lock_guard lk(mu_); + if (device_index < 0 || static_cast(device_index) >= devices_.size()) { + return {}; + } + const DeviceGauges& g = devices_[static_cast(device_index)]; + switch (kind) { + case MetricKind::GpuUtilPct: return g.util; + case MetricKind::GpuPowerMw: return g.power; + case MetricKind::GpuSmClockMhz: return g.sm_clock; + default: return {}; + } +} + +void MetricFeeds::resetForTesting() { + std::lock_guard lk(mu_); + durations_ = DurationFeed{}; + devices_.clear(); + launch_count_.store(0, std::memory_order_relaxed); + launch_last_ns_.store(0, std::memory_order_relaxed); + launch_seeded_.store(false, std::memory_order_release); +} + +// --------------------------------------------------------------- MetricSource + +MetricSource::MetricSource(MetricId id, MetricWindowConfig cfg, MetricFeeds* feeds, + const gpufl_counter_provider_v1* counters) + : id_(std::move(id)), cfg_(cfg), feeds_(feeds), counters_(counters) { + bucket_ns_ = cfg_.bucketIntervalMs() * kNsPerMs; + const auto count = static_cast(cfg_.bucketCount()); + buckets_.assign(count, 0); + if (id_.shape() == MetricShape::Percentile) bucket_durations_.resize(count); + // The ring, not the configured window, is what the rate divides by: the + // bucket count rounds up, so the two differ and using the configured value + // would report a rate the samples do not support. + window_ns_ = static_cast(count) * bucket_ns_; + stale_ns_ = cfg_.stale_after_ms * kNsPerMs; +} + +bool MetricSource::resolveCustomHandle() { + if (handle_ != nullptr) return true; + if (counters_ == nullptr || counters_->lookup == nullptr) return false; + // Lookup, never register. A rule naming a counter asks a question; creating + // the counter here would answer it with itself. + handle_ = counters_->lookup(id_.custom_name.c_str(), id_.custom_name.size()); + return handle_ != nullptr; +} + +double MetricSource::windowRatePerSec() const { + uint64_t total = 0; + for (const uint64_t n : buckets_) total += n; + const double seconds = static_cast(window_ns_) / 1e9; + if (seconds <= 0.0) return 0.0; + return static_cast(total) / seconds; +} + +double MetricSource::windowPercentile() const { + std::vector all; + for (const auto& bucket : bucket_durations_) { + all.insert(all.end(), bucket.begin(), bucket.end()); + } + if (all.empty()) return 0.0; + const size_t mid = all.size() / 2; + std::nth_element(all.begin(), all.begin() + mid, all.end()); + return all[mid]; +} + +void MetricSource::closeBucket(const int64_t boundary_ns) { + head_ = (head_ + 1) % buckets_.size(); + ++buckets_closed_; + ++total_closes_; + + switch (id_.shape()) { + case MetricShape::Rate: { + uint64_t total = 0; + bool source_reported_failure = false; + if (id_.kind == MetricKind::KernelLaunchRate) { + total = feeds_->launchFeed().count; + } else if (resolveCustomHandle()) { + // Since the session baseline, not the raw lifetime total. A + // permanent slot keeps its value across shutdown()/init(), so a + // counter ticked by a PREVIOUS session would otherwise read as + // already-ticked here and a stall rule would arm on evidence + // this run never saw. + total = counters_->load_since_baseline(handle_); + if (total > 0) first_tick_seen_ = true; + // An NVTX-fed counter can report that the APPLICATION failed to + // read it (SAMPLE_UNAVAILABLE). The delta over that stretch is + // unknown, not zero; treating it as zero makes the rate sag and + // fires a stall rule on a workload that never slowed down. + // Returns 0 for counters the bridge does not track, so + // gpufl::counter() metrics never pay this branch. + const uint64_t unavailable = + NvtxCounterBridge::instance().unavailableCountFor( + id_.custom_name); + if (unavailable != last_unavailable_seen_) { + last_unavailable_seen_ = unavailable; + source_reported_failure = true; + } + } + // Unsigned delta: correct across a wrap, which is why the counter is + // allowed to wrap rather than saturate. + const uint64_t delta = baselined_ ? total - last_source_total_ : 0; + last_source_total_ = total; + baselined_ = true; + buckets_[head_] = delta; + if (delta > 0) last_tick_ns_ = boundary_ns; + if (source_reported_failure) { + ++quality_resets_; + last_quality_reason_ = "counter_unavailable"; + // Discard the whole window, not just this bucket: the failure + // is only observed at bucket close, so any bucket since the + // last close may straddle the gap. The window refills from + // post-failure data, the sample reads WarmingUp until it does, + // and the evaluator already treats a non-Fresh sample as + // broken evidence - Pending drops back to Armed. + std::fill(buckets_.begin(), buckets_.end(), 0); + buckets_closed_ = 0; + } + break; + } + case MetricShape::Percentile: { + // Up to THIS boundary only. Draining everything would put a + // catch-up's whole backlog into the oldest bucket and leave the + // rest of the window empty. + MetricFeeds::DurationFeed drained = + feeds_->drainDurationsUpTo(boundary_ns); + auto& slot = bucket_durations_[head_]; + slot.clear(); + // BOTH limits count. The feed refuses at kMaxPendingDurations and + // the bucket trims at kMaxDurationsPerBucket, and a batch between + // the two - which is the common case, since the bucket cap is the + // smaller - passed the feed untouched and was then trimmed here + // without anything recording it. The reading would say the + // percentile was complete while a quarter of the kernels were gone. + durations_truncated_ += drained.dropped; + if (drained.samples.size() > kMaxDurationsPerBucket) { + durations_truncated_ += + drained.samples.size() - kMaxDurationsPerBucket; + drained.samples.resize(kMaxDurationsPerBucket); + } + slot.reserve(drained.samples.size()); + for (const MetricFeeds::DurationSample& s : drained.samples) { + slot.push_back(s.ms); + } + break; + } + case MetricShape::Gauge: + break; + } + + current_.observed_ns = boundary_ns; +} + +MetricSample MetricSource::poll(const int64_t now_ns) { + if (next_boundary_ns_ == 0) { + next_boundary_ns_ = now_ns + bucket_ns_; + current_.observed_ns = now_ns; + } + + // A collector that stalled longer than the whole window has no valid + // evidence left, so jump rather than replaying thousands of empty buckets. + // Replaying them would also be wrong: it would look like a run of measured + // zeros and could fire a stall rule that nothing actually observed. + if (now_ns - next_boundary_ns_ > window_ns_) { + std::fill(buckets_.begin(), buckets_.end(), 0); + for (auto& b : bucket_durations_) b.clear(); + // The FEED as well, not just the local buckets. Durations recorded + // before the stall describe a workload from before the gap; letting the + // next bucket inherit them would fire a rule on evidence that is + // already older than the window it claims to cover. + if (id_.shape() == MetricShape::Percentile) feeds_->drainDurations(); + buckets_closed_ = 0; + baselined_ = false; + next_boundary_ns_ = now_ns + bucket_ns_; + current_.observed_ns = now_ns; + } + + while (now_ns >= next_boundary_ns_) { + closeBucket(next_boundary_ns_); + next_boundary_ns_ += bucket_ns_; + } + + // AFTER the buckets have closed, so a loss recorded by this poll is + // reported by this poll. Reading it first published the previous bucket's + // figure and told the caller the data was whole for one reading longer + // than it was. + current_.truncated_samples = durations_truncated_; + + const bool window_full = buckets_closed_ >= buckets_.size(); + + switch (id_.shape()) { + case MetricShape::Rate: { + int64_t source_ns = 0; + if (id_.kind == MetricKind::KernelLaunchRate) { + const MetricFeeds::LaunchFeed feed = feeds_->launchFeed(); + if (!feed.seeded) { + // Nothing has seeded the launch source, so there is no + // baseline to measure staleness against yet. + current_.state = MetricState::WarmingUp; + current_.observed_ns = now_ns; + current_.sequence = total_closes_; + return current_; + } + source_ns = feed.last_event_ns; + } else { + if (!resolveCustomHandle()) { + // Not registered yet. Resolved lazily on purpose: an env + // rule is parsed during init(), long before application code + // reaches gpufl::counter(), so deciding this at install time + // would reject every counter rule before it could exist. + current_.state = MetricState::Missing; + current_.observed_ns = now_ns; + current_.sequence = total_closes_; + return current_; + } + if (!first_tick_seen_ && + counters_->load_since_baseline(handle_) > 0) { + first_tick_seen_ = true; + // Seed the source clock at the moment the counter is first + // seen to move; ticks before this point have no timestamp. + last_tick_ns_ = now_ns; + } + if (!first_tick_seen_) { + // Registered but never ticked. Not Missing - the counter + // exists - and not Fresh 0 either, because a workload that + // has not started yet must not read as a stalled one. + current_.state = MetricState::WarmingUp; + current_.observed_ns = now_ns; + current_.sequence = total_closes_; + return current_; + } + source_ns = last_tick_ns_; + } + + current_.value = windowRatePerSec(); + current_.last_source_event_ns = source_ns; + // Every closed bucket is a publication, including one with no + // ticks. That is what lets a genuine zero accumulate as evidence + // instead of looking like "no new data" - without it, the stall a + // rule most needs to catch is the one case it never could. + current_.sequence = total_closes_; + if (!window_full) { + current_.state = MetricState::WarmingUp; + } else if (now_ns - source_ns > stale_ns_) { + current_.state = MetricState::Stale; + } else { + current_.state = MetricState::Fresh; + } + break; + } + case MetricShape::Percentile: { + const bool have_samples = + std::any_of(bucket_durations_.begin(), bucket_durations_.end(), + [](const std::vector& b) { return !b.empty(); }); + if (!window_full) { + current_.state = MetricState::WarmingUp; + break; + } + if (!have_samples) { + // An empty window has no percentile, so nothing is published: + // neither a value nor a new sequence. Publishing 0 ms would read + // as instantaneous kernels rather than no kernels, and a rule + // watching for slow kernels would silently never fire. + current_.state = MetricState::Stale; + break; + } + if (total_closes_ != last_published_close_) { + last_published_close_ = total_closes_; + ++sequence_; + } + current_.value = windowPercentile(); + current_.last_source_event_ns = feeds_->durationsLastEventNs(); + current_.sequence = sequence_; + current_.state = MetricState::Fresh; + break; + } + case MetricShape::Gauge: { + const MetricFeeds::GaugeFeed feed = + feeds_->gaugeFeed(id_.kind, id_.device_index); + if (feed.measurements == 0) { + current_.state = MetricState::WarmingUp; + break; + } + current_.value = feed.value; + current_.last_source_event_ns = feed.last_event_ns; + // Measurements, not polls. A sequence that advanced because someone + // asked would let one reading satisfy sustained_ms on its own. + current_.sequence = feed.measurements; + current_.state = now_ns - feed.last_event_ns > stale_ns_ + ? MetricState::Stale + : MetricState::Fresh; + break; + } + } + + return current_; +} + +void MetricSource::resetEpoch(const int64_t now_ns) { + std::fill(buckets_.begin(), buckets_.end(), 0); + for (auto& b : bucket_durations_) b.clear(); + buckets_closed_ = 0; + baselined_ = false; + next_boundary_ns_ = now_ns + bucket_ns_; + current_.observed_ns = now_ns; + current_.state = MetricState::WarmingUp; + // Drop anything the feed accumulated during the window for the same reason + // the buckets are cleared: it describes a contaminated workload. + if (id_.shape() == MetricShape::Percentile) feeds_->drainDurations(); +} + +} // namespace gpufl::detail diff --git a/include/gpufl/core/metric_registry.hpp b/include/gpufl/core/metric_registry.hpp new file mode 100644 index 0000000..df7e056 --- /dev/null +++ b/include/gpufl/core/metric_registry.hpp @@ -0,0 +1,328 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "gpufl/abi/gpufl_counter_abi.h" +#include "gpufl/core/metric_id.hpp" + +namespace gpufl { +struct DeviceSample; +} + +namespace gpufl::detail { + +/** + * @brief Whether a reading may be used as evidence, and if not, why. + * + * The split between Fresh and Stale is what stops a rule satisfying + * `sustained_ms` by re-reading one stale sample. System metrics publish every + * 100-500ms while the evaluator runs at ~1ms, so without this the same reading + * would be counted hundreds of times and every rule would fire on its first + * true sample. + */ +enum class MetricState { + Missing, ///< custom counter never registered, or source unavailable + WarmingUp, ///< registered, but no full window of evidence yet + Fresh, ///< usable, INCLUDING a genuine 0 + Stale, ///< the source stopped producing +}; + +const char* toString(MetricState s); + +/** + * @brief One reading. + * + * Two timestamps, not one. A zero-tick bucket still publishes a new sequence + * and a new `observed_ns`, so publication time alone can never go stale - it + * advances forever even during a total stall, which is exactly the condition a + * rule most needs to detect. Staleness is therefore measured against the + * source: + * + * Stale <=> observed_ns - last_source_event_ns > stale_after_ms + * + * That lets zeros accumulate as fresh evidence and still detects a dead source. + */ +struct MetricSample { + double value = 0.0; + int64_t observed_ns = 0; + int64_t last_source_event_ns = 0; + uint64_t sequence = 0; + MetricState state = MetricState::Missing; + /** + * Completed kernels the feed had to discard before this reading. + * + * Non-zero means the value was computed from a subset. Travels WITH the + * sample rather than sitting in a counter nobody reads: a percentile over + * part of the data looks exactly like one over all of it, and the reader + * deciding what a rule concluded is the one who needs to know. + */ + uint64_t truncated_samples = 0; + + bool usable() const { return state == MetricState::Fresh; } +}; + +/** + * @brief Feeds every metric reads from, owned by the runtime. + * + * Separate from MetricSource because the feeds are process-global while a + * source belongs to one rule. Fed from the collector and sampler threads and + * read from the evaluator, so everything here is under one mutex; the volumes + * involved are per-kernel-launch at worst, not per-sample. + */ +class MetricFeeds { +public: + /** + * @brief A kernel launch was observed at the HOST launch API. + * + * Host launch rate, not GPU execution rate. Naming it `kernel_launch_rate` + * rather than a bare "kernel rate" is deliberate: a launch storm and a slow + * kernel look opposite here, and a caller who confuses the two writes a rule + * that fires on the wrong condition. + */ + void noteKernelLaunch(int64_t ts_ns); + + /** @brief A completed kernel's duration, for the recent_kernel_ms window. */ + void noteKernelDuration(int64_t ts_ns, double duration_ms); + + /** @brief A successful NVML/SMI measurement. Never called on polling. */ + void noteDeviceSample(const DeviceSample& sample, int64_t ts_ns); + + /** + * @brief Seed the launch source's timestamp at runtime startup. + * + * The launch source exists from startup, so its zero is a legitimate Fresh + * reading - but staleness needs something to measure against, and without a + * seed a run with no launches yet would read Stale rather than "0 launches + * per second", which is the opposite verdict. + */ + void seedStartup(int64_t ts_ns); + + /** @brief How many GPUs the device collector reported. 0 until first seen. */ + int deviceCount() const; + + struct LaunchFeed { + uint64_t count = 0; + int64_t last_event_ns = 0; + bool seeded = false; + }; + /** + * One completed kernel. + * + * Carries its own timestamp because a bare duration cannot be placed in a + * bucket. When the collector falls behind and closes several boundaries in + * one poll, an untimestamped batch all lands in the OLDEST bucket and the + * rest come up empty - which skews the percentile and expires the samples + * earlier than their own timestamps say they should. + */ + struct DurationSample { + int64_t ts_ns = 0; + double ms = 0.0; + }; + + struct DurationFeed { + std::vector samples; + int64_t last_event_ns = 0; + /// Durations refused because the buffer was full, so a truncated + /// percentile is visible rather than silently reported as complete. + uint64_t dropped = 0; + }; + + /** + * Cap on undrained durations. + * + * The per-bucket resize only trimmed AFTER draining, which bounds nothing: + * between drains the vector grew with every kernel, and a collector that + * stalls during a launch storm is exactly when it grows fastest. + */ + static constexpr size_t kMaxPendingDurations = 8192; + struct GaugeFeed { + double value = 0.0; + int64_t last_event_ns = 0; + uint64_t measurements = 0; + }; + + LaunchFeed launchFeed() const; + /** + * @brief Take and clear the durations accumulated since the last call. + * + * Single-consumer: whoever drains gets the samples and the next caller sees + * an empty feed. Fine while the MVP allows one rule; a second consumer of + * recent_kernel_ms would silently starve the first. + */ + DurationFeed drainDurations(); + /** + * @brief Take only the durations that completed at or before @p boundary_ns. + * + * What lets a catch-up over several boundaries put each sample in the + * bucket it actually belongs to instead of dumping the batch into the + * first one closed. + */ + DurationFeed drainDurationsUpTo(int64_t boundary_ns); + /** @brief Last duration timestamp WITHOUT draining, for staleness checks. */ + int64_t durationsLastEventNs() const; + GaugeFeed gaugeFeed(MetricKind kind, int device_index) const; + + void resetForTesting(); + +private: + // The launch feed is atomics, not mutex-guarded state. It is written from + // the CUDA launch callback on every launch, and a lock there would put the + // application's launch path behind the collector's polling - changing the + // launch rate the rule is trying to measure. + std::atomic launch_count_{0}; + std::atomic launch_last_ns_{0}; + std::atomic launch_seeded_{false}; + + // Durations and gauges are fed from the activity/sampler threads, not from + // the per-launch path, so a lock is fine here. + mutable std::mutex mu_; + DurationFeed durations_; + struct DeviceGauges { + GaugeFeed util; + GaugeFeed power; + GaugeFeed sm_clock; + }; + std::vector devices_; +}; + +/** + * @brief One metric, sampled on a rate window, for one rule. + * + * `poll()` is called from the collector loop at ~1ms and closes a bucket + * whenever one is due. It publishes a new sequence on every closed bucket even + * when nothing ticked - otherwise a real zero never accumulates as evidence and + * a total stall would read as "no new data" rather than "the rate is 0", and + * the rule that exists to catch stalls would be the one rule that cannot. + */ +class MetricSource { +public: + MetricSource(MetricId id, MetricWindowConfig cfg, MetricFeeds* feeds, + const gpufl_counter_provider_v1* counters); + + /** + * @brief Advance to @p now_ns and return the current reading. + * + * Cheap and idempotent within a bucket: repeated calls between bucket + * boundaries return the same sequence, which is what lets the rule + * evaluator ignore a repeat rather than counting it as new evidence. + */ + MetricSample poll(int64_t now_ns); + + /** + * @brief Discard accumulated evidence and start a fresh window. + * + * Called when a deep window closes. Buckets filled while profiling was + * active describe a contaminated workload, and letting them prove the + * workload recovered is how a rule ends up re-firing on its own overhead. + */ + void resetEpoch(int64_t now_ns); + + const MetricId& id() const { return id_; } + const MetricWindowConfig& config() const { return cfg_; } + /** @brief True once a custom counter has been resolved to a live handle. */ + bool customResolved() const { return handle_ != nullptr; } + + /** @brief Rate windows discarded because the source reported failed reads. */ + uint64_t qualityResets() const { return quality_resets_; } + /** @brief Why the last window was discarded; empty when none ever was. */ + const char* lastQualityReason() const { return last_quality_reason_; } + +private: + void closeBucket(int64_t boundary_ns); + bool resolveCustomHandle(); + double windowRatePerSec() const; + double windowPercentile() const; + + MetricId id_; + MetricWindowConfig cfg_; + MetricFeeds* feeds_ = nullptr; + const gpufl_counter_provider_v1* counters_ = nullptr; + + gpufl_counter_handle handle_ = nullptr; // custom metrics only + + int64_t bucket_ns_ = 0; + int64_t window_ns_ = 0; + int64_t stale_ns_ = 0; + + /** Cap on durations kept per bucket; a launch storm must not grow memory. */ + static constexpr size_t kMaxDurationsPerBucket = 4096; + + // Ring of per-bucket counts. Sized once from the validated config. + std::vector buckets_; + // A percentile does not decompose across buckets, so durations are kept + // per bucket and expire with it rather than being folded into a sum. + std::vector> bucket_durations_; + size_t head_ = 0; + /// Closes in the CURRENT epoch; decides whether the window is full. + uint64_t buckets_closed_ = 0; + /** + * Closes over the whole process, never reset. + * + * The sequence has to be monotonic for the evaluator to tell new evidence + * from a repeat. Deriving it from the per-epoch count would send it + * backwards on every epoch reset, and the evaluator would then ignore real + * samples whose numbers it had already seen. + */ + uint64_t total_closes_ = 0; + + int64_t next_boundary_ns_ = 0; + uint64_t last_source_total_ = 0; ///< counter/launch total at last close + bool baselined_ = false; + bool first_tick_seen_ = false; + /** + * When the custom counter last moved. + * + * The registry stores values, not times, so a custom metric's source + * timestamp has to be observed here - at bucket close, whenever the delta + * is non-zero. Without it there is nothing to measure staleness against and + * a dead counter would read as a steady rate of 0 forever. + */ + int64_t last_tick_ns_ = 0; + /** + * Failed-read count last seen from the NVTX counter bridge. + * + * An UNAVAILABLE sample means the application could not read its own + * counter, so the true delta over that stretch is unknown - NOT zero. A + * rate computed across the gap sags below any threshold, which is a false + * stall. When this advances, the current window is discarded and refills + * from post-failure buckets only. + */ + uint64_t last_unavailable_seen_ = 0; + /// Rate windows this source discarded because its counter reported a + /// failed read. Per source, so a rule only ever wears its own problems. + uint64_t quality_resets_ = 0; + const char* last_quality_reason_ = ""; + uint64_t last_published_close_ = 0; + /// Durations the feed had to refuse. Surfaced so a truncated percentile is + /// not presented as a complete one. + uint64_t durations_truncated_ = 0; + +public: + /** + * @brief How many completed kernels the feed had to discard. + * + * Non-zero means the percentile was computed from a subset. Reported + * rather than used to suppress the metric: at the launch rates that cause + * it - hundreds of thousands per second - suppression would disable the + * metric on exactly the workloads it exists for. + */ + uint64_t durationsTruncated() const { return durations_truncated_; } + + /** + * @brief The per-bucket cap, exposed so a test can target the gap between + * it and the feed's cap rather than hard-coding a number that would drift. + */ + static constexpr size_t kMaxDurationsPerBucketForTesting = kMaxDurationsPerBucket; + +private: + + MetricSample current_; + uint64_t sequence_ = 0; +}; + +} // namespace gpufl::detail diff --git a/include/gpufl/core/model/deep_window_model.cpp b/include/gpufl/core/model/deep_window_model.cpp index 28efc9b..e5b9d17 100644 --- a/include/gpufl/core/model/deep_window_model.cpp +++ b/include/gpufl/core/model/deep_window_model.cpp @@ -15,6 +15,24 @@ std::string DeepWindowModel::buildJson() const { } engines << ']'; + // Omitted entirely when nothing triggered the window, so a manual window + // does not carry an all-zero rule that reads like a real one. + std::ostringstream trigger; + if (e_.trigger.present) { + trigger << ",\"trigger\":{" + << "\"rule_id\":\"" << jsonEscape(e_.trigger.rule_id) << "\"" + << ",\"metric\":\"" << jsonEscape(e_.trigger.metric) << "\"" + << ",\"op\":\"" << jsonEscape(e_.trigger.op) << "\"" + << ",\"threshold\":" << e_.trigger.threshold + << ",\"rearm_threshold\":" << e_.trigger.rearm_threshold + << ",\"observed\":" << e_.trigger.observed + << ",\"rate_window_ms\":" << e_.trigger.rate_window_ms + << ",\"sustained_ms\":" << e_.trigger.sustained_ms + << ",\"first_true_ns\":" << e_.trigger.first_true_ns + << ",\"fired_ns\":" << e_.trigger.fired_ns + << "}"; + } + std::ostringstream oss; oss << "{\"type\":\"deep_window_event\"" << ",\"pid\":" << e_.pid @@ -29,6 +47,56 @@ std::string DeepWindowModel::buildJson() const { << ",\"launches_covered\":" << e_.launches_covered << ",\"requested_duration_ms\":" << e_.requested_duration_ms << ",\"requested_max_launches\":" << e_.requested_max_launches + << trigger.str() + << "}"; + return oss.str(); +} + +std::string DeepWindowRuleSummaryModel::buildJson() const { + std::ostringstream oss; + oss << "{\"type\":\"deep_window_rule_summary\"" + << ",\"pid\":" << e_.pid + << ",\"app\":\"" << jsonEscape(e_.app) << "\"" + << ",\"session_id\":\"" << jsonEscape(e_.session_id) << "\"" + << ",\"rule_id\":\"" << jsonEscape(e_.rule_id) << "\"" + << ",\"expression\":\"" << jsonEscape(e_.expression) << "\"" + << ",\"state\":\"" << jsonEscape(e_.state) << "\"" + << ",\"outcome\":\"" << jsonEscape(e_.outcome) << "\"" + << ",\"reason\":\"" << jsonEscape(e_.reason) << "\"" + << ",\"metric_state\":\"" << jsonEscape(e_.metric_state) << "\"" + << ",\"samples_seen\":" << e_.samples_seen + << ",\"windows_opened\":" << e_.windows_opened + << ",\"truncated_samples\":" << e_.truncated_samples + << ",\"metric_quality_resets\":" << e_.metric_quality_resets + << ",\"last_quality_reason\":\"" << jsonEscape(e_.last_quality_reason) << "\"" + << ",\"state_sequence\":" << e_.state_sequence + << ",\"emitted_ns\":" << e_.emitted_ns; + // Written only when there is one. A null would have to be distinguished + // from 0 downstream, and 0 is a legitimate reading for every metric here. + if (e_.has_last_value) { + oss << ",\"last_value\":" << e_.last_value + << ",\"last_observed_ns\":" << e_.last_observed_ns; + } + oss << "}"; + return oss.str(); +} + +std::string CounterDataQualitySummaryModel::buildJson() const { + std::ostringstream oss; + oss << "{\"type\":\"counter_data_quality_summary\"" + << ",\"pid\":" << e_.pid + << ",\"app\":\"" << jsonEscape(e_.app) << "\"" + << ",\"session_id\":\"" << jsonEscape(e_.session_id) << "\"" + << ",\"source\":\"" << jsonEscape(e_.source) << "\"" + << ",\"schema_version\":" << e_.schema_version + << ",\"tracked_counters\":" << e_.tracked_counters + << ",\"samples_observed\":" << e_.samples_observed + << ",\"registration_rejected\":" << e_.registration_rejected + << ",\"unknown_id_samples\":" << e_.unknown_id_samples + << ",\"unavailable_samples\":" << e_.unavailable_samples + << ",\"negative_delta_samples\":" << e_.negative_delta_samples + << ",\"rate_windows_discarded\":" << e_.rate_windows_discarded + << ",\"emitted_ns\":" << e_.emitted_ns << "}"; return oss.str(); } diff --git a/include/gpufl/core/model/deep_window_model.hpp b/include/gpufl/core/model/deep_window_model.hpp index cafd346..9a7c51e 100644 --- a/include/gpufl/core/model/deep_window_model.hpp +++ b/include/gpufl/core/model/deep_window_model.hpp @@ -20,4 +20,32 @@ struct DeepWindowModel final : IJsonSerializable { const DeepWindowEvent& e_; }; +/** + * JSON serializer for the conditional-rule summary. Same channel as the window + * itself: the two are read together, and a summary that arrived on a different + * channel could be ingested after the windows it explains. + */ +// Lives beside the rule-summary model because its consumer is the same +// conditional-window feature; the event itself is counter data quality. +struct CounterDataQualitySummaryModel final : IJsonSerializable { + explicit CounterDataQualitySummaryModel(const CounterDataQualitySummaryEvent& e) + : e_(e) {} + std::string buildJson() const override; + // Scope channel, like the rule summary it is read next to. + Channel channel() const override { return Channel::Scope; } + + private: + const CounterDataQualitySummaryEvent& e_; +}; + +struct DeepWindowRuleSummaryModel final : IJsonSerializable { + explicit DeepWindowRuleSummaryModel(const DeepWindowRuleSummaryEvent& e) + : e_(e) {} + std::string buildJson() const override; + Channel channel() const override { return Channel::Scope; } + + private: + const DeepWindowRuleSummaryEvent& e_; +}; + } // namespace gpufl::model diff --git a/include/gpufl/core/monitor.cpp b/include/gpufl/core/monitor.cpp index 5fbae3d..92317ed 100644 --- a/include/gpufl/core/monitor.cpp +++ b/include/gpufl/core/monitor.cpp @@ -1,5 +1,7 @@ #include "gpufl/core/monitor.hpp" +#include "gpufl/core/deep_window_rules.hpp" + #include #include #include @@ -13,6 +15,7 @@ #include #include "gpufl/core/activity_record.hpp" +#include "gpufl/core/counter_provider.hpp" #include "gpufl/core/common.hpp" #include "gpufl/core/debug_logger.hpp" #include "gpufl/core/logger/logger.hpp" @@ -155,8 +158,11 @@ struct MonitorState { detail::MonitorBatchManager batches; MetadataManager metadata; - bool suppressOrphanSyntheticKernels = false; - bool drainSyntheticMidRun = false; + // Atomic: set by the backend's start() thread and (since the + // missing-records suppression) by Monitor::Shutdown while the collector + // thread may still be reading it at a drain site. + std::atomic suppressOrphanSyntheticKernels{false}; + std::atomic drainSyntheticMidRun{false}; }; MonitorState g_state; @@ -458,6 +464,9 @@ void CollectorLoop() { // decides how closely a deep window tracks its deadline, and it is a // lock-free check when no window is closing. if (g_state.adapter) g_state.adapter->serviceDeepWindow(); + // Immediately after, so a rule that decides to open is serviced on the + // very next beat rather than a full loop later. + detail::DeepWindowRules::Service(); if (!RecordProcessor::processNext()) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); @@ -491,6 +500,12 @@ void CollectorLoop() { void Monitor::Initialize(const MonitorOptions& opts) { if (g_state.initialized.exchange(true)) return; + // Process-lifetime state must not leak a prior session's capture policy. + // The active backend's start() replaces these defaults before callbacks + // begin. Monitor/no-backend sessions keep both disabled. + SetSuppressOrphanSyntheticKernels(false); + SetDrainSyntheticKernelsMidRun(false); + // The engine AFTER env overrides. InitOptions::profiling_engine is the // pre-override request, and `gpufl trace --passes X` overrides only the // MonitorOptions copy - so anything reporting which engine ran must read @@ -499,6 +514,10 @@ void Monitor::Initialize(const MonitorOptions& opts) { std::memory_order_release); g_monitorBuffer.resetDroppedCount(); + // Counter slots live for the process, so this session has to baseline them + // or it would inherit the previous session's ticks plus anything added + // while gpufl was shut down. + detail::ActiveCounterProvider()->begin_session(); g_state.batches.reset(); g_state.metadata.reset(); g_state.batches.setSourceCollectionEnabled(opts.enable_source_collection); @@ -519,12 +538,38 @@ void Monitor::Shutdown() { if (g_state.adapter) { g_state.adapter->stop(); + // stop() has disabled and flushed activity, so "zero kernel records" + // is final. When a real-record session lost every record (observed: + // Trace+PM adaptive on Linux, ~1/4 of NVTX-counter-target runs - + // enables succeed, sync records flow, kernel records never arrive), + // the teardown drains below would resurrect every launch meta as a + // synthetic row whose "duration" is the host gap to the next launch, + // fabricating a full kernel timeline. Suppress instead: no kernel + // rows, and the capability event already says + // enabled_but_no_records, so the session stays self-describing. + if (IMonitorBackend* b = g_state.adapter->backend(); + b && b->kernelActivityExpectedButMissing()) { + GFL_LOG_ERROR( + "[Monitor] Kernel activity was enabled and launches happened, " + "but no kernel activity record arrived this session. Kernel " + "rows are omitted (a synthetic fallback would carry host-gap " + "durations, not kernel time)."); + } g_state.adapter->shutdown(); } g_state.collectorRunning.store(false); if (g_state.collectorThread.joinable()) g_state.collectorThread.join(); + // AFTER the collector has stopped, and before the logger goes away. Writing + // the summary while the collector still runs would let the evaluator open a + // window the recorded summary never mentions. + // Quality BEFORE Finish: Finish releases the rule session, which destroys + // the metric source whose discard count the quality event reports. The + // collector is already stopped, so nothing advances between the two. + detail::DeepWindowRules::EmitCounterQuality(); + detail::DeepWindowRules::Finish(); + while (RecordProcessor::processNext()) {} if (Runtime* rt = runtime(); rt && rt->logger) { drainSyntheticKernels(rt); @@ -535,11 +580,20 @@ void Monitor::Shutdown() { g_monitorBuffer.resetDroppedCount(); g_state.batches.clearFlushSink(); g_state.adapter.reset(); + // Slots keep their values on purpose: a handle held across this stays + // valid. What stops those adds counting twice is the baseline the next + // Initialize takes, not clearing them here. + detail::ActiveCounterProvider()->end_session(); } void Monitor::DrainAndFinalizeForExit() { if (!g_state.initialized.exchange(false)) return; + // No missing-records suppression here (unlike Shutdown): this path runs + // BEFORE the backend's stop(), so activity was never flushed and + // "zero records seen" may just mean a short run whose only buffer is + // still pending - suppressing would drop the synthetic fallback that is + // this path's whole reason to exist. // The backend's PC sampling cycle thread stops issuing CUPTI reads as soon // as process-exit teardown is flagged (PcSamplingEngine::drainData), so it // sits idle here and can't fault mid-flush. We deliberately do NOT join it @@ -550,6 +604,14 @@ void Monitor::DrainAndFinalizeForExit() { g_state.collectorRunning.store(false); if (g_state.collectorThread.joinable()) g_state.collectorThread.join(); + // Same ordering as Shutdown: the collector is stopped first, so nothing can + // advance the rule past what this summary reports. + // Quality BEFORE Finish: Finish releases the rule session, which destroys + // the metric source whose discard count the quality event reports. The + // collector is already stopped, so nothing advances between the two. + detail::DeepWindowRules::EmitCounterQuality(); + detail::DeepWindowRules::Finish(); + while (RecordProcessor::processNext()) {} if (Runtime* rt = runtime(); rt && rt->logger) { drainSyntheticKernels(rt); @@ -570,6 +632,10 @@ void Monitor::DrainAndFinalizeForExit() { } } g_state.batches.clearFlushSink(); + // Same close-out as Shutdown(). Without it a process that exits through + // this path leaves the session marked active, and an embedded host that + // re-initialised afterwards would inherit its ticks. + detail::ActiveCounterProvider()->end_session(); } void Monitor::ReleaseBackendForExit() { @@ -667,6 +733,14 @@ uint64_t Monitor::AllocateScopeInstanceId() { int Monitor::OpenScopeDepth() { return g_state.batches.openScopeDepth(); } +int64_t Monitor::CaptureScopeCloseTimestamp(uint64_t instance_id) { + return g_state.batches.captureScopeCloseTimestamp(instance_id); +} + +void Monitor::MarkScopeClosePending(uint64_t instance_id, int64_t end_ns) { + g_state.batches.markScopeClosePending(instance_id, end_ns); +} + void Monitor::PushProfileSamples(const std::vector& samples) { if (samples.empty()) return; const uint32_t scope_name_id = g_state.batches.activeScopeNameId(); @@ -711,6 +785,26 @@ void Monitor::PushPmSamples(const std::vector& samples) { g_state.batches.pushPmSamplesResolvingScopes(rows); } +void Monitor::PublishScopeRetentionWatermark(int64_t ts_ns) { + g_state.batches.publishScopeRetentionWatermark(ts_ns); +} + +void Monitor::BeginPmScopeAttribution(int64_t start_ns) { + g_state.batches.beginPmScopeAttribution(start_ns); +} + +void Monitor::EndPmScopeAttribution() { + g_state.batches.endPmScopeAttribution(); +} + +uint64_t Monitor::ScopeAttributionTruncated() { + return g_state.batches.scopeAttributionTruncated(); +} + +uint64_t Monitor::PmSampleRowsSeen() { + return g_state.batches.pmSampleRowsSeen(); +} + void Monitor::EmitPmSamplingConfig(uint32_t device_id, uint32_t interval_us, uint32_t max_samples, const std::string& preset, const std::vector& metrics) { const Runtime* rt = runtime(); if (!(rt && rt->logger)) return; @@ -723,5 +817,8 @@ void Monitor::EmitPmSamplingConfig(uint32_t device_id, uint32_t interval_us, uin void SetSuppressOrphanSyntheticKernels(const bool suppress) { g_state.suppressOrphanSyntheticKernels = suppress; } void SetDrainSyntheticKernelsMidRun(const bool enable) { g_state.drainSyntheticMidRun = enable; } +bool SuppressOrphanSyntheticKernelsForTesting() { + return g_state.suppressOrphanSyntheticKernels.load(std::memory_order_acquire); +} } // namespace gpufl diff --git a/include/gpufl/core/monitor.hpp b/include/gpufl/core/monitor.hpp index 8a5e66c..4986965 100644 --- a/include/gpufl/core/monitor.hpp +++ b/include/gpufl/core/monitor.hpp @@ -231,12 +231,9 @@ struct PmSampleInput { }; // Session-level switch (set by the active backend at start): when true the -// collector DROPS orphaned launch metas at shutdown instead of emitting them as -// synthetic kernel rows. Used for engines where kernel-activity is intentionally -// off and the synthetic host-dispatch durations would mislead - SASS safe mode, -// where real kernel activity deadlocks (NVIDIA CUPTI/driver bug). The Execution -// Signature is accumulated separately, so a multi-pass merge is unaffected. -// Default false: normal / PC modes keep best-effort synthesis. +// collector DROPS unmatched launch metas instead of emitting host launch gaps +// as kernel durations. All real-record modes enable it; callback-derived rows +// remain available only for synthesize-by-design modes such as PC sampling. void SetSuppressOrphanSyntheticKernels(bool suppress); // Session-level switch (set by the active backend at start): when true the @@ -246,6 +243,10 @@ void SetSuppressOrphanSyntheticKernels(bool suppress); // those kernel rows survive. Default false (other modes emit at shutdown). void SetDrainSyntheticKernelsMidRun(bool enable); +// Test seam for the process-lifetime session policy. Production code should +// only set the policy through the two functions above. +bool SuppressOrphanSyntheticKernelsForTesting(); + /** * @brief The central monitoring engine. */ @@ -387,6 +388,10 @@ class Monitor { static uint64_t AllocateScopeInstanceId(); /** @brief Depth a scope opened right now would nest at. */ static int OpenScopeDepth(); + /** @brief Capture and publish a close timestamp before scope-state locking. */ + static int64_t CaptureScopeCloseTimestamp(uint64_t instance_id); + /** @brief Publish an existing close timestamp before pushing its row. */ + static void MarkScopeClosePending(uint64_t instance_id, int64_t end_ns); /** * Push a raw activity record into the monitor ring buffer. @@ -419,6 +424,22 @@ class Monitor { * @brief Push decoded PM sampling time-series rows. */ static void PushPmSamples(const std::vector& samples); + /** + * @brief Release completed scopes that no future sample can reach. + * + * Call only after a decode that SUCCEEDED. A failed or overflowed decode + * means samples were lost rather than delivered, and advancing past them + * would drop the scopes they still need. + */ + static void PublishScopeRetentionWatermark(int64_t ts_ns); + /** @brief Mark the wall-clock boundary from which PM samples may be pending. */ + static void BeginPmScopeAttribution(int64_t start_ns); + /** @brief Mark that the final PM decode completed. */ + static void EndPmScopeAttribution(); + /** @brief Scope history evicted while PM attribution was active. */ + static uint64_t ScopeAttributionTruncated(); + /** @brief PM metric rows that passed through scope attribution. */ + static uint64_t PmSampleRowsSeen(); /** * @brief Emit PM sampling configuration metadata for readers/UI. diff --git a/include/gpufl/core/monitor_backend.hpp b/include/gpufl/core/monitor_backend.hpp index e44a8d2..f002b54 100644 --- a/include/gpufl/core/monitor_backend.hpp +++ b/include/gpufl/core/monitor_backend.hpp @@ -87,6 +87,19 @@ class IMonitorBackend { */ virtual bool IsProfilingOperational() const { return true; } + /** + * @brief True when this session expected REAL kernel activity records + * (kernel activity enabled, not a synthesize-by-design mode), + * launches happened, and yet zero records arrived. Only + * meaningful after stop() has disabled + flushed activity, when + * "zero" is final. Monitor::Shutdown consults this to suppress + * the orphan synthetic-kernel drain: with every record missing, + * that drain would fabricate a full kernel timeline out of + * host launch-to-launch gaps (sleeps included) and present it + * as measured kernel time. Default: false (nothing to suppress). + */ + virtual bool kernelActivityExpectedButMissing() const { return false; } + virtual void OnScopeStart(const char* name) {} virtual void OnScopeStop(const char* name) {} @@ -152,6 +165,29 @@ class IMonitorBackend { */ virtual void ServiceDeepWindow() {} + /** + * @brief Is any engine that a window could arm actually prepared? + * + * The capability gate for conditional windows. Checking the configured + * engine enum is necessary but not sufficient: a Trace-only run resolves to + * a valid engine and still arms nothing inside a window, so a rule would + * spend its whole budget opening windows that collect no deep data. + * + * Answers about what was PREPARED, not what is currently armed - under + * WindowOnly nothing is armed until a window opens, which is precisely when + * the answer is needed. + */ + virtual bool DeepEnginesPrepared() const { return false; } + + /** + * @brief Is preparation waiting for the target's first CUDA context? + * + * Windows injection installs conditional rules before CONTEXT_CREATED. + * Pending is therefore a valid install-time state, but never sufficient to + * open a window; actual opens still require DeepEnginesPrepared(). + */ + virtual bool DeepEnginePreparationPending() const { return false; } + virtual void OnPerfScopeStart(const char* name) {} virtual void OnPerfScopeStop(const char* name) {} // Perf-scope counterparts of OnDeepWindowStart/Stop; see those. diff --git a/include/gpufl/core/monitor_batch_manager.cpp b/include/gpufl/core/monitor_batch_manager.cpp index 966ed38..022ce2c 100644 --- a/include/gpufl/core/monitor_batch_manager.cpp +++ b/include/gpufl/core/monitor_batch_manager.cpp @@ -1,7 +1,12 @@ #include "gpufl/core/monitor_batch_manager.hpp" +#include +#include +#include +#include #include +#include "gpufl/core/common.hpp" #include "gpufl/core/debug_logger.hpp" #include "gpufl/core/logger/logger.hpp" #include "gpufl/core/model/batch_models.hpp" @@ -21,6 +26,15 @@ void MonitorBatchManager::reset() { scopeNameStack_.clear(); openScopeWindows_.clear(); completedScopeWindows_.clear(); + scopeRetentionWatermarkNs_ = 0; + pmScopeAttributionStartNs_ = 0; + scopeHistoryEvictionLogged_ = false; + scopeAttributionTruncated_ = 0; + pmSampleRowsSeen_ = 0; + { + std::lock_guard pending_lk(pendingScopeCloseMu_); + pendingScopeCloseNs_.clear(); + } } syncBatch_.clear(); memAllocBatch_.clear(); @@ -165,6 +179,22 @@ int MonitorBatchManager::openScopeDepth() const { return static_cast(scopeNameStack_.size()); } +int64_t MonitorBatchManager::captureScopeCloseTimestamp(uint64_t instance_id) { + // The timestamp and its publication share the same lock observed by the PM + // snapshot. If snapshot wins, its batch predates this close; if close wins, + // snapshot sees the exact end and cannot provisionally extend past it. + std::lock_guard lk(pendingScopeCloseMu_); + const int64_t end_ns = GetTimestampNs(); + pendingScopeCloseNs_[instance_id] = end_ns; + return end_ns; +} + +void MonitorBatchManager::markScopeClosePending(uint64_t instance_id, int64_t end_ns) { + std::lock_guard lk(pendingScopeCloseMu_); + const auto [it, inserted] = pendingScopeCloseNs_.emplace(instance_id, end_ns); + if (!inserted && end_ns < it->second) it->second = end_ns; +} + bool MonitorBatchManager::pushKernel(const KernelBatchRow& row, const KernelDetailRow* detail) { kernelBatch_.push(row); @@ -187,6 +217,13 @@ void MonitorBatchManager::pushTraceScopeRows(const ScopeBatchRow& begin_row, } void MonitorBatchManager::pushTrackedScopeRow(const ScopeBatchRow& row) { + // Publish a close's already-captured timestamp before waiting for the main + // scope lock. A concurrent PM snapshot can then stop the open interval at + // this timestamp instead of extending it through the whole sample batch. + if (row.event_type != 0) { + markScopeClosePending(row.scope_instance_id, row.ts_ns); + } + std::lock_guard lk(scopeBatchMu_); if (row.event_type == 0) { scopeNameStack_.emplace_back(row.scope_instance_id, row.name_id); @@ -211,7 +248,12 @@ void MonitorBatchManager::pushTrackedScopeRow(const ScopeBatchRow& row) { completedScopeWindows_.push_back( {it->second.start_ns, row.ts_ns, row.scope_instance_id, row.name_id, it->second.depth}); openScopeWindows_.erase(it); + // Bound it here rather than only when PM samples arrive. Nothing + // publishes a retention watermark unless PM is actually sampling, + // so a Trace-only run would grow this for the life of the process. + enforceScopeCapLocked(); } + clearPendingScopeCloseLocked(row.scope_instance_id); } scopeBatch_.push(row); } @@ -230,13 +272,230 @@ void MonitorBatchManager::pushProfileSamples(const std::vector& rows) { - const uint32_t fallback_id = activeScopeNameId_.load(std::memory_order_relaxed); + if (rows.empty()) return; + + std::vector resolved(rows.begin(), rows.end()); + + // Snapshot under the lock, sweep outside it. Holding scopeBatchMu_ for the + // whole sort-and-sweep would block every scope close for the duration, and + // a close that is already holding its end timestamp and waiting here is + // exactly what widens the window below. + std::vector candidates; + { + std::lock_guard lk(scopeBatchMu_); + pmSampleRowsSeen_ += rows.size(); + trimCompletedScopesLocked(); + candidates = snapshotScopeCandidatesLocked(rows); + } + + // No fallback to the currently active scope. A sample no interval covers is + // left unattributed, and that is the point of this path: handing it + // whichever scope happens to be open at DECODE time answers "what is + // running now", not "what was running when this was sampled", and the two + // differ by however long the sample sat in the buffer - exactly the error + // this resolver exists to remove. Open scopes are already candidates, so a + // sample inside a still-running scope resolves properly rather than by luck. + resolveScopeIdsForBatch(candidates, resolved, /*fallback_id=*/0); + + { + std::lock_guard lk(scopeBatchMu_); + for (const auto& row : resolved) pmSampleBatch_.push(row); + } +} + +void MonitorBatchManager::publishScopeRetentionWatermark(int64_t ts_ns) { std::lock_guard lk(scopeBatchMu_); - for (const auto& sample : rows) { - PmSampleBatchRow row = sample; - row.scope_name_id = resolveScopeIdLocked(row.ts_ns); - if (row.scope_name_id == 0) row.scope_name_id = fallback_id; - pmSampleBatch_.push(row); + // Monotonic. A caller that regressed - a decode that failed, a buffer that + // overflowed - must not be able to un-retire scopes it already released. + if (ts_ns > scopeRetentionWatermarkNs_) scopeRetentionWatermarkNs_ = ts_ns; +} + +void MonitorBatchManager::beginPmScopeAttribution(int64_t start_ns) { + std::lock_guard lk(scopeBatchMu_); + pmScopeAttributionStartNs_ = start_ns; +} + +void MonitorBatchManager::endPmScopeAttribution() { + std::lock_guard lk(scopeBatchMu_); + pmScopeAttributionStartNs_ = 0; +} + +uint64_t MonitorBatchManager::scopeAttributionTruncated() const { + std::lock_guard lk(scopeBatchMu_); + return scopeAttributionTruncated_; +} + +uint64_t MonitorBatchManager::pmSampleRowsSeen() const { + std::lock_guard lk(scopeBatchMu_); + return pmSampleRowsSeen_; +} + +void MonitorBatchManager::resolveScopeIdsForTesting(std::vector& rows, + uint32_t fallback_id) { + std::vector candidates; + { + std::lock_guard lk(scopeBatchMu_); + trimCompletedScopesLocked(); + candidates = snapshotScopeCandidatesLocked(rows); + } + resolveScopeIdsForBatch(candidates, rows, fallback_id); +} + +uint32_t MonitorBatchManager::resolveScopeIdForTesting(int64_t ts_ns) const { + std::lock_guard lk(scopeBatchMu_); + return resolveScopeIdLocked(ts_ns); +} + +size_t MonitorBatchManager::retainedCompletedScopesForTesting() const { + std::lock_guard lk(scopeBatchMu_); + return completedScopeWindows_.size(); +} + +void MonitorBatchManager::clearPendingScopeCloseLocked(uint64_t instance_id) { + // Called with scopeBatchMu_ held. Snapshot takes locks in the same order: + // scopeBatchMu_ first, pendingScopeCloseMu_ second. + std::lock_guard lk(pendingScopeCloseMu_); + pendingScopeCloseNs_.erase(instance_id); +} + +void MonitorBatchManager::enforceScopeCapLocked() { + // Runs on every close, not only when PM samples arrive. The watermark is + // the real bound, but nothing publishes one unless PM is actually + // sampling - so a Trace-only run, or one where PM never initialised, would + // otherwise grow this deque for the life of the process with no cap and no + // telemetry to show for it. + if (completedScopeWindows_.size() <= kMaxCompletedScopes) return; + const size_t excess = completedScopeWindows_.size() - kMaxCompletedScopes; + uint64_t attribution_risk = 0; + if (pmScopeAttributionStartNs_ > 0) { + for (size_t i = 0; i < excess; ++i) { + if (completedScopeWindows_[i].end_ns >= pmScopeAttributionStartNs_) { + ++attribution_risk; + } + } + } + completedScopeWindows_.erase(completedScopeWindows_.begin(), + completedScopeWindows_.begin() + static_cast(excess)); + // Trace-only history is unused by PM attribution. Count an eviction as a + // data-quality risk only when it overlaps the current PM collection + // boundary. This also handles mixed sessions: a long Trace warmup cannot + // make a later Deep PM window look partial merely because old, pre-window + // entries are evicted while PM happens to be active. + scopeAttributionTruncated_ += attribution_risk; + if (!scopeHistoryEvictionLogged_) { + scopeHistoryEvictionLogged_ = true; + GFL_LOG_ERROR("[MonitorBatchManager] scope_history_evicted: hard cap reached; ", + excess, + " completed scope record(s) dropped (further messages suppressed)"); + } +} + +void MonitorBatchManager::trimCompletedScopesLocked() { + // The deque is NOT ordered by end_ns, so this cannot stop at the first + // survivor. It is a full pass, but only over what the watermark has not + // already retired, and it runs once per drain rather than once per sample. + if (scopeRetentionWatermarkNs_ > 0) { + const int64_t cutoff = scopeRetentionWatermarkNs_; + const auto it = std::remove_if( + completedScopeWindows_.begin(), completedScopeWindows_.end(), + [cutoff](const ScopeWindow& w) { return w.end_ns < cutoff; }); + completedScopeWindows_.erase(it, completedScopeWindows_.end()); + } + + // Backstop. Reaching this means the watermark is not advancing, so the + // entries dropped here may still have been needed: record it rather than + // let the samples quietly go unattributed. + enforceScopeCapLocked(); +} + +std::vector +MonitorBatchManager::snapshotScopeCandidatesLocked( + const std::vector& rows) const { + // Candidates = closed scopes still retained, PLUS scopes that are still + // open. The open ones matter: PM drains mid-run, so a sample is routinely + // decoded while the scope covering it is still running. Giving them a + // provisional end at the batch's newest sample keeps them eligible for + // every sample in this batch without inventing a close that has not + // happened. + // + int64_t provisional_end = std::numeric_limits::min(); + for (const auto& row : rows) provisional_end = std::max(provisional_end, row.ts_ns); + + std::vector candidates; + candidates.reserve(completedScopeWindows_.size() + openScopeWindows_.size()); + candidates.assign(completedScopeWindows_.begin(), completedScopeWindows_.end()); + { + // Close publishes pending first and never holds this mutex while + // waiting for scopeBatchMu_, so this lock order cannot deadlock. + std::lock_guard pending_lk(pendingScopeCloseMu_); + for (const auto& [instance_id, open] : openScopeWindows_) { + int64_t effective_end = provisional_end; + if (const auto close = pendingScopeCloseNs_.find(instance_id); + close != pendingScopeCloseNs_.end()) { + effective_end = std::min(effective_end, close->second); + } + if (open.start_ns > effective_end) continue; + candidates.push_back(ScopeWindow{open.start_ns, effective_end, instance_id, + open.name_id, open.depth}); + } + } + return candidates; +} + +void MonitorBatchManager::resolveScopeIdsForBatch(std::vector& candidates, + std::vector& rows, + uint32_t fallback_id) { + if (candidates.empty()) { + for (auto& row : rows) row.scope_name_id = fallback_id; + return; + } + + std::sort(candidates.begin(), candidates.end(), + [](const ScopeWindow& a, const ScopeWindow& b) { return a.start_ns < b.start_ns; }); + + std::vector order(rows.size()); + for (size_t i = 0; i < order.size(); ++i) order[i] = i; + std::sort(order.begin(), order.end(), + [&rows](size_t a, size_t b) { return rows[a].ts_ns < rows[b].ts_ns; }); + + // Two structures over the active set rather than one flat list. Rescanning + // it per sample would keep the cost at O(samples x concurrent scopes), + // which is the shape this replaces. `ranked` is ordered so the winner is + // its last element; `expiry` surfaces the soonest end so retirement costs + // a peek instead of a pass. + struct ByRank { + bool operator()(const ScopeWindow* a, const ScopeWindow* b) const { + return b->outranks(*a); + } + }; + struct ByEnd { + bool operator()(const ScopeWindow* a, const ScopeWindow* b) const { + return a->end_ns > b->end_ns; // min-heap on end_ns + } + }; + std::set ranked; + std::priority_queue, ByEnd> expiry; + + size_t next_candidate = 0; + for (const size_t idx : order) { + const int64_t ts = rows[idx].ts_ns; + + // Admit everything that has started. Candidates are start-sorted, so + // each is admitted once across the whole batch. + while (next_candidate < candidates.size() && candidates[next_candidate].start_ns <= ts) { + const ScopeWindow* w = &candidates[next_candidate++]; + ranked.insert(w); + expiry.push(w); + } + // Retire what has ended. Samples are visited in time order, so an + // expired scope can never be wanted again. Both ends are inclusive, + // hence `end_ns < ts` rather than `<=`. + while (!expiry.empty() && expiry.top()->end_ns < ts) { + ranked.erase(expiry.top()); + expiry.pop(); + } + + rows[idx].scope_name_id = ranked.empty() ? fallback_id : (*ranked.rbegin())->name_id; } } @@ -249,19 +508,29 @@ void MonitorBatchManager::pushSynchronization(const SynchronizationEventBatchRow syncBatch_.push(row); } +// Ranking shared by both resolvers. Depth first, then latest start; the +// instance id breaks a remaining tie so the answer does not depend on +// container order. Without it two scopes at the same depth and start could +// resolve differently between the reference and the batch path, since neither +// std::sort nor an unordered_map preserves any order for equal keys. +// +// The tertiary key is also load-bearing for the sweep, not merely tidy: it +// orders a std::set, and a comparator that ever reports equivalence would make +// that set silently keep one of the two scopes and drop the other. Instance ids +// are unique, so no two entries can compare equal. +bool MonitorBatchManager::ScopeWindow::outranks(const ScopeWindow& other) const { + if (depth != other.depth) return depth > other.depth; + if (start_ns != other.start_ns) return start_ns > other.start_ns; + return instance_id > other.instance_id; +} + uint32_t MonitorBatchManager::resolveScopeIdLocked(int64_t ts_ns) const { - uint32_t best_id = 0; - int best_depth = -1; - int64_t best_start = 0; - for (auto it = completedScopeWindows_.rbegin(); it != completedScopeWindows_.rend(); ++it) { - if (ts_ns < it->start_ns || ts_ns > it->end_ns) continue; - if (it->depth > best_depth || (it->depth == best_depth && it->start_ns >= best_start)) { - best_id = it->name_id; - best_depth = it->depth; - best_start = it->start_ns; - } + const ScopeWindow* best = nullptr; + for (const auto& w : completedScopeWindows_) { + if (ts_ns < w.start_ns || ts_ns > w.end_ns) continue; + if (!best || w.outranks(*best)) best = &w; } - return best_id; + return best ? best->name_id : 0; } } // namespace gpufl::detail diff --git a/include/gpufl/core/monitor_batch_manager.hpp b/include/gpufl/core/monitor_batch_manager.hpp index 30d87a2..7d201ff 100644 --- a/include/gpufl/core/monitor_batch_manager.hpp +++ b/include/gpufl/core/monitor_batch_manager.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -44,6 +45,46 @@ class MonitorBatchManager { uint32_t activeScopeNameId() const; /** @brief How many scopes are open right now; the depth a new one nests at. */ int openScopeDepth() const; + /** @brief Capture and publish a close timestamp as one ordered operation. */ + int64_t captureScopeCloseTimestamp(uint64_t instance_id); + /** @brief Publish an already-chosen close timestamp before state transition. */ + void markScopeClosePending(uint64_t instance_id, int64_t end_ns); + + /** + * @brief Publish the point below which no future sample can be attributed. + * + * Contract: no subsequent SUCCESSFUL decode will return a sample with a + * timestamp at or below @p ts_ns. Completed scopes that ended before it can + * therefore never match again and are dropped. + * + * Deliberately NOT wall-clock. A stalled collector lets `now` run on while + * undecoded samples still sit in the buffer, and trimming against it would + * discard the scopes those samples need. It is also not "last decode time" + * nor "oldest decoded timestamp" - both can move ahead of samples that are + * still to come. + * + * Monotonic: a lower value is ignored. The caller must simply not advance it + * when a decode fails or the buffer overflowed, since either means samples + * were lost rather than delivered. + */ + void publishScopeRetentionWatermark(int64_t ts_ns); + + /** @brief Mark the wall-clock boundary from which PM samples may be pending. */ + void beginPmScopeAttribution(int64_t start_ns); + /** @brief Mark that the final PM decode completed. */ + void endPmScopeAttribution(); + /** @brief Completed scope history evicted while PM attribution was active. */ + uint64_t scopeAttributionTruncated() const; + /** @brief PM metric rows that have passed through scope attribution. */ + uint64_t pmSampleRowsSeen() const; + + /** @brief Test seam: resolve a batch exactly as the drain path does. */ + void resolveScopeIdsForTesting(std::vector& rows, uint32_t fallback_id); + /** @brief Test seam: the original per-sample resolver, kept as the reference + * the batch sweep is checked against. */ + uint32_t resolveScopeIdForTesting(int64_t ts_ns) const; + /** @brief Test seam: completed scopes still retained. */ + size_t retainedCompletedScopesForTesting() const; bool pushKernel(const KernelBatchRow& row, const KernelDetailRow* detail = nullptr); bool pushMemcpy(const MemcpyBatchRow& row); @@ -69,6 +110,10 @@ class MonitorBatchManager { uint64_t instance_id = 0; uint32_t name_id = 0; int depth = 0; + + /** @brief True when this scope should win over @p other for a sample + * both contain. Deepest, then latest start, then instance id. */ + bool outranks(const ScopeWindow& other) const; }; struct OpenScopeWindow { @@ -79,6 +124,41 @@ class MonitorBatchManager { uint32_t resolveScopeIdLocked(int64_t ts_ns) const; + /** + * @brief Attribute a whole batch of samples in one pass. + * + * Sorts the samples and a snapshot of the candidate scopes once, then + * sweeps them together, rather than re-scanning every scope for every + * sample. The snapshot includes scopes that are STILL OPEN, given a + * provisional end: PM drains mid-run, so the scope covering a sample is + * routinely still open when that sample is decoded. + * + * Candidates cannot be kept pre-sorted. A scope's close timestamp is taken + * before PushScopeRow acquires the lock, so two threads closing at once + * append out of order. + * + * Selection matches resolveScopeIdLocked exactly: the interval contains the + * timestamp (both ends inclusive), then greatest depth, then latest start. + */ + std::vector snapshotScopeCandidatesLocked( + const std::vector& rows) const; + static void resolveScopeIdsForBatch(std::vector& candidates, + std::vector& rows, + uint32_t fallback_id); + + /** @brief Drop completed scopes that can no longer match any future sample. */ + void trimCompletedScopesLocked(); + + /** + * @brief Apply the hard cap, counting whatever it drops. + * + * Called on every scope close, not only when PM samples arrive: the + * retention watermark is the real bound, but only PM publishes one. A run + * without PM sampling would otherwise never trim at all. + */ + void enforceScopeCapLocked(); + void clearPendingScopeCloseLocked(uint64_t instance_id); + FlushSink flushSink_; DictionaryManager dictManager_; @@ -107,7 +187,26 @@ class MonitorBatchManager { // application scope is open - removes the right one. std::vector> scopeNameStack_; std::unordered_map openScopeWindows_; - std::vector completedScopeWindows_; + // A close timestamp is captured before scopeBatchMu_ can be acquired. + // Publishing it separately lets a concurrent PM snapshot cap an otherwise + // still-open interval at its real end instead of extending it to the batch + // watermark. + mutable std::mutex pendingScopeCloseMu_; + std::unordered_map pendingScopeCloseNs_; + // Closed scopes still needed to attribute samples that have not been + // decoded yet. Unordered - see snapshotScopeCandidatesLocked. + std::deque completedScopeWindows_; + // Below this, no future sample can arrive; see + // publishScopeRetentionWatermark. 0 = nothing published yet, so nothing is + // dropped by the watermark. The hard cap still applies on every close. + int64_t scopeRetentionWatermarkNs_ = 0; + // Backstop only. The watermark is what should bound this; the cap exists so + // a source that never advances it cannot grow the deque without limit. + static constexpr size_t kMaxCompletedScopes = 65536; + int64_t pmScopeAttributionStartNs_ = 0; + bool scopeHistoryEvictionLogged_ = false; + uint64_t scopeAttributionTruncated_ = 0; + uint64_t pmSampleRowsSeen_ = 0; BatchBuffer syncBatch_; BatchBuffer memAllocBatch_; diff --git a/include/gpufl/core/nvtx_counters.cpp b/include/gpufl/core/nvtx_counters.cpp new file mode 100644 index 0000000..931945e --- /dev/null +++ b/include/gpufl/core/nvtx_counters.cpp @@ -0,0 +1,381 @@ +#include "gpufl/core/nvtx_counters.hpp" + +#include +#include + +#include "gpufl/core/counter_provider.hpp" +#include "gpufl/core/counter_registry.hpp" +#include "gpufl/core/debug_logger.hpp" + +namespace gpufl::detail { +namespace { + +// NVTX_COUNTER_SAMPLE_* from nvToolsExtCounters.h. Duplicated as plain +// constants so this file stays free of NVTX headers and can be unit-tested +// without them; the values are part of a released ABI and pinned by a test. +constexpr uint8_t kSampleZero = 0; +constexpr uint8_t kSampleUnchanged = 1; +constexpr uint8_t kSampleUnavailable = 2; + +// Serialises registration only. The sample path never takes it. +std::mutex g_mu; + +std::atomic g_registration_rejected{0}; +std::atomic g_unknown_id_samples{0}; +std::atomic g_unavailable_samples{0}; +std::atomic g_negative_samples{0}; +std::atomic g_samples_observed{0}; + +// Where the previous session's report ended. Guarded by g_mu; the snapshot is +// taken once per session at shutdown, never on a hot path. +uint64_t g_base_registration_rejected = 0; +uint64_t g_base_unknown_id = 0; +uint64_t g_base_unavailable = 0; +uint64_t g_base_negative = 0; +uint64_t g_base_observed = 0; + +bool CharsetOk(const char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '.' || c == '_' || c == '-'; +} + +// Logged once each: a rejected registration usually repeats, and a sample the +// bridge does not know about can arrive on every iteration of a decode loop. +bool g_logged_static_id = false; +bool g_logged_value_type = false; +bool g_logged_bad_name = false; +bool g_logged_collision = false; +bool g_logged_limit = false; +std::atomic g_logged_unknown_id{false}; +std::atomic g_logged_negative{false}; + +} // namespace + +NvtxCounterBridge& NvtxCounterBridge::instance() { + static auto* bridge = new NvtxCounterBridge(); + return *bridge; +} + +std::string NvtxCounterBridge::canonicalName(const std::string& domain_name, + const std::string& counter_name) { + std::string joined; + if (!domain_name.empty()) { + joined = domain_name; + joined += '.'; + } + joined += counter_name; + + std::string out; + out.reserve(joined.size()); + for (const char c : joined) out.push_back(CharsetOk(c) ? c : '_'); + + // A name of nothing but separators would produce a metric like + // `custom.___rate`, which names no counter anybody could have meant. + bool has_alnum = false; + for (const char c : out) { + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9')) { + has_alnum = true; + break; + } + } + if (!has_alnum) return {}; + if (out.size() > CounterRegistry::kMaxNameLength) return {}; + return out; +} + +NvtxCounterBridge::RegisterResult NvtxCounterBridge::registerCounter( + const std::string& domain_name, const std::string& counter_name, + const uint64_t requested_id, const ValueType type) { + RegisterResult result = registerCounterInner(domain_name, counter_name, + requested_id, type); + if (result.status != RegisterStatus::Accepted) { + g_registration_rejected.fetch_add(1, std::memory_order_relaxed); + } + return result; +} + +NvtxCounterBridge::RegisterResult NvtxCounterBridge::registerCounterInner( + const std::string& domain_name, const std::string& counter_name, + const uint64_t requested_id, const ValueType type) { + RegisterResult result; + + // An application-chosen ID is unique only inside its domain, so binding it + // to a slot needs a (domain, id) table. Without one, two domains that both + // use id 123 would share a slot and add their rates together - a wrong + // number that looks like a real one. Refused until that table exists. + if (requested_id != 0) { + result.status = RegisterStatus::StaticIdUnsupported; + std::lock_guard lk(g_mu); + if (!g_logged_static_id) { + g_logged_static_id = true; + GFL_LOG_WARN("[NvtxCounters] counter '", counter_name, + "' uses an application-assigned id; gpufl only " + "tracks tool-assigned ids (pass " + "NVTX_COUNTER_ID_NONE). This counter is not " + "available to rules."); + } + return result; + } + + // The only shape the registry can represent. It accumulates unsigned, so + // an absolute series read as deltas would sum a monotonic curve, and a + // counter with no semantics at all is not enough to tell the two apart. + if (type != ValueType::Delta) { + result.status = RegisterStatus::UnsupportedValueType; + std::lock_guard lk(g_mu); + if (!g_logged_value_type) { + g_logged_value_type = true; + GFL_LOG_WARN("[NvtxCounters] counter '", counter_name, + "' is not a DELTA counter (attach " + "nvtxSemanticsCounter_t with " + "NVTX_COUNTER_FLAG_VALUETYPE_DELTA). Absolute and " + "unspecified counters are not converted, because " + "reading one as deltas produces a plausible wrong " + "rate rather than an obvious failure."); + } + return result; + } + + const std::string name = canonicalName(domain_name, counter_name); + if (name.empty()) { + result.status = RegisterStatus::BadName; + std::lock_guard lk(g_mu); + if (!g_logged_bad_name) { + g_logged_bad_name = true; + GFL_LOG_WARN("[NvtxCounters] counter name has nothing usable in " + "it after canonicalisation; rules address counters " + "as custom._rate over [A-Za-z0-9._-]"); + } + return result; + } + + std::lock_guard lk(g_mu); + const size_t count = count_.load(std::memory_order_relaxed); + + // Canonicalisation maps every out-of-charset byte to '_' and joins the + // domain with '.', so it is not injective: "a b" and "a/b" meet at "a_b", + // and ("a", "b.c") meets ("a.b", "c"). The ORIGINAL pair decides which + // case this is: the same pair again is an idempotent re-registration; a + // different pair is two counters, and merging them would silently add two + // unrelated workloads into a single rate. + for (size_t i = 0; i < count; ++i) { + if (entries_[i].name != name) continue; + if (entries_[i].domain_original == domain_name && + entries_[i].counter_original == counter_name) { + result.id = kDynamicIdBase + i; + result.status = RegisterStatus::Accepted; + result.metric = "custom." + name + "_rate"; + return result; + } + result.status = RegisterStatus::BadName; + if (!g_logged_collision) { + g_logged_collision = true; + GFL_LOG_WARN("[NvtxCounters] counter '", counter_name, + "' in domain '", domain_name, "' canonicalises to '", + name, "', which already belongs to counter '", + entries_[i].counter_original, "' in domain '", + entries_[i].domain_original, + "'. Refused rather than merged; rename one of them."); + } + return result; + } + + if (count >= kMaxTracked) { + result.status = RegisterStatus::LimitReached; + if (!g_logged_limit) { + g_logged_limit = true; + GFL_LOG_WARN("[NvtxCounters] tracking limit (", kMaxTracked, + ") reached; '", name, "' is not available to rules"); + } + return result; + } + + // Through the provider: the evaluator reads whatever + // ActiveCounterProvider() resolves, and registering into this module's own + // registry instead leaves the counter invisible to it wherever a shared + // runtime is bound - proven by mutation on the 3090, where exactly this + // bypass turned a firing rule into custom_metric_never_registered. + const gpufl_counter_provider_v1* provider = ActiveCounterProvider(); + gpufl_counter_handle handle = + provider->register_counter(name.c_str(), name.size()); + if (handle == nullptr) { + result.status = RegisterStatus::LimitReached; + if (!g_logged_limit) { + g_logged_limit = true; + GFL_LOG_WARN("[NvtxCounters] the counter registry refused '", name, + "'; it is not available to rules"); + } + return result; + } + + // Fields first, count last with release: the sample path reads count_ + // with acquire and no lock, so the store below is what publishes the + // entry. The provider travels with the handle it issued - a handle is a + // pointer into that provider's registry, and the pair must never split. + entries_[count].provider = provider; + entries_[count].handle = handle; + entries_[count].name = name; + entries_[count].domain_original = domain_name; + entries_[count].counter_original = counter_name; + entries_[count].unavailable.store(0, std::memory_order_relaxed); + count_.store(count + 1, std::memory_order_release); + + result.id = kDynamicIdBase + count; + result.status = RegisterStatus::Accepted; + result.metric = "custom." + name + "_rate"; + // Printed because nothing else connects the NVTX name the application + // wrote to the metric name a rule has to be written against. + GFL_LOG_INFO("[NvtxCounters] counter '", counter_name, "' -> rule metric ", + result.metric); + return result; +} + +void NvtxCounterBridge::sampleDelta(const uint64_t id, const int64_t value) { + if (id < kDynamicIdBase) { + g_unknown_id_samples.fetch_add(1, std::memory_order_relaxed); + if (!g_logged_unknown_id.exchange(true, std::memory_order_relaxed)) { + GFL_LOG_WARN("[NvtxCounters] sample for a counter this build did " + "not assign an id to; it is not reaching any rule"); + } + return; + } + const uint64_t index = id - kDynamicIdBase; + if (index >= count_.load(std::memory_order_acquire)) { + g_unknown_id_samples.fetch_add(1, std::memory_order_relaxed); + return; + } + if (value < 0) { + // The registry accumulates unsigned. Wrapping it backwards turns the + // very next rate into an enormous number instead of a small one, so a + // negative delta is dropped and counted rather than applied. + g_negative_samples.fetch_add(1, std::memory_order_relaxed); + if (!g_logged_negative.exchange(true, std::memory_order_relaxed)) { + GFL_LOG_WARN("[NvtxCounters] negative delta dropped; a DELTA " + "counter feeding a rate must not go backwards"); + } + return; + } + g_samples_observed.fetch_add(1, std::memory_order_relaxed); + if (value == 0) return; // a real observation of "no traffic"; nothing to add + + const Entry& entry = entries_[index]; + entry.provider->add(entry.handle, static_cast(value)); +} + +void NvtxCounterBridge::sampleNoValue(const uint64_t id, const uint8_t reason) { + if (id < kDynamicIdBase || + id - kDynamicIdBase >= count_.load(std::memory_order_acquire)) { + g_unknown_id_samples.fetch_add(1, std::memory_order_relaxed); + return; + } + switch (reason) { + case kSampleZero: + case kSampleUnchanged: + // Both say the delta is zero. For a rate that is precisely "do + // nothing" - adding 1 here would invent traffic out of a sample + // that exists to report the absence of it. Still a valid + // OBSERVATION, so it counts toward the denominator. + g_samples_observed.fetch_add(1, std::memory_order_relaxed); + return; + case kSampleUnavailable: + default: + // The application could not read its own counter, so the true + // delta over this stretch is unknown - NOT zero. Recorded per + // counter; MetricSource discards the rate window it lands in, + // because a rate computed over the gap sags below any threshold + // and fires the rule on a workload that never slowed down. + entries_[id - kDynamicIdBase].unavailable.fetch_add( + 1, std::memory_order_relaxed); + g_unavailable_samples.fetch_add(1, std::memory_order_relaxed); + return; + } +} + +uint64_t NvtxCounterBridge::unavailableCountFor( + const std::string& canonical_name) const { + // Lock-free on purpose: names are immutable once their entry is published, + // and count_ is acquire-loaded, so this is safe against a concurrent + // registration of a LATER entry. + const size_t count = count_.load(std::memory_order_acquire); + for (size_t i = 0; i < count; ++i) { + if (entries_[i].name == canonical_name) { + return entries_[i].unavailable.load(std::memory_order_relaxed); + } + } + return 0; +} + +NvtxCounterBridge::QualitySnapshot NvtxCounterBridge::takeSessionSnapshot() { + std::lock_guard lk(g_mu); + QualitySnapshot snap; + const uint64_t rr = g_registration_rejected.load(std::memory_order_relaxed); + const uint64_t ui = g_unknown_id_samples.load(std::memory_order_relaxed); + const uint64_t ua = g_unavailable_samples.load(std::memory_order_relaxed); + const uint64_t ng = g_negative_samples.load(std::memory_order_relaxed); + const uint64_t ob = g_samples_observed.load(std::memory_order_relaxed); + snap.registration_rejected = rr - g_base_registration_rejected; + snap.unknown_id_samples = ui - g_base_unknown_id; + snap.unavailable_samples = ua - g_base_unavailable; + snap.negative_delta_samples = ng - g_base_negative; + snap.samples_observed = ob - g_base_observed; + g_base_registration_rejected = rr; + g_base_unknown_id = ui; + g_base_unavailable = ua; + g_base_negative = ng; + g_base_observed = ob; + return snap; +} + +uint64_t NvtxCounterBridge::registrationRejected() const { + return g_registration_rejected.load(std::memory_order_relaxed); +} + +uint64_t NvtxCounterBridge::unknownIdSamples() const { + return g_unknown_id_samples.load(std::memory_order_relaxed); +} + +uint64_t NvtxCounterBridge::unavailableSamples() const { + return g_unavailable_samples.load(std::memory_order_relaxed); +} + +uint64_t NvtxCounterBridge::negativeSamples() const { + return g_negative_samples.load(std::memory_order_relaxed); +} + +size_t NvtxCounterBridge::trackedCount() const { + return count_.load(std::memory_order_acquire); +} + +void NvtxCounterBridge::resetForTesting() { + std::lock_guard lk(g_mu); + const size_t count = count_.load(std::memory_order_relaxed); + for (size_t i = 0; i < count; ++i) { + entries_[i].provider = nullptr; + entries_[i].handle = nullptr; + entries_[i].name.clear(); + entries_[i].domain_original.clear(); + entries_[i].counter_original.clear(); + entries_[i].unavailable.store(0, std::memory_order_relaxed); + } + count_.store(0, std::memory_order_release); + g_registration_rejected.store(0, std::memory_order_relaxed); + g_unknown_id_samples.store(0, std::memory_order_relaxed); + g_unavailable_samples.store(0, std::memory_order_relaxed); + g_negative_samples.store(0, std::memory_order_relaxed); + g_base_registration_rejected = 0; + g_base_unknown_id = 0; + g_base_unavailable = 0; + g_base_negative = 0; + g_samples_observed.store(0, std::memory_order_relaxed); + g_base_observed = 0; + g_logged_static_id = false; + g_logged_value_type = false; + g_logged_bad_name = false; + g_logged_collision = false; + g_logged_limit = false; + g_logged_unknown_id.store(false, std::memory_order_relaxed); + g_logged_negative.store(false, std::memory_order_relaxed); +} + +} // namespace gpufl::detail diff --git a/include/gpufl/core/nvtx_counters.hpp b/include/gpufl/core/nvtx_counters.hpp new file mode 100644 index 0000000..d763410 --- /dev/null +++ b/include/gpufl/core/nvtx_counters.hpp @@ -0,0 +1,226 @@ +#pragma once + +#include +#include +#include + +struct gpufl_counter_provider_v1; + +namespace gpufl::detail { + +/** + * @brief Routes NVTX Counters extension samples into the counter registry. + * + * Lets an application drive a conditional deep window through the STANDARD + * NVIDIA API instead of gpufl's own: `nvtxCounterRegister` + `nvtxCounterSample*` + * reach us as direct calls into the injection library (the launcher already + * owns NVTX_INJECTION64_PATH), so nothing goes through a CUPTI activity + * buffer and the per-sample cost stays one relaxed atomic. + * + * Deliberately free of NVTX types so it is testable without an injected + * process; inject_entry.cpp does the NVTX-shaped translation. + */ +class NvtxCounterBridge { +public: + /** How the application says its samples should be read. */ + enum class ValueType { + /// No counter semantics were attached. NOT assumed to be anything. + Unspecified, + Absolute, + Delta, + DeltaSinceStart, + }; + + /** Why a registration was refused, for the capability report. */ + enum class RegisterStatus { + Accepted, + /// Value type absent or not Delta. Reading an absolute series as + /// deltas would sum a monotonic curve into nonsense, so it is refused + /// rather than guessed. + UnsupportedValueType, + /// An application-chosen (static) ID. Unique only WITHIN its domain, + /// so honouring it needs a (domain, id) table this build does not + /// have; two domains reusing one id would share a slot and silently + /// add their rates together. + StaticIdUnsupported, + /// The name could not be turned into a metric name, or two DIFFERENT + /// NVTX names canonicalise to the same one - merging those would + /// silently add two unrelated workloads into a single rate. + BadName, + /// The registry is full, or this bridge's table is. + LimitReached, + }; + + struct RegisterResult { + /// Non-zero only when Accepted. Encodes the table index, so a sample + /// resolves in constant time with no lookup and no lock. + uint64_t id = 0; + RegisterStatus status = RegisterStatus::BadName; + /// The `custom._rate` a rule would be written against. Reported + /// so the link between the NVTX name and the CLI name is visible + /// rather than something the user has to infer. + std::string metric; + }; + + /** Base of the IDs this bridge hands out. Matches NVTX_COUNTER_ID_DYNAMIC_START. */ + static constexpr uint64_t kDynamicIdBase = static_cast(1) << 32; + /** Bound on tracked counters. The registry has its own, lower, limit. */ + static constexpr size_t kMaxTracked = 64; + + static NvtxCounterBridge& instance(); + + /** + * @brief Bind an NVTX counter to a registry slot. + * + * @param domain_name Domain name, or empty. Prefixed onto the counter name + * so two domains using the same counter name stay distinct. + * @param requested_id The application's counterId; 0 means "assign one". + */ + RegisterResult registerCounter(const std::string& domain_name, + const std::string& counter_name, + uint64_t requested_id, ValueType type); + + /** + * @brief A delta sample. Ignores anything this bridge did not hand out. + * + * Lock-free: the entry holds the provider that issued its handle, taken + * once at registration, so the sample path never resolves the provider - + * resolving it takes a mutex, and a mutex per sample in a decode loop + * changes the throughput the rule is measuring. + * + * Negative deltas are counted and dropped: the registry is an unsigned + * monotonic accumulator, and wrapping one backwards would turn the next + * rate into an astronomically large number rather than a small one. + */ + void sampleDelta(uint64_t id, int64_t value); + + /** + * @brief A sample that carries no value. + * + * `zero` and `unchanged` both mean the delta is 0, which for a rate is + * exactly "do nothing" - NOT an event. `unavailable` means the application + * failed to read its own counter: the true delta is UNKNOWN, not zero, and + * a rate computed over the gap would sag below any threshold and fire the + * rule on a workload that never slowed down. It is recorded per counter so + * the metric layer can discard the affected window. + */ + void sampleNoValue(uint64_t id, uint8_t reason); + + /** + * @brief Failed reads recorded for the counter behind @p canonical_name. + * + * Polled by MetricSource at bucket close: when this advances, the current + * rate window contains a gap of unknown size and is discarded rather than + * evaluated. 0 for names this bridge does not track, so non-NVTX custom + * counters are unaffected. + */ + uint64_t unavailableCountFor(const std::string& canonical_name) const; + + /** + * @brief One session's data-quality tallies, each with ONE meaning. + * + * Kept apart on purpose: "registration was refused" is a configuration + * problem, "an id we never issued" is an application bug, "unavailable" + * is a read the application itself failed, and a negative delta is a + * value that cannot be applied. Folding any two together produces a + * number nobody can act on. + */ + struct QualitySnapshot { + uint64_t registration_rejected = 0; + uint64_t unknown_id_samples = 0; + uint64_t unavailable_samples = 0; + uint64_t negative_delta_samples = 0; + /** + * Valid samples this session: accepted deltas (a zero delta is a real + * observation) plus ZERO/UNCHANGED no-value samples. The denominator + * that makes an all-zero failure row mean something - "0 failures out + * of 12,000 samples" is a clean bill; "0 out of 0" is a session where + * nothing was watched, and without this field the two are the same + * row. + */ + uint64_t samples_observed = 0; + + /// Failures only, on purpose: samples_observed is the denominator, + /// not a problem. + bool any() const { + return registration_rejected != 0 || unknown_id_samples != 0 || + unavailable_samples != 0 || negative_delta_samples != 0; + } + }; + + /** + * @brief This session's tallies, advancing the session baseline. + * + * The tallies are process-lifetime (like the counter slots), and an + * embedded host re-initialises in one process - exporting the raw values + * would re-report session one's problems as session two's. Each call + * returns what accrued since the previous call; the FIRST call returns + * everything since process start, which is where pre-init NVTX + * registrations belong: they happened during this run's startup and no + * other session can report them. + * + * Single consumer: the session-summary emit at shutdown. + */ + QualitySnapshot takeSessionSnapshot(); + + /** @brief Process-lifetime registrations refused, any reason. */ + uint64_t registrationRejected() const; + /** @brief Samples whose id this bridge never issued. */ + uint64_t unknownIdSamples() const; + /** @brief Samples the application itself could not read, all counters. */ + uint64_t unavailableSamples() const; + /** @brief Samples dropped for carrying a negative delta. */ + uint64_t negativeSamples() const; + /** @brief Counters bound so far. */ + size_t trackedCount() const; + + /** @brief Test seam. A real process never drops these bindings. */ + void resetForTesting(); + + /** + * @brief NVTX name to registry name: `.`, canonicalised. + * + * Counter names reach a rule as `custom._rate`, whose charset is + * [A-Za-z0-9._-]; NVTX names are free-form. Everything outside the charset + * becomes '_'. That is not injective - and the domain joins with '.', so + * ("a", "b.c") and ("a.b", "c") also meet - which is why the entry keeps + * the ORIGINAL pair and a collision is refused at registration instead of + * silently merging two counters into one slot. + */ + static std::string canonicalName(const std::string& domain_name, + const std::string& counter_name); + +private: + NvtxCounterBridge() = default; + + /// The actual registration; the public wrapper counts refusals in one + /// place so no refusal path can forget to. + RegisterResult registerCounterInner(const std::string& domain_name, + const std::string& counter_name, + uint64_t requested_id, ValueType type); + + struct Entry { + /// The provider that issued `handle`. A handle is a pointer into the + /// issuing provider's registry, so the pair must never be split. + const gpufl_counter_provider_v1* provider = nullptr; + void* handle = nullptr; // gpufl_counter_handle + std::string name; // canonical registry name + /// The NVTX names as the application wrote them. What tells an + /// idempotent re-registration apart from a canonicalisation collision. + std::string domain_original; + std::string counter_original; + /// Failed reads (SAMPLE_UNAVAILABLE) for this counter. + std::atomic unavailable{0}; + }; + + Entry entries_[kMaxTracked]; + /** + * Published with release AFTER an entry's fields are written, and read + * with acquire on the sample path. The mutex only serialises writers; a + * plain size_t here was a data race with the lock-free readers, and the + * race window is exactly a counter's first samples. + */ + std::atomic count_{0}; +}; + +} // namespace gpufl::detail diff --git a/include/gpufl/core/sampler.cpp b/include/gpufl/core/sampler.cpp index 4900a57..85f9841 100644 --- a/include/gpufl/core/sampler.cpp +++ b/include/gpufl/core/sampler.cpp @@ -1,5 +1,7 @@ #include "gpufl/core/sampler.hpp" +#include "gpufl/core/deep_window_rules.hpp" + #include "gpufl/core/common.hpp" #include "gpufl/core/debug_logger.hpp" #include "gpufl/core/logger/logger.hpp" @@ -106,6 +108,11 @@ void Sampler::runLoop_() { const int64_t ts = detail::GetTimestampNs(); for (const DeviceSample& d : collector_->sampleAll()) { + // A rule reads gauges from here, not by polling: the timestamp has + // to be when the measurement was actually taken, or a metric could + // never be detected as having stopped. + detail::DeepWindowRules::NoteDeviceSample(d, ts); + DeviceMetricBatchRow row; row.ts_ns = ts; row.device_id = d.device_id; diff --git a/include/gpufl/core/trace_type.hpp b/include/gpufl/core/trace_type.hpp index af79da4..5b41495 100644 --- a/include/gpufl/core/trace_type.hpp +++ b/include/gpufl/core/trace_type.hpp @@ -99,8 +99,9 @@ enum class TraceType : uint8_t { // API_ENTER. The collector stores it in g_launchMetaByCorr keyed by // corr_id; the matching KERNEL / MEMCPY / MEMSET activity record later // joins scope path, stack id, and API timestamps from it. Entries with no - // activity record by shutdown become synthetic kernels (drainSynthetic - // Kernels). Fields used on ActivityRecord: corr_id, name (raw), device_id, + // activity record by shutdown are dropped in real-record modes. Only + // synthesize-by-design modes turn them into callback-derived kernel rows. + // Fields used on ActivityRecord: corr_id, name (raw), device_id, // api_start_ns (= API_ENTER ns), user_scope, scope_depth, stack_id, plus // has_details + grid/block/dyn_shared and the precomputed simplified // occupancy (synthetic-kernel modes only). @@ -127,4 +128,4 @@ enum class TraceType : uint8_t { // synthetic rows. Fields used on ActivityRecord: corr_id. KERNEL_META_DISCARD, }; -} \ No newline at end of file +} diff --git a/include/gpufl/gpufl.hpp b/include/gpufl/gpufl.hpp index ff5f5ad..29caf98 100644 --- a/include/gpufl/gpufl.hpp +++ b/include/gpufl/gpufl.hpp @@ -236,6 +236,66 @@ void deepWindow(int64_t max_duration_ms, uint64_t max_launches = 0); // if (tokens_per_sec < 1000) gpufl::deepWindow(spec); void deepWindow(const DeepWindowSpec& spec); +// A named counter the application increments. Rules watch its RATE, so this +// is how something only your code knows - tokens, steps, requests - becomes a +// condition a deep window can trigger on. +// +// auto tokens = gpufl::counter("token"); // once, outside the loop +// for (...) { +// tokens.add(batchSize); // one relaxed atomic add +// } +// +// Prefer this to wrapping a hot loop in a scope. A scope costs two locked +// batch pushes and a row on the wire per iteration; and being one-per, it +// cannot say that a step produced eight tokens. +// +// Registration is the part that validates and allocates, so hoist it out of +// the loop. A handle is safe to keep in a static and across +// shutdown()/init(): the slot behind it lives for the process, and each +// session counts only what accrued after its own start. +class Counter { + public: + Counter() = default; + // Largest single add accepted. Not an overflow guard - the counter is + // 64-bit and rates are unsigned deltas, so a wrap is unreachable in + // practice and correct if it happened. This rejects a value that is not a + // count at all: a pointer, an uninitialised field, a negative cast to + // unsigned. Any of those quietly makes every rate meaningless. + static constexpr int64_t kMaxAddPerCall = 1LL << 40; + + // Ignores n outside (0, kMaxAddPerCall]. Validation happens here, on the + // caller's side of the boundary, so the contract lives in one place. + // + // This is an indirect call plus a relaxed atomic, not an inlined atomic: + // the counter lives in a shared runtime so that a profiled target and an + // injected evaluator share one registry, and passing a std::atomic across + // that boundary by address would tie both sides to one compiler's atomic + // layout. + void add(int64_t n = 1) const; + + // False when the name was rejected, the counter limit was reached, or no + // counter runtime could be bound. add() on such a handle is a no-op rather + // than an error. + bool valid() const { return handle_ != nullptr; } + + private: + friend Counter counter(const std::string&); + Counter(const void* provider, void* handle) + : provider_(provider), handle_(handle) {} + const void* provider_ = nullptr; // gpufl_counter_provider_v1* + void* handle_ = nullptr; // gpufl_counter_handle +}; + +// Registers (or finds) a counter. Names are 1-96 characters of [A-Za-z0-9._-]; +// anything else, or exceeding the counter limit, returns an invalid handle and +// logs why. Calling this repeatedly with the same name is safe and returns the +// same counter, including from several threads at once. +Counter counter(const std::string& name); + +// Convenience wrapper. NOT for tight loops: the name has to be looked up on +// every call, which is exactly the cost the handle exists to avoid. +void tick(const std::string& name, int64_t n = 1); + // Close the current window early. No-op when none is open. void deepWindowClose(); diff --git a/include/gpufl/inject/cuda_interpose_linux.cpp b/include/gpufl/inject/cuda_interpose_linux.cpp new file mode 100644 index 0000000..ed81e29 --- /dev/null +++ b/include/gpufl/inject/cuda_interpose_linux.cpp @@ -0,0 +1,290 @@ +// Linux/LD_PRELOAD CUDA activity-boundary interposition. +// +// Keep this in a separate translation unit from inject_entry.cpp so every +// exported CUDA symbol is compiled against NVIDIA's official CUDA 13 headers. +// That makes the compiler enforce the ABI for opaque handles and 2D/3D +// parameter structures instead of relying on hand-written void* equivalents. + +#if !defined(_WIN32) + +#include +#include + +#include +#include + +extern "C" { +void GpuFlightWaitAtCudaLaunchBoundary(); +void GpuFlightWaitAtCudaSyncBoundary(); +void GpuFlightWaitAtCudaMemoryBoundary(); +} + +namespace { + +template +Fn ResolveNext(const char* symbol) noexcept { + // RTLD_NEXT skips libgpufl_inject itself. Resolution is performed once per + // wrapper by a function-local static below, after the readiness wait; no + // CUDA API is called while the loader is resolving the real symbol. + return reinterpret_cast(dlsym(RTLD_NEXT, symbol)); +} + +} // namespace + +#define GPUFL_CUDA_INTERPOSE(WAIT, RETURN_TYPE, NAME, PARAMS, ARGS, FAILURE) \ + extern "C" __attribute__((visibility("default"))) \ + RETURN_TYPE NAME PARAMS { \ + WAIT(); \ + using Function = RETURN_TYPE (*) PARAMS; \ + static Function resolved_function = ResolveNext(#NAME); \ + return resolved_function ? resolved_function ARGS : FAILURE; \ + } + +// Launch and synchronization boundaries previously lived in inject_entry.cpp +// with ABI-compatible opaque types. Keeping them here means the same official +// headers now validate every interposed CUDA function. +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaLaunchBoundary, cudaError_t, __cudaLaunchKernel, + (const void* function_address, dim3 grid_dim, dim3 block_dim, void** args, + std::size_t shared_mem, cudaStream_t stream), + (function_address, grid_dim, block_dim, args, shared_mem, stream), + cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaLaunchBoundary, cudaError_t, cudaLaunchKernel, + (const void* function_address, dim3 grid_dim, dim3 block_dim, void** args, + std::size_t shared_mem, cudaStream_t stream), + (function_address, grid_dim, block_dim, args, shared_mem, stream), + cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaLaunchBoundary, cudaError_t, cudaLaunchKernelExC, + (const cudaLaunchConfig_t* config, const void* function_address, void** args), + (config, function_address, args), + cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaLaunchBoundary, CUresult, cuLaunchKernel, + (CUfunction function, unsigned int grid_x, unsigned int grid_y, + unsigned int grid_z, unsigned int block_x, unsigned int block_y, + unsigned int block_z, unsigned int shared_mem, CUstream stream, + void** kernel_params, void** extra), + (function, grid_x, grid_y, grid_z, block_x, block_y, block_z, shared_mem, + stream, kernel_params, extra), + CUDA_ERROR_UNKNOWN) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaSyncBoundary, cudaError_t, cudaDeviceSynchronize, + (), (), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaSyncBoundary, cudaError_t, cudaStreamSynchronize, + (cudaStream_t stream), (stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaSyncBoundary, CUresult, cuCtxSynchronize, + (), (), CUDA_ERROR_UNKNOWN) + +// CUDA Runtime API: synchronous, asynchronous, pitched, 3D, and peer copies. +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy, + (void* dst, const void* src, std::size_t bytes, cudaMemcpyKind kind), + (dst, src, bytes, kind), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpyAsync, + (void* dst, const void* src, std::size_t bytes, cudaMemcpyKind kind, + cudaStream_t stream), + (dst, src, bytes, kind, stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpyAsync_ptsz, + (void* dst, const void* src, std::size_t bytes, cudaMemcpyKind kind, + cudaStream_t stream), + (dst, src, bytes, kind, stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy2D, + (void* dst, std::size_t dst_pitch, const void* src, std::size_t src_pitch, + std::size_t width, std::size_t height, cudaMemcpyKind kind), + (dst, dst_pitch, src, src_pitch, width, height, kind), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy2DAsync, + (void* dst, std::size_t dst_pitch, const void* src, std::size_t src_pitch, + std::size_t width, std::size_t height, cudaMemcpyKind kind, + cudaStream_t stream), + (dst, dst_pitch, src, src_pitch, width, height, kind, stream), + cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy2DAsync_ptsz, + (void* dst, std::size_t dst_pitch, const void* src, std::size_t src_pitch, + std::size_t width, std::size_t height, cudaMemcpyKind kind, + cudaStream_t stream), + (dst, dst_pitch, src, src_pitch, width, height, kind, stream), + cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy3D, + (const cudaMemcpy3DParms* params), (params), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy3DAsync, + (const cudaMemcpy3DParms* params, cudaStream_t stream), + (params, stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy3DAsync_ptsz, + (const cudaMemcpy3DParms* params, cudaStream_t stream), + (params, stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpyPeer, + (void* dst, int dst_device, const void* src, int src_device, + std::size_t bytes), + (dst, dst_device, src, src_device, bytes), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpyPeerAsync, + (void* dst, int dst_device, const void* src, int src_device, + std::size_t bytes, cudaStream_t stream), + (dst, dst_device, src, src_device, bytes, stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpyPeerAsync_ptsz, + (void* dst, int dst_device, const void* src, int src_device, + std::size_t bytes, cudaStream_t stream), + (dst, dst_device, src, src_device, bytes, stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy3DPeer, + (const cudaMemcpy3DPeerParms* params), (params), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy3DPeerAsync, + (const cudaMemcpy3DPeerParms* params, cudaStream_t stream), + (params, stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy3DPeerAsync_ptsz, + (const cudaMemcpy3DPeerParms* params, cudaStream_t stream), + (params, stream), cudaErrorUnknown) + +// CUDA Driver API. The non-stream variants have PTDS aliases; asynchronous +// variants have PTSZ aliases when an application opts into per-thread default +// streams. Export both spellings so the readiness barrier cannot be bypassed +// by that compile-time mode. +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, CUresult, cuMemcpy, + (CUdeviceptr dst, CUdeviceptr src, std::size_t bytes), + (dst, src, bytes), CUDA_ERROR_UNKNOWN) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, CUresult, cuMemcpy_ptds, + (CUdeviceptr dst, CUdeviceptr src, std::size_t bytes), + (dst, src, bytes), CUDA_ERROR_UNKNOWN) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, CUresult, cuMemcpyAsync, + (CUdeviceptr dst, CUdeviceptr src, std::size_t bytes, CUstream stream), + (dst, src, bytes, stream), CUDA_ERROR_UNKNOWN) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, CUresult, cuMemcpyAsync_ptsz, + (CUdeviceptr dst, CUdeviceptr src, std::size_t bytes, CUstream stream), + (dst, src, bytes, stream), CUDA_ERROR_UNKNOWN) + +#define GPUFL_DRIVER_COPY_SYNC_PAIR(NAME, PARAMS, ARGS) \ + GPUFL_CUDA_INTERPOSE( \ + GpuFlightWaitAtCudaMemoryBoundary, CUresult, NAME, PARAMS, ARGS, \ + CUDA_ERROR_UNKNOWN) \ + GPUFL_CUDA_INTERPOSE( \ + GpuFlightWaitAtCudaMemoryBoundary, CUresult, NAME##_ptds, PARAMS, \ + ARGS, CUDA_ERROR_UNKNOWN) + +#define GPUFL_DRIVER_COPY_ASYNC_PAIR(NAME, PARAMS, ARGS) \ + GPUFL_CUDA_INTERPOSE( \ + GpuFlightWaitAtCudaMemoryBoundary, CUresult, NAME, PARAMS, ARGS, \ + CUDA_ERROR_UNKNOWN) \ + GPUFL_CUDA_INTERPOSE( \ + GpuFlightWaitAtCudaMemoryBoundary, CUresult, NAME##_ptsz, PARAMS, \ + ARGS, CUDA_ERROR_UNKNOWN) + +GPUFL_DRIVER_COPY_SYNC_PAIR( + cuMemcpyHtoD_v2, + (CUdeviceptr dst, const void* src, std::size_t bytes), + (dst, src, bytes)) + +GPUFL_DRIVER_COPY_SYNC_PAIR( + cuMemcpyDtoH_v2, + (void* dst, CUdeviceptr src, std::size_t bytes), + (dst, src, bytes)) + +GPUFL_DRIVER_COPY_SYNC_PAIR( + cuMemcpyDtoD_v2, + (CUdeviceptr dst, CUdeviceptr src, std::size_t bytes), + (dst, src, bytes)) + +GPUFL_DRIVER_COPY_ASYNC_PAIR( + cuMemcpyHtoDAsync_v2, + (CUdeviceptr dst, const void* src, std::size_t bytes, CUstream stream), + (dst, src, bytes, stream)) + +GPUFL_DRIVER_COPY_ASYNC_PAIR( + cuMemcpyDtoHAsync_v2, + (void* dst, CUdeviceptr src, std::size_t bytes, CUstream stream), + (dst, src, bytes, stream)) + +GPUFL_DRIVER_COPY_ASYNC_PAIR( + cuMemcpyDtoDAsync_v2, + (CUdeviceptr dst, CUdeviceptr src, std::size_t bytes, CUstream stream), + (dst, src, bytes, stream)) + +GPUFL_DRIVER_COPY_SYNC_PAIR( + cuMemcpy2D_v2, + (const CUDA_MEMCPY2D* params), + (params)) + +GPUFL_DRIVER_COPY_ASYNC_PAIR( + cuMemcpy2DAsync_v2, + (const CUDA_MEMCPY2D* params, CUstream stream), + (params, stream)) + +GPUFL_DRIVER_COPY_SYNC_PAIR( + cuMemcpy3D_v2, + (const CUDA_MEMCPY3D* params), + (params)) + +GPUFL_DRIVER_COPY_ASYNC_PAIR( + cuMemcpy3DAsync_v2, + (const CUDA_MEMCPY3D* params, CUstream stream), + (params, stream)) + +GPUFL_DRIVER_COPY_SYNC_PAIR( + cuMemcpyPeer, + (CUdeviceptr dst, CUcontext dst_context, CUdeviceptr src, + CUcontext src_context, std::size_t bytes), + (dst, dst_context, src, src_context, bytes)) + +GPUFL_DRIVER_COPY_ASYNC_PAIR( + cuMemcpyPeerAsync, + (CUdeviceptr dst, CUcontext dst_context, CUdeviceptr src, + CUcontext src_context, std::size_t bytes, CUstream stream), + (dst, dst_context, src, src_context, bytes, stream)) + +GPUFL_DRIVER_COPY_SYNC_PAIR( + cuMemcpy3DPeer, + (const CUDA_MEMCPY3D_PEER* params), + (params)) + +GPUFL_DRIVER_COPY_ASYNC_PAIR( + cuMemcpy3DPeerAsync, + (const CUDA_MEMCPY3D_PEER* params, CUstream stream), + (params, stream)) + +#undef GPUFL_DRIVER_COPY_ASYNC_PAIR +#undef GPUFL_DRIVER_COPY_SYNC_PAIR +#undef GPUFL_CUDA_INTERPOSE + +#endif // !defined(_WIN32) diff --git a/include/gpufl/inject/inject_entry.cpp b/include/gpufl/inject/inject_entry.cpp index 3ac37f3..dc0c645 100644 --- a/include/gpufl/inject/inject_entry.cpp +++ b/include/gpufl/inject/inject_entry.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -53,12 +54,34 @@ #define NVTX_NO_IMPL #endif #include +// Counters extension types (nvtxCounterAttr_t, the CBID list, module id) and +// the injection-side nvtxExtModuleInfo_t it pulls in. NVTX_NO_IMPL above keeps +// this types-only, same as the core header. The semantics header is separate +// and NOT pulled in by the counters one, but it is what says whether a sample +// is a delta or an absolute reading - the difference between a correct rate +// and a plausible wrong one. +// +// Guarded because the header-only NVTX3 distribution CMake falls back to when +// the toolkit ships no NVTX predates the counters extension. Without counters +// the injection simply does not export the extension entry, and NVTX counters +// stay tool-less no-ops in the target. +#if defined(__has_include) +# if __has_include() && \ + __has_include() +# define GPUFL_HAS_NVTX_COUNTERS 1 +# endif +#endif +#ifdef GPUFL_HAS_NVTX_COUNTERS +# include +# include +#endif #include "gpufl/gpufl.hpp" #include "gpufl/core/activity_record.hpp" #include "gpufl/core/common.hpp" #include "gpufl/core/debug_logger.hpp" // GFL_LOG_DEBUG (teardown tracing) #include "gpufl/core/monitor.hpp" +#include "gpufl/core/nvtx_counters.hpp" #include "gpufl/core/teardown_flag.hpp" // setProcessExitTeardown (Windows) #include "gpufl/upload/upload_logs.hpp" // gpufl::uploadLogs for --upload @@ -73,6 +96,11 @@ std::atomic g_deferred_init_finished{false}; std::atomic g_shutdown_started{false}; std::mutex g_deferred_init_mutex; std::condition_variable g_deferred_init_cv; +// A CUDA call made by gpufl's own deferred-init worker must never wait for +// that worker to finish. Most CUPTI setup does not call the runtime copy +// APIs, but the interpose layer must be safe if a driver/toolkit revision +// does: waiting here would be a self-deadlock. +thread_local bool g_is_deferred_init_worker = false; // Captured during init for the atexit upload path (--upload). g_log_path // mirrors InitOptions.log_path; both are read only in shutdownAndSignal, @@ -136,11 +164,11 @@ void endProcessScope() { if (!state.active) return; gpufl::ScopeBatchRow row; - row.ts_ns = gpufl::detail::GetTimestampNs(); row.scope_instance_id = state.instance_id; row.name_id = state.name_id; row.event_type = 1; row.depth = 0; + row.ts_ns = gpufl::Monitor::CaptureScopeCloseTimestamp(state.instance_id); gpufl::Monitor::PushScopeRow(row); gpufl::Monitor::EndProfilerScope(state.name.c_str()); if (state.perf_scope) gpufl::Monitor::EndPerfScope(state.name.c_str()); @@ -242,6 +270,7 @@ void waitForDeferredInit() { // these are compiled out to avoid unused-function diagnostics. void waitForDeferredInitForMs(const int wait_ms) { if (wait_ms <= 0) return; + if (g_is_deferred_init_worker) return; if (g_deferred_init_started.load(std::memory_order_acquire) && !g_deferred_init_finished.load(std::memory_order_acquire)) { std::unique_lock lock(g_deferred_init_mutex); @@ -259,6 +288,14 @@ void waitAtCudaSyncBoundary() { void waitAtCudaLaunchBoundary() { waitForDeferredInitForMs(envIntOrDefault(gpufl::env::kInjectLaunchWaitMs, 15000)); } + +void waitAtCudaMemoryBoundary() { + // A memory transfer is an activity-capture boundary for the same reason a + // launch is: if it crosses the target before CUPTI subscription completes, + // it is gone permanently. Reuse the launch timeout rather than introduce a + // second knob with identical semantics. + waitForDeferredInitForMs(envIntOrDefault(gpufl::env::kInjectLaunchWaitMs, 15000)); +} #endif // !_WIN32 @@ -394,6 +431,101 @@ int DomainRangePop(nvtxDomainHandle_t) { } } // namespace nvtx_injection_impl +// ── NVTX Counters extension (module 4) ──────────────────────────── +// +// The extension loader resolves InitializeInjectionNvtxExtension from the SAME +// NVTX_INJECTION64_PATH the launcher already sets for range injection +// (nvtxDetail/nvtxExtInit.h), so counters arrive as direct function calls into +// this library - no CUPTI activity records, and no second injection tool. That +// is what makes the standard NVIDIA API usable for a rule that fires thousands +// of times a second. +// +// These are the NVTX-shaped half only. Everything that decides what a sample +// MEANS lives in NvtxCounterBridge, which has no NVTX types in it and is unit +// tested without an injected process. +#ifdef GPUFL_HAS_NVTX_COUNTERS +namespace nvtx_counters_impl { + +using gpufl::detail::NvtxCounterBridge; + +// Every struct here was written by the CLIENT's copy of the NVTX headers, +// which can be a different version from ours. structSize gates every read: a +// smaller struct simply does not contain the field, and reading past it is +// reading someone else's memory that happens to parse. +bool AttrReadable(const nvtxCounterAttr_t* attr) { + return attr != nullptr && attr->structSize >= sizeof(nvtxCounterAttr_t); +} + +// Walk the semantics list for the counters extension and read the value type. +// Absent semantics stay Unspecified rather than defaulting to Delta: an +// absolute series accumulated as deltas produces a plausible wrong rate, which +// is worse than refusing the counter. +NvtxCounterBridge::ValueType valueTypeOf(const nvtxCounterAttr_t* attr) { + if (!AttrReadable(attr)) return NvtxCounterBridge::ValueType::Unspecified; + // Bounded walk: the list is application-built, and a cycle or a garbage + // `next` must not hang the register call. Nobody chains more than a + // handful of semantics; 16 is generous. + // + // The HEADER has to be readable before anything else is touched - + // including `next`, which the previous shape read in the loop increment + // even for a node it had just rejected as too short to contain it. A node + // that cannot hold its own header ends the walk rather than being skipped. + int walked = 0; + const nvtxSemanticsHeader_t* h = attr->semantics; + while (h != nullptr && walked++ < 16) { + if (h->structSize < sizeof(nvtxSemanticsHeader_t)) break; + const nvtxSemanticsHeader_t* next = h->next; + + // Exactly the version whose layout this build was compiled against. + // `>=` would read a future version's bytes through today's struct - + // the same plausible-wrong-value failure the DELTA check refuses. + if (h->semanticId == NVTX_SEMANTIC_ID_COUNTERS_V1 && + h->version == NVTX_COUNTER_SEMANTIC_VERSION && + h->structSize >= sizeof(nvtxSemanticsCounter_t)) { + const auto* sem = reinterpret_cast(h); + switch (sem->flags & NVTX_COUNTER_FLAG_VALUETYPE_DELTA_SINCE_START) { + case NVTX_COUNTER_FLAG_VALUETYPE_ABSOLUTE: + return NvtxCounterBridge::ValueType::Absolute; + case NVTX_COUNTER_FLAG_VALUETYPE_DELTA: + return NvtxCounterBridge::ValueType::Delta; + case NVTX_COUNTER_FLAG_VALUETYPE_DELTA_SINCE_START: + return NvtxCounterBridge::ValueType::DeltaSinceStart; + default: + break; // a counters semantic with no value type; keep walking + } + } + h = next; + } + return NvtxCounterBridge::ValueType::Unspecified; +} + +uint64_t NVTX_API CounterRegister(nvtxDomainHandle_t domain, + const nvtxCounterAttr_t* attr) { + const bool readable = AttrReadable(attr); + const char* name = (readable && attr->name != nullptr) ? attr->name : ""; + const uint64_t requested = + readable ? attr->counterId : NVTX_COUNTER_ID_NONE; + const auto result = NvtxCounterBridge::instance().registerCounter( + nvtxDomainName(domain), name, requested, valueTypeOf(attr)); + // 0 is a refusal the application can see: the header documents a register + // that could not be honoured, and every later sample carrying 0 is then + // dropped by the bridge instead of landing on someone else's slot. + return result.id; +} + +void NVTX_API CounterSampleInt64(nvtxDomainHandle_t, const uint64_t id, + const int64_t value) { + NvtxCounterBridge::instance().sampleDelta(id, value); +} + +void NVTX_API CounterSampleNoValue(nvtxDomainHandle_t, const uint64_t id, + const uint8_t reason) { + NvtxCounterBridge::instance().sampleNoValue(id, reason); +} + +} // namespace nvtx_counters_impl +#endif // GPUFL_HAS_NVTX_COUNTERS + void registerDeferredWaitAtexit() { std::call_once(g_deferred_wait_atexit_once, [] { // atexit handlers run in reverse registration order. Register this @@ -546,6 +678,7 @@ void startDeferredInjectInit() { } std::thread([] { + g_is_deferred_init_worker = true; // NVIDIA calls InitializeInjection from inside the CUDA driver's own // initialization. cuptiSubscribe probes driver state, so reaching it // while that initialization is still running faults inside libcuda. @@ -567,6 +700,7 @@ void startDeferredInjectInit() { // Injection must never throw through libcuda's callback path. } markDeferredInitFinished(); + g_is_deferred_init_worker = false; }).detach(); } @@ -574,6 +708,24 @@ void startDeferredInjectInit() { extern "C" { +#ifndef _WIN32 +// Cross-TU boundary hooks used by the typed CUDA interpose translation unit. +// They remain hidden inside libgpufl_inject; only CUDA ABI symbols are public. +__attribute__((visibility("hidden"))) +void GpuFlightWaitAtCudaLaunchBoundary() { + waitAtCudaLaunchBoundary(); +} + +__attribute__((visibility("hidden"))) +void GpuFlightWaitAtCudaSyncBoundary() { + waitAtCudaSyncBoundary(); +} + +__attribute__((visibility("hidden"))) +void GpuFlightWaitAtCudaMemoryBoundary() { + waitAtCudaMemoryBoundary(); +} +#endif GPUFL_INJECT_EXPORT int InitializeInjectionNvtx2( const NvtxGetExportTableFunc_t getExportTable) { @@ -607,6 +759,59 @@ GPUFL_INJECT_EXPORT int InitializeInjectionNvtx2( return 1; } +// NVTX extension-module entry: called once per extension module, on that +// module's first API call in the target, with a slot table to fill. Slots left +// untouched are turned into no-ops by the loader. +// +// Always returns success. Returning 0 makes the loader dlclose an injection +// library it loaded dynamically, and in a no-CUDA target this library can be +// held by exactly that reference while the deferred-init thread is still +// running. A module this build does not recognize gets no slots filled, which +// ends in the same no-ops as a failure return - without the unload. +#ifdef GPUFL_HAS_NVTX_COUNTERS +GPUFL_INJECT_EXPORT +int NVTX_API InitializeInjectionNvtxExtension(nvtxExtModuleInfo_t* module_info) { + // Every field below is written by the CLIENT's copy of the NVTX headers, + // which can be a different version from ours. Nothing is trusted before it + // is checked: structSize first, because it is what says the rest of the + // struct is even there to read. + if (module_info == nullptr || + module_info->structSize < sizeof(nvtxExtModuleInfo_t)) { + return 1; + } + if (module_info->moduleId != NVTX_EXT_COUNTERS_MODULEID || + module_info->compatId != NVTX_EXT_COUNTERS_COMPATID || + module_info->segments == nullptr) { + return 1; + } + + for (size_t s = 0; s < module_info->segmentsCount; ++s) { + nvtxExtModuleSegment_t* segment = module_info->segments + s; + // Counters declares exactly one segment (id 0). Filling slots in a + // segment we cannot name would be writing function pointers into a + // table whose layout we are guessing at. + if (segment->segmentId != 0 || segment->functionSlots == nullptr) { + continue; + } + const auto install = [segment](const size_t cbid, const intptr_t fn) { + if (cbid < segment->slotCount) segment->functionSlots[cbid] = fn; + }; + install(NVTX3EXT_CBID_nvtxCounterRegister, + reinterpret_cast(&nvtx_counters_impl::CounterRegister)); + install(NVTX3EXT_CBID_nvtxCounterSampleInt64, + reinterpret_cast(&nvtx_counters_impl::CounterSampleInt64)); + install(NVTX3EXT_CBID_nvtxCounterSampleNoValue, + reinterpret_cast(&nvtx_counters_impl::CounterSampleNoValue)); + // Float64, by-reference and batch stay unfilled: the registry is an + // unsigned integer accumulator, and the loader turns an unfilled slot + // into a no-op, which is the honest answer for a shape we cannot + // represent. Filling them to "not lose data" is how a float counter + // would start reporting a truncated rate nobody asked for. + } + return 1; +} +#endif // GPUFL_HAS_NVTX_COUNTERS + // First-chance entry: ld.so runs us before main(). // // **Disabled by default.** Phase 0.1 spike (2026-05-11) confirmed @@ -688,7 +893,7 @@ int GpuFlightInitializeInjectionAfterCuda() { return g_init_ok.load(std::memory_order_acquire) ? 1 : 0; } -#ifndef _WIN32 +#if !defined(_WIN32) && !defined(GPUFL_TYPED_CUDA_INTERPOSE) // Launch/sync symbol interposition (wait-for-init + forward) is Linux/glibc // only: it relies on LD_PRELOAD shadowing libcudart's symbols. Windows has // no preload interposition, so these wrappers don't exist there - Windows @@ -772,6 +977,6 @@ int cudaStreamSynchronize(void* stream) { static auto* fn = reinterpret_cast(dlsym(RTLD_NEXT, "cudaStreamSynchronize")); return fn ? fn(stream) : 0; } -#endif // !_WIN32 +#endif // !defined(_WIN32) && !defined(GPUFL_TYPED_CUDA_INTERPOSE) } // extern "C" diff --git a/include/gpufl/upload/upload_logs.cpp b/include/gpufl/upload/upload_logs.cpp index f7b7843..eac0963 100644 --- a/include/gpufl/upload/upload_logs.cpp +++ b/include/gpufl/upload/upload_logs.cpp @@ -1280,14 +1280,12 @@ UploadResult uploadLogs(const UploadOptions& opts) { // "%zu/%zu session(s)" which read like "F of S sessions // complete" but was actually mixing files (numerator) and // sessions (denominator) - fixed by labeling both axes. - std::fprintf(stderr, - "[gpufl::upload] %zu events uploaded (%zu MB), " - "%zu file(s), %zu session(s), %llds elapsed\n", - result.events_uploaded, + GFL_LOG_INFO("[upload] ", result.events_uploaded, + " events uploaded (", result.bytes_uploaded / (1024 * 1024), - result.files_processed, - targets.size(), - static_cast(total_elapsed)); + " MB), ", result.files_processed, " file(s), ", + targets.size(), " session(s), ", total_elapsed, + "s elapsed"); last_progress_time = now; bytes_since_last_progress = 0; }; diff --git a/python/bindings.cpp b/python/bindings.cpp index 19628f9..caf5daa 100644 --- a/python/bindings.cpp +++ b/python/bindings.cpp @@ -208,6 +208,15 @@ PYBIND11_MODULE(_gpufl_client, m) { m.def("deep_window_close", &gpufl::deepWindowClose); m.def("deep_window_active", &gpufl::deepWindowActive); + // Counter is exposed as an object so Python can hoist the lookup out of a + // decode loop the same way C++ does. add() then costs one relaxed atomic + // plus the call, instead of a name lookup per token. + py::class_(m, "Counter") + .def("add", &gpufl::Counter::add, py::arg("n") = 1) + .def("valid", &gpufl::Counter::valid); + m.def("counter", &gpufl::counter, py::arg("name")); + m.def("tick", &gpufl::tick, py::arg("name"), py::arg("n") = 1); + m.def("shutdown", &gpufl::shutdown); // ── Deferred bulk upload ──────────────────────────────────────────── diff --git a/python/gpufl/__init__.py b/python/gpufl/__init__.py index 41e10fb..55264d7 100644 --- a/python/gpufl/__init__.py +++ b/python/gpufl/__init__.py @@ -184,6 +184,7 @@ def _dbg(msg): Scope as _CScope, init, shutdown, system_start, system_stop, deep_window as _c_deep_window, deep_window_close as _c_deep_window_close, deep_window_active as _c_deep_window_active, + counter as _c_counter, tick as _c_tick, BackendKind, InitOptions, ProfilingEngine, upload_logs as _c_upload_logs, UploadOptions, UploadResult, ) @@ -220,6 +221,19 @@ def _c_deep_window_close(): def _c_deep_window_active(): return False + class _StubCounter: + def add(self, n=1): + return None + + def valid(self): + return False + + def _c_counter(name): + return _StubCounter() + + def _c_tick(name, n=1): + return None + class BackendKind: Auto = "Auto" Nvidia = "Nvidia" @@ -604,6 +618,43 @@ def system_stop(name="system"): return _original_system_stop(name) +def counter(name): + """Register (or find) a named counter whose RATE a rule can watch. + + How something only your code knows - tokens, steps, requests - becomes a + condition a deep window can trigger on:: + + tokens = gpufl.counter("token") # once, outside the loop + for step in decode(): + tokens.add(step.token_count) + + Hoist the call out of the loop: registration is the part that validates + and allocates, while ``add`` is a single atomic increment. Prefer this to + wrapping a hot loop in a Scope, which costs two locked batch pushes and a + row on the wire per iteration - and, being one-per, cannot say that a step + produced eight tokens. + + The handle is safe to keep in a module-level variable and across + ``shutdown()``/``init()``: the slot behind it lives for the process, and + each session counts only what accrued after its own start. + + Args: + name: 1-96 characters of ``[A-Za-z0-9._-]``. Anything else, or + exceeding the counter limit, returns a handle whose ``add`` is a + no-op; ``valid()`` reports which you got. + """ + return _c_counter(name) + + +def tick(name, n=1): + """Increment a counter by name. + + Convenience only. NOT for tight loops: the name is looked up on every + call, which is the cost :func:`counter` exists to avoid. + """ + return _c_tick(name, n) + + def deep_window(seconds=0.0, max_launches=0): """Arm deep profiling for a short, self-closing window. @@ -1077,6 +1128,7 @@ def clean_logs(log_path=None, log_prefix=None, *, dry_run=False): "Scope", "init", "shutdown", "session", "clean_logs", "targeting", "system_start", "system_stop", "deep_window", "deep_window_close", "deep_window_active", + "counter", "tick", "BackendKind", "InitOptions", "ProfilingEngine", "upload_logs", "UploadOptions", "UploadResult", ] diff --git a/runtime/counter_runtime.cpp b/runtime/counter_runtime.cpp new file mode 100644 index 0000000..55d44cf --- /dev/null +++ b/runtime/counter_runtime.cpp @@ -0,0 +1,75 @@ +// The shared counter runtime. One instance per process, so a target that ticks +// a counter and an injected evaluator that reads it are talking about the same +// slot - see gpufl_counter_abi.h for why that cannot be assumed otherwise. +// +// Deliberately thin: it owns CounterRegistry and wraps it in C. Keeping the +// logic in the registry means the embedded fallback path and this one cannot +// drift apart in behaviour. + +#include "gpufl/abi/gpufl_counter_abi.h" + +#include + +#include "gpufl/core/counter_registry.hpp" + +namespace { + +using gpufl::detail::CounterRegistry; + +// A handle IS the slot's address. Not a slot id: an id has to be +// bounds-checked against the container, checking it means locking the +// container, and that would put every ticking thread on one mutex - distorting +// the throughput a rule exists to measure. Slots are never freed and a deque +// never relocates them, so the address stays good for the life of the process. +using Slot = CounterRegistry::Slot; + +Slot* AsSlot(const gpufl_counter_handle h) { return static_cast(h); } + +gpufl_counter_handle RegisterCounter(const char* name, const size_t name_length) { + if (name == nullptr) return nullptr; + CounterRegistry& reg = CounterRegistry::instance(); + return reg.slotFor(reg.registerCounter(std::string(name, name_length))); +} + +gpufl_counter_handle Lookup(const char* name, const size_t name_length) { + if (name == nullptr) return nullptr; + CounterRegistry& reg = CounterRegistry::instance(); + return reg.slotFor(reg.findCounter(std::string(name, name_length))); +} + +// Lock-free from here down: one relaxed atomic each. +void Add(const gpufl_counter_handle handle, const uint64_t value) { + CounterRegistry::addRaw(AsSlot(handle), value); +} + +uint64_t Load(const gpufl_counter_handle handle) { + return CounterRegistry::rawValue(AsSlot(handle)); +} + +uint64_t LoadSinceBaseline(const gpufl_counter_handle handle) { + return CounterRegistry::valueSinceBaseline(AsSlot(handle)); +} + +void BeginSession() { CounterRegistry::instance().beginSession(); } +void EndSession() { CounterRegistry::instance().endSession(); } +int SessionActive() { return CounterRegistry::instance().sessionActive() ? 1 : 0; } + +const gpufl_counter_provider_v1 kProvider = { + GPUFL_COUNTER_ABI_VERSION, + sizeof(gpufl_counter_provider_v1), + &RegisterCounter, + &Add, + &Load, + &LoadSinceBaseline, + &BeginSession, + &EndSession, + &SessionActive, + &Lookup, +}; + +} // namespace + +extern "C" GPUFL_COUNTER_EXPORT const gpufl_counter_provider_v1* +gpufl_get_counter_provider_v1(void) { + return &kProvider; +} diff --git a/scripts/counter_cross_module_check.py b/scripts/counter_cross_module_check.py new file mode 100644 index 0000000..8bb74a2 --- /dev/null +++ b/scripts/counter_cross_module_check.py @@ -0,0 +1,136 @@ +# Proves the counter registry is shared across module boundaries. +# +# gpufl is a static library, so the Python extension and the injection library +# each hold their own copy of it. Without the shared runtime they would also +# hold their own counter registries, and a target ticking a counter would be +# invisible to the injected evaluator - the case counters exist for. +# +# Run standalone, or under the launcher: +# +# set GPUFL_REPO= +# python scripts/counter_cross_module_check.py +# gpufl trace -o out --passes Trace -- python scripts/counter_cross_module_check.py +# +# Exits non-zero on any failure, so it is usable as a CTest case rather than +# something a human has to read the output of. A check nobody runs, or one that +# prints a wrong answer and still succeeds, is not evidence of anything. +import ctypes +import os +import pathlib +import sys + +EXPECTED = 42 + +repo = os.environ.get("GPUFL_REPO") +if not repo: + print("XMOD: set GPUFL_REPO to the repository root", file=sys.stderr) + sys.exit(2) +sys.path.insert(0, str(pathlib.Path(repo) / "python")) + +# The extension must be the real one. A stub fallback would tick nothing and +# the check would then "pass" while proving the opposite of what it claims. +if sys.platform == "win32": + ext_dir = pathlib.Path(repo) / "python" / "gpufl" + if ext_dir.is_dir(): + os.add_dll_directory(str(ext_dir)) + +import gpufl # noqa: E402 + +# The package exposes counter() either way, so asking whether the attribute +# exists proves nothing. The extension module itself has to be there - a stub +# ticks nothing, and the check would then report a split registry when the real +# problem was an interpreter that cannot load the extension at all. +if "gpufl._gpufl_client" not in sys.modules: + print("XMOD: the _gpufl_client extension is not loaded - gpufl is running " + "its stub, so nothing was ticked. Check the interpreter matches the " + "one the extension was built for (%s)." % sys.version.split()[0], + file=sys.stderr) + sys.exit(1) + +tokens = gpufl.counter("xmod_token") +tokens.add(41) +tokens.add(1) + + +def runtime_names(): + """Platform library names, most specific first.""" + if sys.platform == "win32": + return ["gpufl_counter_runtime.dll"] + if sys.platform == "darwin": + return ["libgpufl_counter_runtime.dylib"] + return ["libgpufl_counter_runtime.so"] + + +# Beside the extension first. That is where deployment puts it, and resolving +# it any other way would test a layout the product does not ship. +search = [] +ext_file = getattr(sys.modules.get("gpufl._gpufl_client"), "__file__", None) +if ext_file: + search.append(pathlib.Path(ext_file).parent) +search.append(pathlib.Path(repo) / "python" / "gpufl") + +lib = None +for directory in search: + for name in runtime_names(): + candidate = directory / name + if candidate.exists(): + try: + lib = ctypes.CDLL(str(candidate)) + break + except OSError: + continue + if lib is not None: + break +if lib is None: + for name in runtime_names(): + try: + lib = ctypes.CDLL(name) + break + except OSError: + continue +if lib is None: + # Not a pass. The whole point is that this library is reachable; if it is + # not, the two modules are already on separate registries. + print("XMOD: shared runtime not loadable (%s)" % ", ".join(runtime_names()), + file=sys.stderr) + sys.exit(1) + +lib.gpufl_get_counter_provider_v1.restype = ctypes.c_void_p +p = lib.gpufl_get_counter_provider_v1() +if not p: + print("XMOD: no provider", file=sys.stderr) + sys.exit(1) + + +class Provider(ctypes.Structure): + _fields_ = [ + ("abi_version", ctypes.c_uint32), + ("struct_size", ctypes.c_uint32), + ("register_counter", ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t)), + ("add", ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_uint64)), + ("load", ctypes.CFUNCTYPE(ctypes.c_uint64, ctypes.c_void_p)), + ("load_since_baseline", ctypes.CFUNCTYPE(ctypes.c_uint64, ctypes.c_void_p)), + ("begin_session", ctypes.CFUNCTYPE(None)), + ("end_session", ctypes.CFUNCTYPE(None)), + ("session_active", ctypes.CFUNCTYPE(ctypes.c_int)), + ("lookup", ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t)), + ] + + +prov = ctypes.cast(p, ctypes.POINTER(Provider)).contents +name = b"xmod_token" + +# lookup, not register: if the extension had its own registry, the counter +# would be absent here rather than present with a value of 0. Those are +# different failures and the message should say which one happened. +h = prov.lookup(name, len(name)) +if not h: + print("XMOD: counter absent from the shared runtime - the extension is on " + "its own registry", file=sys.stderr) + sys.exit(1) + +value = prov.load(h) +print("XMOD: abi=%d value_via_abi=%d" % (prov.abi_version, value)) +if value != EXPECTED: + print("XMOD: expected %d, got %d" % (EXPECTED, value), file=sys.stderr) + sys.exit(1) diff --git a/scripts/counter_workload_bench.cu b/scripts/counter_workload_bench.cu new file mode 100644 index 0000000..fe49937 --- /dev/null +++ b/scripts/counter_workload_bench.cu @@ -0,0 +1,70 @@ +// Fixed-work decode-shaped target for the workload benchmark. +// +// Every configuration runs THIS binary with THESE arguments; only the rule +// configuration around it changes. Which config fires is decided by the +// threshold, not by reshaping the workload - a benchmark that slows the +// workload down to make the rule fire is measuring two changes at once. +// +// Fixed WORK (iterations), not fixed time: throughput and CPU time are only +// comparable when every run did the same thing. The kernel is deliberately +// small - a high iteration rate is the WORST case for per-iteration overhead, +// which is the thing being measured. +#include +#include +#include +#include + +#include +#include +#include + +__global__ void decode_step(float* x, int inner) { + float v = x[threadIdx.x]; + for (int i = 0; i < inner; ++i) v = v * 1.0001f + 0.5f; + x[threadIdx.x] = v; +} + +int main(int argc, char** argv) { + const long iters = argc > 1 ? std::atol(argv[1]) : 200000L; + const int tokens_per_step = argc > 2 ? std::atoi(argv[2]) : 32; + const int inner = argc > 3 ? std::atoi(argv[3]) : 200; + + nvtxDomainHandle_t domain = nvtxDomainCreateA("bench"); + nvtxSemanticsCounter_t sem = {}; + sem.header.structSize = sizeof(sem); + sem.header.semanticId = NVTX_SEMANTIC_ID_COUNTERS_V1; + sem.header.version = NVTX_COUNTER_SEMANTIC_VERSION; + sem.flags = NVTX_COUNTER_FLAG_VALUETYPE_DELTA; + sem.unit = "tokens"; + sem.unitScaleNumerator = 1; + sem.unitScaleDenominator = 1; + nvtxCounterAttr_t attr = {}; + attr.structSize = sizeof(attr); + attr.name = "tokens"; + attr.counterId = NVTX_COUNTER_ID_NONE; + attr.semantics = &sem.header; + const uint64_t counter = nvtxCounterRegister(domain, &attr); + + float* buf = nullptr; + cudaMalloc(&buf, 64 * sizeof(float)); + // Warm the context and JIT outside the timed region. + for (int i = 0; i < 200; ++i) decode_step<<<1, 64>>>(buf, inner); + cudaDeviceSynchronize(); + + const auto t0 = std::chrono::steady_clock::now(); + for (long i = 0; i < iters; ++i) { + decode_step<<<1, 64>>>(buf, inner); + cudaDeviceSynchronize(); + nvtxCounterSampleInt64(domain, counter, tokens_per_step); + } + const auto t1 = std::chrono::steady_clock::now(); + cudaFree(buf); + + const double secs = + std::chrono::duration_cast>(t1 - t0) + .count(); + std::printf("WL iters=%ld secs=%.3f iters_per_sec=%.1f tokens_per_sec=%.0f\n", + iters, secs, iters / secs, + iters / secs * tokens_per_step); + return 0; +} diff --git a/scripts/counter_workload_bench.py b/scripts/counter_workload_bench.py new file mode 100644 index 0000000..2bad094 --- /dev/null +++ b/scripts/counter_workload_bench.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""Workload-mode benchmark: evaluator cost on a fixed-work decode loop. + +Five configurations, identical target and arguments; only the rule differs. +Which one fires is decided by the threshold, never by reshaping the workload. + + bare no gpufl at all - the floor + no_evaluator gpufl trace, PM prepared+dormant (--deep-after far future), + NO rule: the profiling baseline the evaluator adds onto + rule_missing rule on a counter that never registers: evaluator polls a + Missing source all run + rule_armed rule on the live counter, threshold below any real rate: + full metric pipeline, condition never true + rule_fires threshold above the steady rate: condition true from warmup, + exactly one PM deep window fires mid-run + +Paired randomized blocks: each block runs all five in a shuffled order, and +ratios are computed within the block before aggregating. +""" +import csv +import gzip +import json +import os +import random +import re +import shutil +import statistics +import subprocess +import sys + +HOME = os.path.expanduser("~") +GPUFL = HOME + "/sources/gpufl-client/build/daemon/launcher/gpufl" +TARGET = "/tmp/wl/target" +OUTROOT = "/tmp/wl/runs" +CSV_PATH = "/tmp/wl/results.csv" +BLOCKS = int(sys.argv[1]) if len(sys.argv) > 1 else 10 +ITERS = sys.argv[2] if len(sys.argv) > 2 else "12000" +TOKENS = "32" +RULE_LIVE = "custom.bench.tokens_rate" +DEEP = ["--deep-for", "2s", "--deep-cooldown", "600s"] + + +def config_cmd(name, outdir): + base = ["/usr/bin/time", "-v"] + tgt = [TARGET, ITERS, TOKENS, "400000"] + if name == "bare": + return base + tgt + launch = base + [GPUFL, "trace", "-o", outdir] + if name == "no_evaluator": + return launch + ["--deep-after", "100000s"] + DEEP + ["--"] + tgt + if name == "rule_missing": + return launch + ["--deep-when", "custom.bench.missing_rate<1 for 1s"] + DEEP + ["--"] + tgt + if name == "rule_armed": + return launch + ["--deep-when", RULE_LIVE + "<1 for 1s"] + DEEP + ["--"] + tgt + if name == "rule_fires": + return launch + ["--deep-when", RULE_LIVE + "<999999999 for 1s"] + DEEP + ["--"] + tgt + raise ValueError(name) + + +CONFIGS = ["bare", "no_evaluator", "rule_missing", "rule_armed", "rule_fires"] + + +def parse_time_v(err): + user = sys_t = rss_kb = None + for line in err.splitlines(): + if "User time (seconds):" in line: + user = float(line.split(":")[-1]) + elif "System time (seconds):" in line: + sys_t = float(line.split(":")[-1]) + elif "Maximum resident set size" in line: + rss_kb = int(line.split(":")[-1]) + return user, sys_t, rss_kb + + +def scan_trace(outdir): + windows = 0 + outcome = "" + if not os.path.isdir(outdir): + return windows, outcome + for root, _dirs, files in os.walk(outdir): + for f in files: + if not f.endswith(".log.gz"): + continue + try: + with gzip.open(os.path.join(root, f), "rt", errors="replace") as fh: + for line in fh: + if '"type":"deep_window_event"' in line: + windows += 1 + elif '"type":"deep_window_rule_summary"' in line: + m = re.search(r'"outcome":"([a-z_]+)"', line) + if m: + outcome = m.group(1) + except OSError: + pass + return windows, outcome + + +def one_run(name, block): + outdir = "%s/%s_b%d" % (OUTROOT, name, block) + shutil.rmtree(outdir, ignore_errors=True) + cmd = config_cmd(name, outdir) + proc = subprocess.run(cmd, capture_output=True, text=True) + m = re.search(r"iters_per_sec=([0-9.]+)", proc.stdout) + if proc.returncode != 0 or not m: + print("RUN FAILED", name, "rc=", proc.returncode, file=sys.stderr) + print(proc.stdout[-2000:], file=sys.stderr) + print(proc.stderr[-2000:], file=sys.stderr) + sys.exit(1) + ips = float(m.group(1)) + wall = float(re.search(r"secs=([0-9.]+)", proc.stdout).group(1)) + user, sys_t, rss_kb = parse_time_v(proc.stderr) + windows, outcome = scan_trace(outdir) + shutil.rmtree(outdir, ignore_errors=True) + return { + "block": block, "config": name, "iters_per_sec": ips, "wall_s": wall, + "cpu_s": round((user or 0) + (sys_t or 0), 3), + "maxrss_mb": round((rss_kb or 0) / 1024.0, 1), + "windows": windows, "outcome": outcome, + } + + +def governor(): + try: + with open("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor") as f: + return f.read().strip() + except OSError: + return "unknown" + + +def boot_ci(values, iters=10000): + n = len(values) + meds = sorted(statistics.median(random.choices(values, k=n)) + for _ in range(iters)) + return meds[int(0.025 * iters)], meds[int(0.975 * iters)] + + +def main(): + random.seed(20260728) + os.makedirs(OUTROOT, exist_ok=True) + print("governor:", governor(), flush=True) + rows = [] + for block in range(BLOCKS): + order = CONFIGS[:] + random.shuffle(order) + print("block %d order: %s" % (block, " ".join(order)), flush=True) + for name in order: + row = one_run(name, block) + rows.append(row) + print(" %-13s %9.1f it/s cpu %6.2fs rss %7.1fMB win %d %s" + % (row["config"], row["iters_per_sec"], row["cpu_s"], + row["maxrss_mb"], row["windows"], row["outcome"]), + flush=True) + + # ── gates ─────────────────────────────────────────────────────────── + failures = [] + expected_rows = BLOCKS * len(CONFIGS) + if len(rows) != expected_rows: + failures.append("row count %d != %d" % (len(rows), expected_rows)) + + # windows / outcome each configuration MUST show. no_evaluator and bare + # have no rule, so no rule summary may exist at all. + expect = { + "bare": (0, ""), + "no_evaluator": (0, ""), + "rule_missing": (0, "never_true"), + "rule_armed": (0, "never_true"), + "rule_fires": (1, "fired"), + } + for r in rows: + want_win, want_outcome = expect[r["config"]] + if r["windows"] != want_win or r["outcome"] != want_outcome: + failures.append( + "block %d %s: windows=%d outcome=%r (want %d %r)" % + (r["block"], r["config"], r["windows"], r["outcome"], + want_win, want_outcome)) + + if failures: + for f in failures: + print("GATE FAILED:", f, file=sys.stderr) + sys.exit(1) + + with open(CSV_PATH, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) + + by = {} + for r in rows: + by.setdefault(r["config"], []).append(r) + + print("\n== medians over %d blocks (iters=%s) ==" % (BLOCKS, ITERS)) + print("%-13s %10s %8s %9s %7s %s" % + ("config", "it/s", "cpu_s", "rss_MB", "windows", "outcome")) + for name in CONFIGS: + rs = by.get(name, []) + if not rs: + continue + print("%-13s %10.1f %8.2f %9.1f %7d %s" % ( + name, + statistics.median(x["iters_per_sec"] for x in rs), + statistics.median(x["cpu_s"] for x in rs), + statistics.median(x["maxrss_mb"] for x in rs), + max(x["windows"] for x in rs), + rs[0]["outcome"])) + + print("\n== paired throughput ratios (per block, vs no_evaluator) ==") + base = {r["block"]: r["iters_per_sec"] for r in by.get("no_evaluator", [])} + for name in CONFIGS: + if name == "no_evaluator": + continue + ratios = [r["iters_per_sec"] / base[r["block"]] + for r in by.get(name, []) if r["block"] in base] + if not ratios: + continue + med = statistics.median(ratios) + lo, hi = boot_ci(ratios) + print(" %-13s x%.4f [%.4f, %.4f]" % (name, med, lo, hi)) + + print("\ngovernor after:", governor()) + print("WL_DRIVER_DONE all_gates_passed rows=%d" % len(rows)) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_early_memcpy_capture.py b/scripts/verify_early_memcpy_capture.py new file mode 100644 index 0000000..6688ea4 --- /dev/null +++ b/scripts/verify_early_memcpy_capture.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Verify the injected early-memcpy regression capture from raw NDJSON logs.""" + +from __future__ import annotations + +import argparse +import gzip +import json +from pathlib import Path +from typing import Iterable, TextIO + + +EXPECTED_BYTES = 4_194_304 + + +def open_text(path: Path) -> TextIO: + if path.suffix == ".gz": + return gzip.open(path, "rt", encoding="utf-8") + return path.open("r", encoding="utf-8") + + +def events(root: Path) -> Iterable[dict]: + for path in sorted(root.rglob("*.log")) + sorted(root.rglob("*.log.gz")): + with open_text(path) as stream: + for line_number, line in enumerate(stream, 1): + line = line.strip() + if not line: + continue + try: + yield json.loads(line) + except json.JSONDecodeError as exc: + raise RuntimeError(f"{path}:{line_number}: invalid JSON: {exc}") from exc + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("log_root", type=Path) + args = parser.parse_args() + + matching = {1: [], 2: []} + batch_count = 0 + kernel_rows = 0 + synchronization_rows = 0 + for event in events(args.log_root): + event_type = event.get("type") + if event_type == "kernel_event_batch": + kernel_rows += len(event.get("rows", [])) + continue + if event_type == "synchronization_event_batch": + synchronization_rows += len(event.get("rows", [])) + continue + if event_type != "memcpy_event_batch": + continue + batch_count += 1 + columns = event.get("columns", []) + try: + bytes_index = columns.index("bytes") + kind_index = columns.index("copy_kind") + except ValueError as exc: + raise RuntimeError("memcpy_event_batch is missing bytes/copy_kind") from exc + for row in event.get("rows", []): + kind = int(row[kind_index]) + size = int(row[bytes_index]) + if kind in matching and size == EXPECTED_BYTES: + matching[kind].append(size) + + print( + "memcpy capture:" + f" batches={batch_count}" + f" H2D_count={len(matching[1])} H2D_bytes={sum(matching[1])}" + f" D2H_count={len(matching[2])} D2H_bytes={sum(matching[2])}" + f" kernel_rows={kernel_rows}" + f" synchronization_rows={synchronization_rows}" + ) + if matching[1] != [EXPECTED_BYTES] or matching[2] != [EXPECTED_BYTES]: + print("VERIFY FAIL: expected exactly one 4 MiB H2D and one 4 MiB D2H") + return 1 + if kernel_rows < 1 or synchronization_rows < 1: + print("VERIFY FAIL: expected kernel and synchronization capture to remain active") + return 1 + print("VERIFY PASS") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 684accd..7ca2d86 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -18,6 +18,12 @@ set(GPUFL_TEST_SOURCES core/test_deep_window.cpp core/test_disabled.cpp core/test_wire_contract.cpp + core/test_counter_registry.cpp + core/test_nvtx_counters.cpp + core/test_debug_logger.cpp + core/test_metric_registry.cpp + core/test_deep_window_rule.cpp + core/test_deep_window_rules_install.cpp core/test_monitor.cpp core/test_itanium_demangle.cpp core/test_sampler.cpp @@ -27,8 +33,12 @@ set(GPUFL_TEST_SOURCES # need to depend on the launcher target (which is Linux-only). launcher/test_cli_parse.cpp launcher/test_info_command.cpp + launcher/test_deep_window_env.cpp + launcher/test_agent_launcher.cpp ${CMAKE_SOURCE_DIR}/daemon/launcher/cli_parse.cpp ${CMAKE_SOURCE_DIR}/daemon/launcher/info_command.cpp + ${CMAKE_SOURCE_DIR}/daemon/launcher/deep_window_env.cpp + ${CMAKE_SOURCE_DIR}/daemon/launcher/agent_launcher.cpp ) if(GPUFL_ENABLE_NVIDIA AND GPUFL_HAS_CUPTI) @@ -67,6 +77,18 @@ endif() add_executable(gpufl_tests ${GPUFL_TEST_SOURCES}) +# Standalone target for the Linux injection readiness regression. It +# intentionally performs its H2D copy immediately after two allocations, before +# the first launch boundary, so GPUFL_INJECT_INIT_DELAY_MS exposes any missing +# memcpy wait. The verifier is run explicitly on a CUDA host because this is an +# injected-process E2E, not a hermetic unit test. +if(UNIX AND NOT APPLE AND GPUFL_HAS_CUDA AND TARGET gpufl_inject) + add_executable(gpufl_early_memcpy_target + common/early_memcpy_target.cu + ) + target_link_libraries(gpufl_early_memcpy_target PRIVATE CUDA::cudart) +endif() + # 3. Enable CUDA-specific test target behavior only when CUDA is available if(GPUFL_HAS_CUDA) set_target_properties(gpufl_tests PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON) diff --git a/tests/backends/nvidia/test_nvidia_backend.cpp b/tests/backends/nvidia/test_nvidia_backend.cpp index 8338043..af4bef3 100644 --- a/tests/backends/nvidia/test_nvidia_backend.cpp +++ b/tests/backends/nvidia/test_nvidia_backend.cpp @@ -11,6 +11,52 @@ class CuptiBackendTest : public ::testing::Test { void SetUp() override { SKIP_IF_NO_CUDA(); } }; +// Pure decision for Monitor::Shutdown's orphan-synthesis suppression. No CUDA +// needed. The case this exists for: a real-record session (kernel activity +// enabled, not PC/SASS synthesize-by-design) where launches happened but zero +// activity records arrived - the orphan drain would otherwise fabricate every +// kernel row from host launch-to-launch gaps. +TEST(KernelActivityExpectedButMissing, TruthTable) { + using gpufl::KernelActivityExpectedButMissing; + + // The broken session: real records expected, launches happened, none came. + EXPECT_TRUE(KernelActivityExpectedButMissing(true, false, 100000, 0)); + // One single missing-record launch is still a fully-lost session. + EXPECT_TRUE(KernelActivityExpectedButMissing(true, false, 1, 0)); + + // The final argument is accepted/valid rows, not records merely observed + // before timestamp validation. A rejected record therefore leaves it zero. + EXPECT_TRUE(KernelActivityExpectedButMissing(true, false, 100000, 0)); + // Any valid real row means the all-records-missing diagnostic is false. + // Orphan synthesis is still independently disabled for the whole session. + EXPECT_FALSE(KernelActivityExpectedButMissing(true, false, 100000, 1)); + EXPECT_FALSE(KernelActivityExpectedButMissing(true, false, 100000, 99999)); + + // No launches: nothing was lost (and nothing to suppress). + EXPECT_FALSE(KernelActivityExpectedButMissing(true, false, 0, 0)); + + // Synthesize-by-design modes (PC sampling / SASS safe): synthetic rows + // are the product, never suppressed here. + EXPECT_FALSE(KernelActivityExpectedButMissing(true, true, 100000, 0)); + EXPECT_FALSE(KernelActivityExpectedButMissing(false, true, 100000, 0)); + + // Kernel activity was never enabled: absence of records is expected. + EXPECT_FALSE(KernelActivityExpectedButMissing(false, false, 100000, 0)); +} + +TEST(OrphanKernelSynthesisPolicy, RealRecordModesAlwaysSuppressOrphans) { + using gpufl::ShouldSuppressOrphanKernelSynthesis; + + EXPECT_TRUE(ShouldSuppressOrphanKernelSynthesis(true, false)); + // Even a contradictory caller cannot opt a real-record mode into + // host-gap timing. + EXPECT_TRUE(ShouldSuppressOrphanKernelSynthesis(true, true)); + // SASS metrics-only: neither real rows nor synthetic timing is wanted. + EXPECT_TRUE(ShouldSuppressOrphanKernelSynthesis(false, false)); + // PC sampling / other synthesize-by-design modes. + EXPECT_FALSE(ShouldSuppressOrphanKernelSynthesis(false, true)); +} + TEST_F(CuptiBackendTest, Lifecycle) { gpufl::MonitorOptions opts; opts.enable_debug_output = true; diff --git a/tests/common/early_memcpy_target.cu b/tests/common/early_memcpy_target.cu new file mode 100644 index 0000000..baf0bce --- /dev/null +++ b/tests/common/early_memcpy_target.cu @@ -0,0 +1,69 @@ +#include + +#include +#include +#include + +namespace { + +constexpr std::size_t kElements = 1u << 20; +constexpr std::size_t kBytes = kElements * sizeof(float); + +__global__ void AddOne(const float* input, float* output, std::size_t count) { + const std::size_t index = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (index < count) output[index] = input[index] + 1.0f; +} + +bool Check(cudaError_t result, const char* operation) { + if (result == cudaSuccess) return true; + std::fprintf(stderr, "%s failed: %s\n", operation, cudaGetErrorString(result)); + return false; +} + +} // namespace + +int main() { + std::vector input(kElements, 2.0f); + std::vector output(kElements, 0.0f); + float* device_input = nullptr; + float* device_output = nullptr; + + // The ordering is load-bearing for the regression: + // first allocation -> cuInit/injection callback -> deferred init; + // second allocation -> immediate H2D before any launch/sync wrapper. + if (!Check(cudaMalloc(&device_input, kBytes), "cudaMalloc(input)") || + !Check(cudaMalloc(&device_output, kBytes), "cudaMalloc(output)") || + !Check(cudaMemcpy(device_input, input.data(), kBytes, + cudaMemcpyHostToDevice), + "cudaMemcpy(H2D)")) { + cudaFree(device_input); + cudaFree(device_output); + return 1; + } + + AddOne<<<(kElements + 255) / 256, 256>>>( + device_input, device_output, kElements); + if (!Check(cudaGetLastError(), "AddOne launch") || + !Check(cudaDeviceSynchronize(), "cudaDeviceSynchronize") || + !Check(cudaMemcpy(output.data(), device_output, kBytes, + cudaMemcpyDeviceToHost), + "cudaMemcpy(D2H)")) { + cudaFree(device_input); + cudaFree(device_output); + return 1; + } + + const bool correct = + std::fabs(output.front() - 3.0f) < 1e-6f && + std::fabs(output.back() - 3.0f) < 1e-6f; + cudaFree(device_input); + cudaFree(device_output); + if (!correct) { + std::fprintf(stderr, "VERIFY FAIL\n"); + return 2; + } + + std::printf("VERIFY PASS bytes=%zu\n", kBytes); + return 0; +} diff --git a/tests/core/test_counter_registry.cpp b/tests/core/test_counter_registry.cpp new file mode 100644 index 0000000..6c7e9a5 --- /dev/null +++ b/tests/core/test_counter_registry.cpp @@ -0,0 +1,334 @@ +#include + +#include +#include +#include + +#include "gpufl.hpp" +#include "gpufl/core/counter_registry.hpp" +#include "gpufl/core/counter_provider.hpp" +#include "gpufl/core/monitor.hpp" + +#include +#include + +using gpufl::detail::CounterRegistry; + +namespace { + +class CounterRegistryTest : public ::testing::Test { + protected: + void SetUp() override { CounterRegistry::instance().resetForTesting(); } + void TearDown() override { CounterRegistry::instance().resetForTesting(); } + + CounterRegistry& reg() { return CounterRegistry::instance(); } +}; + +// ── runtime discovery ─────────────────────────────────────────────────────── + +TEST_F(CounterRegistryTest, ResolutionLooksBesideTheInjectionLibrary) { + // The motivating case: an ordinary C++ target links gpufl STATICALLY, so + // the provider's own module directory is the TARGET's directory, which has + // no runtime beside it. If that target calls counter() before its first + // CUDA call - entirely normal - it binds to a local registry and stays + // there, invisible to the evaluator that loads later. + // + // The launcher puts CUDA_INJECTION64_PATH in the child environment before + // exec, so the runtime's location is knowable from the first instruction. + // This checks the candidate list uses it; whether the file is actually + // there is a deployment question the build rules cover. + const auto candidates = gpufl::detail::CounterRuntimeCandidatesForTesting( + "/opt/gpufl/lib/libgpufl_inject.so", nullptr); + + const bool beside_inject = std::any_of( + candidates.begin(), candidates.end(), [](const std::string& c) { + return c.rfind("/opt/gpufl/lib/", 0) == 0; + }); + EXPECT_TRUE(beside_inject) + << "a target that ticks before CUDA init can never find the runtime"; +} + +TEST_F(CounterRegistryTest, AnExplicitRuntimePathIsTriedFirst) { + // Deployment layouts we do not control need a way to say where it is + // rather than have us guess. + const auto candidates = gpufl::detail::CounterRuntimeCandidatesForTesting( + nullptr, "/custom/place/libgpufl_counter_runtime.so"); + ASSERT_FALSE(candidates.empty()); + EXPECT_EQ(candidates.front(), "/custom/place/libgpufl_counter_runtime.so"); +} + +} // namespace + +namespace { + +const gpufl_counter_provider_v1* prov() { + return gpufl::detail::ActiveCounterProvider(); +} + +// Read back through the ACTIVE provider, not the local registry. With a shared +// runtime present, gpufl::counter() writes into the runtime's registry and this +// module's own is untouched - reading the latter would compare two different +// registries and fail for the right reason in the wrong test. +uint64_t Raw(const char* name) { + auto h = prov()->register_counter(name, std::strlen(name)); + return h ? prov()->load(h) : 0; +} + +uint64_t Since(const char* name) { + auto h = prov()->register_counter(name, std::strlen(name)); + return h ? prov()->load_since_baseline(h) : 0; +} + +void BeginSession() { prov()->begin_session(); } +void EndSession() { prov()->end_session(); } + +} // namespace + +TEST_F(CounterRegistryTest, SameNameResolvesToOneSlot) { + const auto a = reg().registerCounter("token"); + const auto b = reg().registerCounter("token"); + EXPECT_EQ(a, b); + EXPECT_EQ(reg().counterCount(), 1u); +} + +TEST_F(CounterRegistryTest, InvalidNamesAreRejected) { + EXPECT_EQ(reg().registerCounter(""), CounterRegistry::kInvalidSlot); + EXPECT_EQ(reg().registerCounter("has space"), CounterRegistry::kInvalidSlot); + EXPECT_EQ(reg().registerCounter("has/slash"), CounterRegistry::kInvalidSlot); + EXPECT_EQ(reg().registerCounter(std::string(200, 'x')), CounterRegistry::kInvalidSlot); + EXPECT_EQ(reg().counterCount(), 0u); +} + +TEST_F(CounterRegistryTest, CardinalityIsCapped) { + for (size_t i = 0; i < CounterRegistry::kMaxCounters; ++i) { + ASSERT_NE(reg().registerCounter("c" + std::to_string(i)), + CounterRegistry::kInvalidSlot); + } + // A permanent slot table is exactly the thing that must not grow without + // limit, so the one past the cap is refused rather than accepted. + EXPECT_EQ(reg().registerCounter("one_too_many"), CounterRegistry::kInvalidSlot); + EXPECT_EQ(reg().counterCount(), CounterRegistry::kMaxCounters); +} + +TEST_F(CounterRegistryTest, NonPositiveAddsAreIgnored) { + auto tokens = gpufl::counter("nonpositive_probe"); + ASSERT_TRUE(tokens.valid()); + tokens.add(5); + tokens.add(0); + tokens.add(-100); // must not wrap the unsigned counter + EXPECT_EQ(Raw("nonpositive_probe"), 5u); +} + +TEST_F(CounterRegistryTest, InvalidHandleAddIsANoOp) { + const auto bad = gpufl::counter("not a valid name"); + EXPECT_FALSE(bad.valid()); + bad.add(1); // must not crash +} + +TEST_F(CounterRegistryTest, AddsBeforeASessionAreExcludedFromIt) { + // Slots outlive the runtime, so anything ticked before init() is already in + // the slot. Without a baseline the new session would count it as its own. + auto tokens = gpufl::counter("pre_session_probe"); + tokens.add(1000); + + EXPECT_EQ(Raw("pre_session_probe"), 1000u); + + BeginSession(); + EXPECT_EQ(Since("pre_session_probe"), 0u) << "pre-init ticks are not this session's"; + + tokens.add(7); + EXPECT_EQ(Since("pre_session_probe"), 7u); +} + +TEST_F(CounterRegistryTest, AddsBetweenSessionsAreExcludedFromTheNext) { + auto tokens = gpufl::counter("token"); + + BeginSession(); + tokens.add(10); + EXPECT_EQ(Since("token"), 10u); + EndSession(); + + // gpufl is down; the handle still works and the slot still accumulates. + tokens.add(500); + + BeginSession(); + EXPECT_EQ(Since("token"), 0u) + << "ticks while no runtime was active belong to no session"; + tokens.add(3); + EXPECT_EQ(Since("token"), 3u); +} + +TEST_F(CounterRegistryTest, HandleSurvivesASessionChange) { + // The point of a process-lifetime slot: a handle held in a static, or by an + // embedded host across shutdown()/init(), keeps working. A generation + // number alone could not make that safe, since it cannot stop the state + // being freed underneath a concurrent add(). + auto tokens = gpufl::counter("token"); + + BeginSession(); + EndSession(); + BeginSession(); + + tokens.add(4); + EXPECT_EQ(Since("token"), 4u); +} + +TEST_F(CounterRegistryTest, CounterRegisteredMidSessionStartsAtZero) { + BeginSession(); + auto late = gpufl::counter("late"); + late.add(9); + EXPECT_EQ(Since("late"), 9u); +} + +TEST_F(CounterRegistryTest, ConcurrentRegistrationOfOneNameYieldsOneSlot) { + constexpr int kThreads = 8; + std::vector threads; + std::vector results(kThreads); + for (int i = 0; i < kThreads; ++i) { + threads.emplace_back([&, i] { results[i] = reg().registerCounter("shared"); }); + } + for (auto& t : threads) t.join(); + + for (const auto slot : results) EXPECT_EQ(slot, results[0]); + EXPECT_EQ(reg().counterCount(), 1u); +} + +TEST_F(CounterRegistryTest, ConcurrentAddsAreNotLost) { + auto tokens = gpufl::counter("token"); + BeginSession(); + + constexpr int kThreads = 8; + constexpr int kPerThread = 10'000; + std::vector threads; + for (int i = 0; i < kThreads; ++i) { + threads.emplace_back([&] { + for (int n = 0; n < kPerThread; ++n) tokens.add(1); + }); + } + for (auto& t : threads) t.join(); + + EXPECT_EQ(Since("token"), + static_cast(kThreads) * kPerThread); +} + +TEST_F(CounterRegistryTest, TickIsEquivalentButLooksThePriceUpEveryCall) { + gpufl::tick("steps"); + gpufl::tick("steps", 4); + EXPECT_EQ(Raw("steps"), 5u); +} + +// ── lifecycle wiring ──────────────────────────────────────────────────────── +// +// The tests above drive the registry directly, so they pass whether or not +// Monitor actually calls it. These go through Monitor::Initialize/Shutdown so +// that deleting the calls in monitor.cpp fails something. + +TEST_F(CounterRegistryTest, MonitorInitializeBaselinesThroughTheRealWiring) { + auto tokens = gpufl::counter("token"); + tokens.add(1000); // before any session exists + + gpufl::MonitorOptions opts; + gpufl::Monitor::Initialize(opts); + EXPECT_EQ(Since("token"), 0u) + << "Monitor::Initialize must baseline the registry"; + EXPECT_TRUE(prov()->session_active() != 0); + + tokens.add(6); + EXPECT_EQ(Since("token"), 6u); + + gpufl::Monitor::Shutdown(); + EXPECT_FALSE(prov()->session_active() != 0) + << "Monitor::Shutdown must close the session"; +} + +TEST_F(CounterRegistryTest, TicksBetweenTwoMonitorSessionsBelongToNeither) { + auto tokens = gpufl::counter("token"); + + gpufl::MonitorOptions opts; + gpufl::Monitor::Initialize(opts); + tokens.add(10); + EXPECT_EQ(Since("token"), 10u); + gpufl::Monitor::Shutdown(); + + // gpufl is down. The handle still works - the slot is process-lifetime - + // but these ticks are nobody's. + tokens.add(500); + + gpufl::Monitor::Initialize(opts); + EXPECT_EQ(Since("token"), 0u); + tokens.add(3); + EXPECT_EQ(Since("token"), 3u); + gpufl::Monitor::Shutdown(); +} + +TEST_F(CounterRegistryTest, ProcessExitPathAlsoClosesTheSession) { + // DrainAndFinalizeForExit is a separate teardown from Shutdown(); an + // embedded host that re-initialised after it would otherwise inherit the + // previous session's ticks. + gpufl::MonitorOptions opts; + gpufl::Monitor::Initialize(opts); + ASSERT_TRUE(prov()->session_active() != 0); + + gpufl::Monitor::DrainAndFinalizeForExit(); + EXPECT_FALSE(prov()->session_active() != 0); +} + +TEST_F(CounterRegistryTest, AddAboveThePerCallBoundIsRefused) { + // Not overflow protection: a value this large is a caller bug - a pointer, + // an uninitialised field - and letting it through makes every later rate + // meaningless. + auto tokens = gpufl::counter("bound_probe"); + tokens.add(gpufl::Counter::kMaxAddPerCall); + tokens.add(gpufl::Counter::kMaxAddPerCall + 1); // refused + EXPECT_EQ(Raw("bound_probe"), + static_cast(gpufl::Counter::kMaxAddPerCall)); +} + +TEST_F(CounterRegistryTest, DeltaIsCorrectAcrossAWrap) { + // Rates are unsigned deltas precisely so a wrap needs no saturation or CAS + // loop. Driven against the in-module registry, since reaching the wrap + // means writing the raw value rather than adding to it. + auto* slot = reg().slotFor(reg().registerCounter("wrap_probe")); + ASSERT_NE(slot, nullptr); + + slot->value.store(std::numeric_limits::max() - 5, + std::memory_order_relaxed); + reg().beginSession(); + CounterRegistry::addRaw(slot, 10); // wraps past zero + + EXPECT_EQ(CounterRegistry::valueSinceBaseline(slot), 10u) + << "unsigned subtraction must stay correct across a wrap"; +} + +// ── shared runtime binding ────────────────────────────────────────────────── + +TEST_F(CounterRegistryTest, BindsToTheSharedRuntimeWhenItIsPresent) { + // The cross-module property this whole ABI exists for cannot be proven from + // inside one executable - that needs the launcher + Python E2E. What can be + // checked here is that binding works at all when the library is reachable, + // and that the fallback is taken (rather than crashing) when it is not. + gpufl::detail::CounterProvider::resetForTesting(); + const auto* provider = gpufl::detail::CounterProvider::get(); + + if (provider == nullptr) { + GTEST_SKIP() << "gpufl_counter_runtime not colocated with the test binary; " + "fallback path exercised by every other test here"; + } + EXPECT_EQ(provider->abi_version, GPUFL_COUNTER_ABI_VERSION); + EXPECT_GE(provider->struct_size, sizeof(gpufl_counter_provider_v1)); + EXPECT_TRUE(gpufl::detail::CounterProvider::isShared()); + + // Round-trip through the C ABI rather than the registry directly. + auto handle = provider->register_counter("abi_probe", 9); + ASSERT_NE(handle, nullptr); + provider->begin_session(); + provider->add(handle, 7); + EXPECT_EQ(provider->load_since_baseline(handle), 7u); + provider->end_session(); +} + +TEST_F(CounterRegistryTest, ActiveProviderIsNeverNull) { + // Callers must not each carry a fallback branch, so this holds whether or + // not the shared runtime is present. + EXPECT_NE(gpufl::detail::ActiveCounterProvider(), nullptr); +} diff --git a/tests/core/test_debug_logger.cpp b/tests/core/test_debug_logger.cpp new file mode 100644 index 0000000..5d7c5d1 --- /dev/null +++ b/tests/core/test_debug_logger.cpp @@ -0,0 +1,42 @@ +#include + +#include "gpufl/core/debug_logger.hpp" + +namespace { + +class DebugLoggerTest : public ::testing::Test { + protected: + void SetUp() override { gpufl::DebugLogger::setEnabled(false); } + void TearDown() override { gpufl::DebugLogger::setEnabled(false); } +}; + +TEST_F(DebugLoggerTest, InfoIsAlwaysVisibleWithoutSourceLocation) { + testing::internal::CaptureStderr(); + GFL_LOG_INFO("prepared ", 3, " engines"); + const std::string output = testing::internal::GetCapturedStderr(); + + EXPECT_EQ(output, "[GPUFL] prepared 3 engines\n"); + EXPECT_EQ(output.find(__FILE__), std::string::npos); +} + +TEST_F(DebugLoggerTest, WarningIsAlwaysVisibleWithoutSourceLocation) { + testing::internal::CaptureStderr(); + GFL_LOG_WARN("sampling returned no data"); + const std::string output = testing::internal::GetCapturedStderr(); + + EXPECT_EQ(output, "[GPUFL-WARN] sampling returned no data\n"); + EXPECT_EQ(output.find(__FILE__), std::string::npos); +} + +TEST_F(DebugLoggerTest, DebugRemainsControlledByTheVerboseFlag) { + testing::internal::CaptureStdout(); + GFL_LOG_DEBUG("hidden"); + EXPECT_TRUE(testing::internal::GetCapturedStdout().empty()); + + gpufl::DebugLogger::setEnabled(true); + testing::internal::CaptureStdout(); + GFL_LOG_DEBUG("visible"); + EXPECT_EQ(testing::internal::GetCapturedStdout(), "[GPUFL] visible\n"); +} + +} // namespace diff --git a/tests/core/test_deep_window.cpp b/tests/core/test_deep_window.cpp index 9311c30..b01d8d0 100644 --- a/tests/core/test_deep_window.cpp +++ b/tests/core/test_deep_window.cpp @@ -10,6 +10,8 @@ // deliberately refuses to report a window as open when gpufl isn't running. #include +#include +#include #include #include @@ -74,6 +76,50 @@ gpufl::DeepWindowSpec Spec(const int64_t ms, const uint64_t launches, return spec; } +TEST_F(DeepWindowTest, ConcurrentRequestsStillLeaveExactlyOneQueued) { + // First-wins has to hold when two threads ask at once. Publishing the + // queued flag after releasing the lock left a gap where the second caller + // took the lock, saw nothing queued, and overwrote the first - the very + // rule this enforces, defeated by concurrency. + // + // What this DOES pin: exactly one winner under contention, so removing the + // first-wins check fails here immediately. + // + // What it does NOT pin: the publish-under-lock ordering. That window is a + // few instructions between unlocking and the atomic store, and a waiter + // woken by the unlock has almost always missed it - 400 gated rounds never + // reproduced it. The ordering is correct by construction rather than by + // demonstration, and pretending otherwise would be worse than saying so. + constexpr int kRounds = 50; + constexpr int kThreads = 4; + + for (int round = 0; round < kRounds; ++round) { + gpufl::DeepWindow::ResetForTesting(); + + std::atomic go{false}; + std::atomic accepted{0}; + std::vector threads; + for (int i = 0; i < kThreads; ++i) { + threads.emplace_back([&] { + while (!go.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + if (gpufl::DeepWindow::RequestOpenTagged(Spec(60000, 0)).status == + gpufl::OpenRequestStatus::Accepted) { + accepted.fetch_add(1, std::memory_order_relaxed); + } + }); + } + go.store(true, std::memory_order_release); + for (auto& t : threads) t.join(); + + ASSERT_EQ(accepted.load(), 1) + << "round " << round << ": more than one request was granted"; + ASSERT_NE(gpufl::DeepWindow::PendingOpenToken(), 0u) + << "round " << round << ": nothing left queued"; + } +} + } // namespace // ── open / close state machine ────────────────────────────────────────────── @@ -277,15 +323,36 @@ TEST_F(DeepWindowTest, RequestOpenCarriesItsBounds) { EXPECT_FALSE(gpufl::DeepWindow::Active()); } -TEST_F(DeepWindowTest, NewestPendingSpecWins) { - gpufl::DeepWindow::RequestOpen(Spec(0, 1)); - gpufl::DeepWindow::RequestOpen(Spec(60000, 0)); +TEST_F(DeepWindowTest, TheFirstQueuedRequestWins) { + // Policy reversed deliberately. "Newest wins" meant that with both + // --deep-after and a conditional rule configured, whichever happened to run + // second silently cancelled the other - decided by call order rather than + // by anything the user asked for. Refusing the second request at least + // leaves the rule able to retry. + gpufl::DeepWindow::RequestOpen(Spec(0, 1)); // budget of one launch + gpufl::DeepWindow::RequestOpen(Spec(60000, 0)); // must be refused Launch(); ASSERT_TRUE(gpufl::DeepWindow::Active()); - // Had the first spec won, this launch would spend its budget of 1. + // The first spec governs, so this launch spends its budget of 1. Launch(); - EXPECT_TRUE(gpufl::DeepWindow::Active()); + gpufl::DeepWindow::ServicePending(); + EXPECT_FALSE(gpufl::DeepWindow::Active()) + << "the second request replaced the first one's bounds"; +} + +TEST_F(DeepWindowTest, ATaggedRequestIsRefusedWhileAnotherIsQueued) { + // The scheduled window is installed at init, before any rule can fire. + gpufl::DeepWindow::ScheduleOpenAfter(/*delay_ms=*/50, Spec(60000, 0)); + + // A rule asking now must be told no, not silently granted a token for a + // window that belongs to the scheduled trigger. `busy` and not one of the + // engine statuses: the rule has to keep retrying, since the scheduled + // window will release the queue. + const gpufl::OpenRequestResult r = + gpufl::DeepWindow::RequestOpenTagged(Spec(1000, 0)); + EXPECT_EQ(r.token, 0u); + EXPECT_EQ(r.status, gpufl::OpenRequestStatus::Busy); } TEST_F(DeepWindowTest, ScheduledOpenWaitsOutItsDelay) { diff --git a/tests/core/test_deep_window_rule.cpp b/tests/core/test_deep_window_rule.cpp new file mode 100644 index 0000000..9f585d5 --- /dev/null +++ b/tests/core/test_deep_window_rule.cpp @@ -0,0 +1,808 @@ +#include + +#include +#include +#include + +#include "gpufl.hpp" +#include "gpufl/core/counter_provider.hpp" +#include "gpufl/core/counter_registry.hpp" +#include "gpufl/core/deep_window_rule.hpp" +#include "gpufl/core/metric_registry.hpp" + +using namespace gpufl::detail; + +namespace { + +constexpr int64_t kMs = 1000000; + +// A stand-in window coordinator. Lets the state machine be driven without a +// GPU, and - more usefully - lets a test refuse an open on purpose, which is +// the branch that decides whether budget is consumed. +struct FakeCoordinator { + bool active = false; + bool refuse = false; + uint64_t next_token = 1; + uint64_t last_opened_token = 0; + uint64_t pending_token = 0; + int opens = 0; + + /// What the coordinator should answer next. Set per test. + gpufl::OpenRequestStatus refusal = gpufl::OpenRequestStatus::Cooldown; + + /// A broken coordinator: says yes and hands back nothing to wait on. + bool accept_without_token = false; + + static gpufl::OpenRequestResult RequestOpen(void* ctx, + const gpufl::DeepWindowSpec&) { + auto* self = static_cast(ctx); + if (self->accept_without_token) { + return {0, gpufl::OpenRequestStatus::Accepted}; + } + if (self->refuse) return {0, self->refusal}; + if (self->active) return {0, gpufl::OpenRequestStatus::Busy}; + self->pending_token = self->next_token++; + return {self->pending_token, gpufl::OpenRequestStatus::Accepted}; + } + static bool Active(void* ctx) { + return static_cast(ctx)->active; + } + static uint64_t LastOpenedToken(void* ctx) { + return static_cast(ctx)->last_opened_token; + } + static uint64_t OpensCompleted(void* ctx) { + return static_cast(static_cast(ctx)->opens); + } + static uint64_t PendingOpenToken(void* ctx) { + return static_cast(ctx)->pending_token; + } + + RuleEvaluator::Hooks hooks() { + RuleEvaluator::Hooks h; + h.request_open = &RequestOpen; + h.window_active = &Active; + h.last_opened_token = &LastOpenedToken; + h.pending_open_token = &PendingOpenToken; + h.opens_completed = &OpensCompleted; + h.ctx = this; + return h; + } + + /** The coordinator refusing the queued request when it gets to it. */ + void dropRequest() { pending_token = 0; } + + /** The coordinator servicing a queued request. */ + void serviceOpen() { + if (pending_token == 0) return; + last_opened_token = pending_token; + pending_token = 0; + active = true; + ++opens; + } + /** A window opened by someone else - manual, or the scheduled trigger. */ + void openManual() { + last_opened_token = 0; + active = true; + ++opens; + } + void close() { active = false; } +}; + +DeepWindowRule makeRule(const char* expr) { + const auto parsed = parseRuleExpression(expr); + EXPECT_TRUE(parsed.ok()) << expr << " -> " << toString(parsed.error); + DeepWindowRule r = parsed.rule; + r.timing.rate_window_ms = 1000; + r.timing.stale_after_ms = 20000; + r.window.max_duration_ms = 500; + r.max_windows = 3; + return r; +} + +// ----------------------------------------------------------------- parsing + +TEST(RuleParseTest, ParsesMetricOperatorThresholdAndDuration) { + const auto r = parseRuleExpression("custom.token_rate<1000 for 2s"); + ASSERT_TRUE(r.ok()) << toString(r.error); + EXPECT_EQ(r.rule.metric.canonical, "custom.token_rate"); + EXPECT_EQ(r.rule.op, Comparison::LessThan); + EXPECT_DOUBLE_EQ(r.rule.threshold, 1000.0); + EXPECT_EQ(r.rule.timing.sustained_ms, 2000); + // No hysteresis unless asked for: rearm degenerates to "condition false". + EXPECT_DOUBLE_EQ(r.rule.rearm_threshold, 1000.0); +} + +TEST(RuleParseTest, AcceptsMillisecondsAndBareNumbers) { + EXPECT_EQ(parseRuleExpression("kernel_launch_rate>50 for 500ms").rule + .timing.sustained_ms, 500); + EXPECT_EQ(parseRuleExpression("kernel_launch_rate>50 for 750").rule + .timing.sustained_ms, 750); + EXPECT_EQ(parseRuleExpression("kernel_launch_rate>50").rule + .timing.sustained_ms, 0); +} + +TEST(RuleParseTest, RejectsGarbage) { + EXPECT_EQ(parseRuleExpression("kernel_launch_rate 50").error, + RuleError::Unparsable); + EXPECT_EQ(parseRuleExpression("kernel_launch_rate>abc").error, + RuleError::Unparsable); + EXPECT_EQ(parseRuleExpression("kernel_launch_rate>50 for 2 parsecs").error, + RuleError::Unparsable); + // The metric reason must survive, not be flattened into "unparsable". + const auto bad = parseRuleExpression("tokne_rate<5"); + EXPECT_EQ(bad.error, RuleError::BadMetric); + EXPECT_EQ(bad.metric_error, MetricParseError::MissingCustomPrefix); +} + +// -------------------------------------------------------------- validation + +TEST(RuleValidateTest, AcceptsAWorkableRule) { + EXPECT_EQ(validateRule(makeRule("custom.token_rate<1000 for 2s")).error, + RuleError::None); +} + +TEST(RuleValidateTest, RejectsRearmOnTheWrongSide) { + DeepWindowRule r = makeRule("custom.token_rate<1000 for 2s"); + r.rearm_threshold = 900; // below the threshold: can never be reached + // Otherwise the rule fires once and then waits forever for a condition + // that cannot occur, which looks identical to a healthy armed rule. + EXPECT_EQ(validateRule(r).error, RuleError::RearmWrongSide); + + DeepWindowRule g = makeRule("kernel_launch_rate>1000 for 2s"); + g.rearm_threshold = 1100; + EXPECT_EQ(validateRule(g).error, RuleError::RearmWrongSide); +} + +TEST(RuleValidateTest, RejectsNonFiniteThreshold) { + DeepWindowRule r = makeRule("custom.token_rate<1000 for 2s"); + r.threshold = std::nan(""); + r.rearm_threshold = r.threshold; + // Every comparison against NaN is false, so the rule would never fire and + // never report a problem. + EXPECT_EQ(validateRule(r).error, RuleError::ThresholdNotFinite); +} + +TEST(RuleValidateTest, RejectsBudgetOutOfRange) { + DeepWindowRule r = makeRule("custom.token_rate<1000 for 2s"); + r.max_windows = 0; + EXPECT_EQ(validateRule(r).error, RuleError::MaxWindowsOutOfRange); + r.max_windows = 100000; + EXPECT_EQ(validateRule(r).error, RuleError::MaxWindowsOutOfRange); +} + +TEST(RuleValidateTest, RejectsAWindowWithNoBound) { + DeepWindowRule r = makeRule("custom.token_rate<1000 for 2s"); + r.window.max_duration_ms = 0; + r.window.max_launches = 0; + // A window that never closes turns a bounded-cost feature into an + // always-on one. + EXPECT_EQ(validateRule(r).error, RuleError::WindowBoundsMissing); +} + +TEST(RuleValidateTest, RejectsTimingThatCanNeverFire) { + DeepWindowRule r = makeRule("custom.token_rate<1000 for 2s"); + r.timing.rate_window_ms = 4000; + r.timing.sustained_ms = 4000; + r.timing.stale_after_ms = 5000; + const auto v = validateRule(r); + EXPECT_EQ(v.error, RuleError::BadTiming); + EXPECT_EQ(v.config_error, ConfigError::StaleBeforeEvidence); + EXPECT_FALSE(v.detail.empty()) << "the arithmetic has to be in the message"; +} + +// ----------------------------------------------------------- state machine + +class RuleEvaluatorTest : public ::testing::Test { + protected: + void SetUp() override { CounterRegistry::instance().resetForTesting(); } + void TearDown() override { CounterRegistry::instance().resetForTesting(); } + + MetricFeeds feeds; + FakeCoordinator coord; + + // Drive the collector beat, feeding `launches` launches per bucket. + void run(RuleEvaluator& ev, MetricSource& src, int64_t from_ns, int64_t to_ns, + int launches_per_beat) { + for (int64_t t = from_ns; t <= to_ns; t += 10 * kMs) { + for (int i = 0; i < launches_per_beat; ++i) feeds.noteKernelLaunch(t); + ev.poll(t); + } + } +}; + +TEST_F(RuleEvaluatorTest, FiresOnlyAfterTheConditionIsSustained) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 2s"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + // Busy: 10 launches per 10ms beat == 1000/s, above the threshold. + run(ev, src, 0, 2000 * kMs, 10); + EXPECT_EQ(ev.state(), RuleState::Armed); + + // Goes quiet. The rule must not fire on the first true reading. + run(ev, src, 2010 * kMs, 4000 * kMs, 0); + EXPECT_EQ(ev.state(), RuleState::Pending) + << "fired before the condition was held for 2s"; + + run(ev, src, 4010 * kMs, 6000 * kMs, 0); + EXPECT_EQ(ev.state(), RuleState::Opening); + EXPECT_EQ(ev.windowsOpened(), 0u) << "budget consumed before a window opened"; +} + +TEST_F(RuleEvaluatorTest, BudgetCountsOnlyWindowsThatActuallyOpened) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + ASSERT_EQ(ev.state(), RuleState::Opening); + EXPECT_EQ(ev.windowsOpened(), 0u); + + coord.serviceOpen(); + ev.poll(4010 * kMs); + EXPECT_EQ(ev.state(), RuleState::Blackout); + EXPECT_EQ(ev.windowsOpened(), 1u); +} + +TEST_F(RuleEvaluatorTest, ADroppedRequestCostsNoBudget) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + ASSERT_EQ(ev.state(), RuleState::Opening); + + // The coordinator discards the request instead of opening. + coord.dropRequest(); + ev.poll(4010 * kMs); + EXPECT_EQ(ev.state(), RuleState::Armed); + EXPECT_EQ(ev.windowsOpened(), 0u) + << "budget bounds what the rule cost, and this cost nothing"; +} + +TEST_F(RuleEvaluatorTest, ARefusedOpenReturnsToArmedRatherThanHoldingPending) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + coord.refuse = true; // cooldown still running + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + EXPECT_EQ(ev.windowsOpened(), 0u); + + // The refusal must discard the evidence, not bank it. Accepting requests + // again is not enough on its own - the rule has to gather a fresh + // sustained run first, or a cooldown would just delay a fire that then + // lands on readings from long before. + coord.refuse = false; + ev.poll(4010 * kMs); + EXPECT_NE(ev.state(), RuleState::Opening) + << "fired on evidence gathered while the coordinator was refusing"; + + run(ev, src, 4020 * kMs, 5000 * kMs, 0); + EXPECT_EQ(ev.state(), RuleState::Opening) + << "never recovered after the refusal"; +} + +TEST_F(RuleEvaluatorTest, AManualWindowBlacksOutButCostsNoBudget) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 2s"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + ASSERT_EQ(ev.state(), RuleState::Armed); + + coord.openManual(); + ev.poll(1510 * kMs); + // Contamination does not care who opened the window. + EXPECT_EQ(ev.state(), RuleState::Blackout); + EXPECT_EQ(ev.windowsOpened(), 0u) + << "a manual window must not spend the rule's budget"; +} + +TEST_F(RuleEvaluatorTest, ContaminatedSamplesDoNotProveRecovery) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + coord.serviceOpen(); + ev.poll(4010 * kMs); + ASSERT_EQ(ev.state(), RuleState::Blackout); + + // Busy traffic while the window is open must not count as recovery. + run(ev, src, 4020 * kMs, 6000 * kMs, 10); + EXPECT_EQ(ev.state(), RuleState::Blackout); + + coord.close(); + ev.poll(6010 * kMs); + EXPECT_EQ(ev.state(), RuleState::Recovery) + << "the clean epoch only starts once the window has closed"; +} + +TEST_F(RuleEvaluatorTest, NoRefireUntilTheWorkloadRecovers) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + coord.serviceOpen(); + ev.poll(4010 * kMs); + coord.close(); + ev.poll(4020 * kMs); + + // Still quiet - the condition is still true. It must NOT reopen. + run(ev, src, 4030 * kMs, 9000 * kMs, 0); + EXPECT_EQ(ev.windowsOpened(), 1u) + << "refired while the condition never stopped holding"; + EXPECT_NE(ev.state(), RuleState::Blackout); + + // Recover, then degrade again: the second window is legitimate. + run(ev, src, 9010 * kMs, 11000 * kMs, 10); + run(ev, src, 11010 * kMs, 14000 * kMs, 0); + coord.serviceOpen(); + ev.poll(14010 * kMs); + EXPECT_EQ(ev.windowsOpened(), 2u); +} + +TEST_F(RuleEvaluatorTest, ExhaustionIsMarkedAtTheOpenThatReachesTheLimit) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + rule.max_windows = 1; + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + coord.serviceOpen(); + ev.poll(4010 * kMs); + ASSERT_EQ(ev.windowsOpened(), 1u); + + // Marked at the transition, not at shutdown, so a crashed run still + // explains itself. + EXPECT_EQ(ev.snapshot(4020 * kMs).outcome, RuleOutcome::Exhausted); + + coord.close(); + ev.poll(4030 * kMs); + // The evaluator has nowhere left to stand, so it goes inactive rather than + // back into recovery. + EXPECT_EQ(ev.state(), RuleState::Inactive); + + run(ev, src, 4040 * kMs, 10000 * kMs, 0); + EXPECT_EQ(ev.windowsOpened(), 1u); +} + +TEST_F(RuleEvaluatorTest, HysteresisNeedsARealRecoveryNotABrushPastTheThreshold) { + DeepWindowRule rule = makeRule("kernel_launch_rate<500 for 500ms"); + rule.rearm_threshold = 900; // must climb back above 900 to rearm + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + coord.serviceOpen(); + ev.poll(4010 * kMs); + coord.close(); + ev.poll(4020 * kMs); + + // ~600/s: above the trigger threshold but below the rearm threshold. + run(ev, src, 4030 * kMs, 7000 * kMs, 6); + EXPECT_EQ(ev.state(), RuleState::WaitingForRearm) + << "rearmed on a value that had not actually recovered"; + + run(ev, src, 7010 * kMs, 10000 * kMs, 10); + EXPECT_EQ(ev.state(), RuleState::Armed); +} + +TEST_F(RuleEvaluatorTest, StaleDataBreaksTheSustainedRun) { + // A gauge, not a rate. For a rate metric the validated timing GUARANTEES a + // stall fires before it goes stale, so a rate could never demonstrate this. + // A gauge stops for its own reasons - NVML no longer answering - while the + // measured quantity is still low, and that is the case worth guarding. + DeepWindowRule rule = makeRule("gpu[0].util_pct<10 for 2s"); + rule.timing.stale_after_ms = 3100; + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + + gpufl::DeviceSample idle; + idle.device_id = 0; + idle.gpu_util = 2; + for (int64_t t = 0; t <= 1000 * kMs; t += 100 * kMs) { + feeds.noteDeviceSample(idle, t); + ev.poll(t); + } + ASSERT_EQ(ev.state(), RuleState::Pending); + + // Measurements stop. Stale is not evidence the condition continued, so the + // run is broken rather than completed by readings nobody took. + for (int64_t t = 1010 * kMs; t <= 9000 * kMs; t += 10 * kMs) ev.poll(t); + EXPECT_EQ(ev.state(), RuleState::Armed); + EXPECT_EQ(ev.windowsOpened(), 0u); +} + +// ----------------------------------------------------------------- gates + +TEST_F(RuleEvaluatorTest, RuleIsRefusedWhenNoDeepEngineIsPrepared) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleCapabilities caps; + caps.deep_engine_prepared = false; + RuleEvaluator ev(rule, "r1", caps, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 4000 * kMs, 0); + EXPECT_EQ(ev.state(), RuleState::Inactive); + EXPECT_EQ(ev.windowsOpened(), 0u) + << "burned budget opening windows that would arm nothing"; + + const RuleSummary s = ev.finish(4000 * kMs); + EXPECT_EQ(s.outcome, RuleOutcome::Unsupported); + EXPECT_EQ(s.reason, "no_deep_engine"); +} + +TEST_F(RuleEvaluatorTest, CustomCounterRuleIsRefusedWhenCountersAreNotShared) { + DeepWindowRule rule = makeRule("custom.token_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleCapabilities caps; + caps.multi_module = true; // injection: target and evaluator differ + caps.counters_shared = false; + RuleEvaluator ev(rule, "r1", caps, &src, coord.hooks()); + + // The target would tick one registry and this evaluator read another, so + // the counter can never be seen. Saying so beats reporting a counter that + // is being ticked as Missing for the whole run. + EXPECT_EQ(ev.state(), RuleState::Inactive); + EXPECT_EQ(ev.finish(1000 * kMs).reason, "counters_not_shared"); +} + +TEST_F(RuleEvaluatorTest, UnsharedCountersAreFineInAnEmbeddedHost) { + DeepWindowRule rule = makeRule("custom.token_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleCapabilities caps; + caps.multi_module = false; // one copy of gpufl in the process + caps.counters_shared = false; + RuleEvaluator ev(rule, "r1", caps, &src, coord.hooks()); + EXPECT_NE(ev.state(), RuleState::Inactive); +} + +TEST_F(RuleEvaluatorTest, GaugeRuleOnAMissingDeviceIsRefusedEagerly) { + DeepWindowRule rule = makeRule("gpu[3].util_pct<10 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleCapabilities caps; + caps.device_count = 1; + RuleEvaluator ev(rule, "r1", caps, &src, coord.hooks()); + // A built-in metric's availability cannot change later, unlike a custom + // counter's, so it is decided now. + EXPECT_EQ(ev.state(), RuleState::Inactive); + EXPECT_EQ(ev.finish(0).reason, "metric_unavailable"); +} + +// --------------------------------------------------------------- summaries + +TEST_F(RuleEvaluatorTest, ARuleThatNeverMatchedStillReportsAnOutcome) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 4000 * kMs, 10); // always busy + const RuleSummary s = ev.finish(4000 * kMs); + // "log once" is invisible in the UI, and absence of a record cannot be + // read as evidence the rule never fired. + EXPECT_EQ(s.outcome, RuleOutcome::NeverTrue); + EXPECT_EQ(s.reason, "condition_never_held"); + EXPECT_GT(s.samples_seen, 0u); + EXPECT_TRUE(s.last_value.has_value()); +} + +TEST_F(RuleEvaluatorTest, StateAndOutcomeAreSeparateFields) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + const RuleSummary mid = ev.snapshot(1500 * kMs); + // `armed` is where the evaluator is standing, not a verdict on the session. + EXPECT_EQ(mid.state, RuleState::Armed); + EXPECT_EQ(mid.outcome, RuleOutcome::None); +} + +TEST_F(RuleEvaluatorTest, ANeverRegisteredCounterIsNamedInTheSummary) { + DeepWindowRule rule = makeRule("custom.never_ticked_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + + for (int64_t t = 0; t <= 4000 * kMs; t += 10 * kMs) ev.poll(t); + const RuleSummary s = ev.finish(4000 * kMs); + EXPECT_EQ(s.outcome, RuleOutcome::NeverTrue); + EXPECT_EQ(s.reason, "custom_metric_never_registered"); + EXPECT_EQ(s.last_metric_state, MetricState::Missing); +} + +TEST_F(RuleEvaluatorTest, StateSequenceIsMonotonic) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + uint64_t previous = 0; + for (int64_t t = 0; t <= 6000 * kMs; t += 10 * kMs) { + if (t < 1500 * kMs) feeds.noteKernelLaunch(t); + ev.poll(t); + const uint64_t seq = ev.snapshot(t).state_sequence; + // A late record must never overwrite a newer one at the backend, and + // that ordering is only as good as this number. + EXPECT_GE(seq, previous); + previous = seq; + } + EXPECT_GT(ev.finish(6000 * kMs).state_sequence, previous); +} + +TEST(RuleRefusedTest, AnInvalidRuleStillProducesAReportableSummary) { + // Configuration is parsed during init(); failing hard there would leave no + // session and no telemetry writer - nowhere to record this very outcome. + const RuleSummary s = RuleEvaluator::refused( + "r1", RuleOutcome::InvalidConfig, "rearm_wrong_side", 42); + EXPECT_EQ(s.state, RuleState::Inactive); + EXPECT_EQ(s.outcome, RuleOutcome::InvalidConfig); + EXPECT_EQ(s.reason, "rearm_wrong_side"); + EXPECT_GT(s.state_sequence, 0u); +} + +TEST_F(RuleEvaluatorTest, AWindowThatOpensAndClosesBetweenBeatsIsNotMissed) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + ASSERT_EQ(ev.state(), RuleState::Armed); + + // A launch-bounded window over a busy loop can open and close inside one + // collector beat. Polling a boolean would see nothing and let the samples + // taken during it count as clean. + coord.openManual(); + coord.close(); + + ev.poll(1510 * kMs); + EXPECT_EQ(ev.state(), RuleState::Recovery) + << "a whole window came and went unnoticed"; +} + + +TEST_F(RuleEvaluatorTest, ATerminalOutcomeIsReportableOnceAtTheTransition) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + rule.max_windows = 1; + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + EXPECT_FALSE(ev.takeTerminalToEmit()) << "nothing terminal has happened yet"; + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + coord.serviceOpen(); + ev.poll(4010 * kMs); + ASSERT_EQ(ev.snapshot(4010 * kMs).outcome, RuleOutcome::Exhausted); + + // Available at the transition, so a run that crashes afterwards still + // explains itself instead of looking like one whose rule never fired. + EXPECT_TRUE(ev.takeTerminalToEmit()); + // Once only: the shutdown summary follows with a higher sequence, and two + // rows claiming the same thing help nobody. + EXPECT_FALSE(ev.takeTerminalToEmit()); +} + +TEST_F(RuleEvaluatorTest, AnUnsupportedRuleIsReportableImmediately) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleCapabilities caps; + caps.deep_engine_prepared = false; + RuleEvaluator ev(rule, "r1", caps, &src, coord.hooks()); + + // Decided at construction, so there is nothing to wait for. + EXPECT_TRUE(ev.takeTerminalToEmit()); + EXPECT_FALSE(ev.takeTerminalToEmit()); +} + +TEST_F(RuleEvaluatorTest, TheShutdownSummaryOutranksTheTransitionOne) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + rule.max_windows = 1; + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + coord.serviceOpen(); + ev.poll(4010 * kMs); + const uint64_t at_transition = ev.snapshot(4010 * kMs).state_sequence; + + // The backend upsert only accepts a strictly greater sequence, so the final + // row must outrank the early one or it would be silently discarded. + EXPECT_GT(ev.finish(5000 * kMs).state_sequence, at_transition); +} + + +TEST_F(RuleEvaluatorTest, TheSummaryReportsHowMuchDataWasDiscarded) { + // A conclusion drawn from a partial percentile must not read like one + // drawn from all of it, and the summary is where that conclusion lands. + DeepWindowRule rule = makeRule("recent_kernel_ms>1000 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + for (size_t i = 0; i < MetricFeeds::kMaxPendingDurations + 500; ++i) { + feeds.noteKernelDuration(10 * kMs, 2.0); + } + for (int64_t t = 0; t <= 2000 * kMs; t += 10 * kMs) ev.poll(t); + + EXPECT_GT(ev.finish(2000 * kMs).truncated_samples, 0u) + << "the session concluded from a subset and never said so"; +} + +// ── a refusal has to say WHY ──────────────────────────────────────────────── +// +// One undifferentiated "refused" made every refusal look temporary, so a rule +// whose deep engine had permanently failed retried until shutdown and then +// reported `never_true` - which says the condition was never met, the exact +// opposite of what happened. + +TEST_F(RuleEvaluatorTest, AFailedEnginePreparationEndsTheRuleAsUnsupported) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + coord.refuse = true; + coord.refusal = gpufl::OpenRequestStatus::EngineUnavailable; + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + + EXPECT_EQ(ev.state(), RuleState::Inactive); + const RuleSummary s = ev.finish(4000 * kMs); + EXPECT_EQ(s.outcome, RuleOutcome::Unsupported) + << "a condition that held but could not be acted on is not never_true"; + EXPECT_EQ(s.reason, "deep_engine_not_prepared"); + EXPECT_EQ(s.windows_opened, 0u); +} + +TEST_F(RuleEvaluatorTest, PreparationPendingIsRetriedRatherThanConcluded) { + // Windows injection installs a rule before CONTEXT_CREATED, so "not ready + // yet" is an ordinary startup state. Treating it as failure would kill + // every rule on that platform. + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + coord.refuse = true; + coord.refusal = gpufl::OpenRequestStatus::PreparationPending; + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + ASSERT_NE(ev.state(), RuleState::Inactive) << "gave up on a pending engine"; + + // Preparation completes; the rule must still be able to fire. + coord.refuse = false; + run(ev, src, 4010 * kMs, 5200 * kMs, 0); + EXPECT_EQ(ev.state(), RuleState::Opening); +} + +TEST_F(RuleEvaluatorTest, ACooldownRefusalIsNotAPermanentFailure) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + coord.refuse = true; + coord.refusal = gpufl::OpenRequestStatus::Cooldown; + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + + EXPECT_NE(ev.state(), RuleState::Inactive); + // Recording an ordinary quiet period as a permanent failure would tell the + // user their engine is broken when the rule is simply waiting its turn. + EXPECT_NE(ev.snapshot(4000 * kMs).outcome, RuleOutcome::Unsupported); +} + +TEST_F(RuleEvaluatorTest, ABusyRefusalStillFiresOnceTheWindowIsFree) { + // The scheduled --deep-after window holds the queue; the rule waits. + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + coord.refuse = true; + coord.refusal = gpufl::OpenRequestStatus::Busy; + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + ASSERT_NE(ev.state(), RuleState::Inactive); + + coord.refuse = false; + run(ev, src, 4010 * kMs, 5200 * kMs, 0); + EXPECT_EQ(ev.state(), RuleState::Opening); + + coord.serviceOpen(); + ev.poll(5210 * kMs); + EXPECT_EQ(ev.windowsOpened(), 1u); +} + +// ── refused to the end is not "never true" ────────────────────────────────── + +TEST_F(RuleEvaluatorTest, ARunRefusedToTheEndReportsBlockedNotNeverTrue) { + // The refusal is temporary and correctly retried, but the run ends before + // it lifts - a manual window held for the rest of the session, or a + // cooldown longer than the time left. The condition held throughout. + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + coord.refuse = true; + coord.refusal = gpufl::OpenRequestStatus::Cooldown; + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + + const RuleSummary s = ev.finish(4000 * kMs); + EXPECT_EQ(s.outcome, RuleOutcome::Blocked) + << "never_true sends the user to their threshold; the threshold was fine"; + EXPECT_EQ(s.reason, "cooldown") << "and it has to name what was in the way"; + EXPECT_EQ(s.windows_opened, 0u); +} + +TEST_F(RuleEvaluatorTest, AConditionThatNeverHeldIsStillNeverTrue) { + // The other side of the same line: nothing was ever asked for, so + // `blocked` would be inventing an obstacle that did not exist. + DeepWindowRule rule = makeRule("kernel_launch_rate>10000 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 4000 * kMs, 1); + + const RuleSummary s = ev.finish(4000 * kMs); + EXPECT_EQ(s.outcome, RuleOutcome::NeverTrue); + EXPECT_EQ(s.reason, "condition_never_held"); +} + +TEST_F(RuleEvaluatorTest, AnAcceptedRequestWithNoTokenIsAFaultNotAWait) { + // A coordinator that answers "accepted, token 0" has a bug. Waiting on it + // parks the evaluator in Opening for the rest of the run, watching a + // request that was never queued - no window, no error, no explanation. + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + coord.accept_without_token = true; + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + + EXPECT_NE(ev.state(), RuleState::Opening) << "stalled on a phantom request"; + const RuleSummary s = ev.finish(4000 * kMs); + EXPECT_EQ(s.outcome, RuleOutcome::Unsupported); + EXPECT_EQ(s.reason, "invalid_open_result"); +} + +TEST_F(RuleEvaluatorTest, ADefaultConstructedResultIsARefusal) { + // Nobody writes {} on purpose; a hook that forgets a return path produces + // it. Defaulting the status to Accepted made that silently mean "yes". + const gpufl::OpenRequestResult unset; + EXPECT_FALSE(unset.accepted()); + EXPECT_NE(unset.status, gpufl::OpenRequestStatus::Accepted); +} + +} // namespace diff --git a/tests/core/test_deep_window_rules_install.cpp b/tests/core/test_deep_window_rules_install.cpp new file mode 100644 index 0000000..972e39f --- /dev/null +++ b/tests/core/test_deep_window_rules_install.cpp @@ -0,0 +1,179 @@ +#include + +#include +#include + +#include "gpufl/core/deep_window_rules.hpp" +#include "gpufl/core/env_vars.hpp" + +using gpufl::detail::DeepWindowRules; + +namespace { + +// Env is process-global, so every test here sets and clears the whole group. +// Leaving one behind would silently change the next test's rule. +void setEnv(const char* name, const char* value) { +#if defined(_WIN32) + _putenv_s(name, value ? value : ""); +#else + if (value) ::setenv(name, value, 1); else ::unsetenv(name); +#endif +} + +class RuleInstallTest : public ::testing::Test { + protected: + void SetUp() override { clearAll(); DeepWindowRules::ResetForTesting(); } + void TearDown() override { clearAll(); DeepWindowRules::ResetForTesting(); } + + static void clearAll() { + for (const char* n : {gpufl::env::kDeepWhen, gpufl::env::kDeepRateWindowMs, + gpufl::env::kDeepStaleAfterMs, gpufl::env::kDeepRearmAt, + gpufl::env::kDeepMaxWindows, gpufl::env::kDeepWindowMs, + gpufl::env::kDeepWindowMaxLaunches}) { + setEnv(n, nullptr); + } + } +}; + +TEST_F(RuleInstallTest, NoRuleWithoutTheEnvVar) { + DeepWindowRules::InstallFromEnv(); + EXPECT_FALSE(DeepWindowRules::Installed()); + EXPECT_FALSE(DeepWindowRules::WantsLaunchFeed()); +} + +TEST_F(RuleInstallTest, AValidRuleInstallsAndAsksForTheLaunchFeed) { + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + DeepWindowRules::InstallFromEnv(); + + EXPECT_TRUE(DeepWindowRules::Installed()); + // The feed costs an atomic per launch, so only a rule that reads it should + // switch it on. + EXPECT_TRUE(DeepWindowRules::WantsLaunchFeed()); +} + +TEST_F(RuleInstallTest, ACustomCounterRuleDoesNotPayForTheLaunchFeed) { + setEnv(gpufl::env::kDeepWhen, "custom.token_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + DeepWindowRules::InstallFromEnv(); + + EXPECT_TRUE(DeepWindowRules::Installed()); + EXPECT_FALSE(DeepWindowRules::WantsLaunchFeed()) + << "every launch would pay for a feed this rule never reads"; +} + +TEST_F(RuleInstallTest, AnInvalidRuleIsInstalledAsRefusedRatherThanIgnored) { + setEnv(gpufl::env::kDeepWhen, "tokne_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + DeepWindowRules::InstallFromEnv(); + + // Installed, so Finish() still has something to report. A rejected rule + // that vanishes looks exactly like one that was simply never true. + EXPECT_TRUE(DeepWindowRules::Installed()); + EXPECT_FALSE(DeepWindowRules::WantsLaunchFeed()); +} + +TEST_F(RuleInstallTest, AWindowWithNoBoundIsRefused) { + // Neither GPUFL_DEEP_WINDOW_MS nor MAX_LAUNCHES: the window would never + // close, which turns a bounded-cost feature into an always-on one. + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 2s"); + DeepWindowRules::InstallFromEnv(); + EXPECT_TRUE(DeepWindowRules::Installed()); + EXPECT_FALSE(DeepWindowRules::WantsLaunchFeed()); +} + +TEST_F(RuleInstallTest, TheDefaultStaleAfterIsOneThatCanActuallyFire) { + // Left unset, stale-after must be derived from the other two rather than + // defaulted to a constant the validator would then reject. + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 30s"); + setEnv(gpufl::env::kDeepRateWindowMs, "5000"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + DeepWindowRules::InstallFromEnv(); + + EXPECT_TRUE(DeepWindowRules::Installed()); + EXPECT_TRUE(DeepWindowRules::WantsLaunchFeed()) + << "a workable rule was refused by its own default stale-after"; +} + +TEST_F(RuleInstallTest, ServiceAndFinishAreSafeWithoutARule) { + // Both run unconditionally from the collector and from shutdown. + DeepWindowRules::Service(); + DeepWindowRules::Finish(); + DeepWindowRules::NoteKernelLaunch(1); + SUCCEED(); +} + +TEST_F(RuleInstallTest, ASecondRuleIsRefusedRatherThanSilentlyReplacing) { + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + DeepWindowRules::InstallFromEnv(); + ASSERT_TRUE(DeepWindowRules::WantsLaunchFeed()); + + setEnv(gpufl::env::kDeepWhen, "custom.token_rate<5 for 2s"); + DeepWindowRules::InstallFromEnv(); + // The first rule stands. Replacing it silently would make which rule ran + // depend on call order. + EXPECT_TRUE(DeepWindowRules::WantsLaunchFeed()); +} + +TEST_F(RuleInstallTest, AMalformedNumericOptionRefusesRatherThanDefaulting) { + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + setEnv(gpufl::env::kDeepMaxWindows, "three"); + DeepWindowRules::InstallFromEnv(); + + // Silently substituting the default would open real windows under a budget + // the user never chose and cannot see. + EXPECT_TRUE(DeepWindowRules::Installed()); + EXPECT_FALSE(DeepWindowRules::WantsLaunchFeed()) + << "a typo'd option was quietly replaced by a default"; +} + +TEST_F(RuleInstallTest, ARuleCanBeInstalledAgainAfterFinish) { + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + DeepWindowRules::InstallFromEnv(); + ASSERT_TRUE(DeepWindowRules::Installed()); + + // An embedded host may shutdown() and init() again in one process. Without + // releasing the session, the second run's rule is refused as a duplicate + // and that run silently has no trigger at all. + DeepWindowRules::Finish(); + DeepWindowRules::InstallFromEnv(); + EXPECT_TRUE(DeepWindowRules::WantsLaunchFeed()) + << "the second session was left with no rule"; +} + +TEST_F(RuleInstallTest, AnOutOfRangeNumericOptionIsRefused) { + // Pins the OUTCOME, not the mechanism. Two independent guards reject this: + // the ERANGE check on the parse, and the range validator downstream. Either + // alone suffices, so removing one does not fail this test - what it fixes + // is which reason gets reported, and that is not observable from here. + // The reason the ERANGE check still exists is the arithmetic in between: + // a saturated LLONG_MAX fed into the derived stale-after sum is signed + // overflow, which is undefined rather than merely wrong. + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + setEnv(gpufl::env::kDeepRateWindowMs, "99999999999999999999999999"); + DeepWindowRules::InstallFromEnv(); + + EXPECT_TRUE(DeepWindowRules::Installed()); + EXPECT_FALSE(DeepWindowRules::WantsLaunchFeed()) + << "a value too large to represent was accepted"; +} + +TEST_F(RuleInstallTest, AMaxWindowsThatDoesNotFitAnIntIsRefused) { + // 4294967297 survives ERANGE - it fits an int64 - and then narrows to 1, + // which the validator accepts. The run would silently use a budget nobody + // configured. + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + setEnv(gpufl::env::kDeepMaxWindows, "4294967297"); + DeepWindowRules::InstallFromEnv(); + + EXPECT_TRUE(DeepWindowRules::Installed()); + EXPECT_FALSE(DeepWindowRules::WantsLaunchFeed()) + << "a value that cannot fit an int was narrowed into a valid one"; +} + +} // namespace diff --git a/tests/core/test_metric_registry.cpp b/tests/core/test_metric_registry.cpp new file mode 100644 index 0000000..0fed401 --- /dev/null +++ b/tests/core/test_metric_registry.cpp @@ -0,0 +1,610 @@ +#include + +#include +#include + +#include "gpufl.hpp" +#include "gpufl/core/counter_provider.hpp" +#include "gpufl/core/counter_registry.hpp" +#include "gpufl/core/events.hpp" +#include "gpufl/core/metric_id.hpp" +#include "gpufl/core/metric_registry.hpp" + +using gpufl::detail::ActiveCounterProvider; +using gpufl::detail::ConfigError; +using gpufl::detail::CounterRegistry; +using gpufl::detail::MetricFeeds; +using gpufl::detail::MetricId; +using gpufl::detail::MetricKind; +using gpufl::detail::MetricParseError; +using gpufl::detail::MetricShape; +using gpufl::detail::MetricSource; +using gpufl::detail::MetricState; +using gpufl::detail::MetricWindowConfig; +using gpufl::detail::parseMetric; +using gpufl::detail::validate; + +namespace { + +constexpr int64_t kMs = 1000000; // ns per ms + +// ------------------------------------------------------------ metric contract + +TEST(MetricContractTest, ParsesBuiltinRates) { + const auto launch = parseMetric("kernel_launch_rate"); + ASSERT_TRUE(launch.ok()); + EXPECT_EQ(launch.id.kind, MetricKind::KernelLaunchRate); + EXPECT_EQ(launch.id.shape(), MetricShape::Rate); + + const auto kernel = parseMetric("recent_kernel_ms"); + ASSERT_TRUE(kernel.ok()); + EXPECT_EQ(kernel.id.shape(), MetricShape::Percentile); +} + +TEST(MetricContractTest, CanonicalisesDeviceIndex) { + const auto a = parseMetric("gpu[0].util_pct"); + const auto b = parseMetric("gpu[00].util_pct"); + ASSERT_TRUE(a.ok()); + ASSERT_TRUE(b.ok()); + // Two spellings of one rule must not hash to two different rule ids. + EXPECT_EQ(a.id.canonical, b.id.canonical); + EXPECT_EQ(a.id.canonical, "gpu[0].util_pct"); +} + +TEST(MetricContractTest, UnknownBuiltinFieldRejectedAtParse) { + const auto r = parseMetric("gpu[0].temperature_pct"); + EXPECT_FALSE(r.ok()); + EXPECT_EQ(r.error, MetricParseError::UnknownBuiltinMetric); +} + +TEST(MetricContractTest, MisspelledMetricWithoutPrefixRejectedAtParse) { + // The whole reason `custom.` exists: without a prefix this is + // indistinguishable from a counter that has not registered yet, and the + // mistake would only surface once the run was over. + const auto r = parseMetric("tokne_rate"); + EXPECT_FALSE(r.ok()); + EXPECT_EQ(r.error, MetricParseError::MissingCustomPrefix); +} + +TEST(MetricContractTest, CustomMetricParsesAndKeepsCounterName) { + const auto r = parseMetric("custom.token_rate"); + ASSERT_TRUE(r.ok()); + EXPECT_EQ(r.id.kind, MetricKind::CustomRate); + EXPECT_EQ(r.id.custom_name, "token"); + EXPECT_TRUE(r.id.resolvesLazily()); +} + +TEST(MetricContractTest, CustomMetricNeedsTheRateSuffix) { + EXPECT_EQ(parseMetric("custom.token").error, + MetricParseError::MalformedCustomMetric); + EXPECT_EQ(parseMetric("custom._rate").error, + MetricParseError::MalformedCustomMetric); +} + +TEST(MetricContractTest, MalformedDeviceIndexRejected) { + EXPECT_EQ(parseMetric("gpu[].util_pct").error, + MetricParseError::MalformedDeviceIndex); + EXPECT_EQ(parseMetric("gpu[x].util_pct").error, + MetricParseError::MalformedDeviceIndex); + EXPECT_EQ(parseMetric("gpu[-1].util_pct").error, + MetricParseError::MalformedDeviceIndex); +} + +TEST(MetricContractTest, EmptyNameRejected) { + EXPECT_EQ(parseMetric("").error, MetricParseError::Empty); +} + +// ------------------------------------------------------------ config validity + +TEST(MetricConfigTest, BucketIntervalIsBoundedAndDerivedOnce) { + EXPECT_EQ((MetricWindowConfig{100, 0, 5000}).bucketIntervalMs(), 10); + EXPECT_EQ((MetricWindowConfig{1000, 0, 5000}).bucketIntervalMs(), 100); + EXPECT_EQ((MetricWindowConfig{60000, 0, 90000}).bucketIntervalMs(), 100); +} + +TEST(MetricConfigTest, AcceptsAWorkableCombination) { + EXPECT_EQ(validate(MetricWindowConfig{1000, 2000, 5000}), ConfigError::None); +} + +TEST(MetricConfigTest, RejectsStaleShorterThanTheEvidenceItNeeds) { + // window=4s, sustained=4s, stale=5s produces its first zero at t=4 and goes + // stale at t=5 - four seconds short of ever firing. Each field on its own + // looks reasonable, which is exactly why the combined check has to exist. + const MetricWindowConfig cfg{4000, 4000, 5000}; + EXPECT_EQ(validate(cfg), ConfigError::StaleBeforeEvidence); + const std::string why = explain(cfg, ConfigError::StaleBeforeEvidence); + EXPECT_NE(why.find("8100"), std::string::npos) << why; // 4000+4000+100 +} + +TEST(MetricConfigTest, RejectsNonPositiveAndOversizedWindows) { + EXPECT_EQ(validate(MetricWindowConfig{0, 0, 100}), + ConfigError::RateWindowNotPositive); + EXPECT_EQ(validate(MetricWindowConfig{MetricWindowConfig::kMaxRateWindowMs + 1, + 0, 100000000}), + ConfigError::RateWindowTooLarge); + EXPECT_EQ(validate(MetricWindowConfig{1000, -1, 5000}), + ConfigError::SustainedNegative); + EXPECT_EQ(validate(MetricWindowConfig{1000, 0, 0}), + ConfigError::StaleAfterNotPositive); +} + +TEST(MetricConfigTest, StaleArithmeticDoesNotOverflow) { + constexpr int64_t kBig = 9223372036854775000LL; + const MetricWindowConfig cfg{1000, kBig, kBig}; + // Must decide, not wrap into an accidental "ok". + EXPECT_EQ(validate(cfg), ConfigError::StaleBeforeEvidence); +} + +// ---------------------------------------------------------------- rate source + +class MetricSourceTest : public ::testing::Test { + protected: + void SetUp() override { CounterRegistry::instance().resetForTesting(); } + void TearDown() override { CounterRegistry::instance().resetForTesting(); } + + MetricFeeds feeds; + + static MetricId parse(const char* text) { + const auto r = parseMetric(text); + EXPECT_TRUE(r.ok()) << text; + return r.id; + } + + // Drive the source forward to `now`, polling on every bucket boundary the + // way the collector loop would. + static gpufl::detail::MetricSample advance(MetricSource& src, int64_t from_ns, + int64_t to_ns, int64_t step_ns) { + gpufl::detail::MetricSample last; + for (int64_t t = from_ns; t <= to_ns; t += step_ns) last = src.poll(t); + return last; + } +}; + +TEST_F(MetricSourceTest, LaunchRateWarmsUpBeforeTheWindowIsFull) { + const MetricWindowConfig cfg{1000, 2000, 5000}; // 100ms buckets, 10 of them + MetricSource src(parse("kernel_launch_rate"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + // Half a window in: no verdict yet. A rule that fired here would be acting + // on a partial window, which reads as a lower rate than reality. + const auto mid = advance(src, 0, 500 * kMs, 10 * kMs); + EXPECT_EQ(mid.state, MetricState::WarmingUp); +} + +TEST_F(MetricSourceTest, LaunchRateWithNoLaunchesIsFreshZeroNotStale) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("kernel_launch_rate"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + const auto s = advance(src, 0, 1100 * kMs, 10 * kMs); + // The launch source exists from startup, so "no launches" is a measurement, + // not an absence. Reporting Stale here would be the opposite verdict. + EXPECT_EQ(s.state, MetricState::Fresh); + EXPECT_DOUBLE_EQ(s.value, 0.0); +} + +TEST_F(MetricSourceTest, EmptyBucketsStillPublishNewSequences) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("kernel_launch_rate"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + const auto a = advance(src, 0, 1100 * kMs, 10 * kMs); + const auto b = advance(src, 1110 * kMs, 1500 * kMs, 10 * kMs); + // Without this a genuine zero never accumulates as evidence and a total + // stall reads as "no new data" - the one case the feature exists for. + EXPECT_GT(b.sequence, a.sequence); + EXPECT_EQ(b.state, MetricState::Fresh); +} + +TEST_F(MetricSourceTest, LaunchRateReflectsObservedLaunches) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("kernel_launch_rate"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + // 500 launches spread over one window == 500/s. + for (int64_t t = 0; t < 1000 * kMs; t += 2 * kMs) { + feeds.noteKernelLaunch(t); + src.poll(t); + } + const auto s = advance(src, 1000 * kMs, 1010 * kMs, 10 * kMs); + EXPECT_EQ(s.state, MetricState::Fresh); + EXPECT_NEAR(s.value, 500.0, 60.0) << "rate=" << s.value; +} + +TEST_F(MetricSourceTest, PollingBetweenBucketsDoesNotAdvanceTheSequence) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("kernel_launch_rate"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + const auto warm = advance(src, 0, 1100 * kMs, 10 * kMs); + + const auto a = src.poll(1105 * kMs); + const auto b = src.poll(1110 * kMs); + const auto c = src.poll(1115 * kMs); + // The evaluator runs ~100x faster than a bucket closes. If every poll + // looked like new evidence, one reading would satisfy any sustained_ms. + EXPECT_EQ(a.sequence, b.sequence); + EXPECT_EQ(b.sequence, c.sequence); + EXPECT_GE(a.sequence, warm.sequence); +} + +TEST_F(MetricSourceTest, CustomCounterIsMissingUntilTheAppRegistersIt) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("custom.metric_absent_rate"), cfg, &feeds, + ActiveCounterProvider()); + + const auto before = advance(src, 0, 1100 * kMs, 10 * kMs); + EXPECT_EQ(before.state, MetricState::Missing); + EXPECT_FALSE(src.customResolved()); + + // Asking about a counter must not create it - that would make "never + // registered" indistinguishable from "registered but idle". + // + // Checked through the ACTIVE provider, which is what the source consults. + // With a shared runtime present the local registry is a different registry + // entirely, and asserting against it would pass without proving anything. + EXPECT_EQ(ActiveCounterProvider()->lookup("metric_absent", + std::strlen("metric_absent")), + nullptr); +} + +TEST_F(MetricSourceTest, CustomCounterResolvesLazilyAfterRegistration) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("custom.metric_lazy_rate"), cfg, &feeds, + ActiveCounterProvider()); + EXPECT_EQ(src.poll(0).state, MetricState::Missing); + + // An env rule is parsed during init(), long before application code reaches + // gpufl::counter(). Rejecting it at install time would kill every such rule. + auto tokens = gpufl::counter("metric_lazy"); + const auto registered = advance(src, 10 * kMs, 1100 * kMs, 10 * kMs); + EXPECT_EQ(registered.state, MetricState::WarmingUp) + << "registered but never ticked is not Missing, and not Fresh 0 either"; + + for (int64_t t = 1100 * kMs; t < 2200 * kMs; t += 10 * kMs) { + tokens.add(10); + src.poll(t); + } + const auto s = src.poll(2200 * kMs); + EXPECT_EQ(s.state, MetricState::Fresh); + EXPECT_GT(s.value, 0.0); +} + +TEST_F(MetricSourceTest, CustomCounterGoesStaleWhenTicksStop) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("custom.metric_stall_rate"), cfg, &feeds, + ActiveCounterProvider()); + // Unique to this test: slots are permanent by design, so a name shared with + // another test makes the result depend on execution order. + auto c = gpufl::counter("metric_stall"); + + for (int64_t t = 0; t < 2000 * kMs; t += 10 * kMs) { + c.add(5); + src.poll(t); + } + ASSERT_EQ(src.poll(2000 * kMs).state, MetricState::Fresh); + + // Zeros first accumulate as fresh evidence... + const auto zeros = advance(src, 2010 * kMs, 4000 * kMs, 10 * kMs); + EXPECT_EQ(zeros.state, MetricState::Fresh); + EXPECT_DOUBLE_EQ(zeros.value, 0.0); + + // ...and only later does the source itself read as dead. Both halves matter: + // a rule must be able to fire on the stall before staleness hides it. + const auto dead = advance(src, 4010 * kMs, 9000 * kMs, 10 * kMs); + EXPECT_EQ(dead.state, MetricState::Stale); +} + +TEST_F(MetricSourceTest, PercentilePublishesNothingForAnEmptyWindow) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + const auto s = advance(src, 0, 1500 * kMs, 10 * kMs); + // 0 ms would read as instantaneous kernels rather than no kernels, and a + // rule watching for slow kernels would silently never fire. + EXPECT_NE(s.state, MetricState::Fresh); + EXPECT_DOUBLE_EQ(s.value, 0.0); +} + +TEST_F(MetricSourceTest, PercentileReportsTheMedianOverTheWindow) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + for (int64_t t = 0; t < 1200 * kMs; t += 10 * kMs) { + feeds.noteKernelDuration(t, 4.0); + src.poll(t); + } + const auto s = src.poll(1200 * kMs); + EXPECT_EQ(s.state, MetricState::Fresh); + EXPECT_NEAR(s.value, 4.0, 0.001); +} + +TEST_F(MetricSourceTest, GaugeAdvancesOnMeasurementsNotPolls) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("gpu[0].util_pct"), cfg, &feeds, ActiveCounterProvider()); + + EXPECT_EQ(src.poll(0).state, MetricState::WarmingUp); + + gpufl::DeviceSample sample; + sample.device_id = 0; + sample.gpu_util = 91; + feeds.noteDeviceSample(sample, 100 * kMs); + + const auto a = src.poll(110 * kMs); + EXPECT_EQ(a.state, MetricState::Fresh); + EXPECT_DOUBLE_EQ(a.value, 91.0); + + // NVML publishes every 100-500ms; the evaluator polls every ~1ms. A + // sequence that moved on polling would let one measurement satisfy any + // sustained_ms all by itself. + const auto b = src.poll(300 * kMs); + EXPECT_EQ(a.sequence, b.sequence); + + feeds.noteDeviceSample(sample, 400 * kMs); + EXPECT_GT(src.poll(410 * kMs).sequence, b.sequence); +} + +TEST_F(MetricSourceTest, GaugeGoesStaleWhenMeasurementsStop) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("gpu[0].util_pct"), cfg, &feeds, ActiveCounterProvider()); + + gpufl::DeviceSample sample; + sample.device_id = 0; + sample.gpu_util = 50; + feeds.noteDeviceSample(sample, 0); + + EXPECT_EQ(src.poll(1000 * kMs).state, MetricState::Fresh); + EXPECT_EQ(src.poll(9000 * kMs).state, MetricState::Stale); +} + +TEST_F(MetricSourceTest, MissingDeviceReadsAsWarmingUpNotZero) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("gpu[3].util_pct"), cfg, &feeds, ActiveCounterProvider()); + + gpufl::DeviceSample sample; + sample.device_id = 0; + sample.gpu_util = 77; + feeds.noteDeviceSample(sample, 0); + + // A rule on a GPU that does not exist must not read as 0% utilisation, + // which is a perfectly firable value for a "GPU went idle" rule. + EXPECT_EQ(src.poll(1000 * kMs).state, MetricState::WarmingUp); +} + +TEST_F(MetricSourceTest, EpochResetDiscardsEvidenceButKeepsSequenceMonotonic) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("kernel_launch_rate"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + for (int64_t t = 0; t < 1200 * kMs; t += 10 * kMs) { + feeds.noteKernelLaunch(t); + src.poll(t); + } + const auto before = src.poll(1200 * kMs); + ASSERT_EQ(before.state, MetricState::Fresh); + + src.resetEpoch(1200 * kMs); + const auto after = src.poll(1210 * kMs); + // Buckets filled while profiling was active describe a contaminated + // workload; letting them prove recovery is how a rule re-fires on its own + // overhead. + EXPECT_EQ(after.state, MetricState::WarmingUp); + // The sequence must not go backwards, or the evaluator would ignore real + // samples whose numbers it had already seen. + EXPECT_GE(after.sequence, before.sequence); +} + +TEST_F(MetricSourceTest, ALongCollectorStallDoesNotReplayFabricatedZeros) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("kernel_launch_rate"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + for (int64_t t = 0; t < 1200 * kMs; t += 10 * kMs) { + feeds.noteKernelLaunch(t); + src.poll(t); + } + ASSERT_EQ(src.poll(1200 * kMs).state, MetricState::Fresh); + + // The collector was blocked for a minute. Replaying 600 empty buckets would + // look like a measured run of zeros and could fire a stall rule that + // nothing actually observed. + const auto after = src.poll(61200 * kMs); + EXPECT_EQ(after.state, MetricState::WarmingUp); +} + +TEST_F(MetricSourceTest, CustomCounterIgnoresTicksFromAnEarlierSession) { + // Slots are permanent by design, so a counter ticked by a previous session + // still holds that value. Reading the raw total would make the new session + // believe the counter had already moved, arm on evidence it never saw, and + // fire a stall rule on a workload that had not started. + auto c = gpufl::counter("metric_prev_session"); + c.add(5000); // "previous session" traffic + + ActiveCounterProvider()->begin_session(); // this session's baseline + + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("custom.metric_prev_session_rate"), cfg, &feeds, + ActiveCounterProvider()); + + const auto s = advance(src, 0, 3000 * kMs, 10 * kMs); + EXPECT_EQ(s.state, MetricState::WarmingUp) + << "counted a previous session's ticks as this session's first tick"; + ActiveCounterProvider()->end_session(); +} + +TEST_F(MetricSourceTest, LaunchFeedIsVisibleWithoutTakingALock) { + // The launch path writes atomics only. This does not prove the absence of a + // lock, but it does pin the visibility contract the atomics have to keep: + // seeded is released last, so observing it means count and timestamp are + // already visible. + MetricFeeds f; + f.noteKernelLaunch(1234); + const auto feed = f.launchFeed(); + EXPECT_TRUE(feed.seeded); + EXPECT_EQ(feed.count, 1u); + EXPECT_EQ(feed.last_event_ns, 1234); +} + +TEST_F(MetricSourceTest, TheDurationFeedIsBoundedBeforeItIsDrained) { + // The per-bucket trim only ran AFTER a drain, which bounds nothing: between + // drains the buffer grew with every kernel, and the case where it grows + // fastest - a launch storm under a stalled collector - is exactly the case + // where the drain is late. + MetricFeeds f; + for (size_t i = 0; i < MetricFeeds::kMaxPendingDurations + 5000; ++i) { + f.noteKernelDuration(static_cast(i), 1.0); + } + const auto drained = f.drainDurations(); + EXPECT_LE(drained.samples.size(), MetricFeeds::kMaxPendingDurations); + EXPECT_GT(drained.dropped, 0u) + << "samples were discarded without saying so"; + // The source is alive even while samples are refused, so freezing this + // would report a busy workload as a dead one. + EXPECT_EQ(f.durationsLastEventNs(), + static_cast(MetricFeeds::kMaxPendingDurations + 4999)); +} + +TEST_F(MetricSourceTest, ALongStallDiscardsDurationsRecordedBeforeIt) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + for (int64_t t = 0; t < 1200 * kMs; t += 10 * kMs) { + feeds.noteKernelDuration(t, 4.0); + src.poll(t); + } + ASSERT_EQ(src.poll(1200 * kMs).state, MetricState::Fresh); + + // Kernels recorded just before the collector stalls, then a minute-long gap. + for (int i = 0; i < 50; ++i) feeds.noteKernelDuration(1250 * kMs, 900.0); + const auto after = src.poll(61200 * kMs); + + // Those 900ms durations describe a workload from before the gap. Letting + // the next bucket inherit them would fire a slow-kernel rule on evidence + // older than the window it claims to cover. + EXPECT_NE(after.state, MetricState::Fresh); + EXPECT_LT(after.value, 900.0); +} + +TEST_F(MetricSourceTest, ACatchUpPutsDurationsInTheBucketTheyBelongTo) { + // The collector falls behind and closes several boundaries in one poll. + // With untimestamped samples the whole backlog lands in the OLDEST bucket + // and the rest come up empty, so the window holds one dense bucket instead + // of the spread that actually happened. + const MetricWindowConfig cfg{1000, 2000, 5000}; // 100ms buckets + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + src.poll(0); + + // Four buckets' worth of kernels, one per bucket, recorded while the + // collector is asleep. The last is 100ms; the rest are 1ms. + feeds.noteKernelDuration(50 * kMs, 1.0); + feeds.noteKernelDuration(150 * kMs, 1.0); + feeds.noteKernelDuration(250 * kMs, 1.0); + feeds.noteKernelDuration(350 * kMs, 100.0); + + // One poll that closes all four boundaries at once. + src.poll(400 * kMs); + + // Keep the window full without adding anything, then read. + const auto s = advance(src, 410 * kMs, 1100 * kMs, 10 * kMs); + ASSERT_EQ(s.state, MetricState::Fresh); + // Median of {1,1,1,100} is 1ms. If all four had been dumped into one + // bucket they would still be together here, so the sharper check is that + // the samples survive the catch-up at all rather than being expired by + // buckets that never held them. + EXPECT_LT(s.value, 50.0) << "value=" << s.value; +} + +TEST_F(MetricSourceTest, TruncationIsCountedWhereSomeoneCanSeeIt) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + src.poll(0); + + for (size_t i = 0; i < MetricFeeds::kMaxPendingDurations + 2000; ++i) { + feeds.noteKernelDuration(10 * kMs, 5.0); + } + src.poll(200 * kMs); + + // A percentile computed from a subset is not the percentile. Counting the + // loss is the minimum; suppressing the metric instead would disable it at + // exactly the launch rates it exists for. + EXPECT_GT(src.durationsTruncated(), 0u) + << "samples were discarded and nothing recorded it"; +} + +TEST_F(MetricSourceTest, TruncationTravelsWithTheReading) { + // An internal counter nobody reads cannot stop a partial percentile being + // presented as a complete one. The number has to reach whoever draws the + // conclusion, so it rides on the sample. + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + src.poll(0); + + for (size_t i = 0; i < MetricFeeds::kMaxPendingDurations + 1000; ++i) { + feeds.noteKernelDuration(10 * kMs, 5.0); + } + const auto s = advance(src, 10 * kMs, 1500 * kMs, 10 * kMs); + EXPECT_GT(s.truncated_samples, 0u) + << "the reading claims to be complete when it is not"; +} + +TEST_F(MetricSourceTest, AnUntruncatedReadingSaysSoWithZero) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + for (int64_t t = 0; t < 1200 * kMs; t += 10 * kMs) { + feeds.noteKernelDuration(t, 3.0); + src.poll(t); + } + const auto s = src.poll(1200 * kMs); + ASSERT_EQ(s.state, MetricState::Fresh); + EXPECT_EQ(s.truncated_samples, 0u); +} + +TEST_F(MetricSourceTest, BucketLevelTrimmingIsCountedToo) { + // The gap the feed-level test could not see. There are TWO limits and the + // bucket's is the smaller, so this batch - above the bucket cap, below the + // feed cap - passes the feed untouched and is trimmed only when the bucket + // closes. Counting just the feed's refusals reports a complete percentile + // while a fifth of the kernels are gone. + static_assert(MetricSource::kMaxDurationsPerBucketForTesting < + MetricFeeds::kMaxPendingDurations, + "this test only means something while the bucket cap is smaller"); + constexpr size_t kBatch = + MetricSource::kMaxDurationsPerBucketForTesting + 900; + + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + src.poll(0); + + for (size_t i = 0; i < kBatch; ++i) feeds.noteKernelDuration(10 * kMs, 2.0); + const auto s = src.poll(200 * kMs); + + EXPECT_EQ(s.truncated_samples, + kBatch - MetricSource::kMaxDurationsPerBucketForTesting) + << "the bucket trimmed silently"; +} + +TEST_F(MetricSourceTest, ALossIsVisibleOnThePollThatCausedIt) { + // Read before the buckets closed, the count belonged to the previous + // bucket - so the reading that dropped the samples still claimed the data + // was whole, and only the next one admitted it. + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + src.poll(0); + + for (size_t i = 0; i < MetricSource::kMaxDurationsPerBucketForTesting + 500; + ++i) { + feeds.noteKernelDuration(10 * kMs, 2.0); + } + // The very first poll that closes a bucket must already say so. + EXPECT_GT(src.poll(200 * kMs).truncated_samples, 0u); +} + +} // namespace diff --git a/tests/core/test_monitor.cpp b/tests/core/test_monitor.cpp index 60e67cd..9770cf8 100644 --- a/tests/core/test_monitor.cpp +++ b/tests/core/test_monitor.cpp @@ -77,6 +77,30 @@ TEST_F(MonitorTest, MultipleInitialize) { gpufl::Monitor::Shutdown(); // Should be safe } +TEST_F(MonitorTest, InitializeClearsThePreviousSessionsSynthesisPolicy) { + gpufl::SetSuppressOrphanSyntheticKernels(true); + + gpufl::MonitorOptions opts; + opts.profiling_engine = gpufl::ProfilingEngine::Monitor; + opts.backend_kind = gpufl::MonitorBackendKind::None; + gpufl::Monitor::Initialize(opts); + + EXPECT_FALSE(gpufl::SuppressOrphanSyntheticKernelsForTesting()) + << "a prior session's suppression state must not leak into a new one"; +} + +TEST_F(MonitorTest, TraceEnablesTrustworthyOrphanPolicyBeforeCollection) { + SKIP_IF_NO_CUDA(); + + gpufl::MonitorOptions opts; + opts.profiling_engine = gpufl::ProfilingEngine::Trace; + gpufl::Monitor::Initialize(opts); + gpufl::Monitor::Start(); + + EXPECT_TRUE(gpufl::SuppressOrphanSyntheticKernelsForTesting()) + << "Trace must drop unmatched launches instead of inventing GPU time"; +} + // ── scope name stack ──────────────────────────────────────────────────────── // // The active scope name is what stamps every profile and PM sample. It has to @@ -163,3 +187,282 @@ TEST(ScopeNameStackTest, DepthReportsWhereANewScopeWouldNest) { m.pushTrackedScopeRow(ScopeRow(2, inner, 1)); EXPECT_EQ(m.openScopeDepth(), 1); } + +// ── PM sample scope attribution ───────────────────────────────────────────── +// +// Attribution used to rescan every completed scope of the run for every PM +// sample, over a list that was never trimmed. These cover the replacement: a +// retention watermark driven by decode progress, and a per-drain sort-and-sweep +// that must agree with the original resolver exactly. + +namespace { + +gpufl::PmSampleBatchRow PmSample(int64_t ts_ns) { + gpufl::PmSampleBatchRow row; + row.ts_ns = ts_ns; + return row; +} + +gpufl::ScopeBatchRow ScopeEdge(uint64_t instance_id, uint32_t name_id, int64_t ts_ns, + uint8_t event_type, int depth) { + gpufl::ScopeBatchRow row; + row.ts_ns = ts_ns; + row.scope_instance_id = instance_id; + row.name_id = name_id; + row.event_type = event_type; + row.depth = depth; + return row; +} + +// Open then close a scope over [start_ns, end_ns] at the given depth. +void RecordScope(gpufl::detail::MonitorBatchManager& m, uint64_t instance_id, + uint32_t name_id, int64_t start_ns, int64_t end_ns, int depth) { + m.pushTrackedScopeRow(ScopeEdge(instance_id, name_id, start_ns, 0, depth)); + m.pushTrackedScopeRow(ScopeEdge(instance_id, name_id, end_ns, 1, depth)); +} + +} // namespace + +TEST(ScopeAttributionTest, BatchSweepMatchesThePerSampleResolver) { + // Both must pick the same scope for every timestamp. The sweep is only an + // optimisation, so any disagreement is a regression in attribution. + gpufl::detail::MonitorBatchManager m; + const uint32_t outer = m.internScopeName("process:app"); + const uint32_t mid = m.internScopeName("epoch"); + const uint32_t inner = m.internScopeName("step"); + + // Nested, overlapping, and sharing boundaries. Closed out of order on + // purpose: real closes are timestamped before they take the lock. + RecordScope(m, 3, inner, 300, 400, 2); + RecordScope(m, 2, mid, 200, 500, 1); + RecordScope(m, 1, outer, 100, 900, 0); + RecordScope(m, 4, inner, 500, 500, 2); // zero-width, boundary shared with mid + // Overlapping siblings at the SAME depth. Without these the depth + // comparison alone decides every sample and the start_ns tie-break is + // never exercised - two threads each running their own scope is exactly + // how this arises. + RecordScope(m, 5, mid, 600, 800, 1); + RecordScope(m, 6, inner, 700, 850, 1); + + std::vector timestamps; + for (int64_t ts = 50; ts <= 950; ts += 7) timestamps.push_back(ts); + for (int64_t ts : {100LL, 200LL, 300LL, 400LL, 500LL, 900LL}) timestamps.push_back(ts); + + std::vector rows; + for (int64_t ts : timestamps) rows.push_back(PmSample(ts)); + m.resolveScopeIdsForTesting(rows, /*fallback_id=*/0); + + for (const auto& row : rows) { + const uint32_t reference = m.resolveScopeIdForTesting(row.ts_ns); + EXPECT_EQ(row.scope_name_id, reference) + << "sweep and per-sample resolver disagree at ts=" << row.ts_ns; + } +} + +TEST(ScopeAttributionTest, SampleInsideAStillOpenScopeIsAttributedToIt) { + // PM drains mid-run, so the scope covering a sample is routinely still + // open. Before this, such samples fell back to whatever was on top at drain + // time - which is not the same question. + gpufl::detail::MonitorBatchManager m; + const uint32_t outer = m.internScopeName("process:app"); + const uint32_t open = m.internScopeName("deep_window"); + + RecordScope(m, 1, outer, 100, 900, 0); + m.pushTrackedScopeRow(ScopeEdge(2, open, 400, 0, 1)); // never closed + + std::vector rows{PmSample(300), PmSample(500)}; + m.resolveScopeIdsForTesting(rows, /*fallback_id=*/0); + + EXPECT_EQ(rows[0].scope_name_id, outer) << "before the open scope started"; + EXPECT_EQ(rows[1].scope_name_id, open) << "inside the still-open scope"; +} + +TEST(ScopeAttributionTest, PendingCloseCapsAnOpenScopeAtItsCapturedTimestamp) { + // A close captures its timestamp before waiting for the scope-state lock. + // The snapshot must observe that pending timestamp instead of extending the + // still-open map entry through the entire PM batch. + gpufl::detail::MonitorBatchManager m; + const uint32_t name = m.internScopeName("closing"); + m.pushTrackedScopeRow(ScopeEdge(1, name, 100, 0, 0)); + m.markScopeClosePending(1, 200); + + std::vector rows{PmSample(150), PmSample(250)}; + m.resolveScopeIdsForTesting(rows, /*fallback_id=*/0); + + EXPECT_EQ(rows[0].scope_name_id, name); + EXPECT_EQ(rows[1].scope_name_id, 0u) + << "a pending close must prevent provisional extension past its timestamp"; +} + +TEST(ScopeAttributionTest, RetentionDropsOnlyWhatTheWatermarkReleases) { + gpufl::detail::MonitorBatchManager m; + const uint32_t name = m.internScopeName("step"); + RecordScope(m, 1, name, 100, 200, 0); + RecordScope(m, 2, name, 300, 400, 0); + RecordScope(m, 3, name, 500, 600, 0); + EXPECT_EQ(m.retainedCompletedScopesForTesting(), 3u); + + // Nothing published yet: a run without PM sampling must never lose scopes. + std::vector probe{PmSample(150)}; + m.resolveScopeIdsForTesting(probe, 0); + EXPECT_EQ(m.retainedCompletedScopesForTesting(), 3u); + + m.publishScopeRetentionWatermark(450); + m.resolveScopeIdsForTesting(probe, 0); + EXPECT_EQ(m.retainedCompletedScopesForTesting(), 1u) + << "only the scope ending after the watermark survives"; +} + +TEST(ScopeAttributionTest, WatermarkNeverMovesBackwards) { + // A failed decode or an overflow must not un-retire scopes already + // released, so a lower value is ignored rather than applied. + gpufl::detail::MonitorBatchManager m; + const uint32_t name = m.internScopeName("step"); + RecordScope(m, 1, name, 100, 200, 0); + RecordScope(m, 2, name, 300, 400, 0); + + m.publishScopeRetentionWatermark(350); + m.publishScopeRetentionWatermark(50); // ignored + + std::vector probe{PmSample(380)}; + m.resolveScopeIdsForTesting(probe, 0); + EXPECT_EQ(m.retainedCompletedScopesForTesting(), 1u); +} + +TEST(ScopeAttributionTest, DelayedDrainKeepsScopesThatWallClockWouldHaveDropped) { + // The watermark is event time, not wall clock. A collector stalled for + // seconds must still attribute the samples it eventually decodes. + gpufl::detail::MonitorBatchManager m; + const uint32_t name = m.internScopeName("step"); + RecordScope(m, 1, name, 1'000'000'000, 1'100'000'000, 0); + + // Simulate a long stall: no watermark is published because nothing decoded. + std::vector rows{PmSample(1'050'000'000)}; + m.resolveScopeIdsForTesting(rows, /*fallback_id=*/7); + + EXPECT_EQ(rows[0].scope_name_id, name) + << "a scope that ended 2s ago is still needed by an undecoded sample"; +} + +TEST(ScopeAttributionTest, EqualDepthAndStartResolveIdenticallyInBothPaths) { + // std::sort is not stable and open scopes come out of an unordered_map, so + // without a tertiary key two scopes sharing depth AND start could resolve + // differently between the batch sweep and the reference resolver. + gpufl::detail::MonitorBatchManager m; + const uint32_t a = m.internScopeName("thread_a"); + const uint32_t b = m.internScopeName("thread_b"); + + // Same depth and start, DIFFERENT ends. If the ranking ever reports the two + // as equivalent, the sweep's ordered set keeps only one of them - and a + // sample after the shorter one ends then resolves to nothing instead of to + // the scope still covering it. Equal ends would hide that entirely. + RecordScope(m, 10, a, 100, 200, 1); + RecordScope(m, 11, b, 100, 400, 1); + + std::vector rows{PmSample(150), PmSample(250), PmSample(350)}; + m.resolveScopeIdsForTesting(rows, /*fallback_id=*/0); + for (const auto& row : rows) { + EXPECT_EQ(row.scope_name_id, m.resolveScopeIdForTesting(row.ts_ns)); + } +} + +TEST(ScopeAttributionTest, UncoveredSampleIsUnattributedNotGivenTheLiveScope) { + // The old fallback handed an unmatched sample whatever scope was open at + // DECODE time. That is the temporal error this resolver exists to remove: + // a sample can sit in the buffer while scopes come and go. + gpufl::detail::MonitorBatchManager m; + const uint32_t earlier = m.internScopeName("earlier"); + const uint32_t live = m.internScopeName("live_now"); + + RecordScope(m, 1, earlier, 100, 200, 0); + // A different scope is open by the time the batch is decoded. + m.pushTrackedScopeRow(ScopeEdge(2, live, 900, 0, 0)); + + std::vector rows{PmSample(500)}; // covered by neither + m.resolveScopeIdsForTesting(rows, /*fallback_id=*/0); + + EXPECT_EQ(rows[0].scope_name_id, 0u) + << "an uncovered sample must stay unattributed, not inherit the live scope"; +} + +TEST(ScopeAttributionTest, ManyOverlappingScopesStillResolveCorrectly) { + // Cross-thread workloads keep a large active set. The sweep has to stay + // correct as that grows, not just when a couple of scopes nest. + gpufl::detail::MonitorBatchManager m; + + // A distinct name per scope. With one shared name the assertion could not + // tell a correct pick from a wrong one - every answer would compare equal. + constexpr int kScopes = 400; + std::vector names; + names.reserve(kScopes); + for (int i = 0; i < kScopes; ++i) { + names.push_back(m.internScopeName("worker_" + std::to_string(i))); + } + for (int i = 0; i < kScopes; ++i) { + RecordScope(m, static_cast(i + 1), names[i], + /*start*/ i, /*end*/ 10'000 + i, /*depth*/ i % 4); + } + + std::vector rows; + for (int64_t ts = 0; ts < 10'000; ts += 337) rows.push_back(PmSample(ts)); + m.resolveScopeIdsForTesting(rows, /*fallback_id=*/0); + + for (const auto& row : rows) { + EXPECT_EQ(row.scope_name_id, m.resolveScopeIdForTesting(row.ts_ns)) + << "large active set diverges at ts=" << row.ts_ns; + } +} + +TEST(ScopeAttributionTest, CapAppliesWithoutAnyPmSamples) { + // The retention watermark is the real bound, but only PM sampling ever + // publishes one. A Trace-only run closes scopes and never decodes a sample, + // so if the cap only ran on the PM path this deque would grow for the life + // of the process - unbounded, and with nothing recorded to say so. + gpufl::detail::MonitorBatchManager m; + const uint32_t name = m.internScopeName("step"); + + constexpr int kScopes = 70'000; // over kMaxCompletedScopes + for (int i = 0; i < kScopes; ++i) { + RecordScope(m, static_cast(i + 1), name, i, i + 1, 0); + } + + EXPECT_LE(m.retainedCompletedScopesForTesting(), 65536u); + EXPECT_EQ(m.scopeAttributionTruncated(), 0u) + << "evicting unused Trace-only history is not PM attribution loss"; + EXPECT_EQ(m.pmSampleRowsSeen(), 0u) + << "evicting unused Trace-only history is not PM attribution loss"; +} + +TEST(ScopeAttributionTest, CapCountsEvictionWhilePmSamplingIsActive) { + gpufl::detail::MonitorBatchManager m; + const uint32_t name = m.internScopeName("step"); + m.beginPmScopeAttribution(100); + + constexpr int kScopes = 70'000; + for (int i = 0; i < kScopes; ++i) { + RecordScope(m, static_cast(i + 1), name, i, i + 1, 0); + } + + EXPECT_GT(m.scopeAttributionTruncated(), 0u) + << "eviction while PM can have buffered samples is an attribution risk"; +} + +TEST(ScopeAttributionTest, OldTraceHistoryEvictedDuringPmIsNotPartialAttribution) { + gpufl::detail::MonitorBatchManager m; + const uint32_t name = m.internScopeName("trace_step"); + + constexpr int kScopes = 70'000; + for (int i = 0; i < kScopes; ++i) { + RecordScope(m, static_cast(i + 1), name, i, i + 1, 0); + } + m.beginPmScopeAttribution(1'000'000); + + // Force more evictions after PM starts. Every evicted entry still predates + // the PM boundary, so none can be needed by a PM sample. + for (int i = 0; i < 100; ++i) { + const int64_t ts = 1'000'000 + i; + RecordScope(m, static_cast(kScopes + i + 1), name, ts, ts + 1, 0); + } + + EXPECT_EQ(m.scopeAttributionTruncated(), 0u); +} diff --git a/tests/core/test_nvtx_counters.cpp b/tests/core/test_nvtx_counters.cpp new file mode 100644 index 0000000..7bfd1dd --- /dev/null +++ b/tests/core/test_nvtx_counters.cpp @@ -0,0 +1,453 @@ +#include + +#include + +#include "gpufl/core/counter_provider.hpp" +#include "gpufl/core/counter_registry.hpp" +#include "gpufl/core/metric_id.hpp" +#include "gpufl/core/metric_registry.hpp" +#include "gpufl/core/model/deep_window_model.hpp" +#include "gpufl/core/nvtx_counters.hpp" + +using gpufl::detail::ActiveCounterProvider; +using gpufl::detail::CounterRegistry; +using gpufl::detail::NvtxCounterBridge; +using ValueType = NvtxCounterBridge::ValueType; +using RegisterStatus = NvtxCounterBridge::RegisterStatus; + +namespace { + +// NVTX_COUNTER_SAMPLE_* from nvToolsExtCounters.h, restated so a change to the +// released ABI shows up here rather than as a counter that quietly stops +// matching. +constexpr uint8_t kZero = 0; +constexpr uint8_t kUnchanged = 1; +constexpr uint8_t kUnavailable = 2; + +class NvtxCounterBridgeTest : public ::testing::Test { + protected: + void SetUp() override { + CounterRegistry::instance().resetForTesting(); + NvtxCounterBridge::instance().resetForTesting(); + } + void TearDown() override { + NvtxCounterBridge::instance().resetForTesting(); + CounterRegistry::instance().resetForTesting(); + } + + NvtxCounterBridge& bridge() { return NvtxCounterBridge::instance(); } + + /// What a rule would read: the value accrued since this session started. + uint64_t observed(const std::string& name) { + const auto* provider = ActiveCounterProvider(); + gpufl_counter_handle h = provider->lookup(name.c_str(), name.size()); + return h == nullptr ? 0 : provider->load_since_baseline(h); + } + + NvtxCounterBridge::RegisterResult acceptDelta(const char* domain, + const char* name) { + auto r = bridge().registerCounter(domain, name, 0, ValueType::Delta); + EXPECT_EQ(r.status, RegisterStatus::Accepted); + return r; + } +}; + +// ── what a sample MEANS ───────────────────────────────────────────────────── + +TEST_F(NvtxCounterBridgeTest, DeltaSamplesAccumulateIntoTheRegistry) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + + bridge().sampleDelta(r.id, 8); + bridge().sampleDelta(r.id, 16); + + // A rate is derived from this by the metric registry; what the bridge owes + // is the running total the application actually reported. + EXPECT_EQ(observed("inference.tokens"), 24u); +} + +TEST_F(NvtxCounterBridgeTest, ANoValueSampleIsNotAnEvent) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + bridge().sampleDelta(r.id, 10); + + bridge().sampleNoValue(r.id, kZero); + bridge().sampleNoValue(r.id, kUnchanged); + + // ZERO and UNCHANGED both say the delta was zero. Counting either as +1 + // would invent traffic out of a sample whose entire purpose is to report + // that there was none - and would make an idle workload look busy to the + // rule watching it. + EXPECT_EQ(observed("inference.tokens"), 10u); + EXPECT_EQ(bridge().unavailableSamples(), 0u); +} + +TEST_F(NvtxCounterBridgeTest, AnUnavailableSampleIsRecordedNotCounted) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + bridge().sampleDelta(r.id, 10); + + bridge().sampleNoValue(r.id, kUnavailable); + + // The application could not read its own counter. That must not move the + // total, but it must not vanish either: the resulting rate rests on less + // data than its sample count suggests. + EXPECT_EQ(observed("inference.tokens"), 10u); + EXPECT_EQ(bridge().unavailableSamples(), 1u); +} + +TEST_F(NvtxCounterBridgeTest, ANegativeDeltaIsDroppedRatherThanWrapped) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + bridge().sampleDelta(r.id, 10); + + bridge().sampleDelta(r.id, -4); + + // The registry accumulates unsigned and rates are unsigned deltas, so + // subtracting here would not read as -4; the very next bucket would report + // a rate near 2^64. + EXPECT_EQ(observed("inference.tokens"), 10u); + EXPECT_EQ(bridge().negativeSamples(), 1u); +} + +// ── what is refused, and why ──────────────────────────────────────────────── + +TEST_F(NvtxCounterBridgeTest, ACounterWithNoSemanticsIsRefused) { + // nvtxCounterSampleInt64 says the value is an int64; it does not say + // whether it is a delta or an absolute reading. Accumulating an absolute + // series would produce a rate that looks reasonable and is wrong. + const auto r = bridge().registerCounter("d", "c", 0, ValueType::Unspecified); + EXPECT_EQ(r.status, RegisterStatus::UnsupportedValueType); + EXPECT_EQ(r.id, 0u); +} + +TEST_F(NvtxCounterBridgeTest, AnAbsoluteCounterIsRefused) { + EXPECT_EQ(bridge().registerCounter("d", "c", 0, ValueType::Absolute).status, + RegisterStatus::UnsupportedValueType); + EXPECT_EQ(bridge() + .registerCounter("d", "c", 0, ValueType::DeltaSinceStart) + .status, + RegisterStatus::UnsupportedValueType); +} + +TEST_F(NvtxCounterBridgeTest, AnApplicationAssignedIdIsRefused) { + // An NVTX counter id is unique only WITHIN its domain. Binding one to a + // slot without a (domain, id) table would let two domains that both chose + // id 123 share a slot and add their rates together. + const auto r = bridge().registerCounter("d", "c", 1u << 24, ValueType::Delta); + EXPECT_EQ(r.status, RegisterStatus::StaticIdUnsupported); + EXPECT_EQ(r.id, 0u); +} + +TEST_F(NvtxCounterBridgeTest, SamplesForAnIdWeNeverIssuedGoNowhere) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + + // A refused registration returns 0, and the application keeps sampling. + bridge().sampleDelta(0, 99); + bridge().sampleDelta(1u << 24, 99); // a static id + bridge().sampleDelta(r.id + 1000, 99); // past the table + + EXPECT_EQ(observed("inference.tokens"), 0u) + << "an unknown id landed on someone else's slot"; + EXPECT_EQ(bridge().unknownIdSamples(), 3u); +} + +// ── naming ────────────────────────────────────────────────────────────────── + +TEST_F(NvtxCounterBridgeTest, TheDomainIsPartOfTheName) { + // Two teams each calling their counter "tokens" in their own domain are + // two counters, and a rule has to be able to name one of them. + const auto a = acceptDelta("inference", "tokens"); + const auto b = acceptDelta("training", "tokens"); + EXPECT_NE(a.id, b.id); + EXPECT_EQ(a.metric, "custom.inference.tokens_rate"); + EXPECT_EQ(b.metric, "custom.training.tokens_rate"); +} + +TEST_F(NvtxCounterBridgeTest, AnUndomainedCounterKeepsItsBareName) { + EXPECT_EQ(NvtxCounterBridge::canonicalName("", "tokens"), "tokens"); +} + +TEST_F(NvtxCounterBridgeTest, OutOfCharsetBytesAreMappedNotDropped) { + // Rules address counters as custom._rate over [A-Za-z0-9._-], and + // NVTX names are free-form. + EXPECT_EQ(NvtxCounterBridge::canonicalName("My Server", "tokens/sec"), + "My_Server.tokens_sec"); + // Nothing usable left is a refusal, not a metric called custom.___rate. + EXPECT_TRUE(NvtxCounterBridge::canonicalName("", "///").empty()); + EXPECT_TRUE(NvtxCounterBridge::canonicalName("", "").empty()); +} + +TEST_F(NvtxCounterBridgeTest, TwoNamesThatCanonicaliseAlikeAreRefused) { + ActiveCounterProvider()->begin_session(); + // "a b" and "a/b" both canonicalise to "a_b", but they are DIFFERENT NVTX + // counters. Sharing a binding would silently add two unrelated workloads + // into one rate - a wrong number that looks like a real one - so the + // second registration is refused and told to rename. + const auto a = acceptDelta("", "a b"); + const auto b = bridge().registerCounter("", "a/b", 0, ValueType::Delta); + EXPECT_EQ(b.status, RegisterStatus::BadName); + EXPECT_EQ(b.id, 0u); + EXPECT_EQ(bridge().trackedCount(), 1u); + + bridge().sampleDelta(a.id, 3); + EXPECT_EQ(observed("a_b"), 3u) << "the refused counter leaked into the slot"; +} + +TEST_F(NvtxCounterBridgeTest, TheDomainJoinIsACollisionAxisToo) { + // The domain joins with '.', so ("a", "b.c") and ("a.b", "c") meet at + // "a.b.c" without any out-of-charset byte involved. Only the original + // (domain, counter) pair can tell them apart. + const auto first = acceptDelta("a", "b.c"); + const auto second = bridge().registerCounter("a.b", "c", 0, ValueType::Delta); + EXPECT_EQ(first.metric, "custom.a.b.c_rate"); + EXPECT_EQ(second.status, RegisterStatus::BadName); + EXPECT_EQ(bridge().trackedCount(), 1u); +} + +TEST_F(NvtxCounterBridgeTest, RegisteringTheSameCounterTwiceIsIdempotent) { + // Same ORIGINAL pair - not merely the same canonical name - is the one + // case that returns the existing binding. + const auto first = acceptDelta("inference", "tokens"); + const auto again = acceptDelta("inference", "tokens"); + EXPECT_EQ(first.id, again.id); + EXPECT_EQ(bridge().trackedCount(), 1u); +} + +// ── the failed-read contract, all the way to the metric layer ─────────────── + +TEST_F(NvtxCounterBridgeTest, AnUnavailableSampleDiscardsTheRateWindow) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + + const auto parsed = gpufl::detail::parseMetric("custom.inference.tokens_rate"); + ASSERT_EQ(parsed.error, gpufl::detail::MetricParseError::None); + gpufl::detail::MetricWindowConfig cfg; + cfg.rate_window_ms = 100; // bucket = 10ms, 10 buckets + cfg.stale_after_ms = 20000; + gpufl::detail::MetricFeeds feeds; + gpufl::detail::MetricSource src(parsed.id, cfg, &feeds, + ActiveCounterProvider()); + + constexpr int64_t kStep = 10 * 1000000; // one bucket interval + int64_t t = 0; + gpufl::detail::MetricSample s; + + // Steady traffic until the window is full and the metric is usable. + int polls = 0; + do { + bridge().sampleDelta(r.id, 5); + s = src.poll(t); + t += kStep; + } while (s.state != gpufl::detail::MetricState::Fresh && ++polls < 40); + ASSERT_EQ(s.state, gpufl::detail::MetricState::Fresh) + << "never reached a usable reading; the fixture is wrong"; + + // The application fails to read its counter. Real throughput has NOT + // dropped - only the observation did. Without the discard, the missing + // deltas read as a rate collapse and a stall rule fires on a workload + // that never slowed down. + bridge().sampleNoValue(r.id, kUnavailable); + s = src.poll(t); + t += kStep; + EXPECT_EQ(s.state, gpufl::detail::MetricState::WarmingUp) + << "a window containing a failed read was presented as evidence"; + + // Recovery: the window must refill from post-failure data before the + // metric is usable again - a new baseline, not a resumed one. + int refill_polls = 0; + bool fresh_before_refill = false; + do { + bridge().sampleDelta(r.id, 5); + s = src.poll(t); + t += kStep; + ++refill_polls; + if (s.state == gpufl::detail::MetricState::Fresh && refill_polls < 10) { + fresh_before_refill = true; + } + } while (s.state != gpufl::detail::MetricState::Fresh && refill_polls < 40); + EXPECT_EQ(s.state, gpufl::detail::MetricState::Fresh) + << "the metric never recovered after the failed read"; + EXPECT_FALSE(fresh_before_refill) + << "usable again before a full window of post-failure data"; +} + +// ── the routing that keeps the evaluator and the target together ──────────── + +TEST_F(NvtxCounterBridgeTest, TheCounterIsVisibleThroughTheActiveProvider) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + bridge().sampleDelta(r.id, 5); + + // The evaluator reads whatever ActiveCounterProvider() resolves. Writing + // straight to this module's CounterRegistry instead would split the two + // apart wherever a shared runtime is present - being in the same injection + // module as the evaluator is not enough to assume otherwise. + const std::string name = "inference.tokens"; + gpufl_counter_handle via_provider = + ActiveCounterProvider()->lookup(name.c_str(), name.size()); + ASSERT_NE(via_provider, nullptr); + EXPECT_EQ(ActiveCounterProvider()->load_since_baseline(via_provider), 5u); +} + +TEST_F(NvtxCounterBridgeTest, TicksBeforeTheSessionStartsAreNotCounted) { + // Registration and the first samples can legitimately arrive before + // gpufl::init(): NVTX fires on the application's first NVTX call, which + // may precede any CUDA call. + const auto r = acceptDelta("inference", "tokens"); + bridge().sampleDelta(r.id, 100); + + ActiveCounterProvider()->begin_session(); + bridge().sampleDelta(r.id, 7); + + EXPECT_EQ(observed("inference.tokens"), 7u) + << "pre-session traffic leaked into this session's rate"; +} + +// ── the session data-quality contract ─────────────────────────────────────── + +TEST_F(NvtxCounterBridgeTest, EveryRefusedRegistrationIsCountedOnce) { + bridge().registerCounter("d", "c1", 1u << 24, ValueType::Delta); // static id + bridge().registerCounter("d", "c2", 0, ValueType::Absolute); // value type + bridge().registerCounter("", "///", 0, ValueType::Delta); // bad name + acceptDelta("", "a b"); + bridge().registerCounter("", "a/b", 0, ValueType::Delta); // collision + EXPECT_EQ(bridge().registrationRejected(), 4u) + << "a refusal path forgot to count itself"; + // The accepted one, and its idempotent repeat, are not refusals. + acceptDelta("", "a b"); + EXPECT_EQ(bridge().registrationRejected(), 4u); +} + +TEST_F(NvtxCounterBridgeTest, ASessionSnapshotReportsOnlyItsOwnSession) { + // The tallies live for the process, like the counter slots; an embedded + // host re-initialises in one process, and exporting raw totals would + // re-report session one's problems as session two's. + bridge().registerCounter("d", "c", 0, ValueType::Absolute); + const auto r = acceptDelta("inference", "tokens"); + bridge().sampleDelta(r.id, -1); + bridge().sampleNoValue(r.id, kUnavailable); + + auto first = bridge().takeSessionSnapshot(); + EXPECT_EQ(first.registration_rejected, 1u); + EXPECT_EQ(first.negative_delta_samples, 1u); + EXPECT_EQ(first.unavailable_samples, 1u); + EXPECT_TRUE(first.any()); + + // Session two: only what happened after the previous report. + bridge().sampleDelta(0, 5); // unknown id + auto second = bridge().takeSessionSnapshot(); + EXPECT_EQ(second.registration_rejected, 0u) + << "session one's refusal was reported twice"; + EXPECT_EQ(second.unknown_id_samples, 1u); + EXPECT_EQ(second.unavailable_samples, 0u); + + // Session three: nothing happened, nothing to say. + EXPECT_FALSE(bridge().takeSessionSnapshot().any()); +} + +TEST_F(NvtxCounterBridgeTest, PreInitEventsBelongToTheFirstSession) { + // NVTX registration legitimately runs before gpufl::init() - proven on + // hardware - so a refusal during startup happens before any session + // exists. It belongs to the first session that reports: no other session + // can, and dropping it hides exactly the config error the event is for. + bridge().registerCounter("d", "c", 0, ValueType::Unspecified); + + // ... gpufl::init() happens here ... + const auto snap = bridge().takeSessionSnapshot(); + EXPECT_EQ(snap.registration_rejected, 1u) + << "a startup config error was attributed to no session at all"; +} + +// ── the wire shape ────────────────────────────────────────────────────────── +// +// The backend parses these by field name from hand-built JSON; nothing in the +// type system connects the two sides. Pinned here so a renamed field fails a +// test instead of silently landing as an empty column. + +TEST(CounterDataQualityWireTest, TheSummaryCarriesEveryTallyByName) { + gpufl::CounterDataQualitySummaryEvent ev; + ev.pid = 7; + ev.app = "serve"; + ev.session_id = "s1"; + ev.tracked_counters = 1; + ev.samples_observed = 12000; + ev.registration_rejected = 2; + ev.unknown_id_samples = 3; + ev.unavailable_samples = 4; + ev.negative_delta_samples = 1; + ev.rate_windows_discarded = 2; + ev.emitted_ns = 42; + + const std::string json = + gpufl::model::CounterDataQualitySummaryModel(ev).buildJson(); + EXPECT_NE(json.find("\"type\":\"counter_data_quality_summary\""), + std::string::npos) << json; + // Only "nvtx" is observed today; a generic-looking row would claim + // coverage of gpufl::counter() failures it does not have. + EXPECT_NE(json.find("\"source\":\"nvtx\""), std::string::npos); + EXPECT_NE(json.find("\"schema_version\":1"), std::string::npos); + // The denominators that make an all-zero row readable at all. + EXPECT_NE(json.find("\"tracked_counters\":1"), std::string::npos); + EXPECT_NE(json.find("\"samples_observed\":12000"), std::string::npos); + EXPECT_NE(json.find("\"registration_rejected\":2"), std::string::npos); + EXPECT_NE(json.find("\"unknown_id_samples\":3"), std::string::npos); + EXPECT_NE(json.find("\"unavailable_samples\":4"), std::string::npos); + EXPECT_NE(json.find("\"negative_delta_samples\":1"), std::string::npos); + EXPECT_NE(json.find("\"rate_windows_discarded\":2"), std::string::npos); +} + +TEST(CounterDataQualityWireTest, TheRuleSummaryCarriesItsOwnQualityFields) { + gpufl::DeepWindowRuleSummaryEvent ev; + ev.session_id = "s1"; + ev.outcome = "never_true"; + ev.metric_quality_resets = 2; + ev.last_quality_reason = "counter_unavailable"; + + const std::string json = + gpufl::model::DeepWindowRuleSummaryModel(ev).buildJson(); + EXPECT_NE(json.find("\"metric_quality_resets\":2"), std::string::npos) + << json; + EXPECT_NE(json.find("\"last_quality_reason\":\"counter_unavailable\""), + std::string::npos); +} + +TEST_F(NvtxCounterBridgeTest, SamplesObservedCountsValidObservationsOnly) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + + bridge().sampleDelta(r.id, 8); + bridge().sampleDelta(r.id, 0); // real observation of "no traffic" + bridge().sampleNoValue(r.id, kZero); // ditto + bridge().sampleNoValue(r.id, kUnchanged); // ditto + bridge().sampleNoValue(r.id, kUnavailable); // a FAILURE, not an observation + bridge().sampleDelta(r.id, -1); // ditto + bridge().sampleDelta(0, 5); // unknown id: ditto + + const auto snap = bridge().takeSessionSnapshot(); + // 8 + 0 + zero + unchanged = 4 observations; the three failures are + // tallied on their own axes, or the failure RATE would be diluted by its + // own failures. + EXPECT_EQ(snap.samples_observed, 4u); + EXPECT_EQ(snap.unavailable_samples, 1u); + EXPECT_EQ(snap.negative_delta_samples, 1u); + EXPECT_EQ(snap.unknown_id_samples, 1u); +} + +TEST_F(NvtxCounterBridgeTest, SamplesObservedIsSessionScopedToo) { + // The stale-clean-row problem: a counter registered in session one must + // not make every later session claim it observed something. + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + bridge().sampleDelta(r.id, 8); + EXPECT_EQ(bridge().takeSessionSnapshot().samples_observed, 1u); + + // Session two: the counter is still tracked, but nothing sampled it. + const auto second = bridge().takeSessionSnapshot(); + EXPECT_EQ(second.samples_observed, 0u) + << "an idle session inherited the previous session's denominator"; + EXPECT_FALSE(second.any()); +} + +} // namespace diff --git a/tests/launcher/test_agent_launcher.cpp b/tests/launcher/test_agent_launcher.cpp new file mode 100644 index 0000000..a777764 --- /dev/null +++ b/tests/launcher/test_agent_launcher.cpp @@ -0,0 +1,42 @@ +#include + +#include +#include + +#include "agent_launcher.hpp" + +namespace { + +std::vector exitCommand(const int code) { +#ifdef _WIN32 + return {"cmd.exe", "/d", "/s", "/c", "exit " + std::to_string(code)}; +#else + return {"/bin/sh", "-c", "exit " + std::to_string(code)}; +#endif +} + +} // namespace + +TEST(AgentProcessTest, AZeroExitIsACompletedUpload) { + gpufl::launcher::AgentProcess process; + std::string error; + ASSERT_TRUE(process.start(exitCommand(0), error)) << error; + + const auto result = process.waitForExit(5000); + + EXPECT_TRUE(result.exited); + EXPECT_EQ(result.exit_code, 0); + EXPECT_TRUE(result.succeeded()); +} + +TEST(AgentProcessTest, ANonzeroExitIsNotACompletedUpload) { + gpufl::launcher::AgentProcess process; + std::string error; + ASSERT_TRUE(process.start(exitCommand(7), error)) << error; + + const auto result = process.waitForExit(5000); + + EXPECT_TRUE(result.exited); + EXPECT_EQ(result.exit_code, 7); + EXPECT_FALSE(result.succeeded()); +} diff --git a/tests/launcher/test_cli_parse.cpp b/tests/launcher/test_cli_parse.cpp index 5a34d84..a73a5a2 100644 --- a/tests/launcher/test_cli_parse.cpp +++ b/tests/launcher/test_cli_parse.cpp @@ -666,3 +666,163 @@ TEST(CliParseTopLevel, UnknownSubcommand) { auto p = parseTopLevel(2, argv); EXPECT_EQ(p.sub, Subcommand::Unknown); } + +// ── --passes and --deep-* are different execution models ──────────────────── +// +// They cannot be combined: the deep engines are fixed before the first CUDA +// call, so a --passes list either already holds what the window would arm, or +// does not - and `--passes=Trace --deep-after=30s` silently opened a window +// that armed nothing. Rejected before the target runs, in either flag order. + +namespace { + +gpufl::launcher::TraceParseResult parseTrace(std::vector argv) { + return gpufl::launcher::parseTraceArgs(argv); +} + +bool rejectsPassesWithDeep(const std::vector& argv) { + const auto r = parseTrace(argv); + return !r.args.has_value() && + r.error.find("--passes cannot be combined") != std::string::npos; +} + +} // namespace + +TEST(CliParseDeepModeTest, PassesBeforeDeepFlagIsRejected) { + EXPECT_TRUE(rejectsPassesWithDeep( + {"--passes=Trace", "--deep-after=30s", "--deep-for=5s", "--", "app"})); +} + +TEST(CliParseDeepModeTest, DeepFlagBeforePassesIsRejected) { + // Order must not decide: a user who writes the flags the other way round + // is making the same mistake. + EXPECT_TRUE(rejectsPassesWithDeep( + {"--deep-when=custom.token_rate<100", "--deep-for=5s", + "--passes=Deep", "--", "app"})); +} + +TEST(CliParseDeepModeTest, SpaceSeparatedFormIsRejectedToo) { + EXPECT_TRUE(rejectsPassesWithDeep( + {"--passes", "PcSampling", "--deep-for", "5s", "--", "app"})); +} + +TEST(CliParseDeepModeTest, EveryDeepFlagTriggersTheRejection) { + // --deep-after is included deliberately, with no grandfather clause: two + // rules ("--deep-when refuses, --deep-after tolerates") would be harder to + // explain than the break. + EXPECT_TRUE(rejectsPassesWithDeep( + {"--passes=Trace", "--deep-after=1s", "--deep-for=1s", "--", "app"})); + EXPECT_TRUE(rejectsPassesWithDeep( + {"--passes=Trace", "--deep-for=1s", "--", "app"})); + EXPECT_TRUE(rejectsPassesWithDeep( + {"--passes=Trace", "--deep-launches=500", "--", "app"})); + EXPECT_TRUE(rejectsPassesWithDeep( + {"--passes=Trace", "--deep-when=kernel_launch_rate<10", + "--deep-for=1s", "--", "app"})); +} + +TEST(CliParseDeepModeTest, TheRejectionNamesTheWayOut) { + const auto r = parseTrace( + {"--passes=Trace", "--deep-for=5s", "--", "app"}); + ASSERT_FALSE(r.args.has_value()); + // An error that only says "no" leaves the user guessing which flag to drop. + EXPECT_NE(r.error.find("Drop --passes"), std::string::npos) << r.error; +} + +TEST(CliParseDeepModeTest, TheTwoWindowTriggersAreMutuallyExclusive) { + // Both set is not "either may open it". Measured on the 3090: the + // scheduled window opens at t=0, the rule is refused behind it, and the + // summary then reports `never_true` for a condition that held all run. + const auto r = parseTrace({"--deep-when=kernel_launch_rate<10", + "--deep-after=1s", "--deep-for=2s", "--", "app"}); + ASSERT_FALSE(r.args.has_value()); + EXPECT_NE(r.error.find("--deep-when"), std::string::npos) << r.error; + EXPECT_NE(r.error.find("--deep-after"), std::string::npos) << r.error; +} + +TEST(CliParseDeepModeTest, AConditionalRunDoesNotCarryATimeTrigger) { + const auto r = parseTrace({"--deep-when=kernel_launch_rate<10", + "--deep-for=2s", "--", "app"}); + ASSERT_TRUE(r.args.has_value()) << r.error; + // deep_after_ms still holds its default 0; what must not happen is the + // launcher treating that default as a request. GPUFL_DEEP_AFTER_MS installs + // the time trigger by being present at all, so the flag being unset has to + // survive as far as the environment. + EXPECT_FALSE(r.args->deep_after_set); +} + +TEST(CliParseDeepModeTest, ADeepRunResolvesToExactlyOneAdaptivePass) { + const auto r = parseTrace({"--deep-for=5s", "--", "app"}); + ASSERT_TRUE(r.args.has_value()) << r.error; + + const auto plan = gpufl::launcher::resolvePassPlan(*r.args); + // One pass, not one per engine: a window triggered by a live condition + // cannot be reproduced across relaunches, so splitting it would change + // what is being measured. + ASSERT_EQ(plan.size(), 1u); + EXPECT_NE(plan[0].find("Trace"), std::string::npos) << plan[0]; + EXPECT_NE(plan[0].find('+'), std::string::npos) + << "the deep engine should be prepared alongside the base: " << plan[0]; +} + +TEST(CliParseDeepModeTest, TheAdaptivePlanPinsTheBaseAndArmsOnlyInTheWindow) { + const auto r = parseTrace({"--deep-for=5s", "--", "app"}); + ASSERT_TRUE(r.args.has_value()) << r.error; + + const auto plan = gpufl::launcher::resolveAdaptivePlan(*r.args); + // Trace is pinned rather than left to the deep engine's own policy: + // kernel_launch_rate and recent_kernel_ms come from its completed-kernel + // records, and a rule that loses its metric reads as "never held". + EXPECT_EQ(plan.base, "Trace"); + EXPECT_TRUE(plan.arm_window_only); + ASSERT_FALSE(plan.selected_deep.empty()); + // PM only for now. PC and SASS join once their dormant cost has been + // measured - picking the deepest engine is not the same as picking the + // deepest one that fits an overhead budget. + EXPECT_EQ(plan.selected_deep.size(), 1u); + EXPECT_EQ(plan.selected_deep[0], "PmSampling"); +} + +TEST(CliParseDeepModeTest, WithoutADeepFlagThereIsNoAdaptivePlan) { + const auto r = parseTrace({"--passes=Trace,PcSampling", "--", "app"}); + ASSERT_TRUE(r.args.has_value()) << r.error; + + EXPECT_TRUE(gpufl::launcher::resolveAdaptivePlan(*r.args).selected_deep.empty()); + // The explicit list is honoured untouched. + EXPECT_EQ(gpufl::launcher::resolvePassPlan(*r.args).size(), 2u); +} + +TEST(CliParseDeepModeTest, PlainTraceIsStillTheDefault) { + const auto r = parseTrace({"--", "app"}); + ASSERT_TRUE(r.args.has_value()) << r.error; + const auto plan = gpufl::launcher::resolvePassPlan(*r.args); + ASSERT_EQ(plan.size(), 1u); + EXPECT_EQ(plan[0], "Trace"); +} + +TEST(CliParseDeepModeTest, ProgrammaticMixedModeFailsSharedValidation) { + gpufl::launcher::TraceArgs args; + args.passes = {"Trace"}; + args.deep_requested = true; + + const std::string error = + gpufl::launcher::validateTraceExecutionMode(args); + EXPECT_NE(error.find("--passes cannot be combined"), std::string::npos); + EXPECT_NE(error.find("Drop --passes"), std::string::npos); +} + +TEST(CliParseDeepModeTest, SharedValidationAcceptsEachModeSeparately) { + gpufl::launcher::TraceArgs explicit_args; + explicit_args.passes = {"Trace", "PmSampling"}; + EXPECT_TRUE( + gpufl::launcher::validateTraceExecutionMode(explicit_args).empty()); + + gpufl::launcher::TraceArgs adaptive_args; + adaptive_args.deep_requested = true; + EXPECT_TRUE( + gpufl::launcher::validateTraceExecutionMode(adaptive_args).empty()); + EXPECT_EQ(gpufl::launcher::resolveCaptureMode(explicit_args), + gpufl::launcher::CaptureMode::ExplicitPasses); + EXPECT_EQ(gpufl::launcher::resolveCaptureMode(adaptive_args), + gpufl::launcher::CaptureMode::AdaptiveDeepWindow); +} diff --git a/tests/launcher/test_deep_window_env.cpp b/tests/launcher/test_deep_window_env.cpp new file mode 100644 index 0000000..5415ea3 --- /dev/null +++ b/tests/launcher/test_deep_window_env.cpp @@ -0,0 +1,166 @@ +#include + +#include +#include +#include + +#include "cli_parse.hpp" +#include "gpufl/core/env_vars.hpp" +#include "trace_command_common.hpp" + +using gpufl::launcher::TraceArgs; +using gpufl::launcher::TracePlatform; +using gpufl::launcher::applyDeepWindowEnv; +namespace env = gpufl::env; + +namespace { + +// Records what the launcher did to the environment instead of doing it. The +// point of the recording is the REMOVALS: a test that only inspects what was +// set cannot tell "we left it alone" from "we took it away", and those two +// differ by whether a trigger the parent shell exported survives. +class RecordingPlatform final : public TracePlatform { + public: + /// The environment the target would inherit. Seed it to model a parent + /// shell that already had a GPUFL variable exported. + std::map env; + std::vector removed; + + bool has(const char* key) const { return env.count(key) != 0; } + std::string get(const char* key) const { + const auto it = env.find(key); + return it == env.end() ? std::string() : it->second; + } + + bool setEnv(const char* key, const std::string& value, + std::string&) const override { + const_cast(this)->env[key] = value; + return true; + } + bool unsetEnv(const char* key, std::string&) const override { + auto* self = const_cast(this); + self->env.erase(key); + self->removed.emplace_back(key); + return true; + } + + // Not reached: applyDeepWindowEnv only touches the environment. + const char* platformName() const override { return "recording"; } + const char* injectLibraryName() const override { return "none"; } + gpufl::launcher::fs::path selfExe() const override { return {}; } + std::vector injectLibCandidates( + const gpufl::launcher::fs::path&) const override { return {}; } + gpufl::launcher::fs::path defaultOutputDir( + const std::string&) const override { return {}; } + std::string defaultAppName(const std::string&) const override { return {}; } + bool prepareInjectionEnv(const gpufl::launcher::fs::path&, + std::string&) const override { return true; } + gpufl::launcher::TraceProcessResult runProcess( + const std::vector&, + const gpufl::launcher::RunOptions&) const override { return {}; } +}; + +TraceArgs conditionalRun() { + TraceArgs a; + a.deep_requested = true; + a.deep_when = "kernel_launch_rate<100 for 500ms"; + a.deep_for_ms = 2000; + return a; +} + +TraceArgs scheduledRun() { + TraceArgs a; + a.deep_requested = true; + a.deep_after_ms = 3000; + a.deep_after_set = true; + a.deep_for_ms = 2000; + return a; +} + +// ── trigger ownership ─────────────────────────────────────────────────────── +// +// GPUFL_DEEP_AFTER_MS and GPUFL_DEEP_WHEN install their triggers by EXISTING, +// whatever their value. Not setting one is therefore not the same as the +// target not seeing one, and every test here is about that difference. + +TEST(DeepWindowEnvTest, AConditionalRunRemovesAnInheritedTimeTrigger) { + RecordingPlatform p; + p.env[env::kDeepAfterMs] = "0"; // exported by the parent shell + + ASSERT_TRUE(applyDeepWindowEnv(conditionalRun(), p)); + + // Left in place, this opens a window at t=0 that the rule is then refused + // behind - and the rule reports never_true for a condition that held. + EXPECT_FALSE(p.has(env::kDeepAfterMs)); + EXPECT_EQ(p.get(env::kDeepWhen), "kernel_launch_rate<100 for 500ms"); +} + +TEST(DeepWindowEnvTest, AScheduledRunRemovesAnInheritedRule) { + RecordingPlatform p; + p.env[env::kDeepWhen] = "custom.token_rate<10 for 1s"; + + ASSERT_TRUE(applyDeepWindowEnv(scheduledRun(), p)); + + // Otherwise --deep-after silently installs a rule nobody asked for, which + // then competes with the window the user did ask for. + EXPECT_FALSE(p.has(env::kDeepWhen)); + EXPECT_EQ(p.get(env::kDeepAfterMs), "3000"); +} + +TEST(DeepWindowEnvTest, EachModeRemovesTheOtherTriggerEvenWhenUnset) { + // The removal is unconditional, not "only if we saw one". The launcher + // cannot see the parent environment of a target it has not spawned yet on + // every platform, so it does not try to decide. + RecordingPlatform cond; + ASSERT_TRUE(applyDeepWindowEnv(conditionalRun(), cond)); + EXPECT_NE(std::find(cond.removed.begin(), cond.removed.end(), + env::kDeepAfterMs), cond.removed.end()); + + RecordingPlatform sched; + ASSERT_TRUE(applyDeepWindowEnv(scheduledRun(), sched)); + EXPECT_NE(std::find(sched.removed.begin(), sched.removed.end(), + env::kDeepWhen), sched.removed.end()); +} + +TEST(DeepWindowEnvTest, ARunWithNoDeepFlagsTouchesNeitherTrigger) { + // `--passes PcSampling` with GPUFL_DEEP_WHEN exported is the supported way + // to reach an engine the adaptive plan does not select yet. Scrubbing here + // would take that away. + RecordingPlatform p; + p.env[env::kDeepWhen] = "custom.token_rate<10 for 1s"; + p.env[env::kDeepAfterMs] = "5000"; + + TraceArgs plain; // deep_requested stays false + ASSERT_TRUE(applyDeepWindowEnv(plain, p)); + + EXPECT_TRUE(p.removed.empty()); + EXPECT_EQ(p.get(env::kDeepWhen), "custom.token_rate<10 for 1s"); + EXPECT_EQ(p.get(env::kDeepAfterMs), "5000"); + EXPECT_FALSE(p.has(env::kDeepArm)) << "window-only arming was not asked for"; +} + +TEST(DeepWindowEnvTest, TheWindowBoundsAndArmModeStillTravel) { + RecordingPlatform p; + TraceArgs a = conditionalRun(); + a.deep_launches = 500; + a.deep_cooldown_ms = 1500; + + ASSERT_TRUE(applyDeepWindowEnv(a, p)); + + EXPECT_EQ(p.get(env::kDeepArm), "window"); + EXPECT_EQ(p.get(env::kDeepWindowMs), "2000"); + EXPECT_EQ(p.get(env::kDeepWindowMaxLaunches), "500"); + EXPECT_EQ(p.get(env::kDeepWindowCooldownMs), "1500"); +} + +TEST(DeepWindowEnvTest, AnUnsetBoundIsNotPublishedAsZero) { + // 0 means "no bound" to the client, and publishing it would override a + // bound the parent environment had legitimately set. + RecordingPlatform p; + ASSERT_TRUE(applyDeepWindowEnv(conditionalRun(), p)); + + EXPECT_FALSE(p.has(env::kDeepWindowMaxLaunches)); + EXPECT_FALSE(p.has(env::kDeepWindowCooldownMs)); +} + +} // namespace diff --git a/tests/package/consumer/CMakeLists.txt b/tests/package/consumer/CMakeLists.txt new file mode 100644 index 0000000..549aa2a --- /dev/null +++ b/tests/package/consumer/CMakeLists.txt @@ -0,0 +1,13 @@ +# The documented consumer, verbatim. This project is configured against a +# scratch INSTALL tree by tests/package/counters_consumer_check.cmake - it is +# the fixture that keeps `find_package(gpufl_client)` + `gpufl::counters` +# honest. If EXPORT_NAME, the Config file install, or the counters export +# set's isolation from the SDK's dependencies regresses, this is the test +# that fails; the unit suite links the build tree and cannot see any of it. +cmake_minimum_required(VERSION 3.20) +project(gpufl_counters_consumer CXX) + +find_package(gpufl_client 1.2 REQUIRED) + +add_executable(app main.cpp) +target_link_libraries(app PRIVATE gpufl::counters) diff --git a/tests/package/consumer/main.cpp b/tests/package/consumer/main.cpp new file mode 100644 index 0000000..0ec4b53 --- /dev/null +++ b/tests/package/consumer/main.cpp @@ -0,0 +1,13 @@ +// Ticks a counter through the INSTALLED package. The check script greps the +// line below, so a silent link against a stub or a handle that failed to +// register fails the test rather than passing quietly. +#include + +#include "gpufl.hpp" + +int main() { + auto tokens = gpufl::counter("tokens"); + for (int i = 0; i < 1000; ++i) tokens.add(8); + std::printf("consumer ok valid=%d\n", tokens.valid() ? 1 : 0); + return tokens.valid() ? 0 : 1; +} diff --git a/tests/package/counters_consumer_check.cmake b/tests/package/counters_consumer_check.cmake new file mode 100644 index 0000000..6cab723 --- /dev/null +++ b/tests/package/counters_consumer_check.cmake @@ -0,0 +1,102 @@ +# Steps the counters package through the exact flow a user follows: +# +# cmake --build --target gpufl_counters +# cmake --install --prefix --component counters +# cmake -S consumer -DCMAKE_PREFIX_PATH= +# cmake --build consumer +# ./app -> "consumer ok valid=1" +# +# Run by CTest (see the add_test in the root CMakeLists), not by hand: the one +# manual run that validated this flow proved nothing about the NEXT change. +# Every step's failure names the step, so a regression reads as "install-tree +# consumer broke at configure" rather than a bare non-zero exit. +# +# The install is the COUNTERS COMPONENT, and the build is the counters target, +# so this test responds to exactly what the package promises - the archive, +# the public headers, the Config/Version files, the gpufl::counters export and +# its Threads/dl propagation - and to nothing else. A full-tree install tied +# it to zlib's install rules, httplib's export set and whether +# gpufl_counter_runtime happened to be built in this configuration; a Debug +# run failed on that last one without a single counters file being wrong. + +foreach(var GPUFL_BINARY_DIR GPUFL_SOURCE_DIR GPUFL_CONFIG GPUFL_GENERATOR) + if(NOT DEFINED ${var}) + message(FATAL_ERROR "counters_consumer_check: ${var} not set") + endif() +endforeach() + +set(scratch "${GPUFL_BINARY_DIR}/package_check") +file(REMOVE_RECURSE "${scratch}") + +# Single-config generators expand $ to an empty string, and +# `--config ""` is an error rather than a no-op. The flag exists only when +# there is a configuration to name. +set(config_args) +if(GPUFL_CONFIG) + set(config_args --config "${GPUFL_CONFIG}") +endif() + +function(run_step name) + execute_process(COMMAND ${ARGN} + RESULT_VARIABLE rc + OUTPUT_VARIABLE out + ERROR_VARIABLE err) + if(NOT rc EQUAL 0) + message(FATAL_ERROR + "install-tree consumer broke at ${name} (exit ${rc}):\n${out}\n${err}") + endif() + set(step_output "${out}" PARENT_SCOPE) +endfunction() + +# Built here rather than assumed: CTest promises nothing about what ran +# before this test, and an install of a never-built target is a confusing +# missing-file error instead of a compile error. +run_step("counters build" + ${CMAKE_COMMAND} --build "${GPUFL_BINARY_DIR}" + --target gpufl_counters ${config_args}) + +run_step("install" + ${CMAKE_COMMAND} --install "${GPUFL_BINARY_DIR}" + --prefix "${scratch}/prefix" --component counters ${config_args}) + +# The parent's platform/toolset travel along when they exist: a VS parent +# configured for a specific -A/-T must not have its consumer silently probe a +# different one. +set(gen_args) +if(DEFINED GPUFL_PLATFORM AND GPUFL_PLATFORM) + list(APPEND gen_args -A "${GPUFL_PLATFORM}") +endif() +if(DEFINED GPUFL_TOOLSET AND GPUFL_TOOLSET) + list(APPEND gen_args -T "${GPUFL_TOOLSET}") +endif() + +run_step("consumer configure" + ${CMAKE_COMMAND} -G "${GPUFL_GENERATOR}" ${gen_args} + -S "${GPUFL_SOURCE_DIR}/tests/package/consumer" + -B "${scratch}/consumer" + "-DCMAKE_PREFIX_PATH=${scratch}/prefix" + "-DCMAKE_BUILD_TYPE=${GPUFL_CONFIG}") + +run_step("consumer build" + ${CMAKE_COMMAND} --build "${scratch}/consumer" ${config_args}) + +# Single-config generators put the binary at the top; multi-config nests it. +set(app "${scratch}/consumer/app") +foreach(candidate "${scratch}/consumer/app" + "${scratch}/consumer/app.exe" + "${scratch}/consumer/${GPUFL_CONFIG}/app.exe" + "${scratch}/consumer/${GPUFL_CONFIG}/app") + if(EXISTS "${candidate}") + set(app "${candidate}") + break() + endif() +endforeach() + +run_step("consumer run" "${app}") +if(NOT step_output MATCHES "consumer ok valid=1") + message(FATAL_ERROR + "install-tree consumer ran but did not report a valid counter:\n" + "${step_output}") +endif() + +message(STATUS "install-tree counters consumer: ok")