From 8d7f3a5588100ec5e2402543b81b976013419cfb Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Sun, 26 Jul 2026 18:43:21 -0700 Subject: [PATCH 01/28] fix(scope): bound and index PM sample attribution Every PM sample rescanned every completed scope of the run, over a deque that was only ever cleared at session end. A 2s window produces ~19,000 samples and a long run leaves tens of thousands of scopes, so the product ran under a mutex on the collector thread and the deque grew for the life of the process. Resolve a whole drain at once instead: sort the samples and a snapshot of the candidate scopes, then sweep with a min-heap for expiry and an ordered set for the winner, so neither is rescanned per sample. Selection is unchanged - interval contains the timestamp, greatest depth, latest start - and a property test checks the sweep against the original resolver over nested, cross-thread and shared-boundary intervals. Ranking gains the scope instance id as a tertiary key. That is not tidiness: the sweep orders a std::set, and a comparator that ever reported two scopes equivalent would make the set keep one and silently drop the other. Retention is keyed to an event-time watermark the PM engine publishes only when CUPTI reports END_OF_RECORDS with no overflow - the one stop reason that proves the hardware buffer was exhausted. Inferring a boundary from the buffer span instead would be an assumption about capacity, wrong precisely when records were left behind. The watermark is monotonic and never advances on a failed or truncated decode, so a lost batch cannot strand the scopes its leftovers still need. Wall clock would be wrong for the same reason in reverse: a stalled collector lets now run on while undecoded samples wait. The candidate snapshot now includes scopes that are still OPEN. PM drains mid-run, so the scope covering a sample is routinely still running when that sample is decoded. Those samples used to fall back to activeScopeNameId_, which answers what is on top NOW rather than what covered the sample, and cannot represent nesting or cross-thread scopes at all. That fallback is gone: a sample no interval covers is left unattributed rather than credited to whatever happened to be open. A close timestamp is taken before the batch lock can be acquired, so a scope could still look open to a snapshot that had already been given its end. Closes now capture and publish that timestamp under one mutex, and the snapshot caps an open interval at any pending close it finds. Deep window close is the deliberate exception: its end is taken before the engine disarm so the recorded window measures the requested boundary, and its pending close is published after the final drain, which leaves a handful of drain samples - 7 of 18,981 in a 2s window, within 0.7ms - attributed to the window they were collected by but past its recorded end. The hard cap runs on every scope close, not only when PM samples arrive: the watermark is the real bound but only PM publishes one, so a Trace-only run had no limit at all. Eviction is counted every time and logged once, since past the cap every close would otherwise emit a line. It reaches the dashboard through the capability matrix rather than the local log alone, and only when PM rows actually exist - evicting scope history on a run with no PM sampling is a memory backstop firing, not attribution being lost. Not covered: the overflow and COUNTER_DATA_FULL branches have no real-GPU run behind them yet. --- .../nvidia/cupti_capture_capabilities.cpp | 22 +- .../nvidia/engine/pm_sampling_engine.cpp | 34 +- include/gpufl/core/deep_window.cpp | 4 + include/gpufl/core/gpufl.cpp | 4 +- include/gpufl/core/monitor.cpp | 28 ++ include/gpufl/core/monitor.hpp | 20 ++ include/gpufl/core/monitor_batch_manager.cpp | 303 +++++++++++++++++- include/gpufl/core/monitor_batch_manager.hpp | 101 +++++- include/gpufl/inject/inject_entry.cpp | 2 +- tests/core/test_monitor.cpp | 279 ++++++++++++++++ 10 files changed, 773 insertions(+), 24 deletions(-) diff --git a/include/gpufl/backends/nvidia/cupti_capture_capabilities.cpp b/include/gpufl/backends/nvidia/cupti_capture_capabilities.cpp index 322bc02..9a467ae 100644 --- a/include/gpufl/backends/nvidia/cupti_capture_capabilities.cpp +++ b/include/gpufl/backends/nvidia/cupti_capture_capabilities.cpp @@ -86,7 +86,27 @@ void CuptiBackend::EmitCaptureCapabilities_() const { std::fprintf(stderr, "%s\n", warning.c_str()); } - const CaptureCapabilitiesEvent evt = BuildCaptureCapabilitiesEvent(input); + CaptureCapabilitiesEvent evt = BuildCaptureCapabilitiesEvent(input); + + // Scope attribution gave up on some samples. Reported through the + // capability matrix rather than the local log alone: a log line is invisible + // to whoever reads the session later, and the samples still upload - they + // just carry no scope. Without this the dashboard presents partial + // attribution as though it were complete. + const uint64_t truncated = Monitor::ScopeAttributionTruncated(); + if (truncated > 0 && Monitor::PmSampleRowsSeen() > 0) { + CaptureCapability cap; + cap.feature = "scope_attribution"; + cap.requested = true; + cap.status = "partial"; + cap.reason_code = "scope_attribution_truncated"; + cap.message = "Evicted " + std::to_string(truncated) + + " completed scope records after the retention cap was reached; " + "PM scope attribution may be incomplete. Raise pm_sampling_max_samples " + "or lengthen the sampling interval so decodes keep up."; + evt.capabilities.push_back(std::move(cap)); + } + rt->logger->write(model::CaptureCapabilitiesModel(evt)); } diff --git a/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp b/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp index e412da8..e96ccef 100644 --- a/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp +++ b/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp @@ -6,6 +6,7 @@ #include #endif +#include #include #include #include @@ -407,9 +408,17 @@ void PmSamplingEngine::DecodeAndEmit_() { LogCuptiErrorIfFailed(this->name(), "cuptiPmSamplingDecodeData", res); return; } - if (decode.overflow || res == CUPTI_ERROR_OUT_OF_MEMORY) { + const bool overflowed = decode.overflow || res == CUPTI_ERROR_OUT_OF_MEMORY; + if (overflowed) { GFL_LOG_ERROR("[PmSamplingEngine] PM sampling hardware buffer overflow; increase pm_sampling_max_samples or interval_us"); } + // CUPTI states outright whether the hardware buffer was exhausted. + // END_OF_RECORDS is the only reason that proves nothing older is left to + // come; COUNTER_DATA_FULL means records remain because the destination + // image filled first. + const bool buffer_exhausted = + res == CUPTI_SUCCESS && !overflowed && + decode.decodeStopReason == CUPTI_PM_SAMPLING_DECODE_STOP_REASON_END_OF_RECORDS; CUpti_PmSampling_GetCounterDataInfo_Params info = { CUpti_PmSampling_GetCounterDataInfo_Params_STRUCT_SIZE}; @@ -447,7 +456,7 @@ void PmSamplingEngine::DecodeAndEmit_() { if (LogCuptiErrorIfFailed(this->name(), "cuptiProfilerHostEvaluateToGpuValues", res)) continue; const uint64_t mid = sampleInfo.startTimestamp + - ((sampleInfo.endTimestamp - sampleInfo.startTimestamp) / 2ull); + (sampleInfo.endTimestamp - sampleInfo.startTimestamp) / 2ull; // CUPTI sample ts -> wall-clock via the kernel anchor, so PM lines up // with the kernel timeline. const int64_t mid_wall_ns = ctx_.base_cpu_ns + @@ -470,6 +479,22 @@ void PmSamplingEngine::DecodeAndEmit_() { if (!rows.empty()) { Monitor::PushPmSamples(rows); produced_data_.store(true, std::memory_order_relaxed); + + // Release the scopes no later sample can reach - but only when CUPTI + // has told us the hardware buffer is actually empty. Sampling is time + // ordered, so once it is, every future record is newer than everything + // just decoded and the newest timestamp here is a real boundary. + // + // Deriving one from the buffer span instead would be an inference about + // capacity, not a guarantee, and it would be wrong precisely when + // records were left behind. On overflow or COUNTER_DATA_FULL the + // watermark simply stays where it is, so a truncated decode cannot + // strand the scopes its leftovers still need. + if (buffer_exhausted) { + int64_t newest_ns = std::numeric_limits::min(); + for (const auto& row : rows) newest_ns = std::max(newest_ns, row.ts_ns); + Monitor::PublishScopeRetentionWatermark(newest_ns); + } } // Hand the image back empty so the next decode starts from a clean slate @@ -528,6 +553,9 @@ void PmSamplingEngine::StartPmSampling_() { return; } + // Capture before the CUPTI call so the boundary is conservative: even a + // sample produced during a slow successful start is covered. + const int64_t attribution_start_ns = detail::GetTimestampNs(); CUpti_PmSampling_Start_Params start = {CUpti_PmSampling_Start_Params_STRUCT_SIZE}; start.pPmSamplingObject = pm_object_; CUptiResult res = cuptiPmSamplingStart(&start); @@ -536,6 +564,7 @@ void PmSamplingEngine::StartPmSampling_() { return; } running_ = true; + Monitor::BeginPmScopeAttribution(attribution_start_ns); operational_.store(true, std::memory_order_relaxed); GFL_LOG_DEBUG("[PmSamplingEngine] >>> STARTED (Scope Begin) <<<"); } @@ -551,6 +580,7 @@ void PmSamplingEngine::StopPmSampling_() { GFL_LOG_DEBUG("[PmSamplingEngine] <<< COLLECTING (Scope End) >>>"); DecodeAndEmit_(); } + Monitor::EndPmScopeAttribution(); running_ = false; } #endif diff --git a/include/gpufl/core/deep_window.cpp b/include/gpufl/core/deep_window.cpp index bec910d..6b05f0c 100644 --- a/include/gpufl/core/deep_window.cpp +++ b/include/gpufl/core/deep_window.cpp @@ -296,6 +296,10 @@ void DeepWindow::Close(const DeepWindowClose reason) { close_row.name_id = Monitor::InternScopeName(name); close_row.event_type = 1; close_row.depth = 0; // ignored on close; the open row carries it + // end_ns intentionally precedes engine disarm so the window event + // measures the requested boundary. Publish it only after the final + // drain above, immediately before the scope-state transition. + Monitor::MarkScopeClosePending(g_scope_instance_id, end_ns); Monitor::PushScopeRow(close_row); g_scope_instance_id = 0; } diff --git a/include/gpufl/core/gpufl.cpp b/include/gpufl/core/gpufl.cpp index f8e64f3..9236d46 100644 --- a/include/gpufl/core/gpufl.cpp +++ b/include/gpufl/core/gpufl.cpp @@ -848,14 +848,14 @@ ScopedMonitor::~ScopedMonitor() { auto& stack = getThreadScopeStack(); if (!stack.empty()) stack.pop_back(); const int depth = static_cast(stack.size()); - const int64_t end_ns = detail::GetTimestampNs(); ScopeBatchRow row; - row.ts_ns = end_ns; row.scope_instance_id = scope_id_; row.name_id = Monitor::InternScopeName(name_); row.event_type = 1; // end row.depth = depth; + const int64_t end_ns = Monitor::CaptureScopeCloseTimestamp(scope_id_); + row.ts_ns = end_ns; Monitor::PushScopeRow(row); // Scopes are recorded via scope_event only - we no longer echo each diff --git a/include/gpufl/core/monitor.cpp b/include/gpufl/core/monitor.cpp index 5fbae3d..6dc8c86 100644 --- a/include/gpufl/core/monitor.cpp +++ b/include/gpufl/core/monitor.cpp @@ -667,6 +667,14 @@ uint64_t Monitor::AllocateScopeInstanceId() { int Monitor::OpenScopeDepth() { return g_state.batches.openScopeDepth(); } +int64_t Monitor::CaptureScopeCloseTimestamp(uint64_t instance_id) { + return g_state.batches.captureScopeCloseTimestamp(instance_id); +} + +void Monitor::MarkScopeClosePending(uint64_t instance_id, int64_t end_ns) { + g_state.batches.markScopeClosePending(instance_id, end_ns); +} + void Monitor::PushProfileSamples(const std::vector& samples) { if (samples.empty()) return; const uint32_t scope_name_id = g_state.batches.activeScopeNameId(); @@ -711,6 +719,26 @@ void Monitor::PushPmSamples(const std::vector& samples) { g_state.batches.pushPmSamplesResolvingScopes(rows); } +void Monitor::PublishScopeRetentionWatermark(int64_t ts_ns) { + g_state.batches.publishScopeRetentionWatermark(ts_ns); +} + +void Monitor::BeginPmScopeAttribution(int64_t start_ns) { + g_state.batches.beginPmScopeAttribution(start_ns); +} + +void Monitor::EndPmScopeAttribution() { + g_state.batches.endPmScopeAttribution(); +} + +uint64_t Monitor::ScopeAttributionTruncated() { + return g_state.batches.scopeAttributionTruncated(); +} + +uint64_t Monitor::PmSampleRowsSeen() { + return g_state.batches.pmSampleRowsSeen(); +} + void Monitor::EmitPmSamplingConfig(uint32_t device_id, uint32_t interval_us, uint32_t max_samples, const std::string& preset, const std::vector& metrics) { const Runtime* rt = runtime(); if (!(rt && rt->logger)) return; diff --git a/include/gpufl/core/monitor.hpp b/include/gpufl/core/monitor.hpp index 8a5e66c..c091515 100644 --- a/include/gpufl/core/monitor.hpp +++ b/include/gpufl/core/monitor.hpp @@ -387,6 +387,10 @@ class Monitor { static uint64_t AllocateScopeInstanceId(); /** @brief Depth a scope opened right now would nest at. */ static int OpenScopeDepth(); + /** @brief Capture and publish a close timestamp before scope-state locking. */ + static int64_t CaptureScopeCloseTimestamp(uint64_t instance_id); + /** @brief Publish an existing close timestamp before pushing its row. */ + static void MarkScopeClosePending(uint64_t instance_id, int64_t end_ns); /** * Push a raw activity record into the monitor ring buffer. @@ -419,6 +423,22 @@ class Monitor { * @brief Push decoded PM sampling time-series rows. */ static void PushPmSamples(const std::vector& samples); + /** + * @brief Release completed scopes that no future sample can reach. + * + * Call only after a decode that SUCCEEDED. A failed or overflowed decode + * means samples were lost rather than delivered, and advancing past them + * would drop the scopes they still need. + */ + static void PublishScopeRetentionWatermark(int64_t ts_ns); + /** @brief Mark the wall-clock boundary from which PM samples may be pending. */ + static void BeginPmScopeAttribution(int64_t start_ns); + /** @brief Mark that the final PM decode completed. */ + static void EndPmScopeAttribution(); + /** @brief Scope history evicted while PM attribution was active. */ + static uint64_t ScopeAttributionTruncated(); + /** @brief PM metric rows that passed through scope attribution. */ + static uint64_t PmSampleRowsSeen(); /** * @brief Emit PM sampling configuration metadata for readers/UI. diff --git a/include/gpufl/core/monitor_batch_manager.cpp b/include/gpufl/core/monitor_batch_manager.cpp index 966ed38..022ce2c 100644 --- a/include/gpufl/core/monitor_batch_manager.cpp +++ b/include/gpufl/core/monitor_batch_manager.cpp @@ -1,7 +1,12 @@ #include "gpufl/core/monitor_batch_manager.hpp" +#include +#include +#include +#include #include +#include "gpufl/core/common.hpp" #include "gpufl/core/debug_logger.hpp" #include "gpufl/core/logger/logger.hpp" #include "gpufl/core/model/batch_models.hpp" @@ -21,6 +26,15 @@ void MonitorBatchManager::reset() { scopeNameStack_.clear(); openScopeWindows_.clear(); completedScopeWindows_.clear(); + scopeRetentionWatermarkNs_ = 0; + pmScopeAttributionStartNs_ = 0; + scopeHistoryEvictionLogged_ = false; + scopeAttributionTruncated_ = 0; + pmSampleRowsSeen_ = 0; + { + std::lock_guard pending_lk(pendingScopeCloseMu_); + pendingScopeCloseNs_.clear(); + } } syncBatch_.clear(); memAllocBatch_.clear(); @@ -165,6 +179,22 @@ int MonitorBatchManager::openScopeDepth() const { return static_cast(scopeNameStack_.size()); } +int64_t MonitorBatchManager::captureScopeCloseTimestamp(uint64_t instance_id) { + // The timestamp and its publication share the same lock observed by the PM + // snapshot. If snapshot wins, its batch predates this close; if close wins, + // snapshot sees the exact end and cannot provisionally extend past it. + std::lock_guard lk(pendingScopeCloseMu_); + const int64_t end_ns = GetTimestampNs(); + pendingScopeCloseNs_[instance_id] = end_ns; + return end_ns; +} + +void MonitorBatchManager::markScopeClosePending(uint64_t instance_id, int64_t end_ns) { + std::lock_guard lk(pendingScopeCloseMu_); + const auto [it, inserted] = pendingScopeCloseNs_.emplace(instance_id, end_ns); + if (!inserted && end_ns < it->second) it->second = end_ns; +} + bool MonitorBatchManager::pushKernel(const KernelBatchRow& row, const KernelDetailRow* detail) { kernelBatch_.push(row); @@ -187,6 +217,13 @@ void MonitorBatchManager::pushTraceScopeRows(const ScopeBatchRow& begin_row, } void MonitorBatchManager::pushTrackedScopeRow(const ScopeBatchRow& row) { + // Publish a close's already-captured timestamp before waiting for the main + // scope lock. A concurrent PM snapshot can then stop the open interval at + // this timestamp instead of extending it through the whole sample batch. + if (row.event_type != 0) { + markScopeClosePending(row.scope_instance_id, row.ts_ns); + } + std::lock_guard lk(scopeBatchMu_); if (row.event_type == 0) { scopeNameStack_.emplace_back(row.scope_instance_id, row.name_id); @@ -211,7 +248,12 @@ void MonitorBatchManager::pushTrackedScopeRow(const ScopeBatchRow& row) { completedScopeWindows_.push_back( {it->second.start_ns, row.ts_ns, row.scope_instance_id, row.name_id, it->second.depth}); openScopeWindows_.erase(it); + // Bound it here rather than only when PM samples arrive. Nothing + // publishes a retention watermark unless PM is actually sampling, + // so a Trace-only run would grow this for the life of the process. + enforceScopeCapLocked(); } + clearPendingScopeCloseLocked(row.scope_instance_id); } scopeBatch_.push(row); } @@ -230,13 +272,230 @@ void MonitorBatchManager::pushProfileSamples(const std::vector& rows) { - const uint32_t fallback_id = activeScopeNameId_.load(std::memory_order_relaxed); + if (rows.empty()) return; + + std::vector resolved(rows.begin(), rows.end()); + + // Snapshot under the lock, sweep outside it. Holding scopeBatchMu_ for the + // whole sort-and-sweep would block every scope close for the duration, and + // a close that is already holding its end timestamp and waiting here is + // exactly what widens the window below. + std::vector candidates; + { + std::lock_guard lk(scopeBatchMu_); + pmSampleRowsSeen_ += rows.size(); + trimCompletedScopesLocked(); + candidates = snapshotScopeCandidatesLocked(rows); + } + + // No fallback to the currently active scope. A sample no interval covers is + // left unattributed, and that is the point of this path: handing it + // whichever scope happens to be open at DECODE time answers "what is + // running now", not "what was running when this was sampled", and the two + // differ by however long the sample sat in the buffer - exactly the error + // this resolver exists to remove. Open scopes are already candidates, so a + // sample inside a still-running scope resolves properly rather than by luck. + resolveScopeIdsForBatch(candidates, resolved, /*fallback_id=*/0); + + { + std::lock_guard lk(scopeBatchMu_); + for (const auto& row : resolved) pmSampleBatch_.push(row); + } +} + +void MonitorBatchManager::publishScopeRetentionWatermark(int64_t ts_ns) { std::lock_guard lk(scopeBatchMu_); - for (const auto& sample : rows) { - PmSampleBatchRow row = sample; - row.scope_name_id = resolveScopeIdLocked(row.ts_ns); - if (row.scope_name_id == 0) row.scope_name_id = fallback_id; - pmSampleBatch_.push(row); + // Monotonic. A caller that regressed - a decode that failed, a buffer that + // overflowed - must not be able to un-retire scopes it already released. + if (ts_ns > scopeRetentionWatermarkNs_) scopeRetentionWatermarkNs_ = ts_ns; +} + +void MonitorBatchManager::beginPmScopeAttribution(int64_t start_ns) { + std::lock_guard lk(scopeBatchMu_); + pmScopeAttributionStartNs_ = start_ns; +} + +void MonitorBatchManager::endPmScopeAttribution() { + std::lock_guard lk(scopeBatchMu_); + pmScopeAttributionStartNs_ = 0; +} + +uint64_t MonitorBatchManager::scopeAttributionTruncated() const { + std::lock_guard lk(scopeBatchMu_); + return scopeAttributionTruncated_; +} + +uint64_t MonitorBatchManager::pmSampleRowsSeen() const { + std::lock_guard lk(scopeBatchMu_); + return pmSampleRowsSeen_; +} + +void MonitorBatchManager::resolveScopeIdsForTesting(std::vector& rows, + uint32_t fallback_id) { + std::vector candidates; + { + std::lock_guard lk(scopeBatchMu_); + trimCompletedScopesLocked(); + candidates = snapshotScopeCandidatesLocked(rows); + } + resolveScopeIdsForBatch(candidates, rows, fallback_id); +} + +uint32_t MonitorBatchManager::resolveScopeIdForTesting(int64_t ts_ns) const { + std::lock_guard lk(scopeBatchMu_); + return resolveScopeIdLocked(ts_ns); +} + +size_t MonitorBatchManager::retainedCompletedScopesForTesting() const { + std::lock_guard lk(scopeBatchMu_); + return completedScopeWindows_.size(); +} + +void MonitorBatchManager::clearPendingScopeCloseLocked(uint64_t instance_id) { + // Called with scopeBatchMu_ held. Snapshot takes locks in the same order: + // scopeBatchMu_ first, pendingScopeCloseMu_ second. + std::lock_guard lk(pendingScopeCloseMu_); + pendingScopeCloseNs_.erase(instance_id); +} + +void MonitorBatchManager::enforceScopeCapLocked() { + // Runs on every close, not only when PM samples arrive. The watermark is + // the real bound, but nothing publishes one unless PM is actually + // sampling - so a Trace-only run, or one where PM never initialised, would + // otherwise grow this deque for the life of the process with no cap and no + // telemetry to show for it. + if (completedScopeWindows_.size() <= kMaxCompletedScopes) return; + const size_t excess = completedScopeWindows_.size() - kMaxCompletedScopes; + uint64_t attribution_risk = 0; + if (pmScopeAttributionStartNs_ > 0) { + for (size_t i = 0; i < excess; ++i) { + if (completedScopeWindows_[i].end_ns >= pmScopeAttributionStartNs_) { + ++attribution_risk; + } + } + } + completedScopeWindows_.erase(completedScopeWindows_.begin(), + completedScopeWindows_.begin() + static_cast(excess)); + // Trace-only history is unused by PM attribution. Count an eviction as a + // data-quality risk only when it overlaps the current PM collection + // boundary. This also handles mixed sessions: a long Trace warmup cannot + // make a later Deep PM window look partial merely because old, pre-window + // entries are evicted while PM happens to be active. + scopeAttributionTruncated_ += attribution_risk; + if (!scopeHistoryEvictionLogged_) { + scopeHistoryEvictionLogged_ = true; + GFL_LOG_ERROR("[MonitorBatchManager] scope_history_evicted: hard cap reached; ", + excess, + " completed scope record(s) dropped (further messages suppressed)"); + } +} + +void MonitorBatchManager::trimCompletedScopesLocked() { + // The deque is NOT ordered by end_ns, so this cannot stop at the first + // survivor. It is a full pass, but only over what the watermark has not + // already retired, and it runs once per drain rather than once per sample. + if (scopeRetentionWatermarkNs_ > 0) { + const int64_t cutoff = scopeRetentionWatermarkNs_; + const auto it = std::remove_if( + completedScopeWindows_.begin(), completedScopeWindows_.end(), + [cutoff](const ScopeWindow& w) { return w.end_ns < cutoff; }); + completedScopeWindows_.erase(it, completedScopeWindows_.end()); + } + + // Backstop. Reaching this means the watermark is not advancing, so the + // entries dropped here may still have been needed: record it rather than + // let the samples quietly go unattributed. + enforceScopeCapLocked(); +} + +std::vector +MonitorBatchManager::snapshotScopeCandidatesLocked( + const std::vector& rows) const { + // Candidates = closed scopes still retained, PLUS scopes that are still + // open. The open ones matter: PM drains mid-run, so a sample is routinely + // decoded while the scope covering it is still running. Giving them a + // provisional end at the batch's newest sample keeps them eligible for + // every sample in this batch without inventing a close that has not + // happened. + // + int64_t provisional_end = std::numeric_limits::min(); + for (const auto& row : rows) provisional_end = std::max(provisional_end, row.ts_ns); + + std::vector candidates; + candidates.reserve(completedScopeWindows_.size() + openScopeWindows_.size()); + candidates.assign(completedScopeWindows_.begin(), completedScopeWindows_.end()); + { + // Close publishes pending first and never holds this mutex while + // waiting for scopeBatchMu_, so this lock order cannot deadlock. + std::lock_guard pending_lk(pendingScopeCloseMu_); + for (const auto& [instance_id, open] : openScopeWindows_) { + int64_t effective_end = provisional_end; + if (const auto close = pendingScopeCloseNs_.find(instance_id); + close != pendingScopeCloseNs_.end()) { + effective_end = std::min(effective_end, close->second); + } + if (open.start_ns > effective_end) continue; + candidates.push_back(ScopeWindow{open.start_ns, effective_end, instance_id, + open.name_id, open.depth}); + } + } + return candidates; +} + +void MonitorBatchManager::resolveScopeIdsForBatch(std::vector& candidates, + std::vector& rows, + uint32_t fallback_id) { + if (candidates.empty()) { + for (auto& row : rows) row.scope_name_id = fallback_id; + return; + } + + std::sort(candidates.begin(), candidates.end(), + [](const ScopeWindow& a, const ScopeWindow& b) { return a.start_ns < b.start_ns; }); + + std::vector order(rows.size()); + for (size_t i = 0; i < order.size(); ++i) order[i] = i; + std::sort(order.begin(), order.end(), + [&rows](size_t a, size_t b) { return rows[a].ts_ns < rows[b].ts_ns; }); + + // Two structures over the active set rather than one flat list. Rescanning + // it per sample would keep the cost at O(samples x concurrent scopes), + // which is the shape this replaces. `ranked` is ordered so the winner is + // its last element; `expiry` surfaces the soonest end so retirement costs + // a peek instead of a pass. + struct ByRank { + bool operator()(const ScopeWindow* a, const ScopeWindow* b) const { + return b->outranks(*a); + } + }; + struct ByEnd { + bool operator()(const ScopeWindow* a, const ScopeWindow* b) const { + return a->end_ns > b->end_ns; // min-heap on end_ns + } + }; + std::set ranked; + std::priority_queue, ByEnd> expiry; + + size_t next_candidate = 0; + for (const size_t idx : order) { + const int64_t ts = rows[idx].ts_ns; + + // Admit everything that has started. Candidates are start-sorted, so + // each is admitted once across the whole batch. + while (next_candidate < candidates.size() && candidates[next_candidate].start_ns <= ts) { + const ScopeWindow* w = &candidates[next_candidate++]; + ranked.insert(w); + expiry.push(w); + } + // Retire what has ended. Samples are visited in time order, so an + // expired scope can never be wanted again. Both ends are inclusive, + // hence `end_ns < ts` rather than `<=`. + while (!expiry.empty() && expiry.top()->end_ns < ts) { + ranked.erase(expiry.top()); + expiry.pop(); + } + + rows[idx].scope_name_id = ranked.empty() ? fallback_id : (*ranked.rbegin())->name_id; } } @@ -249,19 +508,29 @@ void MonitorBatchManager::pushSynchronization(const SynchronizationEventBatchRow syncBatch_.push(row); } +// Ranking shared by both resolvers. Depth first, then latest start; the +// instance id breaks a remaining tie so the answer does not depend on +// container order. Without it two scopes at the same depth and start could +// resolve differently between the reference and the batch path, since neither +// std::sort nor an unordered_map preserves any order for equal keys. +// +// The tertiary key is also load-bearing for the sweep, not merely tidy: it +// orders a std::set, and a comparator that ever reports equivalence would make +// that set silently keep one of the two scopes and drop the other. Instance ids +// are unique, so no two entries can compare equal. +bool MonitorBatchManager::ScopeWindow::outranks(const ScopeWindow& other) const { + if (depth != other.depth) return depth > other.depth; + if (start_ns != other.start_ns) return start_ns > other.start_ns; + return instance_id > other.instance_id; +} + uint32_t MonitorBatchManager::resolveScopeIdLocked(int64_t ts_ns) const { - uint32_t best_id = 0; - int best_depth = -1; - int64_t best_start = 0; - for (auto it = completedScopeWindows_.rbegin(); it != completedScopeWindows_.rend(); ++it) { - if (ts_ns < it->start_ns || ts_ns > it->end_ns) continue; - if (it->depth > best_depth || (it->depth == best_depth && it->start_ns >= best_start)) { - best_id = it->name_id; - best_depth = it->depth; - best_start = it->start_ns; - } + const ScopeWindow* best = nullptr; + for (const auto& w : completedScopeWindows_) { + if (ts_ns < w.start_ns || ts_ns > w.end_ns) continue; + if (!best || w.outranks(*best)) best = &w; } - return best_id; + return best ? best->name_id : 0; } } // namespace gpufl::detail diff --git a/include/gpufl/core/monitor_batch_manager.hpp b/include/gpufl/core/monitor_batch_manager.hpp index 30d87a2..7d201ff 100644 --- a/include/gpufl/core/monitor_batch_manager.hpp +++ b/include/gpufl/core/monitor_batch_manager.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -44,6 +45,46 @@ class MonitorBatchManager { uint32_t activeScopeNameId() const; /** @brief How many scopes are open right now; the depth a new one nests at. */ int openScopeDepth() const; + /** @brief Capture and publish a close timestamp as one ordered operation. */ + int64_t captureScopeCloseTimestamp(uint64_t instance_id); + /** @brief Publish an already-chosen close timestamp before state transition. */ + void markScopeClosePending(uint64_t instance_id, int64_t end_ns); + + /** + * @brief Publish the point below which no future sample can be attributed. + * + * Contract: no subsequent SUCCESSFUL decode will return a sample with a + * timestamp at or below @p ts_ns. Completed scopes that ended before it can + * therefore never match again and are dropped. + * + * Deliberately NOT wall-clock. A stalled collector lets `now` run on while + * undecoded samples still sit in the buffer, and trimming against it would + * discard the scopes those samples need. It is also not "last decode time" + * nor "oldest decoded timestamp" - both can move ahead of samples that are + * still to come. + * + * Monotonic: a lower value is ignored. The caller must simply not advance it + * when a decode fails or the buffer overflowed, since either means samples + * were lost rather than delivered. + */ + void publishScopeRetentionWatermark(int64_t ts_ns); + + /** @brief Mark the wall-clock boundary from which PM samples may be pending. */ + void beginPmScopeAttribution(int64_t start_ns); + /** @brief Mark that the final PM decode completed. */ + void endPmScopeAttribution(); + /** @brief Completed scope history evicted while PM attribution was active. */ + uint64_t scopeAttributionTruncated() const; + /** @brief PM metric rows that have passed through scope attribution. */ + uint64_t pmSampleRowsSeen() const; + + /** @brief Test seam: resolve a batch exactly as the drain path does. */ + void resolveScopeIdsForTesting(std::vector& rows, uint32_t fallback_id); + /** @brief Test seam: the original per-sample resolver, kept as the reference + * the batch sweep is checked against. */ + uint32_t resolveScopeIdForTesting(int64_t ts_ns) const; + /** @brief Test seam: completed scopes still retained. */ + size_t retainedCompletedScopesForTesting() const; bool pushKernel(const KernelBatchRow& row, const KernelDetailRow* detail = nullptr); bool pushMemcpy(const MemcpyBatchRow& row); @@ -69,6 +110,10 @@ class MonitorBatchManager { uint64_t instance_id = 0; uint32_t name_id = 0; int depth = 0; + + /** @brief True when this scope should win over @p other for a sample + * both contain. Deepest, then latest start, then instance id. */ + bool outranks(const ScopeWindow& other) const; }; struct OpenScopeWindow { @@ -79,6 +124,41 @@ class MonitorBatchManager { uint32_t resolveScopeIdLocked(int64_t ts_ns) const; + /** + * @brief Attribute a whole batch of samples in one pass. + * + * Sorts the samples and a snapshot of the candidate scopes once, then + * sweeps them together, rather than re-scanning every scope for every + * sample. The snapshot includes scopes that are STILL OPEN, given a + * provisional end: PM drains mid-run, so the scope covering a sample is + * routinely still open when that sample is decoded. + * + * Candidates cannot be kept pre-sorted. A scope's close timestamp is taken + * before PushScopeRow acquires the lock, so two threads closing at once + * append out of order. + * + * Selection matches resolveScopeIdLocked exactly: the interval contains the + * timestamp (both ends inclusive), then greatest depth, then latest start. + */ + std::vector snapshotScopeCandidatesLocked( + const std::vector& rows) const; + static void resolveScopeIdsForBatch(std::vector& candidates, + std::vector& rows, + uint32_t fallback_id); + + /** @brief Drop completed scopes that can no longer match any future sample. */ + void trimCompletedScopesLocked(); + + /** + * @brief Apply the hard cap, counting whatever it drops. + * + * Called on every scope close, not only when PM samples arrive: the + * retention watermark is the real bound, but only PM publishes one. A run + * without PM sampling would otherwise never trim at all. + */ + void enforceScopeCapLocked(); + void clearPendingScopeCloseLocked(uint64_t instance_id); + FlushSink flushSink_; DictionaryManager dictManager_; @@ -107,7 +187,26 @@ class MonitorBatchManager { // application scope is open - removes the right one. std::vector> scopeNameStack_; std::unordered_map openScopeWindows_; - std::vector completedScopeWindows_; + // A close timestamp is captured before scopeBatchMu_ can be acquired. + // Publishing it separately lets a concurrent PM snapshot cap an otherwise + // still-open interval at its real end instead of extending it to the batch + // watermark. + mutable std::mutex pendingScopeCloseMu_; + std::unordered_map pendingScopeCloseNs_; + // Closed scopes still needed to attribute samples that have not been + // decoded yet. Unordered - see snapshotScopeCandidatesLocked. + std::deque completedScopeWindows_; + // Below this, no future sample can arrive; see + // publishScopeRetentionWatermark. 0 = nothing published yet, so nothing is + // dropped by the watermark. The hard cap still applies on every close. + int64_t scopeRetentionWatermarkNs_ = 0; + // Backstop only. The watermark is what should bound this; the cap exists so + // a source that never advances it cannot grow the deque without limit. + static constexpr size_t kMaxCompletedScopes = 65536; + int64_t pmScopeAttributionStartNs_ = 0; + bool scopeHistoryEvictionLogged_ = false; + uint64_t scopeAttributionTruncated_ = 0; + uint64_t pmSampleRowsSeen_ = 0; BatchBuffer syncBatch_; BatchBuffer memAllocBatch_; diff --git a/include/gpufl/inject/inject_entry.cpp b/include/gpufl/inject/inject_entry.cpp index 3ac37f3..9a44f51 100644 --- a/include/gpufl/inject/inject_entry.cpp +++ b/include/gpufl/inject/inject_entry.cpp @@ -136,11 +136,11 @@ void endProcessScope() { if (!state.active) return; gpufl::ScopeBatchRow row; - row.ts_ns = gpufl::detail::GetTimestampNs(); row.scope_instance_id = state.instance_id; row.name_id = state.name_id; row.event_type = 1; row.depth = 0; + row.ts_ns = gpufl::Monitor::CaptureScopeCloseTimestamp(state.instance_id); gpufl::Monitor::PushScopeRow(row); gpufl::Monitor::EndProfilerScope(state.name.c_str()); if (state.perf_scope) gpufl::Monitor::EndPerfScope(state.name.c_str()); diff --git a/tests/core/test_monitor.cpp b/tests/core/test_monitor.cpp index 60e67cd..2e494ca 100644 --- a/tests/core/test_monitor.cpp +++ b/tests/core/test_monitor.cpp @@ -163,3 +163,282 @@ TEST(ScopeNameStackTest, DepthReportsWhereANewScopeWouldNest) { m.pushTrackedScopeRow(ScopeRow(2, inner, 1)); EXPECT_EQ(m.openScopeDepth(), 1); } + +// ── PM sample scope attribution ───────────────────────────────────────────── +// +// Attribution used to rescan every completed scope of the run for every PM +// sample, over a list that was never trimmed. These cover the replacement: a +// retention watermark driven by decode progress, and a per-drain sort-and-sweep +// that must agree with the original resolver exactly. + +namespace { + +gpufl::PmSampleBatchRow PmSample(int64_t ts_ns) { + gpufl::PmSampleBatchRow row; + row.ts_ns = ts_ns; + return row; +} + +gpufl::ScopeBatchRow ScopeEdge(uint64_t instance_id, uint32_t name_id, int64_t ts_ns, + uint8_t event_type, int depth) { + gpufl::ScopeBatchRow row; + row.ts_ns = ts_ns; + row.scope_instance_id = instance_id; + row.name_id = name_id; + row.event_type = event_type; + row.depth = depth; + return row; +} + +// Open then close a scope over [start_ns, end_ns] at the given depth. +void RecordScope(gpufl::detail::MonitorBatchManager& m, uint64_t instance_id, + uint32_t name_id, int64_t start_ns, int64_t end_ns, int depth) { + m.pushTrackedScopeRow(ScopeEdge(instance_id, name_id, start_ns, 0, depth)); + m.pushTrackedScopeRow(ScopeEdge(instance_id, name_id, end_ns, 1, depth)); +} + +} // namespace + +TEST(ScopeAttributionTest, BatchSweepMatchesThePerSampleResolver) { + // Both must pick the same scope for every timestamp. The sweep is only an + // optimisation, so any disagreement is a regression in attribution. + gpufl::detail::MonitorBatchManager m; + const uint32_t outer = m.internScopeName("process:app"); + const uint32_t mid = m.internScopeName("epoch"); + const uint32_t inner = m.internScopeName("step"); + + // Nested, overlapping, and sharing boundaries. Closed out of order on + // purpose: real closes are timestamped before they take the lock. + RecordScope(m, 3, inner, 300, 400, 2); + RecordScope(m, 2, mid, 200, 500, 1); + RecordScope(m, 1, outer, 100, 900, 0); + RecordScope(m, 4, inner, 500, 500, 2); // zero-width, boundary shared with mid + // Overlapping siblings at the SAME depth. Without these the depth + // comparison alone decides every sample and the start_ns tie-break is + // never exercised - two threads each running their own scope is exactly + // how this arises. + RecordScope(m, 5, mid, 600, 800, 1); + RecordScope(m, 6, inner, 700, 850, 1); + + std::vector timestamps; + for (int64_t ts = 50; ts <= 950; ts += 7) timestamps.push_back(ts); + for (int64_t ts : {100LL, 200LL, 300LL, 400LL, 500LL, 900LL}) timestamps.push_back(ts); + + std::vector rows; + for (int64_t ts : timestamps) rows.push_back(PmSample(ts)); + m.resolveScopeIdsForTesting(rows, /*fallback_id=*/0); + + for (const auto& row : rows) { + const uint32_t reference = m.resolveScopeIdForTesting(row.ts_ns); + EXPECT_EQ(row.scope_name_id, reference) + << "sweep and per-sample resolver disagree at ts=" << row.ts_ns; + } +} + +TEST(ScopeAttributionTest, SampleInsideAStillOpenScopeIsAttributedToIt) { + // PM drains mid-run, so the scope covering a sample is routinely still + // open. Before this, such samples fell back to whatever was on top at drain + // time - which is not the same question. + gpufl::detail::MonitorBatchManager m; + const uint32_t outer = m.internScopeName("process:app"); + const uint32_t open = m.internScopeName("deep_window"); + + RecordScope(m, 1, outer, 100, 900, 0); + m.pushTrackedScopeRow(ScopeEdge(2, open, 400, 0, 1)); // never closed + + std::vector rows{PmSample(300), PmSample(500)}; + m.resolveScopeIdsForTesting(rows, /*fallback_id=*/0); + + EXPECT_EQ(rows[0].scope_name_id, outer) << "before the open scope started"; + EXPECT_EQ(rows[1].scope_name_id, open) << "inside the still-open scope"; +} + +TEST(ScopeAttributionTest, PendingCloseCapsAnOpenScopeAtItsCapturedTimestamp) { + // A close captures its timestamp before waiting for the scope-state lock. + // The snapshot must observe that pending timestamp instead of extending the + // still-open map entry through the entire PM batch. + gpufl::detail::MonitorBatchManager m; + const uint32_t name = m.internScopeName("closing"); + m.pushTrackedScopeRow(ScopeEdge(1, name, 100, 0, 0)); + m.markScopeClosePending(1, 200); + + std::vector rows{PmSample(150), PmSample(250)}; + m.resolveScopeIdsForTesting(rows, /*fallback_id=*/0); + + EXPECT_EQ(rows[0].scope_name_id, name); + EXPECT_EQ(rows[1].scope_name_id, 0u) + << "a pending close must prevent provisional extension past its timestamp"; +} + +TEST(ScopeAttributionTest, RetentionDropsOnlyWhatTheWatermarkReleases) { + gpufl::detail::MonitorBatchManager m; + const uint32_t name = m.internScopeName("step"); + RecordScope(m, 1, name, 100, 200, 0); + RecordScope(m, 2, name, 300, 400, 0); + RecordScope(m, 3, name, 500, 600, 0); + EXPECT_EQ(m.retainedCompletedScopesForTesting(), 3u); + + // Nothing published yet: a run without PM sampling must never lose scopes. + std::vector probe{PmSample(150)}; + m.resolveScopeIdsForTesting(probe, 0); + EXPECT_EQ(m.retainedCompletedScopesForTesting(), 3u); + + m.publishScopeRetentionWatermark(450); + m.resolveScopeIdsForTesting(probe, 0); + EXPECT_EQ(m.retainedCompletedScopesForTesting(), 1u) + << "only the scope ending after the watermark survives"; +} + +TEST(ScopeAttributionTest, WatermarkNeverMovesBackwards) { + // A failed decode or an overflow must not un-retire scopes already + // released, so a lower value is ignored rather than applied. + gpufl::detail::MonitorBatchManager m; + const uint32_t name = m.internScopeName("step"); + RecordScope(m, 1, name, 100, 200, 0); + RecordScope(m, 2, name, 300, 400, 0); + + m.publishScopeRetentionWatermark(350); + m.publishScopeRetentionWatermark(50); // ignored + + std::vector probe{PmSample(380)}; + m.resolveScopeIdsForTesting(probe, 0); + EXPECT_EQ(m.retainedCompletedScopesForTesting(), 1u); +} + +TEST(ScopeAttributionTest, DelayedDrainKeepsScopesThatWallClockWouldHaveDropped) { + // The watermark is event time, not wall clock. A collector stalled for + // seconds must still attribute the samples it eventually decodes. + gpufl::detail::MonitorBatchManager m; + const uint32_t name = m.internScopeName("step"); + RecordScope(m, 1, name, 1'000'000'000, 1'100'000'000, 0); + + // Simulate a long stall: no watermark is published because nothing decoded. + std::vector rows{PmSample(1'050'000'000)}; + m.resolveScopeIdsForTesting(rows, /*fallback_id=*/7); + + EXPECT_EQ(rows[0].scope_name_id, name) + << "a scope that ended 2s ago is still needed by an undecoded sample"; +} + +TEST(ScopeAttributionTest, EqualDepthAndStartResolveIdenticallyInBothPaths) { + // std::sort is not stable and open scopes come out of an unordered_map, so + // without a tertiary key two scopes sharing depth AND start could resolve + // differently between the batch sweep and the reference resolver. + gpufl::detail::MonitorBatchManager m; + const uint32_t a = m.internScopeName("thread_a"); + const uint32_t b = m.internScopeName("thread_b"); + + // Same depth and start, DIFFERENT ends. If the ranking ever reports the two + // as equivalent, the sweep's ordered set keeps only one of them - and a + // sample after the shorter one ends then resolves to nothing instead of to + // the scope still covering it. Equal ends would hide that entirely. + RecordScope(m, 10, a, 100, 200, 1); + RecordScope(m, 11, b, 100, 400, 1); + + std::vector rows{PmSample(150), PmSample(250), PmSample(350)}; + m.resolveScopeIdsForTesting(rows, /*fallback_id=*/0); + for (const auto& row : rows) { + EXPECT_EQ(row.scope_name_id, m.resolveScopeIdForTesting(row.ts_ns)); + } +} + +TEST(ScopeAttributionTest, UncoveredSampleIsUnattributedNotGivenTheLiveScope) { + // The old fallback handed an unmatched sample whatever scope was open at + // DECODE time. That is the temporal error this resolver exists to remove: + // a sample can sit in the buffer while scopes come and go. + gpufl::detail::MonitorBatchManager m; + const uint32_t earlier = m.internScopeName("earlier"); + const uint32_t live = m.internScopeName("live_now"); + + RecordScope(m, 1, earlier, 100, 200, 0); + // A different scope is open by the time the batch is decoded. + m.pushTrackedScopeRow(ScopeEdge(2, live, 900, 0, 0)); + + std::vector rows{PmSample(500)}; // covered by neither + m.resolveScopeIdsForTesting(rows, /*fallback_id=*/0); + + EXPECT_EQ(rows[0].scope_name_id, 0u) + << "an uncovered sample must stay unattributed, not inherit the live scope"; +} + +TEST(ScopeAttributionTest, ManyOverlappingScopesStillResolveCorrectly) { + // Cross-thread workloads keep a large active set. The sweep has to stay + // correct as that grows, not just when a couple of scopes nest. + gpufl::detail::MonitorBatchManager m; + + // A distinct name per scope. With one shared name the assertion could not + // tell a correct pick from a wrong one - every answer would compare equal. + constexpr int kScopes = 400; + std::vector names; + names.reserve(kScopes); + for (int i = 0; i < kScopes; ++i) { + names.push_back(m.internScopeName("worker_" + std::to_string(i))); + } + for (int i = 0; i < kScopes; ++i) { + RecordScope(m, static_cast(i + 1), names[i], + /*start*/ i, /*end*/ 10'000 + i, /*depth*/ i % 4); + } + + std::vector rows; + for (int64_t ts = 0; ts < 10'000; ts += 337) rows.push_back(PmSample(ts)); + m.resolveScopeIdsForTesting(rows, /*fallback_id=*/0); + + for (const auto& row : rows) { + EXPECT_EQ(row.scope_name_id, m.resolveScopeIdForTesting(row.ts_ns)) + << "large active set diverges at ts=" << row.ts_ns; + } +} + +TEST(ScopeAttributionTest, CapAppliesWithoutAnyPmSamples) { + // The retention watermark is the real bound, but only PM sampling ever + // publishes one. A Trace-only run closes scopes and never decodes a sample, + // so if the cap only ran on the PM path this deque would grow for the life + // of the process - unbounded, and with nothing recorded to say so. + gpufl::detail::MonitorBatchManager m; + const uint32_t name = m.internScopeName("step"); + + constexpr int kScopes = 70'000; // over kMaxCompletedScopes + for (int i = 0; i < kScopes; ++i) { + RecordScope(m, static_cast(i + 1), name, i, i + 1, 0); + } + + EXPECT_LE(m.retainedCompletedScopesForTesting(), 65536u); + EXPECT_EQ(m.scopeAttributionTruncated(), 0u) + << "evicting unused Trace-only history is not PM attribution loss"; + EXPECT_EQ(m.pmSampleRowsSeen(), 0u) + << "evicting unused Trace-only history is not PM attribution loss"; +} + +TEST(ScopeAttributionTest, CapCountsEvictionWhilePmSamplingIsActive) { + gpufl::detail::MonitorBatchManager m; + const uint32_t name = m.internScopeName("step"); + m.beginPmScopeAttribution(100); + + constexpr int kScopes = 70'000; + for (int i = 0; i < kScopes; ++i) { + RecordScope(m, static_cast(i + 1), name, i, i + 1, 0); + } + + EXPECT_GT(m.scopeAttributionTruncated(), 0u) + << "eviction while PM can have buffered samples is an attribution risk"; +} + +TEST(ScopeAttributionTest, OldTraceHistoryEvictedDuringPmIsNotPartialAttribution) { + gpufl::detail::MonitorBatchManager m; + const uint32_t name = m.internScopeName("trace_step"); + + constexpr int kScopes = 70'000; + for (int i = 0; i < kScopes; ++i) { + RecordScope(m, static_cast(i + 1), name, i, i + 1, 0); + } + m.beginPmScopeAttribution(1'000'000); + + // Force more evictions after PM starts. Every evicted entry still predates + // the PM boundary, so none can be needed by a PM sample. + for (int i = 0; i < 100; ++i) { + const int64_t ts = 1'000'000 + i; + RecordScope(m, static_cast(kScopes + i + 1), name, ts, ts + 1, 0); + } + + EXPECT_EQ(m.scopeAttributionTruncated(), 0u); +} From 81aa157506a90bb3609a0720a96577149ba98863 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Sun, 26 Jul 2026 19:22:49 -0700 Subject: [PATCH 02/28] feat(counter): named counters for rate-based triggers A rule needs to watch something only the application knows - tokens, steps, requests - and neither existing primitive fits. A scope costs two locked batch pushes and a wire row per iteration, which rules it out of a decode loop, and being one-per it cannot say that a step produced eight tokens. auto tokens = gpufl::counter("token"); // once, outside the loop tokens.add(batchSize); // one relaxed atomic add The handle holds the slot's address, taken once at registration, so add() never touches the registry container. It could not do so safely in any case: reading a deque's size or indexing it races a concurrent registration, even though the addresses of existing elements stay valid. Slots live for the process and are never freed, so a handle kept in a static or held by an embedded host across shutdown()/init() stays valid. A generation number could not have made that safe on its own - it cannot stop state being freed while another thread is inside add() - and with permanent slots there is nothing left to rebind to, so no generation check reaches the hot path either. What separates sessions instead is a baseline taken at Initialize: ticks from a previous session, or from while gpufl was down, belong to neither and are excluded rather than counted twice. Both teardown paths close the session, including the process-exit one. add() validates before the atomic, since a single atomic add cannot also validate: non-positive values and anything above a per-call bound are dropped. That bound is not overflow protection - the counter is 64-bit and rates are unsigned deltas, correct across a wrap - it catches a caller passing something that is not a count, which would quietly poison every later rate. Names are checked against an explicit ASCII set rather than std::isalnum, whose answer depends on the host program's locale. Rejections log once and truncate the offending name, because a bad name usually fails on every call and tick() in a loop would otherwise write a line per iteration. tick(name, n) stays as a convenience wrapper, documented as not for tight loops: it looks the name up every call, which is the cost the handle exists to avoid. Python gets a Counter object for the same reason. Lifecycle tests drive Monitor::Initialize/Shutdown rather than the registry directly, so deleting the wiring fails a test. Known gap: gpufl is a static library linked separately into the injection DLL, the Python extension and any host application, so each holds its own registry. Counters therefore work in embedded mode but not yet under `gpufl trace`, where the target ticks one registry and the evaluator would read another. Fixed separately by moving the registry behind a versioned C ABI in its own shared runtime. --- CMakeLists.txt | 2 + include/gpufl/core/counter_api.cpp | 21 ++ include/gpufl/core/counter_registry.cpp | 132 +++++++++++++ include/gpufl/core/counter_registry.hpp | 136 +++++++++++++ include/gpufl/core/monitor.cpp | 13 ++ include/gpufl/gpufl.hpp | 53 +++++ python/bindings.cpp | 9 + python/gpufl/__init__.py | 52 +++++ tests/CMakeLists.txt | 1 + tests/core/test_counter_registry.cpp | 247 ++++++++++++++++++++++++ 10 files changed, 666 insertions(+) create mode 100644 include/gpufl/core/counter_api.cpp create mode 100644 include/gpufl/core/counter_registry.cpp create mode 100644 include/gpufl/core/counter_registry.hpp create mode 100644 tests/core/test_counter_registry.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fb0fc1c..ad9381e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,6 +107,8 @@ target_sources(gpufl PRIVATE include/gpufl/core/runtime.cpp include/gpufl/core/backend_factory.cpp include/gpufl/core/monitor_adapter.cpp + include/gpufl/core/counter_registry.cpp + include/gpufl/core/counter_api.cpp include/gpufl/core/monitor_batch_manager.cpp include/gpufl/core/monitor_record_builders.cpp include/gpufl/core/monitor.cpp diff --git a/include/gpufl/core/counter_api.cpp b/include/gpufl/core/counter_api.cpp new file mode 100644 index 0000000..1019258 --- /dev/null +++ b/include/gpufl/core/counter_api.cpp @@ -0,0 +1,21 @@ +#include "gpufl.hpp" +#include "gpufl/core/counter_registry.hpp" + +namespace gpufl { + +Counter counter(const std::string& name) { + auto& registry = detail::CounterRegistry::instance(); + const auto slot = registry.registerCounter(name); + if (slot == detail::CounterRegistry::kInvalidSlot) return Counter{}; + // The address, not the id: add() must not touch the container, or it would + // race a concurrent registration. + return Counter{registry.valueSlot(slot)}; +} + +void tick(const std::string& name, int64_t n) { + // Deliberately does the lookup every call. Documented as not for tight + // loops; anyone who cares holds a Counter instead. + counter(name).add(n); +} + +} // namespace gpufl diff --git a/include/gpufl/core/counter_registry.cpp b/include/gpufl/core/counter_registry.cpp new file mode 100644 index 0000000..4d7d15e --- /dev/null +++ b/include/gpufl/core/counter_registry.cpp @@ -0,0 +1,132 @@ +#include "gpufl/core/counter_registry.hpp" + +#include + +#include "gpufl/core/debug_logger.hpp" + +namespace gpufl::detail { + +CounterRegistry& CounterRegistry::instance() { + // Function-local static: the table has to outlive every runtime, and this + // is the only storage duration that guarantees it without an init order + // dependency on whoever registers first. + static CounterRegistry registry; + return registry; +} + +bool CounterRegistry::nameIsValid(const std::string& name) { + if (name.empty() || name.size() > kMaxNameLength) return false; + // Explicit ASCII, not std::isalnum: that one answers according to the + // current locale, and the accepted set here is part of a wire contract + // rather than something a host program's locale gets to widen. + return std::all_of(name.begin(), name.end(), [](const char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_' || c == '.' || c == '-'; + }); +} + +std::string CounterRegistry::forLog(const std::string& name) { + // A rejected name is attacker- or bug-supplied and can be arbitrarily long. + // Show enough to identify it, never enough to bloat the log. + constexpr size_t kMaxLogged = 32; + if (name.size() <= kMaxLogged) return name; + return name.substr(0, kMaxLogged) + "...(" + std::to_string(name.size()) + " chars)"; +} + +CounterRegistry::SlotId CounterRegistry::registerCounter(const std::string& name) { + if (!nameIsValid(name)) { + std::lock_guard lk(mu_); + if (!loggedInvalidName_) { + loggedInvalidName_ = true; + GFL_LOG_ERROR("[CounterRegistry] rejected counter name '", forLog(name), + "': must be 1-", kMaxNameLength, + " characters of [A-Za-z0-9._-] " + "(further name rejections suppressed)"); + } + return kInvalidSlot; + } + + std::lock_guard lk(mu_); + // Same name from several threads resolves to one slot; the map lookup + // under the lock is what makes that true. + if (const auto it = byName_.find(name); it != byName_.end()) return it->second; + + if (slots_.size() >= kMaxCounters) { + if (!loggedLimitReached_) { + loggedLimitReached_ = true; + GFL_LOG_ERROR("[CounterRegistry] counter limit reached (", kMaxCounters, + "); '", forLog(name), + "' not registered (further rejections suppressed)"); + } + return kInvalidSlot; + } + + const auto slot = static_cast(slots_.size()); + slots_.emplace_back(); + slots_.back().name = name; + // Registered mid-generation: baseline at the current value so ticks from a + // previous session, or from before init(), do not land in this one. + slots_.back().baseline = slots_.back().value.load(std::memory_order_relaxed); + byName_.emplace(name, slot); + return slot; +} + +std::atomic* CounterRegistry::valueSlot(SlotId slot) { + std::lock_guard lk(mu_); + if (slot >= slots_.size()) return nullptr; + // Safe to hand out and keep: a deque never relocates existing elements, and + // slots are never erased outside resetForTesting(). + return &slots_[slot].value; +} + +uint64_t CounterRegistry::rawValue(SlotId slot) const { + std::lock_guard lk(mu_); + if (slot >= slots_.size()) return 0; + return slots_[slot].value.load(std::memory_order_relaxed); +} + +uint64_t CounterRegistry::valueSinceBaseline(SlotId slot) const { + std::lock_guard lk(mu_); + if (slot >= slots_.size()) return 0; + // Unsigned subtraction, deliberately: correct across a wrap, where a signed + // difference would go negative. + return slots_[slot].value.load(std::memory_order_relaxed) - slots_[slot].baseline; +} + +const std::string& CounterRegistry::name(SlotId slot) const { + static const std::string kEmpty; + std::lock_guard lk(mu_); + if (slot >= slots_.size()) return kEmpty; + return slots_[slot].name; +} + +size_t CounterRegistry::counterCount() const { + std::lock_guard lk(mu_); + return slots_.size(); +} + +void CounterRegistry::beginSession() { + std::lock_guard lk(mu_); + for (auto& slot : slots_) { + slot.baseline = slot.value.load(std::memory_order_relaxed); + } + sessionActive_.store(true, std::memory_order_release); +} + +void CounterRegistry::endSession() { + // Values stay. A handle that outlives this keeps pointing at a live slot; + // it is the baseline taken by the next beginSession that stops those adds + // from being counted twice. + sessionActive_.store(false, std::memory_order_release); +} + +void CounterRegistry::resetForTesting() { + std::lock_guard lk(mu_); + slots_.clear(); + byName_.clear(); + sessionActive_.store(false, std::memory_order_release); + loggedInvalidName_ = false; + loggedLimitReached_ = false; +} + +} // namespace gpufl::detail diff --git a/include/gpufl/core/counter_registry.hpp b/include/gpufl/core/counter_registry.hpp new file mode 100644 index 0000000..fb1ab58 --- /dev/null +++ b/include/gpufl/core/counter_registry.hpp @@ -0,0 +1,136 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace gpufl::detail { + +/** + * @brief Named counters the application increments, read as rates. + * + * Exists so a rule can watch something only the application knows - tokens, + * steps, requests - without the application computing a rate or reaching for a + * scope. A scope would cost two locked batch pushes and a wire row per + * iteration, which rules it out of a decode loop; this costs one relaxed atomic + * add, and unlike a scope it can carry a count, so a step that produced eight + * tokens says so. + * + * ## Slots outlive the runtime, deliberately + * + * A handle can be stored in a static, or held across a shutdown()/init() cycle + * by an embedded host. A generation number alone would not make that safe: it + * cannot stop shutdown() freeing state while another thread is inside add(). + * So the slot itself is never freed. init()/shutdown() only change which + * runtime reads it, and nothing add() touches can disappear underneath it. + * + * That also removes any need to rebind a stale handle. There is nothing to + * rebind to - the slot a handle points at is still the right one - so the hot + * path carries no generation check either. + * + * The cost is a value that survives sessions, so each runtime records a + * baseline at init() and reports only what accrued since. Adds made while no + * runtime is active land in the slot and are excluded from the next session + * rather than counted twice. + */ +class CounterRegistry { +public: + /** @brief Index into the permanent slot table. kInvalidSlot on rejection. */ + using SlotId = uint32_t; + static constexpr SlotId kInvalidSlot = 0xFFFFFFFFu; + + /** Bounds. A permanent table is exactly what must not grow without limit. */ + static constexpr size_t kMaxCounters = 256; + static constexpr size_t kMaxNameLength = 96; + /** + * Largest single add accepted. + * + * Not an overflow guard - the counter is 64-bit and rates are unsigned + * deltas, so a wrap is both unreachable in practice and handled if it + * happened. This catches a caller passing something that is not a count at + * all: a pointer, an uninitialised value, a negative cast to unsigned. One + * of those silently makes every rate meaningless, which is worse than being + * told the value was refused. + */ + static constexpr int64_t kMaxAddPerCall = 1LL << 40; // ~1.1e12 + + static CounterRegistry& instance(); + + /** + * @brief Find or create the slot for @p name. + * + * Rejects a name that is empty, over kMaxNameLength, contains anything + * outside [A-Za-z0-9._-], or would exceed kMaxCounters. Registering the + * same name twice returns the same slot, including from several threads at + * once. + */ + SlotId registerCounter(const std::string& name); + + /** + * @brief Stable address of a slot's value, or nullptr for kInvalidSlot. + * + * Handed to the caller once, at registration, so add() never touches the + * container. Reading slots_.size() or indexing it without the mutex would + * race a concurrent registration even though a deque keeps references to + * existing elements valid. + */ + std::atomic* valueSlot(SlotId slot); + + /** @brief Raw slot value, including anything added before this runtime. */ + uint64_t rawValue(SlotId slot) const; + + /** @brief Value accrued since the current runtime took its baseline. */ + uint64_t valueSinceBaseline(SlotId slot) const; + + const std::string& name(SlotId slot) const; + size_t counterCount() const; + + /** + * @brief Start a session, baselining every slot at its current value. + * + * What keeps a permanent slot from leaking one session's ticks into the + * next. Anything added while no session was active is excluded here. + */ + void beginSession(); + /** @brief End the current session. Slots keep their values on purpose. */ + void endSession(); + /** @brief Whether a session is currently active. */ + bool sessionActive() const { return sessionActive_.load(std::memory_order_acquire); } + + /** @brief Test seam. Drops every slot, which a real process never does. */ + void resetForTesting(); + +private: + CounterRegistry() = default; + + struct Slot { + std::atomic value{0}; + uint64_t baseline = 0; // guarded by mu_ + std::string name; // guarded by mu_, never changes once set + }; + + static bool nameIsValid(const std::string& name); + /** @brief Truncate an untrusted name so a rejection cannot bloat the log. */ + static std::string forLog(const std::string& name); + + // Guards registration and the name/baseline fields. Never taken by add(). + mutable std::mutex mu_; + // Deque, not vector: a handle holds a Slot* and a vector would invalidate + // it on growth. Slots are never erased. + std::deque slots_; + std::unordered_map byName_; + // A flag, not a counter. Nothing needs to tell one session from another - + // baselines already separate them - and a monotonic epoch that no reader + // consults is a thing that drifts out of date without anyone noticing. + std::atomic sessionActive_{false}; + // Rejections are logged once each. A name that fails validation usually + // fails on every call, and tick() in a decode loop would otherwise write a + // line per iteration. + bool loggedInvalidName_ = false; + bool loggedLimitReached_ = false; +}; + +} // namespace gpufl::detail diff --git a/include/gpufl/core/monitor.cpp b/include/gpufl/core/monitor.cpp index 6dc8c86..689c83e 100644 --- a/include/gpufl/core/monitor.cpp +++ b/include/gpufl/core/monitor.cpp @@ -13,6 +13,7 @@ #include #include "gpufl/core/activity_record.hpp" +#include "gpufl/core/counter_registry.hpp" #include "gpufl/core/common.hpp" #include "gpufl/core/debug_logger.hpp" #include "gpufl/core/logger/logger.hpp" @@ -499,6 +500,10 @@ void Monitor::Initialize(const MonitorOptions& opts) { std::memory_order_release); g_monitorBuffer.resetDroppedCount(); + // Counter slots live for the process, so this session has to baseline them + // or it would inherit the previous session's ticks plus anything added + // while gpufl was shut down. + detail::CounterRegistry::instance().beginSession(); g_state.batches.reset(); g_state.metadata.reset(); g_state.batches.setSourceCollectionEnabled(opts.enable_source_collection); @@ -535,6 +540,10 @@ void Monitor::Shutdown() { g_monitorBuffer.resetDroppedCount(); g_state.batches.clearFlushSink(); g_state.adapter.reset(); + // Slots keep their values on purpose: a handle held across this stays + // valid. What stops those adds counting twice is the baseline the next + // Initialize takes, not clearing them here. + detail::CounterRegistry::instance().endSession(); } void Monitor::DrainAndFinalizeForExit() { @@ -570,6 +579,10 @@ void Monitor::DrainAndFinalizeForExit() { } } g_state.batches.clearFlushSink(); + // Same close-out as Shutdown(). Without it a process that exits through + // this path leaves the session marked active, and an embedded host that + // re-initialised afterwards would inherit its ticks. + detail::CounterRegistry::instance().endSession(); } void Monitor::ReleaseBackendForExit() { diff --git a/include/gpufl/gpufl.hpp b/include/gpufl/gpufl.hpp index ff5f5ad..7cecce1 100644 --- a/include/gpufl/gpufl.hpp +++ b/include/gpufl/gpufl.hpp @@ -236,6 +236,59 @@ void deepWindow(int64_t max_duration_ms, uint64_t max_launches = 0); // if (tokens_per_sec < 1000) gpufl::deepWindow(spec); void deepWindow(const DeepWindowSpec& spec); +// A named counter the application increments. Rules watch its RATE, so this +// is how something only your code knows - tokens, steps, requests - becomes a +// condition a deep window can trigger on. +// +// auto tokens = gpufl::counter("token"); // once, outside the loop +// for (...) { +// tokens.add(batchSize); // one relaxed atomic add +// } +// +// Prefer this to wrapping a hot loop in a scope. A scope costs two locked +// batch pushes and a row on the wire per iteration; and being one-per, it +// cannot say that a step produced eight tokens. +// +// Registration is the part that validates and allocates, so hoist it out of +// the loop. A handle is safe to keep in a static and across +// shutdown()/init(): the slot behind it lives for the process, and each +// session counts only what accrued after its own start. +class Counter { + public: + Counter() = default; + // Largest single add accepted. Not an overflow guard - the counter is + // 64-bit and rates are unsigned deltas, so a wrap is unreachable in + // practice and correct if it happened. This rejects a value that is not a + // count at all: a pointer, an uninitialised field, a negative cast to + // unsigned. Any of those quietly makes every rate meaningless. + static constexpr int64_t kMaxAddPerCall = 1LL << 40; + // Ignores n outside (0, kMaxAddPerCall]. Two compares before one relaxed + // atomic; a single atomic add could not validate anything itself. + void add(int64_t n = 1) const { + if (slot_ && n > 0 && n <= kMaxAddPerCall) { + slot_->fetch_add(static_cast(n), std::memory_order_relaxed); + } + } + // False when the name was rejected or the counter limit was reached. add() + // on such a handle is a no-op rather than an error. + bool valid() const { return slot_ != nullptr; } + + private: + friend Counter counter(const std::string&); + explicit Counter(std::atomic* slot) : slot_(slot) {} + std::atomic* slot_ = nullptr; +}; + +// Registers (or finds) a counter. Names are 1-96 characters of [A-Za-z0-9._-]; +// anything else, or exceeding the counter limit, returns an invalid handle and +// logs why. Calling this repeatedly with the same name is safe and returns the +// same counter, including from several threads at once. +Counter counter(const std::string& name); + +// Convenience wrapper. NOT for tight loops: the name has to be looked up on +// every call, which is exactly the cost the handle exists to avoid. +void tick(const std::string& name, int64_t n = 1); + // Close the current window early. No-op when none is open. void deepWindowClose(); diff --git a/python/bindings.cpp b/python/bindings.cpp index 19628f9..caf5daa 100644 --- a/python/bindings.cpp +++ b/python/bindings.cpp @@ -208,6 +208,15 @@ PYBIND11_MODULE(_gpufl_client, m) { m.def("deep_window_close", &gpufl::deepWindowClose); m.def("deep_window_active", &gpufl::deepWindowActive); + // Counter is exposed as an object so Python can hoist the lookup out of a + // decode loop the same way C++ does. add() then costs one relaxed atomic + // plus the call, instead of a name lookup per token. + py::class_(m, "Counter") + .def("add", &gpufl::Counter::add, py::arg("n") = 1) + .def("valid", &gpufl::Counter::valid); + m.def("counter", &gpufl::counter, py::arg("name")); + m.def("tick", &gpufl::tick, py::arg("name"), py::arg("n") = 1); + m.def("shutdown", &gpufl::shutdown); // ── Deferred bulk upload ──────────────────────────────────────────── diff --git a/python/gpufl/__init__.py b/python/gpufl/__init__.py index 41e10fb..55264d7 100644 --- a/python/gpufl/__init__.py +++ b/python/gpufl/__init__.py @@ -184,6 +184,7 @@ def _dbg(msg): Scope as _CScope, init, shutdown, system_start, system_stop, deep_window as _c_deep_window, deep_window_close as _c_deep_window_close, deep_window_active as _c_deep_window_active, + counter as _c_counter, tick as _c_tick, BackendKind, InitOptions, ProfilingEngine, upload_logs as _c_upload_logs, UploadOptions, UploadResult, ) @@ -220,6 +221,19 @@ def _c_deep_window_close(): def _c_deep_window_active(): return False + class _StubCounter: + def add(self, n=1): + return None + + def valid(self): + return False + + def _c_counter(name): + return _StubCounter() + + def _c_tick(name, n=1): + return None + class BackendKind: Auto = "Auto" Nvidia = "Nvidia" @@ -604,6 +618,43 @@ def system_stop(name="system"): return _original_system_stop(name) +def counter(name): + """Register (or find) a named counter whose RATE a rule can watch. + + How something only your code knows - tokens, steps, requests - becomes a + condition a deep window can trigger on:: + + tokens = gpufl.counter("token") # once, outside the loop + for step in decode(): + tokens.add(step.token_count) + + Hoist the call out of the loop: registration is the part that validates + and allocates, while ``add`` is a single atomic increment. Prefer this to + wrapping a hot loop in a Scope, which costs two locked batch pushes and a + row on the wire per iteration - and, being one-per, cannot say that a step + produced eight tokens. + + The handle is safe to keep in a module-level variable and across + ``shutdown()``/``init()``: the slot behind it lives for the process, and + each session counts only what accrued after its own start. + + Args: + name: 1-96 characters of ``[A-Za-z0-9._-]``. Anything else, or + exceeding the counter limit, returns a handle whose ``add`` is a + no-op; ``valid()`` reports which you got. + """ + return _c_counter(name) + + +def tick(name, n=1): + """Increment a counter by name. + + Convenience only. NOT for tight loops: the name is looked up on every + call, which is the cost :func:`counter` exists to avoid. + """ + return _c_tick(name, n) + + def deep_window(seconds=0.0, max_launches=0): """Arm deep profiling for a short, self-closing window. @@ -1077,6 +1128,7 @@ def clean_logs(log_path=None, log_prefix=None, *, dry_run=False): "Scope", "init", "shutdown", "session", "clean_logs", "targeting", "system_start", "system_stop", "deep_window", "deep_window_close", "deep_window_active", + "counter", "tick", "BackendKind", "InitOptions", "ProfilingEngine", "upload_logs", "UploadOptions", "UploadResult", ] diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 684accd..1005aef 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -18,6 +18,7 @@ set(GPUFL_TEST_SOURCES core/test_deep_window.cpp core/test_disabled.cpp core/test_wire_contract.cpp + core/test_counter_registry.cpp core/test_monitor.cpp core/test_itanium_demangle.cpp core/test_sampler.cpp diff --git a/tests/core/test_counter_registry.cpp b/tests/core/test_counter_registry.cpp new file mode 100644 index 0000000..e48e9f4 --- /dev/null +++ b/tests/core/test_counter_registry.cpp @@ -0,0 +1,247 @@ +#include + +#include +#include + +#include "gpufl.hpp" +#include "gpufl/core/counter_registry.hpp" +#include "gpufl/core/monitor.hpp" + +#include + +using gpufl::detail::CounterRegistry; + +namespace { + +class CounterRegistryTest : public ::testing::Test { + protected: + void SetUp() override { CounterRegistry::instance().resetForTesting(); } + void TearDown() override { CounterRegistry::instance().resetForTesting(); } + + CounterRegistry& reg() { return CounterRegistry::instance(); } +}; + +} // namespace + +TEST_F(CounterRegistryTest, SameNameResolvesToOneSlot) { + const auto a = reg().registerCounter("token"); + const auto b = reg().registerCounter("token"); + EXPECT_EQ(a, b); + EXPECT_EQ(reg().counterCount(), 1u); +} + +TEST_F(CounterRegistryTest, InvalidNamesAreRejected) { + EXPECT_EQ(reg().registerCounter(""), CounterRegistry::kInvalidSlot); + EXPECT_EQ(reg().registerCounter("has space"), CounterRegistry::kInvalidSlot); + EXPECT_EQ(reg().registerCounter("has/slash"), CounterRegistry::kInvalidSlot); + EXPECT_EQ(reg().registerCounter(std::string(200, 'x')), CounterRegistry::kInvalidSlot); + EXPECT_EQ(reg().counterCount(), 0u); +} + +TEST_F(CounterRegistryTest, CardinalityIsCapped) { + for (size_t i = 0; i < CounterRegistry::kMaxCounters; ++i) { + ASSERT_NE(reg().registerCounter("c" + std::to_string(i)), + CounterRegistry::kInvalidSlot); + } + // A permanent slot table is exactly the thing that must not grow without + // limit, so the one past the cap is refused rather than accepted. + EXPECT_EQ(reg().registerCounter("one_too_many"), CounterRegistry::kInvalidSlot); + EXPECT_EQ(reg().counterCount(), CounterRegistry::kMaxCounters); +} + +TEST_F(CounterRegistryTest, NonPositiveAddsAreIgnored) { + auto tokens = gpufl::counter("token"); + ASSERT_TRUE(tokens.valid()); + tokens.add(5); + tokens.add(0); + tokens.add(-100); // must not wrap the unsigned counter + EXPECT_EQ(reg().rawValue(reg().registerCounter("token")), 5u); +} + +TEST_F(CounterRegistryTest, InvalidHandleAddIsANoOp) { + const auto bad = gpufl::counter("not a valid name"); + EXPECT_FALSE(bad.valid()); + bad.add(1); // must not crash +} + +TEST_F(CounterRegistryTest, AddsBeforeASessionAreExcludedFromIt) { + // Slots outlive the runtime, so anything ticked before init() is already in + // the slot. Without a baseline the new session would count it as its own. + auto tokens = gpufl::counter("token"); + tokens.add(1000); + + const auto slot = reg().registerCounter("token"); + EXPECT_EQ(reg().rawValue(slot), 1000u); + + reg().beginSession(); + EXPECT_EQ(reg().valueSinceBaseline(slot), 0u) << "pre-init ticks are not this session's"; + + tokens.add(7); + EXPECT_EQ(reg().valueSinceBaseline(slot), 7u); +} + +TEST_F(CounterRegistryTest, AddsBetweenSessionsAreExcludedFromTheNext) { + auto tokens = gpufl::counter("token"); + const auto slot = reg().registerCounter("token"); + + reg().beginSession(); + tokens.add(10); + EXPECT_EQ(reg().valueSinceBaseline(slot), 10u); + reg().endSession(); + + // gpufl is down; the handle still works and the slot still accumulates. + tokens.add(500); + + reg().beginSession(); + EXPECT_EQ(reg().valueSinceBaseline(slot), 0u) + << "ticks while no runtime was active belong to no session"; + tokens.add(3); + EXPECT_EQ(reg().valueSinceBaseline(slot), 3u); +} + +TEST_F(CounterRegistryTest, HandleSurvivesASessionChange) { + // The point of a process-lifetime slot: a handle held in a static, or by an + // embedded host across shutdown()/init(), keeps working. A generation + // number alone could not make that safe, since it cannot stop the state + // being freed underneath a concurrent add(). + auto tokens = gpufl::counter("token"); + const auto slot = reg().registerCounter("token"); + + reg().beginSession(); + reg().endSession(); + reg().beginSession(); + + tokens.add(4); + EXPECT_EQ(reg().valueSinceBaseline(slot), 4u); +} + +TEST_F(CounterRegistryTest, CounterRegisteredMidSessionStartsAtZero) { + reg().beginSession(); + auto late = gpufl::counter("late"); + late.add(9); + EXPECT_EQ(reg().valueSinceBaseline(reg().registerCounter("late")), 9u); +} + +TEST_F(CounterRegistryTest, ConcurrentRegistrationOfOneNameYieldsOneSlot) { + constexpr int kThreads = 8; + std::vector threads; + std::vector results(kThreads); + for (int i = 0; i < kThreads; ++i) { + threads.emplace_back([&, i] { results[i] = reg().registerCounter("shared"); }); + } + for (auto& t : threads) t.join(); + + for (const auto slot : results) EXPECT_EQ(slot, results[0]); + EXPECT_EQ(reg().counterCount(), 1u); +} + +TEST_F(CounterRegistryTest, ConcurrentAddsAreNotLost) { + auto tokens = gpufl::counter("token"); + const auto slot = reg().registerCounter("token"); + reg().beginSession(); + + constexpr int kThreads = 8; + constexpr int kPerThread = 10'000; + std::vector threads; + for (int i = 0; i < kThreads; ++i) { + threads.emplace_back([&] { + for (int n = 0; n < kPerThread; ++n) tokens.add(1); + }); + } + for (auto& t : threads) t.join(); + + EXPECT_EQ(reg().valueSinceBaseline(slot), + static_cast(kThreads) * kPerThread); +} + +TEST_F(CounterRegistryTest, TickIsEquivalentButLooksThePriceUpEveryCall) { + gpufl::tick("steps"); + gpufl::tick("steps", 4); + EXPECT_EQ(reg().rawValue(reg().registerCounter("steps")), 5u); +} + +// ── lifecycle wiring ──────────────────────────────────────────────────────── +// +// The tests above drive the registry directly, so they pass whether or not +// Monitor actually calls it. These go through Monitor::Initialize/Shutdown so +// that deleting the calls in monitor.cpp fails something. + +TEST_F(CounterRegistryTest, MonitorInitializeBaselinesThroughTheRealWiring) { + auto tokens = gpufl::counter("token"); + const auto slot = reg().registerCounter("token"); + tokens.add(1000); // before any session exists + + gpufl::MonitorOptions opts; + gpufl::Monitor::Initialize(opts); + EXPECT_EQ(reg().valueSinceBaseline(slot), 0u) + << "Monitor::Initialize must baseline the registry"; + EXPECT_TRUE(reg().sessionActive()); + + tokens.add(6); + EXPECT_EQ(reg().valueSinceBaseline(slot), 6u); + + gpufl::Monitor::Shutdown(); + EXPECT_FALSE(reg().sessionActive()) + << "Monitor::Shutdown must close the session"; +} + +TEST_F(CounterRegistryTest, TicksBetweenTwoMonitorSessionsBelongToNeither) { + auto tokens = gpufl::counter("token"); + const auto slot = reg().registerCounter("token"); + + gpufl::MonitorOptions opts; + gpufl::Monitor::Initialize(opts); + tokens.add(10); + EXPECT_EQ(reg().valueSinceBaseline(slot), 10u); + gpufl::Monitor::Shutdown(); + + // gpufl is down. The handle still works - the slot is process-lifetime - + // but these ticks are nobody's. + tokens.add(500); + + gpufl::Monitor::Initialize(opts); + EXPECT_EQ(reg().valueSinceBaseline(slot), 0u); + tokens.add(3); + EXPECT_EQ(reg().valueSinceBaseline(slot), 3u); + gpufl::Monitor::Shutdown(); +} + +TEST_F(CounterRegistryTest, ProcessExitPathAlsoClosesTheSession) { + // DrainAndFinalizeForExit is a separate teardown from Shutdown(); an + // embedded host that re-initialised after it would otherwise inherit the + // previous session's ticks. + gpufl::MonitorOptions opts; + gpufl::Monitor::Initialize(opts); + ASSERT_TRUE(reg().sessionActive()); + + gpufl::Monitor::DrainAndFinalizeForExit(); + EXPECT_FALSE(reg().sessionActive()); +} + +TEST_F(CounterRegistryTest, AddAboveThePerCallBoundIsRefused) { + // Not overflow protection: a value this large is a caller bug - a pointer, + // an uninitialised field - and letting it through makes every later rate + // meaningless. + auto tokens = gpufl::counter("token"); + const auto slot = reg().registerCounter("token"); + tokens.add(gpufl::Counter::kMaxAddPerCall); + tokens.add(gpufl::Counter::kMaxAddPerCall + 1); // refused + EXPECT_EQ(reg().rawValue(slot), + static_cast(gpufl::Counter::kMaxAddPerCall)); +} + +TEST_F(CounterRegistryTest, DeltaIsCorrectAcrossAWrap) { + // Rates are unsigned deltas precisely so a wrap needs no saturation or CAS + // loop. Drive the counter to just below the wrap and cross it. + auto tokens = gpufl::counter("token"); + const auto slot = reg().registerCounter("token"); + auto* raw = reg().valueSlot(slot); + ASSERT_NE(raw, nullptr); + + raw->store(std::numeric_limits::max() - 5, std::memory_order_relaxed); + reg().beginSession(); + tokens.add(10); // wraps past zero + + EXPECT_EQ(reg().valueSinceBaseline(slot), 10u) + << "unsigned subtraction must stay correct across a wrap"; +} From eeaf636339adc375242f9149c3a92767725d30b5 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Sun, 26 Jul 2026 19:39:46 -0700 Subject: [PATCH 03/28] feat(counter): share the registry across modules via a C ABI runtime gpufl is a static library, linked separately into gpufl_inject.dll, the Python extension, and any host application. Each copy held its own registry, so under `gpufl trace -- python server.py` the target would tick one and the injected evaluator would read another: the counter reads as Missing forever, which is exactly the case counters were added for. Counters worked embedded and silently did nothing under the launcher. Move the registry into a small shared runtime and bind every module to it through a versioned C ABI. Only C types cross the boundary - no std:: types, no atomic passed by address, nothing that ties the two sides to one compiler's layout - so add() becomes an indirect call plus a relaxed atomic rather than an inlined one. Measure that in the overhead pass before optimising it. Loaded explicitly by path rather than through an import table. Injection arrives via CUDA_INJECTION64_PATH, so the driver maps gpufl_inject.dll into a target process whose DLL search path has no reason to include our bin directory, and an import entry would simply fail there. The path is derived from THIS module's own location, which works wherever it was loaded from. An already-mapped copy is preferred over any path. Deployment colocates the runtime with each consumer - beside the injection DLL, inside the wheel - so several copies exist on disk, and loading them by full path would map them as separate modules with separate registries: the very split this exists to prevent. Binding to whatever is already loaded makes the number of copies irrelevant, and makes the order between Python import and CUDA init irrelevant too. When no runtime is found the module falls back to its own registry. That is right for an embedded host, which holds the only copy of gpufl in the process, and wrong under injection. CounterProvider::isShared() reports which happened so the rule evaluator can refuse a custom counter rule it cannot honour, rather than reporting a counter that is being ticked as Missing. The tests now read back through the active provider rather than the local registry. Colocating the runtime with the test binary made twelve of them fail, which was the mechanism working: writes went to the runtime's registry while assertions read this module's. They pass both with and without the runtime present. Still open: the cross-module property cannot be proven from inside one executable. It needs the launcher plus Python E2E, which in turn needs the deployment rules that put the runtime beside gpufl_inject.dll and the extension. --- CMakeLists.txt | 16 ++ include/gpufl/abi/gpufl_counter_abi.h | 86 +++++++++++ include/gpufl/core/counter_api.cpp | 83 +++++++++- include/gpufl/core/counter_provider.cpp | 194 ++++++++++++++++++++++++ include/gpufl/core/counter_provider.hpp | 53 +++++++ include/gpufl/core/counter_registry.cpp | 6 + include/gpufl/core/counter_registry.hpp | 9 ++ include/gpufl/core/monitor.cpp | 8 +- include/gpufl/gpufl.hpp | 31 ++-- runtime/counter_runtime.cpp | 80 ++++++++++ tests/core/test_counter_registry.cpp | 141 +++++++++++------ 11 files changed, 640 insertions(+), 67 deletions(-) create mode 100644 include/gpufl/abi/gpufl_counter_abi.h create mode 100644 include/gpufl/core/counter_provider.cpp create mode 100644 include/gpufl/core/counter_provider.hpp create mode 100644 runtime/counter_runtime.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ad9381e..0d7c233 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,21 @@ option(BUILD_TESTING "Build the testing tree." ON) add_library(gpufl STATIC) add_library(gpufl::gpufl ALIAS gpufl) +# One counter registry per PROCESS, not per module. `gpufl` is static and is +# linked separately into gpufl_inject, the Python extension and any host +# application, so without this each would hold its own registry - a profiled +# target would tick one and the injected evaluator would read another. Only a C +# ABI crosses this boundary; see include/gpufl/abi/gpufl_counter_abi.h. +add_library(gpufl_counter_runtime SHARED + runtime/counter_runtime.cpp + include/gpufl/core/counter_registry.cpp +) +target_include_directories(gpufl_counter_runtime PUBLIC include) +set_target_properties(gpufl_counter_runtime PROPERTIES + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON +) + target_include_directories(gpufl PUBLIC $ @@ -109,6 +124,7 @@ target_sources(gpufl PRIVATE include/gpufl/core/monitor_adapter.cpp include/gpufl/core/counter_registry.cpp include/gpufl/core/counter_api.cpp + include/gpufl/core/counter_provider.cpp include/gpufl/core/monitor_batch_manager.cpp include/gpufl/core/monitor_record_builders.cpp include/gpufl/core/monitor.cpp diff --git a/include/gpufl/abi/gpufl_counter_abi.h b/include/gpufl/abi/gpufl_counter_abi.h new file mode 100644 index 0000000..04c0669 --- /dev/null +++ b/include/gpufl/abi/gpufl_counter_abi.h @@ -0,0 +1,86 @@ +#ifndef GPUFL_COUNTER_ABI_H +#define GPUFL_COUNTER_ABI_H + +/* + * Counter registry ABI. + * + * Exists because `gpufl` is a STATIC library, linked separately into + * gpufl_inject.dll, the Python extension, and any host application. Each copy + * would hold its own registry, so a Python server under `gpufl trace` would + * tick one and the injected evaluator would read another - the counter would + * read as Missing forever, which is exactly the case counters were added for. + * + * A small shared runtime owns the registry and everyone binds to it. Only C + * types cross the boundary: no std:: types, no atomics passed by address, no + * layout that depends on the compiler that built either side. + * + * Versioning: `abi_version` gates compatibility and `struct_size` allows + * appending members. A consumer must check both before calling anything, and + * must not assume a member exists because its own header declares it. + */ + +#include +#include + +#if defined(_WIN32) +# define GPUFL_COUNTER_EXPORT __declspec(dllexport) +#else +# define GPUFL_COUNTER_EXPORT __attribute__((visibility("default"))) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define GPUFL_COUNTER_ABI_VERSION 1u + +/* Opaque. Stable for the life of the process once returned. */ +typedef void* gpufl_counter_handle; + +typedef struct gpufl_counter_provider_v1 { + uint32_t abi_version; + uint32_t struct_size; + + /* + * Find or create the counter named [name, name + name_length). + * Returns NULL when the name is invalid or the counter limit is reached. + * The same name always returns the same handle, including across modules + * and from several threads at once. + */ + gpufl_counter_handle (*register_counter)(const char* name, size_t name_length); + + /* + * Add to a counter. The caller validates its own input; this ignores a + * NULL handle. Values wrap, and readers take unsigned deltas, which stay + * correct across a wrap. + */ + void (*add)(gpufl_counter_handle handle, uint64_t value); + + /* Raw value, including anything added before the current session. */ + uint64_t (*load)(gpufl_counter_handle handle); + + /* Value accrued since the current session's baseline. */ + uint64_t (*load_since_baseline)(gpufl_counter_handle handle); + + /* + * Baseline every counter at its current value. Called when a runtime + * initialises, so ticks from a previous session - or from while no runtime + * was active - are not counted as this one's. + */ + void (*begin_session)(void); + void (*end_session)(void); + int (*session_active)(void); +} gpufl_counter_provider_v1; + +/* + * Entry point the shared runtime exports. Never returns NULL from a runtime + * that loaded successfully. + */ +GPUFL_COUNTER_EXPORT const gpufl_counter_provider_v1* +gpufl_get_counter_provider_v1(void); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* GPUFL_COUNTER_ABI_H */ diff --git a/include/gpufl/core/counter_api.cpp b/include/gpufl/core/counter_api.cpp index 1019258..48b481d 100644 --- a/include/gpufl/core/counter_api.cpp +++ b/include/gpufl/core/counter_api.cpp @@ -1,18 +1,87 @@ #include "gpufl.hpp" + +#include "gpufl/abi/gpufl_counter_abi.h" +#include "gpufl/core/counter_provider.hpp" #include "gpufl/core/counter_registry.hpp" namespace gpufl { +namespace { + +// Fallback provider, used when no shared runtime is present. Correct for an +// embedded host, which holds the only copy of gpufl in the process. Under +// injection it is NOT correct - the target and the evaluator are separate +// modules and would each get their own registry - which is why +// CounterProvider::isShared() exists for the evaluator to gate on. +// +// Implemented as a provider rather than a separate branch in add() so both +// paths go through one code shape and cannot drift. +using detail::CounterRegistry; + +gpufl_counter_handle LocalRegister(const char* name, size_t len) { + if (name == nullptr) return nullptr; + const auto slot = CounterRegistry::instance().registerCounter(std::string(name, len)); + if (slot == CounterRegistry::kInvalidSlot) return nullptr; + return reinterpret_cast(static_cast(slot) + 1u); +} + +bool LocalSlot(gpufl_counter_handle h, CounterRegistry::SlotId* out) { + if (h == nullptr) return false; + *out = static_cast(reinterpret_cast(h) - 1u); + return true; +} + +void LocalAdd(gpufl_counter_handle h, uint64_t v) { + CounterRegistry::SlotId slot; + if (LocalSlot(h, &slot)) CounterRegistry::instance().addRaw(slot, v); +} + +uint64_t LocalLoad(gpufl_counter_handle h) { + CounterRegistry::SlotId slot; + return LocalSlot(h, &slot) ? CounterRegistry::instance().rawValue(slot) : 0; +} + +uint64_t LocalLoadSinceBaseline(gpufl_counter_handle h) { + CounterRegistry::SlotId slot; + return LocalSlot(h, &slot) ? CounterRegistry::instance().valueSinceBaseline(slot) : 0; +} + +void LocalBeginSession() { CounterRegistry::instance().beginSession(); } +void LocalEndSession() { CounterRegistry::instance().endSession(); } +int LocalSessionActive() { return CounterRegistry::instance().sessionActive() ? 1 : 0; } + +const gpufl_counter_provider_v1 kLocalProvider = { + GPUFL_COUNTER_ABI_VERSION, + sizeof(gpufl_counter_provider_v1), + &LocalRegister, &LocalAdd, &LocalLoad, &LocalLoadSinceBaseline, + &LocalBeginSession, &LocalEndSession, &LocalSessionActive, +}; + +} // namespace + +namespace detail { + +const gpufl_counter_provider_v1* ActiveCounterProvider() { + if (const auto* shared = CounterProvider::get()) return shared; + return &kLocalProvider; +} + +} // namespace detail + +void Counter::add(const int64_t n) const { + if (handle_ == nullptr || n <= 0 || n > kMaxAddPerCall) return; + static_cast(provider_) + ->add(handle_, static_cast(n)); +} Counter counter(const std::string& name) { - auto& registry = detail::CounterRegistry::instance(); - const auto slot = registry.registerCounter(name); - if (slot == detail::CounterRegistry::kInvalidSlot) return Counter{}; - // The address, not the id: add() must not touch the container, or it would - // race a concurrent registration. - return Counter{registry.valueSlot(slot)}; + const gpufl_counter_provider_v1* provider = detail::ActiveCounterProvider(); + gpufl_counter_handle handle = + provider->register_counter(name.c_str(), name.size()); + if (handle == nullptr) return Counter{}; + return Counter{provider, handle}; } -void tick(const std::string& name, int64_t n) { +void tick(const std::string& name, const int64_t n) { // Deliberately does the lookup every call. Documented as not for tight // loops; anyone who cares holds a Counter instead. counter(name).add(n); diff --git a/include/gpufl/core/counter_provider.cpp b/include/gpufl/core/counter_provider.cpp new file mode 100644 index 0000000..16650a2 --- /dev/null +++ b/include/gpufl/core/counter_provider.cpp @@ -0,0 +1,194 @@ +#include "gpufl/core/counter_provider.hpp" + +#include +#include +#include + +#if defined(_WIN32) +# include +#else +# include +#endif + +#include "gpufl/core/counter_registry.hpp" +#include "gpufl/core/debug_logger.hpp" + +namespace gpufl::detail { +namespace { + +#if defined(_WIN32) +constexpr const char* kRuntimeName = "gpufl_counter_runtime.dll"; +#elif defined(__APPLE__) +constexpr const char* kRuntimeName = "libgpufl_counter_runtime.dylib"; +#else +constexpr const char* kRuntimeName = "libgpufl_counter_runtime.so"; +#endif + +std::mutex g_mu; +bool g_resolved = false; +const gpufl_counter_provider_v1* g_provider = nullptr; +bool g_shared = false; + +/** + * Directory of the module this code is linked into - the injection DLL, the + * Python extension, or the host executable. NOT the process directory: under + * CUDA_INJECTION64_PATH the process is the profiled target, which has no + * reason to sit next to our files. + */ +std::string ThisModuleDir() { +#if defined(_WIN32) + HMODULE module = nullptr; + if (!GetModuleHandleExA( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(&ThisModuleDir), &module)) { + return {}; + } + char path[MAX_PATH] = {}; + const DWORD n = GetModuleFileNameA(module, path, sizeof(path)); + if (n == 0 || n >= sizeof(path)) return {}; + std::string full(path, n); + const auto slash = full.find_last_of("\\/"); + return slash == std::string::npos ? std::string{} : full.substr(0, slash); +#else + Dl_info info{}; + if (dladdr(reinterpret_cast(&ThisModuleDir), &info) == 0 || + info.dli_fname == nullptr) { + return {}; + } + const std::string full(info.dli_fname); + const auto slash = full.find_last_of('/'); + return slash == std::string::npos ? std::string{} : full.substr(0, slash); +#endif +} + +/** + * A copy already mapped into this process, or nullptr. + * + * Tried before any path, and it is what actually delivers the single-instance + * property. Deployment colocates the runtime with each consumer - beside the + * injection DLL, inside the Python wheel - so several copies exist on disk, and + * loading them by full path would map them as separate modules with separate + * registries. That is the very split this ABI exists to prevent. Binding to + * whatever is already loaded makes the number of copies on disk irrelevant. + */ +void* AlreadyLoaded() { +#if defined(_WIN32) + return reinterpret_cast(GetModuleHandleA(kRuntimeName)); +#else + // NOLOAD returns a handle only if the object is already mapped. + return dlopen(kRuntimeName, RTLD_NOW | RTLD_NOLOAD); +#endif +} + +void* OpenLibrary(const std::string& path) { +#if defined(_WIN32) + return LoadLibraryA(path.c_str()); +#else + // RTLD_GLOBAL so a second binder resolves to this same object rather than + // mapping its own copy. + return dlopen(path.c_str(), RTLD_NOW | RTLD_GLOBAL); +#endif +} + +void* FindSymbol(void* handle, const char* name) { +#if defined(_WIN32) + return reinterpret_cast( + GetProcAddress(static_cast(handle), name)); +#else + return dlsym(handle, name); +#endif +} + +/** Candidate locations, most specific first. */ +std::vector RuntimeCandidates() { + std::vector out; + const std::string dir = ThisModuleDir(); + if (!dir.empty()) { + out.push_back(dir + "/" + kRuntimeName); + // Build trees put the shared runtime beside the config directory + // rather than next to every consumer. + out.push_back(dir + "/../" + kRuntimeName); + out.push_back(dir + "/../bin/" + kRuntimeName); + } + // Last resort: the platform loader's own search. Also the path that finds + // an already-loaded copy by name, which is the common case for whichever + // module binds second. + out.emplace_back(kRuntimeName); + return out; +} + +void Resolve() { + // Whatever is already mapped wins, before any path is considered. See + // AlreadyLoaded(): several copies of this library legitimately exist on + // disk, and binding by path would defeat the point. + std::vector candidates = RuntimeCandidates(); + candidates.insert(candidates.begin(), std::string{}); // "" = already loaded + + for (const std::string& candidate : candidates) { + void* handle = candidate.empty() ? AlreadyLoaded() : OpenLibrary(candidate); + if (handle == nullptr) continue; + + using GetProviderFn = const gpufl_counter_provider_v1* (*)(); + const auto entry = reinterpret_cast( + FindSymbol(handle, "gpufl_get_counter_provider_v1")); + if (entry == nullptr) continue; + + const gpufl_counter_provider_v1* provider = entry(); + if (provider == nullptr) continue; + // Check both before calling through: a newer runtime may have appended + // members this build does not know about, and an older one may lack + // members this build assumes. + if (provider->abi_version != GPUFL_COUNTER_ABI_VERSION || + provider->struct_size < sizeof(gpufl_counter_provider_v1)) { + GFL_LOG_ERROR("[CounterProvider] ", candidate, + " reports ABI ", provider->abi_version, + " (size ", provider->struct_size, "); this build needs ", + GPUFL_COUNTER_ABI_VERSION, " (size ", + sizeof(gpufl_counter_provider_v1), ")"); + continue; + } + + g_provider = provider; + g_shared = true; + GFL_LOG_DEBUG("[CounterProvider] bound to shared runtime: ", + candidate.empty() ? "(already loaded)" : candidate.c_str()); + return; + } + + // No shared runtime. Correct for an embedded host, which holds the only + // copy of gpufl in the process; wrong under injection, where the target and + // the evaluator are separate modules and would each get their own registry. + // isShared() is what lets the evaluator refuse a rule it cannot honour, + // rather than reporting a counter that is being ticked as Missing. + g_provider = nullptr; + g_shared = false; + GFL_LOG_DEBUG("[CounterProvider] ", kRuntimeName, + " not found; counters stay local to this module"); +} + +} // namespace + +const gpufl_counter_provider_v1* CounterProvider::get() { + std::lock_guard lk(g_mu); + if (!g_resolved) { + g_resolved = true; + Resolve(); + } + return g_provider; +} + +bool CounterProvider::isShared() { + get(); + std::lock_guard lk(g_mu); + return g_shared; +} + +void CounterProvider::resetForTesting() { + std::lock_guard lk(g_mu); + g_resolved = false; + g_provider = nullptr; + g_shared = false; +} + +} // namespace gpufl::detail diff --git a/include/gpufl/core/counter_provider.hpp b/include/gpufl/core/counter_provider.hpp new file mode 100644 index 0000000..733038d --- /dev/null +++ b/include/gpufl/core/counter_provider.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include "gpufl/abi/gpufl_counter_abi.h" + +namespace gpufl::detail { + +/** + * @brief Binds this module to the process-wide counter runtime. + * + * Every module holding a copy of the static `gpufl` library calls this and + * ends up on the same registry, which is the whole point: without it a Python + * target ticks the extension's registry while the injected evaluator reads the + * injection DLL's, and the counter reads as Missing forever. + * + * Loaded explicitly by absolute path rather than through an import table. + * Injection arrives via CUDA_INJECTION64_PATH, so the driver loads + * gpufl_inject.dll into a process whose DLL search path has no reason to + * include our bin directory - an import entry would fail there. Resolving the + * path from THIS module's own location works wherever the module was loaded + * from. + * + * Whoever binds first loads the library; everyone after gets the same instance, + * which is what makes ordering between Python import and CUDA init irrelevant. + */ +class CounterProvider { +public: + /** @brief The provider, or nullptr if the runtime could not be loaded. */ + static const gpufl_counter_provider_v1* get(); + + /** + * @brief True when counters are shared across modules. + * + * False means the shared runtime was not found and this module fell back to + * its own in-process registry. That is correct for an embedded host, which + * has exactly one copy of gpufl, and wrong under injection, where the + * target and the evaluator are different modules. Callers that care - the + * rule evaluator - must refuse custom counter rules when this is false and + * more than one module is in play. + */ + static bool isShared(); + + /** @brief Test seam: forget the binding so the next get() resolves again. */ + static void resetForTesting(); +}; + +/** + * @brief The provider in use: the shared runtime, or this module's own. + * + * Never null, so callers do not each need a fallback branch. + */ +const gpufl_counter_provider_v1* ActiveCounterProvider(); + +} // namespace gpufl::detail diff --git a/include/gpufl/core/counter_registry.cpp b/include/gpufl/core/counter_registry.cpp index 4d7d15e..17ff1a3 100644 --- a/include/gpufl/core/counter_registry.cpp +++ b/include/gpufl/core/counter_registry.cpp @@ -79,6 +79,12 @@ std::atomic* CounterRegistry::valueSlot(SlotId slot) { return &slots_[slot].value; } +void CounterRegistry::addRaw(SlotId slot, uint64_t value) { + std::lock_guard lk(mu_); + if (slot >= slots_.size()) return; + slots_[slot].value.fetch_add(value, std::memory_order_relaxed); +} + uint64_t CounterRegistry::rawValue(SlotId slot) const { std::lock_guard lk(mu_); if (slot >= slots_.size()) return 0; diff --git a/include/gpufl/core/counter_registry.hpp b/include/gpufl/core/counter_registry.hpp index fb1ab58..966cb5d 100644 --- a/include/gpufl/core/counter_registry.hpp +++ b/include/gpufl/core/counter_registry.hpp @@ -79,6 +79,15 @@ class CounterRegistry { */ std::atomic* valueSlot(SlotId slot); + /** + * @brief Add without validating. For the C ABI boundary only. + * + * gpufl::Counter::add already rejects non-positive and absurd values before + * it gets here, and re-checking across the boundary would just duplicate a + * contract in two places that could then disagree. + */ + void addRaw(SlotId slot, uint64_t value); + /** @brief Raw slot value, including anything added before this runtime. */ uint64_t rawValue(SlotId slot) const; diff --git a/include/gpufl/core/monitor.cpp b/include/gpufl/core/monitor.cpp index 689c83e..e03b598 100644 --- a/include/gpufl/core/monitor.cpp +++ b/include/gpufl/core/monitor.cpp @@ -13,7 +13,7 @@ #include #include "gpufl/core/activity_record.hpp" -#include "gpufl/core/counter_registry.hpp" +#include "gpufl/core/counter_provider.hpp" #include "gpufl/core/common.hpp" #include "gpufl/core/debug_logger.hpp" #include "gpufl/core/logger/logger.hpp" @@ -503,7 +503,7 @@ void Monitor::Initialize(const MonitorOptions& opts) { // Counter slots live for the process, so this session has to baseline them // or it would inherit the previous session's ticks plus anything added // while gpufl was shut down. - detail::CounterRegistry::instance().beginSession(); + detail::ActiveCounterProvider()->begin_session(); g_state.batches.reset(); g_state.metadata.reset(); g_state.batches.setSourceCollectionEnabled(opts.enable_source_collection); @@ -543,7 +543,7 @@ void Monitor::Shutdown() { // Slots keep their values on purpose: a handle held across this stays // valid. What stops those adds counting twice is the baseline the next // Initialize takes, not clearing them here. - detail::CounterRegistry::instance().endSession(); + detail::ActiveCounterProvider()->end_session(); } void Monitor::DrainAndFinalizeForExit() { @@ -582,7 +582,7 @@ void Monitor::DrainAndFinalizeForExit() { // Same close-out as Shutdown(). Without it a process that exits through // this path leaves the session marked active, and an embedded host that // re-initialised afterwards would inherit its ticks. - detail::CounterRegistry::instance().endSession(); + detail::ActiveCounterProvider()->end_session(); } void Monitor::ReleaseBackendForExit() { diff --git a/include/gpufl/gpufl.hpp b/include/gpufl/gpufl.hpp index 7cecce1..29caf98 100644 --- a/include/gpufl/gpufl.hpp +++ b/include/gpufl/gpufl.hpp @@ -262,21 +262,28 @@ class Counter { // count at all: a pointer, an uninitialised field, a negative cast to // unsigned. Any of those quietly makes every rate meaningless. static constexpr int64_t kMaxAddPerCall = 1LL << 40; - // Ignores n outside (0, kMaxAddPerCall]. Two compares before one relaxed - // atomic; a single atomic add could not validate anything itself. - void add(int64_t n = 1) const { - if (slot_ && n > 0 && n <= kMaxAddPerCall) { - slot_->fetch_add(static_cast(n), std::memory_order_relaxed); - } - } - // False when the name was rejected or the counter limit was reached. add() - // on such a handle is a no-op rather than an error. - bool valid() const { return slot_ != nullptr; } + + // Ignores n outside (0, kMaxAddPerCall]. Validation happens here, on the + // caller's side of the boundary, so the contract lives in one place. + // + // This is an indirect call plus a relaxed atomic, not an inlined atomic: + // the counter lives in a shared runtime so that a profiled target and an + // injected evaluator share one registry, and passing a std::atomic across + // that boundary by address would tie both sides to one compiler's atomic + // layout. + void add(int64_t n = 1) const; + + // False when the name was rejected, the counter limit was reached, or no + // counter runtime could be bound. add() on such a handle is a no-op rather + // than an error. + bool valid() const { return handle_ != nullptr; } private: friend Counter counter(const std::string&); - explicit Counter(std::atomic* slot) : slot_(slot) {} - std::atomic* slot_ = nullptr; + Counter(const void* provider, void* handle) + : provider_(provider), handle_(handle) {} + const void* provider_ = nullptr; // gpufl_counter_provider_v1* + void* handle_ = nullptr; // gpufl_counter_handle }; // Registers (or finds) a counter. Names are 1-96 characters of [A-Za-z0-9._-]; diff --git a/runtime/counter_runtime.cpp b/runtime/counter_runtime.cpp new file mode 100644 index 0000000..e6d4a2b --- /dev/null +++ b/runtime/counter_runtime.cpp @@ -0,0 +1,80 @@ +// The shared counter runtime. One instance per process, so a target that ticks +// a counter and an injected evaluator that reads it are talking about the same +// slot - see gpufl_counter_abi.h for why that cannot be assumed otherwise. +// +// Deliberately thin: it owns CounterRegistry and wraps it in C. Keeping the +// logic in the registry means the embedded fallback path and this one cannot +// drift apart in behaviour. + +#include "gpufl/abi/gpufl_counter_abi.h" + +#include + +#include "gpufl/core/counter_registry.hpp" + +namespace { + +using gpufl::detail::CounterRegistry; + +// Handles are slot ids biased by one, cast to a pointer, rather than addresses +// into the registry. An id survives anything the container does, and NULL then +// falls out naturally as "invalid" without needing a sentinel address. +gpufl_counter_handle ToHandle(const CounterRegistry::SlotId slot) { + if (slot == CounterRegistry::kInvalidSlot) return nullptr; + return reinterpret_cast( + static_cast(slot) + 1u); +} + +bool FromHandle(const gpufl_counter_handle handle, CounterRegistry::SlotId* out) { + if (handle == nullptr) return false; + *out = static_cast( + reinterpret_cast(handle) - 1u); + return true; +} + +gpufl_counter_handle RegisterCounter(const char* name, const size_t name_length) { + if (name == nullptr) return nullptr; + return ToHandle(CounterRegistry::instance().registerCounter( + std::string(name, name_length))); +} + +void Add(const gpufl_counter_handle handle, const uint64_t value) { + CounterRegistry::SlotId slot; + if (!FromHandle(handle, &slot)) return; + CounterRegistry::instance().addRaw(slot, value); +} + +uint64_t Load(const gpufl_counter_handle handle) { + CounterRegistry::SlotId slot; + if (!FromHandle(handle, &slot)) return 0; + return CounterRegistry::instance().rawValue(slot); +} + +uint64_t LoadSinceBaseline(const gpufl_counter_handle handle) { + CounterRegistry::SlotId slot; + if (!FromHandle(handle, &slot)) return 0; + return CounterRegistry::instance().valueSinceBaseline(slot); +} + +void BeginSession() { CounterRegistry::instance().beginSession(); } +void EndSession() { CounterRegistry::instance().endSession(); } +int SessionActive() { return CounterRegistry::instance().sessionActive() ? 1 : 0; } + +const gpufl_counter_provider_v1 kProvider = { + GPUFL_COUNTER_ABI_VERSION, + sizeof(gpufl_counter_provider_v1), + &RegisterCounter, + &Add, + &Load, + &LoadSinceBaseline, + &BeginSession, + &EndSession, + &SessionActive, +}; + +} // namespace + +extern "C" GPUFL_COUNTER_EXPORT const gpufl_counter_provider_v1* +gpufl_get_counter_provider_v1(void) { + return &kProvider; +} diff --git a/tests/core/test_counter_registry.cpp b/tests/core/test_counter_registry.cpp index e48e9f4..e4851ff 100644 --- a/tests/core/test_counter_registry.cpp +++ b/tests/core/test_counter_registry.cpp @@ -5,8 +5,10 @@ #include "gpufl.hpp" #include "gpufl/core/counter_registry.hpp" +#include "gpufl/core/counter_provider.hpp" #include "gpufl/core/monitor.hpp" +#include #include using gpufl::detail::CounterRegistry; @@ -23,6 +25,31 @@ class CounterRegistryTest : public ::testing::Test { } // namespace +namespace { + +const gpufl_counter_provider_v1* prov() { + return gpufl::detail::ActiveCounterProvider(); +} + +// Read back through the ACTIVE provider, not the local registry. With a shared +// runtime present, gpufl::counter() writes into the runtime's registry and this +// module's own is untouched - reading the latter would compare two different +// registries and fail for the right reason in the wrong test. +uint64_t Raw(const char* name) { + auto h = prov()->register_counter(name, std::strlen(name)); + return h ? prov()->load(h) : 0; +} + +uint64_t Since(const char* name) { + auto h = prov()->register_counter(name, std::strlen(name)); + return h ? prov()->load_since_baseline(h) : 0; +} + +void BeginSession() { prov()->begin_session(); } +void EndSession() { prov()->end_session(); } + +} // namespace + TEST_F(CounterRegistryTest, SameNameResolvesToOneSlot) { const auto a = reg().registerCounter("token"); const auto b = reg().registerCounter("token"); @@ -50,12 +77,12 @@ TEST_F(CounterRegistryTest, CardinalityIsCapped) { } TEST_F(CounterRegistryTest, NonPositiveAddsAreIgnored) { - auto tokens = gpufl::counter("token"); + auto tokens = gpufl::counter("nonpositive_probe"); ASSERT_TRUE(tokens.valid()); tokens.add(5); tokens.add(0); tokens.add(-100); // must not wrap the unsigned counter - EXPECT_EQ(reg().rawValue(reg().registerCounter("token")), 5u); + EXPECT_EQ(Raw("nonpositive_probe"), 5u); } TEST_F(CounterRegistryTest, InvalidHandleAddIsANoOp) { @@ -67,36 +94,34 @@ TEST_F(CounterRegistryTest, InvalidHandleAddIsANoOp) { TEST_F(CounterRegistryTest, AddsBeforeASessionAreExcludedFromIt) { // Slots outlive the runtime, so anything ticked before init() is already in // the slot. Without a baseline the new session would count it as its own. - auto tokens = gpufl::counter("token"); + auto tokens = gpufl::counter("pre_session_probe"); tokens.add(1000); - const auto slot = reg().registerCounter("token"); - EXPECT_EQ(reg().rawValue(slot), 1000u); + EXPECT_EQ(Raw("pre_session_probe"), 1000u); - reg().beginSession(); - EXPECT_EQ(reg().valueSinceBaseline(slot), 0u) << "pre-init ticks are not this session's"; + BeginSession(); + EXPECT_EQ(Since("pre_session_probe"), 0u) << "pre-init ticks are not this session's"; tokens.add(7); - EXPECT_EQ(reg().valueSinceBaseline(slot), 7u); + EXPECT_EQ(Since("pre_session_probe"), 7u); } TEST_F(CounterRegistryTest, AddsBetweenSessionsAreExcludedFromTheNext) { auto tokens = gpufl::counter("token"); - const auto slot = reg().registerCounter("token"); - reg().beginSession(); + BeginSession(); tokens.add(10); - EXPECT_EQ(reg().valueSinceBaseline(slot), 10u); - reg().endSession(); + EXPECT_EQ(Since("token"), 10u); + EndSession(); // gpufl is down; the handle still works and the slot still accumulates. tokens.add(500); - reg().beginSession(); - EXPECT_EQ(reg().valueSinceBaseline(slot), 0u) + BeginSession(); + EXPECT_EQ(Since("token"), 0u) << "ticks while no runtime was active belong to no session"; tokens.add(3); - EXPECT_EQ(reg().valueSinceBaseline(slot), 3u); + EXPECT_EQ(Since("token"), 3u); } TEST_F(CounterRegistryTest, HandleSurvivesASessionChange) { @@ -105,21 +130,20 @@ TEST_F(CounterRegistryTest, HandleSurvivesASessionChange) { // number alone could not make that safe, since it cannot stop the state // being freed underneath a concurrent add(). auto tokens = gpufl::counter("token"); - const auto slot = reg().registerCounter("token"); - reg().beginSession(); - reg().endSession(); - reg().beginSession(); + BeginSession(); + EndSession(); + BeginSession(); tokens.add(4); - EXPECT_EQ(reg().valueSinceBaseline(slot), 4u); + EXPECT_EQ(Since("token"), 4u); } TEST_F(CounterRegistryTest, CounterRegisteredMidSessionStartsAtZero) { - reg().beginSession(); + BeginSession(); auto late = gpufl::counter("late"); late.add(9); - EXPECT_EQ(reg().valueSinceBaseline(reg().registerCounter("late")), 9u); + EXPECT_EQ(Since("late"), 9u); } TEST_F(CounterRegistryTest, ConcurrentRegistrationOfOneNameYieldsOneSlot) { @@ -137,8 +161,7 @@ TEST_F(CounterRegistryTest, ConcurrentRegistrationOfOneNameYieldsOneSlot) { TEST_F(CounterRegistryTest, ConcurrentAddsAreNotLost) { auto tokens = gpufl::counter("token"); - const auto slot = reg().registerCounter("token"); - reg().beginSession(); + BeginSession(); constexpr int kThreads = 8; constexpr int kPerThread = 10'000; @@ -150,14 +173,14 @@ TEST_F(CounterRegistryTest, ConcurrentAddsAreNotLost) { } for (auto& t : threads) t.join(); - EXPECT_EQ(reg().valueSinceBaseline(slot), + EXPECT_EQ(Since("token"), static_cast(kThreads) * kPerThread); } TEST_F(CounterRegistryTest, TickIsEquivalentButLooksThePriceUpEveryCall) { gpufl::tick("steps"); gpufl::tick("steps", 4); - EXPECT_EQ(reg().rawValue(reg().registerCounter("steps")), 5u); + EXPECT_EQ(Raw("steps"), 5u); } // ── lifecycle wiring ──────────────────────────────────────────────────────── @@ -168,31 +191,29 @@ TEST_F(CounterRegistryTest, TickIsEquivalentButLooksThePriceUpEveryCall) { TEST_F(CounterRegistryTest, MonitorInitializeBaselinesThroughTheRealWiring) { auto tokens = gpufl::counter("token"); - const auto slot = reg().registerCounter("token"); tokens.add(1000); // before any session exists gpufl::MonitorOptions opts; gpufl::Monitor::Initialize(opts); - EXPECT_EQ(reg().valueSinceBaseline(slot), 0u) + EXPECT_EQ(Since("token"), 0u) << "Monitor::Initialize must baseline the registry"; - EXPECT_TRUE(reg().sessionActive()); + EXPECT_TRUE(prov()->session_active() != 0); tokens.add(6); - EXPECT_EQ(reg().valueSinceBaseline(slot), 6u); + EXPECT_EQ(Since("token"), 6u); gpufl::Monitor::Shutdown(); - EXPECT_FALSE(reg().sessionActive()) + EXPECT_FALSE(prov()->session_active() != 0) << "Monitor::Shutdown must close the session"; } TEST_F(CounterRegistryTest, TicksBetweenTwoMonitorSessionsBelongToNeither) { auto tokens = gpufl::counter("token"); - const auto slot = reg().registerCounter("token"); gpufl::MonitorOptions opts; gpufl::Monitor::Initialize(opts); tokens.add(10); - EXPECT_EQ(reg().valueSinceBaseline(slot), 10u); + EXPECT_EQ(Since("token"), 10u); gpufl::Monitor::Shutdown(); // gpufl is down. The handle still works - the slot is process-lifetime - @@ -200,9 +221,9 @@ TEST_F(CounterRegistryTest, TicksBetweenTwoMonitorSessionsBelongToNeither) { tokens.add(500); gpufl::Monitor::Initialize(opts); - EXPECT_EQ(reg().valueSinceBaseline(slot), 0u); + EXPECT_EQ(Since("token"), 0u); tokens.add(3); - EXPECT_EQ(reg().valueSinceBaseline(slot), 3u); + EXPECT_EQ(Since("token"), 3u); gpufl::Monitor::Shutdown(); } @@ -212,36 +233,68 @@ TEST_F(CounterRegistryTest, ProcessExitPathAlsoClosesTheSession) { // previous session's ticks. gpufl::MonitorOptions opts; gpufl::Monitor::Initialize(opts); - ASSERT_TRUE(reg().sessionActive()); + ASSERT_TRUE(prov()->session_active() != 0); gpufl::Monitor::DrainAndFinalizeForExit(); - EXPECT_FALSE(reg().sessionActive()); + EXPECT_FALSE(prov()->session_active() != 0); } TEST_F(CounterRegistryTest, AddAboveThePerCallBoundIsRefused) { // Not overflow protection: a value this large is a caller bug - a pointer, // an uninitialised field - and letting it through makes every later rate // meaningless. - auto tokens = gpufl::counter("token"); - const auto slot = reg().registerCounter("token"); + auto tokens = gpufl::counter("bound_probe"); tokens.add(gpufl::Counter::kMaxAddPerCall); tokens.add(gpufl::Counter::kMaxAddPerCall + 1); // refused - EXPECT_EQ(reg().rawValue(slot), + EXPECT_EQ(Raw("bound_probe"), static_cast(gpufl::Counter::kMaxAddPerCall)); } TEST_F(CounterRegistryTest, DeltaIsCorrectAcrossAWrap) { // Rates are unsigned deltas precisely so a wrap needs no saturation or CAS - // loop. Drive the counter to just below the wrap and cross it. - auto tokens = gpufl::counter("token"); - const auto slot = reg().registerCounter("token"); + // loop. Driven against the in-module registry: reaching the wrap needs the + // slot's address, which the C ABI deliberately does not hand out. + const auto slot = reg().registerCounter("wrap_probe"); auto* raw = reg().valueSlot(slot); ASSERT_NE(raw, nullptr); raw->store(std::numeric_limits::max() - 5, std::memory_order_relaxed); reg().beginSession(); - tokens.add(10); // wraps past zero + reg().addRaw(slot, 10); // wraps past zero EXPECT_EQ(reg().valueSinceBaseline(slot), 10u) << "unsigned subtraction must stay correct across a wrap"; } + +// ── shared runtime binding ────────────────────────────────────────────────── + +TEST_F(CounterRegistryTest, BindsToTheSharedRuntimeWhenItIsPresent) { + // The cross-module property this whole ABI exists for cannot be proven from + // inside one executable - that needs the launcher + Python E2E. What can be + // checked here is that binding works at all when the library is reachable, + // and that the fallback is taken (rather than crashing) when it is not. + gpufl::detail::CounterProvider::resetForTesting(); + const auto* provider = gpufl::detail::CounterProvider::get(); + + if (provider == nullptr) { + GTEST_SKIP() << "gpufl_counter_runtime not colocated with the test binary; " + "fallback path exercised by every other test here"; + } + EXPECT_EQ(provider->abi_version, GPUFL_COUNTER_ABI_VERSION); + EXPECT_GE(provider->struct_size, sizeof(gpufl_counter_provider_v1)); + EXPECT_TRUE(gpufl::detail::CounterProvider::isShared()); + + // Round-trip through the C ABI rather than the registry directly. + auto handle = provider->register_counter("abi_probe", 9); + ASSERT_NE(handle, nullptr); + provider->begin_session(); + provider->add(handle, 7); + EXPECT_EQ(provider->load_since_baseline(handle), 7u); + provider->end_session(); +} + +TEST_F(CounterRegistryTest, ActiveProviderIsNeverNull) { + // Callers must not each carry a fallback branch, so this holds whether or + // not the shared runtime is present. + EXPECT_NE(gpufl::detail::ActiveCounterProvider(), nullptr); +} From 467fe7b1efc503b6becc2dd4792152188c0a3602 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Sun, 26 Jul 2026 19:45:43 -0700 Subject: [PATCH 04/28] build(counter): ship the counter runtime beside every consumer The provider resolves the runtime from its OWN module's directory, so the library has to be there. Injection is where this matters: the CUDA driver maps gpufl_inject.dll into the profiled target via CUDA_INJECTION64_PATH, and that process's DLL search path has no reason to include our bin directory. Copies now land beside gpufl_inject.dll, beside gpufl.exe, and beside the Python extension, and the runtime is installed into the wheel. Several copies on disk are harmless: the provider prefers an already-mapped module over any path, so whichever consumer binds first loads it for the process and the rest attach to that one. Verified in a real process rather than by inspection. The Python extension ticks a counter and a separate binder - ctypes loading the runtime and calling only through the C ABI - reads back the same value, which a private per-module registry could not produce. Under the launcher, an injected gpufl_inject.dll resolves the runtime from its own directory inside a foreign process, which is the Windows search-path concern that ruled out an import-table dependency. scripts/counter_cross_module_check.py is that check, kept so it can be re-run on other machines. Still open: nothing on the injected side reads counters yet, so the full target-ticks / evaluator-reads path only closes once the rule evaluator exists. --- CMakeLists.txt | 21 +++++++++++++ daemon/launcher/CMakeLists.txt | 12 ++++++++ scripts/counter_cross_module_check.py | 43 +++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 scripts/counter_cross_module_check.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d7c233..fb35227 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -654,6 +654,17 @@ if(BUILD_GPUFL_INJECT AND ((UNIX AND NOT APPLE) OR WIN32)) include/gpufl/inject/inject_entry.cpp ) target_link_libraries(gpufl_inject PRIVATE gpufl::gpufl) + + # The injected DLL is loaded by the CUDA driver into the profiled target, + # whose DLL search path has no reason to include our bin directory. The + # provider resolves the runtime from the injection DLL's own location, so + # it has to be there. + add_custom_command(TARGET gpufl_inject POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMENT "Colocating gpufl_counter_runtime with gpufl_inject" + ) if(CUDAToolkit_INCLUDE_DIRS) target_include_directories(gpufl_inject PRIVATE ${CUDAToolkit_INCLUDE_DIRS}) endif() @@ -776,6 +787,16 @@ if(BUILD_PYTHON) target_link_libraries(_gpufl_client PRIVATE gpufl::gpufl) + # Same for the Python extension: it may be the first module to bind, in + # which case it is the one that loads the runtime for the whole process. + add_custom_command(TARGET _gpufl_client POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMENT "Colocating gpufl_counter_runtime with _gpufl_client" + ) + install(TARGETS gpufl_counter_runtime DESTINATION gpufl) + # If CUDA is available, link it to the Python module if(GPUFL_HAS_CUDA) target_link_libraries(_gpufl_client PRIVATE CUDA::cudart) diff --git a/daemon/launcher/CMakeLists.txt b/daemon/launcher/CMakeLists.txt index e95b5cd..cbf9059 100644 --- a/daemon/launcher/CMakeLists.txt +++ b/daemon/launcher/CMakeLists.txt @@ -74,6 +74,18 @@ if(WIN32 AND TARGET gpufl_inject) COMMENT "Colocating gpufl_inject.dll next to gpufl.exe") endif() +# The counter runtime travels with the inject DLL. Once injected, gpufl_inject +# resolves it from its OWN directory - the target process's search path is not +# ours to rely on - so it has to be wherever the inject DLL ended up. +if(WIN32 AND TARGET gpufl_counter_runtime) + add_dependencies(gpufl_launcher gpufl_counter_runtime) + add_custom_command(TARGET gpufl_launcher POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMENT "Colocating gpufl_counter_runtime.dll next to gpufl.exe") +endif() + # On Windows, copy CUPTI / NVPERF DLLs next to gpufl.exe (= beside the # colocated gpufl_inject.dll) so the inject DLL's dependencies resolve when # the driver loads it into the target. trace_command_win.cpp also prepends diff --git a/scripts/counter_cross_module_check.py b/scripts/counter_cross_module_check.py new file mode 100644 index 0000000..a49aeea --- /dev/null +++ b/scripts/counter_cross_module_check.py @@ -0,0 +1,43 @@ +# Proves the counter registry is shared across module boundaries. +# +# gpufl is a static library, so the Python extension and gpufl_inject.dll each +# hold their own copy of it. Without the shared runtime they would also hold +# their own counter registries, and a target ticking a counter would be +# invisible to the injected evaluator - the case counters exist for. +# +# Run standalone, or under the launcher: +# +# set GPUFL_REPO= +# python scripts/counter_cross_module_check.py +# gpufl trace -o out --passes Trace -- python scripts/counter_cross_module_check.py +# +# Expected: "value_via_abi=42". A private registry in the extension reads 0. +import ctypes, os, sys, pathlib +sys.path.insert(0, str(pathlib.Path(os.environ["GPUFL_REPO"]) / "python")) +import gpufl + +tokens = gpufl.counter("xmod_token") +tokens.add(41) +tokens.add(1) + +# Read the slot back through the runtime DLL directly - a third binder, using +# only the C ABI. If the extension had its own registry this would read 0. +lib = ctypes.CDLL("gpufl_counter_runtime.dll") +lib.gpufl_get_counter_provider_v1.restype = ctypes.c_void_p +p = lib.gpufl_get_counter_provider_v1() +if not p: + print("XMOD: no provider"); sys.exit(1) + +class Provider(ctypes.Structure): + _fields_ = [("abi_version", ctypes.c_uint32), ("struct_size", ctypes.c_uint32), + ("register_counter", ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t)), + ("add", ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_uint64)), + ("load", ctypes.CFUNCTYPE(ctypes.c_uint64, ctypes.c_void_p)), + ("load_since_baseline", ctypes.CFUNCTYPE(ctypes.c_uint64, ctypes.c_void_p)), + ("begin_session", ctypes.CFUNCTYPE(None)), + ("end_session", ctypes.CFUNCTYPE(None)), + ("session_active", ctypes.CFUNCTYPE(ctypes.c_int))] +prov = ctypes.cast(p, ctypes.POINTER(Provider)).contents +name = b"xmod_token" +h = prov.register_counter(name, len(name)) +print("XMOD: abi=%d value_via_abi=%d" % (prov.abi_version, prov.load(h))) From 0b75ec7e7f0f3a5eb06a30ae3aab45b8b771398d Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Sun, 26 Jul 2026 20:12:02 -0700 Subject: [PATCH 05/28] feat(metrics): freshness-aware metric sources for rule triggers Rules need readings they can treat as evidence. System metrics publish every 100-500ms while the evaluator runs at ~1ms, so a naive read would count one sample hundreds of times and every rule would fire on its first true reading. Each sample carries two timestamps. Publication time alone can never go stale - it advances forever even during a total stall, which is the condition a rule most needs to catch - so staleness is measured against the source instead. That lets zeros accumulate as fresh evidence and still detects a dead source. An empty window does not mean zero for every metric. A rate over no events is a real 0; a percentile over nothing is not, and publishing 0 ms would read as instantaneous kernels rather than no kernels. Config is validated as a combination, not field by field: stale_after must outlast rate_window + sustained + bucket, or the rule goes stale before it can ever accumulate the evidence to fire. The `custom.` prefix makes a typo a parse error rather than a rule that waits forever for a counter that will never exist. Counter lookup is added to the ABI so a rule naming a counter does not create it - otherwise "never registered" and "registered but idle" become indistinguishable, and they need different answers. Seven of the tests were each confirmed to fail with their own fix reverted. --- CMakeLists.txt | 2 + include/gpufl/abi/gpufl_counter_abi.h | 12 + include/gpufl/core/counter_api.cpp | 8 + include/gpufl/core/counter_registry.cpp | 6 + include/gpufl/core/counter_registry.hpp | 10 + include/gpufl/core/metric_id.cpp | 271 ++++++++++++++++ include/gpufl/core/metric_id.hpp | 150 +++++++++ include/gpufl/core/metric_registry.cpp | 347 ++++++++++++++++++++ include/gpufl/core/metric_registry.hpp | 234 ++++++++++++++ runtime/counter_runtime.cpp | 7 + tests/CMakeLists.txt | 1 + tests/core/test_metric_registry.cpp | 406 ++++++++++++++++++++++++ 12 files changed, 1454 insertions(+) create mode 100644 include/gpufl/core/metric_id.cpp create mode 100644 include/gpufl/core/metric_id.hpp create mode 100644 include/gpufl/core/metric_registry.cpp create mode 100644 include/gpufl/core/metric_registry.hpp create mode 100644 tests/core/test_metric_registry.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fb35227..5fe59e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -125,6 +125,8 @@ target_sources(gpufl PRIVATE include/gpufl/core/counter_registry.cpp include/gpufl/core/counter_api.cpp include/gpufl/core/counter_provider.cpp + include/gpufl/core/metric_id.cpp + include/gpufl/core/metric_registry.cpp include/gpufl/core/monitor_batch_manager.cpp include/gpufl/core/monitor_record_builders.cpp include/gpufl/core/monitor.cpp diff --git a/include/gpufl/abi/gpufl_counter_abi.h b/include/gpufl/abi/gpufl_counter_abi.h index 04c0669..649cd67 100644 --- a/include/gpufl/abi/gpufl_counter_abi.h +++ b/include/gpufl/abi/gpufl_counter_abi.h @@ -70,6 +70,18 @@ typedef struct gpufl_counter_provider_v1 { void (*begin_session)(void); void (*end_session)(void); int (*session_active)(void); + + /* + * Find WITHOUT creating. Returns NULL when the application has not + * registered this counter. + * + * A rule naming a counter must not bring it into existence: doing so makes + * "the application never registered it" indistinguishable from "registered + * but never ticked", and those need different answers - the first is a + * config mistake worth reporting, the second is a workload that is simply + * idle. + */ + gpufl_counter_handle (*lookup)(const char* name, size_t name_length); } gpufl_counter_provider_v1; /* diff --git a/include/gpufl/core/counter_api.cpp b/include/gpufl/core/counter_api.cpp index 48b481d..187091c 100644 --- a/include/gpufl/core/counter_api.cpp +++ b/include/gpufl/core/counter_api.cpp @@ -45,6 +45,13 @@ uint64_t LocalLoadSinceBaseline(gpufl_counter_handle h) { return LocalSlot(h, &slot) ? CounterRegistry::instance().valueSinceBaseline(slot) : 0; } +gpufl_counter_handle LocalLookup(const char* name, size_t len) { + if (name == nullptr) return nullptr; + const auto slot = CounterRegistry::instance().findCounter(std::string(name, len)); + if (slot == CounterRegistry::kInvalidSlot) return nullptr; + return reinterpret_cast(static_cast(slot) + 1u); +} + void LocalBeginSession() { CounterRegistry::instance().beginSession(); } void LocalEndSession() { CounterRegistry::instance().endSession(); } int LocalSessionActive() { return CounterRegistry::instance().sessionActive() ? 1 : 0; } @@ -54,6 +61,7 @@ const gpufl_counter_provider_v1 kLocalProvider = { sizeof(gpufl_counter_provider_v1), &LocalRegister, &LocalAdd, &LocalLoad, &LocalLoadSinceBaseline, &LocalBeginSession, &LocalEndSession, &LocalSessionActive, + &LocalLookup, }; } // namespace diff --git a/include/gpufl/core/counter_registry.cpp b/include/gpufl/core/counter_registry.cpp index 17ff1a3..6f4b416 100644 --- a/include/gpufl/core/counter_registry.cpp +++ b/include/gpufl/core/counter_registry.cpp @@ -33,6 +33,12 @@ std::string CounterRegistry::forLog(const std::string& name) { return name.substr(0, kMaxLogged) + "...(" + std::to_string(name.size()) + " chars)"; } +CounterRegistry::SlotId CounterRegistry::findCounter(const std::string& name) const { + std::lock_guard lk(mu_); + const auto it = byName_.find(name); + return it == byName_.end() ? kInvalidSlot : it->second; +} + CounterRegistry::SlotId CounterRegistry::registerCounter(const std::string& name) { if (!nameIsValid(name)) { std::lock_guard lk(mu_); diff --git a/include/gpufl/core/counter_registry.hpp b/include/gpufl/core/counter_registry.hpp index 966cb5d..67499b0 100644 --- a/include/gpufl/core/counter_registry.hpp +++ b/include/gpufl/core/counter_registry.hpp @@ -69,6 +69,16 @@ class CounterRegistry { */ SlotId registerCounter(const std::string& name); + /** + * @brief Find @p name WITHOUT creating it. kInvalidSlot if absent. + * + * For readers that must not bring a counter into existence by asking about + * it - a rule naming a counter is a question, not a registration, and + * conflating the two hides the difference between a config typo and an idle + * workload. + */ + SlotId findCounter(const std::string& name) const; + /** * @brief Stable address of a slot's value, or nullptr for kInvalidSlot. * diff --git a/include/gpufl/core/metric_id.cpp b/include/gpufl/core/metric_id.cpp new file mode 100644 index 0000000..6a3cff2 --- /dev/null +++ b/include/gpufl/core/metric_id.cpp @@ -0,0 +1,271 @@ +#include "gpufl/core/metric_id.hpp" + +#include +#include +#include + +#include "gpufl/core/counter_registry.hpp" + +namespace gpufl::detail { +namespace { + +constexpr const char* kCustomPrefix = "custom."; +constexpr const char* kRateSuffix = "_rate"; + +bool startsWith(const std::string& s, const char* prefix) { + const size_t n = std::char_traits::length(prefix); + return s.size() >= n && s.compare(0, n, prefix) == 0; +} + +bool endsWith(const std::string& s, const char* suffix) { + const size_t n = std::char_traits::length(suffix); + return s.size() >= n && s.compare(s.size() - n, n, suffix) == 0; +} + +/** + * Same charset the counter registry enforces. Shared on purpose: a name the + * registry would refuse must not parse into a rule that then waits forever for + * a counter that can never exist. + */ +bool customNameCharsetOk(const std::string& name) { + for (const char c : name) { + const bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '.' || c == '_' || c == '-'; + if (!ok) return false; + } + return true; +} + +/** Parse `gpu[N].`. Returns false if the shape does not match at all. */ +bool parseDeviceMetric(const std::string& text, MetricParseResult* out) { + if (!startsWith(text, "gpu[")) return false; + + const size_t close = text.find(']'); + if (close == std::string::npos || close == 4 || text.size() <= close + 1 || + text[close + 1] != '.') { + out->error = MetricParseError::MalformedDeviceIndex; + return true; + } + + int64_t index = 0; + for (size_t i = 4; i < close; ++i) { + const char c = text[i]; + if (c < '0' || c > '9') { + out->error = MetricParseError::MalformedDeviceIndex; + return true; + } + index = index * 10 + (c - '0'); + if (index > 4095) { // no plausible host, and keeps the parse bounded + out->error = MetricParseError::MalformedDeviceIndex; + return true; + } + } + + const std::string field = text.substr(close + 2); + if (field == "util_pct") { + out->id.kind = MetricKind::GpuUtilPct; + } else if (field == "power_mw") { + out->id.kind = MetricKind::GpuPowerMw; + } else if (field == "sm_clock_mhz") { + out->id.kind = MetricKind::GpuSmClockMhz; + } else { + out->error = MetricParseError::UnknownBuiltinMetric; + return true; + } + + out->id.device_index = static_cast(index); + // Canonicalised so gpu[00] and gpu[0] cannot hash to two different rule ids. + char buf[64]; + std::snprintf(buf, sizeof(buf), "gpu[%d].%s", out->id.device_index, field.c_str()); + out->id.canonical = buf; + return true; +} + +} // namespace + +MetricShape MetricId::shape() const { + switch (kind) { + case MetricKind::KernelLaunchRate: + case MetricKind::CustomRate: + return MetricShape::Rate; + case MetricKind::RecentKernelMs: + return MetricShape::Percentile; + case MetricKind::GpuUtilPct: + case MetricKind::GpuPowerMw: + case MetricKind::GpuSmClockMhz: + break; + } + return MetricShape::Gauge; +} + +const char* toString(MetricKind k) { + switch (k) { + case MetricKind::GpuUtilPct: return "gpu_util_pct"; + case MetricKind::GpuPowerMw: return "gpu_power_mw"; + case MetricKind::GpuSmClockMhz: return "gpu_sm_clock_mhz"; + case MetricKind::KernelLaunchRate: return "kernel_launch_rate"; + case MetricKind::RecentKernelMs: return "recent_kernel_ms"; + case MetricKind::CustomRate: return "custom_rate"; + } + return "unknown"; +} + +const char* toString(MetricParseError e) { + switch (e) { + case MetricParseError::None: return "ok"; + case MetricParseError::Empty: return "empty_metric_name"; + case MetricParseError::UnknownBuiltinMetric: return "unknown_builtin_metric"; + case MetricParseError::MissingCustomPrefix: return "missing_custom_prefix"; + case MetricParseError::MalformedDeviceIndex: return "malformed_device_index"; + case MetricParseError::MalformedCustomMetric: return "malformed_custom_metric"; + case MetricParseError::CustomNameTooLong: return "custom_name_too_long"; + case MetricParseError::CustomNameCharset: return "custom_name_charset"; + } + return "unknown"; +} + +MetricParseResult parseMetric(const std::string& text) { + MetricParseResult out; + + if (text.empty()) { + out.error = MetricParseError::Empty; + return out; + } + + if (parseDeviceMetric(text, &out)) return out; + + if (text == "kernel_launch_rate") { + out.id.kind = MetricKind::KernelLaunchRate; + out.id.canonical = text; + return out; + } + if (text == "recent_kernel_ms") { + out.id.kind = MetricKind::RecentKernelMs; + out.id.canonical = text; + return out; + } + + if (startsWith(text, kCustomPrefix)) { + const std::string body = + text.substr(std::char_traits::length(kCustomPrefix)); + if (!endsWith(body, kRateSuffix)) { + out.error = MetricParseError::MalformedCustomMetric; + return out; + } + const std::string name = + body.substr(0, body.size() - std::char_traits::length(kRateSuffix)); + if (name.empty()) { + out.error = MetricParseError::MalformedCustomMetric; + return out; + } + if (name.size() > CounterRegistry::kMaxNameLength) { + out.error = MetricParseError::CustomNameTooLong; + return out; + } + if (!customNameCharsetOk(name)) { + out.error = MetricParseError::CustomNameCharset; + return out; + } + out.id.kind = MetricKind::CustomRate; + out.id.custom_name = name; + out.id.canonical = std::string(kCustomPrefix) + name + kRateSuffix; + return out; + } + + // Anything left is either a misspelled built-in or a custom counter written + // without its prefix. Both are config errors, and both are caught here + // rather than at shutdown, which is the entire reason the prefix exists. + out.error = MetricParseError::MissingCustomPrefix; + return out; +} + +int64_t MetricWindowConfig::bucketIntervalMs() const { + if (rate_window_ms <= 0) return 10; + return std::max(10, std::min(100, rate_window_ms / 10)); +} + +int64_t MetricWindowConfig::bucketCount() const { + const int64_t bucket = bucketIntervalMs(); + if (rate_window_ms <= 0) return 1; + return std::max(1, (rate_window_ms + bucket - 1) / bucket); +} + +const char* toString(ConfigError e) { + switch (e) { + case ConfigError::None: return "ok"; + case ConfigError::RateWindowNotPositive: return "rate_window_not_positive"; + case ConfigError::RateWindowTooLarge: return "rate_window_too_large"; + case ConfigError::SustainedNegative: return "sustained_negative"; + case ConfigError::StaleAfterNotPositive: return "stale_after_not_positive"; + case ConfigError::StaleBeforeEvidence: return "stale_before_evidence"; + } + return "unknown"; +} + +ConfigError validate(const MetricWindowConfig& cfg) { + if (cfg.rate_window_ms <= 0) return ConfigError::RateWindowNotPositive; + if (cfg.rate_window_ms > MetricWindowConfig::kMaxRateWindowMs) { + return ConfigError::RateWindowTooLarge; + } + if (cfg.sustained_ms < 0) return ConfigError::SustainedNegative; + if (cfg.stale_after_ms <= 0) return ConfigError::StaleAfterNotPositive; + + // Overflow-safe: the sum is the point of the check, so it must not be the + // thing that breaks it. + constexpr int64_t kMax = std::numeric_limits::max(); + int64_t need = cfg.rate_window_ms; + if (cfg.sustained_ms > kMax - need) return ConfigError::StaleBeforeEvidence; + need += cfg.sustained_ms; + const int64_t bucket = cfg.bucketIntervalMs(); + if (bucket > kMax - need) return ConfigError::StaleBeforeEvidence; + need += bucket; + + if (cfg.stale_after_ms < need) return ConfigError::StaleBeforeEvidence; + return ConfigError::None; +} + +std::string explain(const MetricWindowConfig& cfg, const ConfigError e) { + char buf[320]; + switch (e) { + case ConfigError::None: + return "ok"; + case ConfigError::RateWindowNotPositive: + std::snprintf(buf, sizeof(buf), + "rate window must be > 0 (got %lldms)", + static_cast(cfg.rate_window_ms)); + break; + case ConfigError::RateWindowTooLarge: + std::snprintf(buf, sizeof(buf), + "rate window %lldms exceeds the %lldms limit", + static_cast(cfg.rate_window_ms), + static_cast(MetricWindowConfig::kMaxRateWindowMs)); + break; + case ConfigError::SustainedNegative: + std::snprintf(buf, sizeof(buf), + "sustained must be >= 0 (got %lldms)", + static_cast(cfg.sustained_ms)); + break; + case ConfigError::StaleAfterNotPositive: + std::snprintf(buf, sizeof(buf), + "stale-after must be > 0 (got %lldms)", + static_cast(cfg.stale_after_ms)); + break; + case ConfigError::StaleBeforeEvidence: + // Spell out the arithmetic: the fields are individually reasonable + // and the reader has no way to see why the combination cannot fire. + std::snprintf( + buf, sizeof(buf), + "stale-after %lldms is too short to ever fire: needs >= rate " + "window %lld + sustained %lld + bucket %lld = %lldms", + static_cast(cfg.stale_after_ms), + static_cast(cfg.rate_window_ms), + static_cast(cfg.sustained_ms), + static_cast(cfg.bucketIntervalMs()), + static_cast(cfg.rate_window_ms + cfg.sustained_ms + + cfg.bucketIntervalMs())); + break; + } + return buf; +} + +} // namespace gpufl::detail diff --git a/include/gpufl/core/metric_id.hpp b/include/gpufl/core/metric_id.hpp new file mode 100644 index 0000000..6560d6c --- /dev/null +++ b/include/gpufl/core/metric_id.hpp @@ -0,0 +1,150 @@ +#pragma once + +#include +#include + +namespace gpufl::detail { + +/** + * @brief What a rule is allowed to watch. + * + * Deliberately small. Every entry is something gpufl knows and the application + * does not - otherwise the caller may as well write the `if` themselves, which + * is the alternative this feature has to beat. + */ +enum class MetricKind { + GpuUtilPct, ///< gpu[N].util_pct + GpuPowerMw, ///< gpu[N].power_mw + GpuSmClockMhz, ///< gpu[N].sm_clock_mhz + KernelLaunchRate, ///< kernel_launch_rate, launches/s at the HOST API + RecentKernelMs, ///< recent_kernel_ms, p50 kernel duration over the window + CustomRate, ///< custom._rate, ticks/s +}; + +/** @brief How a metric behaves, which decides warm-up and empty-window rules. */ +enum class MetricShape { + /// Events per second. An empty window is a real 0. + Rate, + /// A percentile over samples. An empty window has no value at all. + Percentile, + /// Last successful measurement. Advances only on a real measurement. + Gauge, +}; + +/** + * @brief A parsed, canonicalised metric name. + * + * `canonical` is what the rule id hashes over, so `gpu[00].util_pct` and + * `gpu[0].util_pct` cannot produce two different ids for one rule. + */ +struct MetricId { + MetricKind kind = MetricKind::KernelLaunchRate; + int device_index = 0; ///< only meaningful for gpu[N].* metrics + std::string custom_name; ///< the in custom._rate + std::string canonical; + + MetricShape shape() const; + /** @brief True when the metric cannot be resolved until the app registers it. */ + bool resolvesLazily() const { return kind == MetricKind::CustomRate; } +}; + +/** + * @brief Why a metric name or a rule config was refused. + * + * Every value here surfaces as `outcome=invalid_config` with this string as the + * reason. A rejected rule that leaves no trace is indistinguishable from one + * that was simply never true, so nothing may fail silently. + */ +enum class MetricParseError { + None, + Empty, + UnknownBuiltinMetric, ///< looks built-in, is not: gpu[0].temperature_pct + MissingCustomPrefix, ///< bare name that is not built-in: tokne_rate + MalformedDeviceIndex, ///< gpu[].util_pct, gpu[-1].util_pct, gpu[x].util_pct + MalformedCustomMetric, ///< custom._rate with a bad or missing name + CustomNameTooLong, + CustomNameCharset, +}; + +const char* toString(MetricParseError e); +const char* toString(MetricKind k); + +struct MetricParseResult { + MetricId id; + MetricParseError error = MetricParseError::None; + bool ok() const { return error == MetricParseError::None; } +}; + +/** + * @brief Parse a metric name from config. + * + * The `custom.` prefix is what makes a typo detectable here rather than at + * shutdown. Without it `tokne_rate` is syntactically indistinguishable from a + * counter that has simply not registered yet, and the mistake only surfaces + * once the run is over. + */ +MetricParseResult parseMetric(const std::string& text); + +/** + * @brief Timing configuration shared by the metric source and the rule. + * + * Held together because the three fields are not independently valid: a + * combination that can never produce evidence is a config error even though + * each field on its own looks reasonable. + */ +struct MetricWindowConfig { + /** + * Upper bound on the rate window. + * + * Property 4 of the design is that no setting is unbounded. The window also + * sizes the bucket ring, so an unbounded value is an unbounded allocation + * driven by a config string. + */ + static constexpr int64_t kMaxRateWindowMs = 5 * 60 * 1000; // 5 minutes + + int64_t rate_window_ms = 1000; + int64_t sustained_ms = 2000; + int64_t stale_after_ms = 5000; + + /** + * @brief Bucket width, derived so the validator and the producer agree. + * + * Internal and not configurable. Defined once here precisely because two + * independent definitions would eventually disagree and the validator would + * start accepting configurations the producer cannot satisfy. + */ + int64_t bucketIntervalMs() const; + + /** @brief Number of buckets covering the rate window (>= 1). */ + int64_t bucketCount() const; +}; + +enum class ConfigError { + None, + RateWindowNotPositive, + RateWindowTooLarge, + SustainedNegative, + StaleAfterNotPositive, + /// stale_after_ms < rate_window + sustained + bucket: the rule goes stale + /// before it can ever accumulate the evidence needed to fire. + StaleBeforeEvidence, +}; + +const char* toString(ConfigError e); + +/** + * @brief Reject a configuration that cannot work, with the arithmetic named. + * + * The interesting case is StaleBeforeEvidence. A total stall only fires if the + * metric stays Fresh long enough to accumulate `sustained_ms` of zero readings, + * and the first zero does not appear until a full rate window has elapsed. The + * weaker pairwise checks (`stale > window`, `stale >= sustained`) accept + * `window=4s, sustained=4s, stale=5s`, which produces its first zero at t=4 and + * goes stale at t=5 - four seconds short of ever firing. + */ +ConfigError validate(const MetricWindowConfig& cfg); + +/** @brief Human-readable explanation including the numbers, for the summary. */ +std::string explain(const MetricWindowConfig& cfg, ConfigError e); + +} // namespace gpufl::detail diff --git a/include/gpufl/core/metric_registry.cpp b/include/gpufl/core/metric_registry.cpp new file mode 100644 index 0000000..5ce12c2 --- /dev/null +++ b/include/gpufl/core/metric_registry.cpp @@ -0,0 +1,347 @@ +#include "gpufl/core/metric_registry.hpp" + +#include +#include + +#include "gpufl/core/events.hpp" + +namespace gpufl::detail { +namespace { + +constexpr int64_t kNsPerMs = 1000000; + +} // namespace + +const char* toString(const MetricState s) { + switch (s) { + case MetricState::Missing: return "missing"; + case MetricState::WarmingUp: return "warming_up"; + case MetricState::Fresh: return "fresh"; + case MetricState::Stale: return "stale"; + } + return "unknown"; +} + +// ---------------------------------------------------------------- MetricFeeds + +void MetricFeeds::noteKernelLaunch(const int64_t ts_ns) { + std::lock_guard lk(mu_); + ++launch_.count; + launch_.last_event_ns = ts_ns; + launch_.seeded = true; +} + +void MetricFeeds::noteKernelDuration(const int64_t ts_ns, const double duration_ms) { + std::lock_guard lk(mu_); + durations_.samples.push_back(duration_ms); + durations_.last_event_ns = ts_ns; +} + +void MetricFeeds::noteDeviceSample(const DeviceSample& sample, const int64_t ts_ns) { + if (sample.device_id < 0) return; + std::lock_guard lk(mu_); + const auto index = static_cast(sample.device_id); + if (index >= devices_.size()) devices_.resize(index + 1); + DeviceGauges& g = devices_[index]; + + // The measurement timestamp advances only here, never on a poll. A metric + // whose sequence advanced because someone asked about it could never be + // detected as dead. + g.util.value = static_cast(sample.gpu_util); + g.util.last_event_ns = ts_ns; + ++g.util.measurements; + + g.power.value = static_cast(sample.power_mw); + g.power.last_event_ns = ts_ns; + ++g.power.measurements; + + g.sm_clock.value = static_cast(sample.clock_sm); + g.sm_clock.last_event_ns = ts_ns; + ++g.sm_clock.measurements; +} + +void MetricFeeds::seedStartup(const int64_t ts_ns) { + std::lock_guard lk(mu_); + if (launch_.seeded) return; + launch_.last_event_ns = ts_ns; + launch_.seeded = true; + durations_.last_event_ns = ts_ns; +} + +int MetricFeeds::deviceCount() const { + std::lock_guard lk(mu_); + return static_cast(devices_.size()); +} + +MetricFeeds::LaunchFeed MetricFeeds::launchFeed() const { + std::lock_guard lk(mu_); + return launch_; +} + +MetricFeeds::DurationFeed MetricFeeds::drainDurations() { + std::lock_guard lk(mu_); + DurationFeed out; + out.samples = std::move(durations_.samples); + out.last_event_ns = durations_.last_event_ns; + durations_.samples.clear(); + return out; +} + +int64_t MetricFeeds::durationsLastEventNs() const { + std::lock_guard lk(mu_); + return durations_.last_event_ns; +} + +MetricFeeds::GaugeFeed MetricFeeds::gaugeFeed(const MetricKind kind, + const int device_index) const { + std::lock_guard lk(mu_); + if (device_index < 0 || static_cast(device_index) >= devices_.size()) { + return {}; + } + const DeviceGauges& g = devices_[static_cast(device_index)]; + switch (kind) { + case MetricKind::GpuUtilPct: return g.util; + case MetricKind::GpuPowerMw: return g.power; + case MetricKind::GpuSmClockMhz: return g.sm_clock; + default: return {}; + } +} + +void MetricFeeds::resetForTesting() { + std::lock_guard lk(mu_); + launch_ = LaunchFeed{}; + durations_ = DurationFeed{}; + devices_.clear(); +} + +// --------------------------------------------------------------- MetricSource + +MetricSource::MetricSource(MetricId id, MetricWindowConfig cfg, MetricFeeds* feeds, + const gpufl_counter_provider_v1* counters) + : id_(std::move(id)), cfg_(cfg), feeds_(feeds), counters_(counters) { + bucket_ns_ = cfg_.bucketIntervalMs() * kNsPerMs; + const auto count = static_cast(cfg_.bucketCount()); + buckets_.assign(count, 0); + if (id_.shape() == MetricShape::Percentile) bucket_durations_.resize(count); + // The ring, not the configured window, is what the rate divides by: the + // bucket count rounds up, so the two differ and using the configured value + // would report a rate the samples do not support. + window_ns_ = static_cast(count) * bucket_ns_; + stale_ns_ = cfg_.stale_after_ms * kNsPerMs; +} + +bool MetricSource::resolveCustomHandle() { + if (handle_ != nullptr) return true; + if (counters_ == nullptr || counters_->lookup == nullptr) return false; + // Lookup, never register. A rule naming a counter asks a question; creating + // the counter here would answer it with itself. + handle_ = counters_->lookup(id_.custom_name.c_str(), id_.custom_name.size()); + return handle_ != nullptr; +} + +double MetricSource::windowRatePerSec() const { + uint64_t total = 0; + for (const uint64_t n : buckets_) total += n; + const double seconds = static_cast(window_ns_) / 1e9; + if (seconds <= 0.0) return 0.0; + return static_cast(total) / seconds; +} + +double MetricSource::windowPercentile() const { + std::vector all; + for (const auto& bucket : bucket_durations_) { + all.insert(all.end(), bucket.begin(), bucket.end()); + } + if (all.empty()) return 0.0; + const size_t mid = all.size() / 2; + std::nth_element(all.begin(), all.begin() + mid, all.end()); + return all[mid]; +} + +void MetricSource::closeBucket(const int64_t boundary_ns) { + head_ = (head_ + 1) % buckets_.size(); + ++buckets_closed_; + ++total_closes_; + + switch (id_.shape()) { + case MetricShape::Rate: { + uint64_t total = 0; + if (id_.kind == MetricKind::KernelLaunchRate) { + total = feeds_->launchFeed().count; + } else if (resolveCustomHandle()) { + total = counters_->load(handle_); + if (total > 0) first_tick_seen_ = true; + } + // Unsigned delta: correct across a wrap, which is why the counter is + // allowed to wrap rather than saturate. + const uint64_t delta = baselined_ ? total - last_source_total_ : 0; + last_source_total_ = total; + baselined_ = true; + buckets_[head_] = delta; + if (delta > 0) last_tick_ns_ = boundary_ns; + break; + } + case MetricShape::Percentile: { + MetricFeeds::DurationFeed drained = feeds_->drainDurations(); + auto& slot = bucket_durations_[head_]; + slot.clear(); + if (drained.samples.size() > kMaxDurationsPerBucket) { + drained.samples.resize(kMaxDurationsPerBucket); + } + slot = std::move(drained.samples); + break; + } + case MetricShape::Gauge: + break; + } + + current_.observed_ns = boundary_ns; +} + +MetricSample MetricSource::poll(const int64_t now_ns) { + if (next_boundary_ns_ == 0) { + next_boundary_ns_ = now_ns + bucket_ns_; + current_.observed_ns = now_ns; + } + + // A collector that stalled longer than the whole window has no valid + // evidence left, so jump rather than replaying thousands of empty buckets. + // Replaying them would also be wrong: it would look like a run of measured + // zeros and could fire a stall rule that nothing actually observed. + if (now_ns - next_boundary_ns_ > window_ns_) { + std::fill(buckets_.begin(), buckets_.end(), 0); + for (auto& b : bucket_durations_) b.clear(); + buckets_closed_ = 0; + baselined_ = false; + next_boundary_ns_ = now_ns + bucket_ns_; + current_.observed_ns = now_ns; + } + + while (now_ns >= next_boundary_ns_) { + closeBucket(next_boundary_ns_); + next_boundary_ns_ += bucket_ns_; + } + + const bool window_full = buckets_closed_ >= buckets_.size(); + + switch (id_.shape()) { + case MetricShape::Rate: { + int64_t source_ns = 0; + if (id_.kind == MetricKind::KernelLaunchRate) { + const MetricFeeds::LaunchFeed feed = feeds_->launchFeed(); + if (!feed.seeded) { + // Nothing has seeded the launch source, so there is no + // baseline to measure staleness against yet. + current_.state = MetricState::WarmingUp; + current_.observed_ns = now_ns; + current_.sequence = total_closes_; + return current_; + } + source_ns = feed.last_event_ns; + } else { + if (!resolveCustomHandle()) { + // Not registered yet. Resolved lazily on purpose: an env + // rule is parsed during init(), long before application code + // reaches gpufl::counter(), so deciding this at install time + // would reject every counter rule before it could exist. + current_.state = MetricState::Missing; + current_.observed_ns = now_ns; + current_.sequence = total_closes_; + return current_; + } + if (!first_tick_seen_ && counters_->load(handle_) > 0) { + first_tick_seen_ = true; + // Seed the source clock at the moment the counter is first + // seen to move; ticks before this point have no timestamp. + last_tick_ns_ = now_ns; + } + if (!first_tick_seen_) { + // Registered but never ticked. Not Missing - the counter + // exists - and not Fresh 0 either, because a workload that + // has not started yet must not read as a stalled one. + current_.state = MetricState::WarmingUp; + current_.observed_ns = now_ns; + current_.sequence = total_closes_; + return current_; + } + source_ns = last_tick_ns_; + } + + current_.value = windowRatePerSec(); + current_.last_source_event_ns = source_ns; + // Every closed bucket is a publication, including one with no + // ticks. That is what lets a genuine zero accumulate as evidence + // instead of looking like "no new data" - without it, the stall a + // rule most needs to catch is the one case it never could. + current_.sequence = total_closes_; + if (!window_full) { + current_.state = MetricState::WarmingUp; + } else if (now_ns - source_ns > stale_ns_) { + current_.state = MetricState::Stale; + } else { + current_.state = MetricState::Fresh; + } + break; + } + case MetricShape::Percentile: { + const bool have_samples = + std::any_of(bucket_durations_.begin(), bucket_durations_.end(), + [](const std::vector& b) { return !b.empty(); }); + if (!window_full) { + current_.state = MetricState::WarmingUp; + break; + } + if (!have_samples) { + // An empty window has no percentile, so nothing is published: + // neither a value nor a new sequence. Publishing 0 ms would read + // as instantaneous kernels rather than no kernels, and a rule + // watching for slow kernels would silently never fire. + current_.state = MetricState::Stale; + break; + } + if (total_closes_ != last_published_close_) { + last_published_close_ = total_closes_; + ++sequence_; + } + current_.value = windowPercentile(); + current_.last_source_event_ns = feeds_->durationsLastEventNs(); + current_.sequence = sequence_; + current_.state = MetricState::Fresh; + break; + } + case MetricShape::Gauge: { + const MetricFeeds::GaugeFeed feed = + feeds_->gaugeFeed(id_.kind, id_.device_index); + if (feed.measurements == 0) { + current_.state = MetricState::WarmingUp; + break; + } + current_.value = feed.value; + current_.last_source_event_ns = feed.last_event_ns; + // Measurements, not polls. A sequence that advanced because someone + // asked would let one reading satisfy sustained_ms on its own. + current_.sequence = feed.measurements; + current_.state = now_ns - feed.last_event_ns > stale_ns_ + ? MetricState::Stale + : MetricState::Fresh; + break; + } + } + + return current_; +} + +void MetricSource::resetEpoch(const int64_t now_ns) { + std::fill(buckets_.begin(), buckets_.end(), 0); + for (auto& b : bucket_durations_) b.clear(); + buckets_closed_ = 0; + baselined_ = false; + next_boundary_ns_ = now_ns + bucket_ns_; + current_.observed_ns = now_ns; + current_.state = MetricState::WarmingUp; + // Drop anything the feed accumulated during the window for the same reason + // the buckets are cleared: it describes a contaminated workload. + if (id_.shape() == MetricShape::Percentile) feeds_->drainDurations(); +} + +} // namespace gpufl::detail diff --git a/include/gpufl/core/metric_registry.hpp b/include/gpufl/core/metric_registry.hpp new file mode 100644 index 0000000..0580b4b --- /dev/null +++ b/include/gpufl/core/metric_registry.hpp @@ -0,0 +1,234 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "gpufl/abi/gpufl_counter_abi.h" +#include "gpufl/core/metric_id.hpp" + +namespace gpufl { +struct DeviceSample; +} + +namespace gpufl::detail { + +/** + * @brief Whether a reading may be used as evidence, and if not, why. + * + * The split between Fresh and Stale is what stops a rule satisfying + * `sustained_ms` by re-reading one stale sample. System metrics publish every + * 100-500ms while the evaluator runs at ~1ms, so without this the same reading + * would be counted hundreds of times and every rule would fire on its first + * true sample. + */ +enum class MetricState { + Missing, ///< custom counter never registered, or source unavailable + WarmingUp, ///< registered, but no full window of evidence yet + Fresh, ///< usable, INCLUDING a genuine 0 + Stale, ///< the source stopped producing +}; + +const char* toString(MetricState s); + +/** + * @brief One reading. + * + * Two timestamps, not one. A zero-tick bucket still publishes a new sequence + * and a new `observed_ns`, so publication time alone can never go stale - it + * advances forever even during a total stall, which is exactly the condition a + * rule most needs to detect. Staleness is therefore measured against the + * source: + * + * Stale <=> observed_ns - last_source_event_ns > stale_after_ms + * + * That lets zeros accumulate as fresh evidence and still detects a dead source. + */ +struct MetricSample { + double value = 0.0; + int64_t observed_ns = 0; + int64_t last_source_event_ns = 0; + uint64_t sequence = 0; + MetricState state = MetricState::Missing; + + bool usable() const { return state == MetricState::Fresh; } +}; + +/** + * @brief Feeds every metric reads from, owned by the runtime. + * + * Separate from MetricSource because the feeds are process-global while a + * source belongs to one rule. Fed from the collector and sampler threads and + * read from the evaluator, so everything here is under one mutex; the volumes + * involved are per-kernel-launch at worst, not per-sample. + */ +class MetricFeeds { +public: + /** + * @brief A kernel launch was observed at the HOST launch API. + * + * Host launch rate, not GPU execution rate. Naming it `kernel_launch_rate` + * rather than a bare "kernel rate" is deliberate: a launch storm and a slow + * kernel look opposite here, and a caller who confuses the two writes a rule + * that fires on the wrong condition. + */ + void noteKernelLaunch(int64_t ts_ns); + + /** @brief A completed kernel's duration, for the recent_kernel_ms window. */ + void noteKernelDuration(int64_t ts_ns, double duration_ms); + + /** @brief A successful NVML/SMI measurement. Never called on polling. */ + void noteDeviceSample(const DeviceSample& sample, int64_t ts_ns); + + /** + * @brief Seed the launch source's timestamp at runtime startup. + * + * The launch source exists from startup, so its zero is a legitimate Fresh + * reading - but staleness needs something to measure against, and without a + * seed a run with no launches yet would read Stale rather than "0 launches + * per second", which is the opposite verdict. + */ + void seedStartup(int64_t ts_ns); + + /** @brief How many GPUs the device collector reported. 0 until first seen. */ + int deviceCount() const; + + struct LaunchFeed { + uint64_t count = 0; + int64_t last_event_ns = 0; + bool seeded = false; + }; + struct DurationFeed { + std::vector samples; // durations since the last drain + int64_t last_event_ns = 0; + }; + struct GaugeFeed { + double value = 0.0; + int64_t last_event_ns = 0; + uint64_t measurements = 0; + }; + + LaunchFeed launchFeed() const; + /** + * @brief Take and clear the durations accumulated since the last call. + * + * Single-consumer: whoever drains gets the samples and the next caller sees + * an empty feed. Fine while the MVP allows one rule; a second consumer of + * recent_kernel_ms would silently starve the first. + */ + DurationFeed drainDurations(); + /** @brief Last duration timestamp WITHOUT draining, for staleness checks. */ + int64_t durationsLastEventNs() const; + GaugeFeed gaugeFeed(MetricKind kind, int device_index) const; + + void resetForTesting(); + +private: + mutable std::mutex mu_; + LaunchFeed launch_; + DurationFeed durations_; + struct DeviceGauges { + GaugeFeed util; + GaugeFeed power; + GaugeFeed sm_clock; + }; + std::vector devices_; +}; + +/** + * @brief One metric, sampled on a rate window, for one rule. + * + * `poll()` is called from the collector loop at ~1ms and closes a bucket + * whenever one is due. It publishes a new sequence on every closed bucket even + * when nothing ticked - otherwise a real zero never accumulates as evidence and + * a total stall would read as "no new data" rather than "the rate is 0", and + * the rule that exists to catch stalls would be the one rule that cannot. + */ +class MetricSource { +public: + MetricSource(MetricId id, MetricWindowConfig cfg, MetricFeeds* feeds, + const gpufl_counter_provider_v1* counters); + + /** + * @brief Advance to @p now_ns and return the current reading. + * + * Cheap and idempotent within a bucket: repeated calls between bucket + * boundaries return the same sequence, which is what lets the rule + * evaluator ignore a repeat rather than counting it as new evidence. + */ + MetricSample poll(int64_t now_ns); + + /** + * @brief Discard accumulated evidence and start a fresh window. + * + * Called when a deep window closes. Buckets filled while profiling was + * active describe a contaminated workload, and letting them prove the + * workload recovered is how a rule ends up re-firing on its own overhead. + */ + void resetEpoch(int64_t now_ns); + + const MetricId& id() const { return id_; } + const MetricWindowConfig& config() const { return cfg_; } + /** @brief True once a custom counter has been resolved to a live handle. */ + bool customResolved() const { return handle_ != nullptr; } + +private: + void closeBucket(int64_t boundary_ns); + bool resolveCustomHandle(); + double windowRatePerSec() const; + double windowPercentile() const; + + MetricId id_; + MetricWindowConfig cfg_; + MetricFeeds* feeds_ = nullptr; + const gpufl_counter_provider_v1* counters_ = nullptr; + + gpufl_counter_handle handle_ = nullptr; // custom metrics only + + int64_t bucket_ns_ = 0; + int64_t window_ns_ = 0; + int64_t stale_ns_ = 0; + + /** Cap on durations kept per bucket; a launch storm must not grow memory. */ + static constexpr size_t kMaxDurationsPerBucket = 4096; + + // Ring of per-bucket counts. Sized once from the validated config. + std::vector buckets_; + // A percentile does not decompose across buckets, so durations are kept + // per bucket and expire with it rather than being folded into a sum. + std::vector> bucket_durations_; + size_t head_ = 0; + /// Closes in the CURRENT epoch; decides whether the window is full. + uint64_t buckets_closed_ = 0; + /** + * Closes over the whole process, never reset. + * + * The sequence has to be monotonic for the evaluator to tell new evidence + * from a repeat. Deriving it from the per-epoch count would send it + * backwards on every epoch reset, and the evaluator would then ignore real + * samples whose numbers it had already seen. + */ + uint64_t total_closes_ = 0; + + int64_t next_boundary_ns_ = 0; + uint64_t last_source_total_ = 0; ///< counter/launch total at last close + bool baselined_ = false; + bool first_tick_seen_ = false; + /** + * When the custom counter last moved. + * + * The registry stores values, not times, so a custom metric's source + * timestamp has to be observed here - at bucket close, whenever the delta + * is non-zero. Without it there is nothing to measure staleness against and + * a dead counter would read as a steady rate of 0 forever. + */ + int64_t last_tick_ns_ = 0; + uint64_t last_published_close_ = 0; + + MetricSample current_; + uint64_t sequence_ = 0; +}; + +} // namespace gpufl::detail diff --git a/runtime/counter_runtime.cpp b/runtime/counter_runtime.cpp index e6d4a2b..c647e74 100644 --- a/runtime/counter_runtime.cpp +++ b/runtime/counter_runtime.cpp @@ -56,6 +56,12 @@ uint64_t LoadSinceBaseline(const gpufl_counter_handle handle) { return CounterRegistry::instance().valueSinceBaseline(slot); } +gpufl_counter_handle Lookup(const char* name, const size_t name_length) { + if (name == nullptr) return nullptr; + return ToHandle(CounterRegistry::instance().findCounter( + std::string(name, name_length))); +} + void BeginSession() { CounterRegistry::instance().beginSession(); } void EndSession() { CounterRegistry::instance().endSession(); } int SessionActive() { return CounterRegistry::instance().sessionActive() ? 1 : 0; } @@ -70,6 +76,7 @@ const gpufl_counter_provider_v1 kProvider = { &BeginSession, &EndSession, &SessionActive, + &Lookup, }; } // namespace diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1005aef..645ae61 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -19,6 +19,7 @@ set(GPUFL_TEST_SOURCES core/test_disabled.cpp core/test_wire_contract.cpp core/test_counter_registry.cpp + core/test_metric_registry.cpp core/test_monitor.cpp core/test_itanium_demangle.cpp core/test_sampler.cpp diff --git a/tests/core/test_metric_registry.cpp b/tests/core/test_metric_registry.cpp new file mode 100644 index 0000000..7219ee9 --- /dev/null +++ b/tests/core/test_metric_registry.cpp @@ -0,0 +1,406 @@ +#include + +#include +#include + +#include "gpufl.hpp" +#include "gpufl/core/counter_provider.hpp" +#include "gpufl/core/counter_registry.hpp" +#include "gpufl/core/events.hpp" +#include "gpufl/core/metric_id.hpp" +#include "gpufl/core/metric_registry.hpp" + +using gpufl::detail::ActiveCounterProvider; +using gpufl::detail::ConfigError; +using gpufl::detail::CounterRegistry; +using gpufl::detail::MetricFeeds; +using gpufl::detail::MetricId; +using gpufl::detail::MetricKind; +using gpufl::detail::MetricParseError; +using gpufl::detail::MetricShape; +using gpufl::detail::MetricSource; +using gpufl::detail::MetricState; +using gpufl::detail::MetricWindowConfig; +using gpufl::detail::parseMetric; +using gpufl::detail::validate; + +namespace { + +constexpr int64_t kMs = 1000000; // ns per ms + +// ------------------------------------------------------------ metric contract + +TEST(MetricContractTest, ParsesBuiltinRates) { + const auto launch = parseMetric("kernel_launch_rate"); + ASSERT_TRUE(launch.ok()); + EXPECT_EQ(launch.id.kind, MetricKind::KernelLaunchRate); + EXPECT_EQ(launch.id.shape(), MetricShape::Rate); + + const auto kernel = parseMetric("recent_kernel_ms"); + ASSERT_TRUE(kernel.ok()); + EXPECT_EQ(kernel.id.shape(), MetricShape::Percentile); +} + +TEST(MetricContractTest, CanonicalisesDeviceIndex) { + const auto a = parseMetric("gpu[0].util_pct"); + const auto b = parseMetric("gpu[00].util_pct"); + ASSERT_TRUE(a.ok()); + ASSERT_TRUE(b.ok()); + // Two spellings of one rule must not hash to two different rule ids. + EXPECT_EQ(a.id.canonical, b.id.canonical); + EXPECT_EQ(a.id.canonical, "gpu[0].util_pct"); +} + +TEST(MetricContractTest, UnknownBuiltinFieldRejectedAtParse) { + const auto r = parseMetric("gpu[0].temperature_pct"); + EXPECT_FALSE(r.ok()); + EXPECT_EQ(r.error, MetricParseError::UnknownBuiltinMetric); +} + +TEST(MetricContractTest, MisspelledMetricWithoutPrefixRejectedAtParse) { + // The whole reason `custom.` exists: without a prefix this is + // indistinguishable from a counter that has not registered yet, and the + // mistake would only surface once the run was over. + const auto r = parseMetric("tokne_rate"); + EXPECT_FALSE(r.ok()); + EXPECT_EQ(r.error, MetricParseError::MissingCustomPrefix); +} + +TEST(MetricContractTest, CustomMetricParsesAndKeepsCounterName) { + const auto r = parseMetric("custom.token_rate"); + ASSERT_TRUE(r.ok()); + EXPECT_EQ(r.id.kind, MetricKind::CustomRate); + EXPECT_EQ(r.id.custom_name, "token"); + EXPECT_TRUE(r.id.resolvesLazily()); +} + +TEST(MetricContractTest, CustomMetricNeedsTheRateSuffix) { + EXPECT_EQ(parseMetric("custom.token").error, + MetricParseError::MalformedCustomMetric); + EXPECT_EQ(parseMetric("custom._rate").error, + MetricParseError::MalformedCustomMetric); +} + +TEST(MetricContractTest, MalformedDeviceIndexRejected) { + EXPECT_EQ(parseMetric("gpu[].util_pct").error, + MetricParseError::MalformedDeviceIndex); + EXPECT_EQ(parseMetric("gpu[x].util_pct").error, + MetricParseError::MalformedDeviceIndex); + EXPECT_EQ(parseMetric("gpu[-1].util_pct").error, + MetricParseError::MalformedDeviceIndex); +} + +TEST(MetricContractTest, EmptyNameRejected) { + EXPECT_EQ(parseMetric("").error, MetricParseError::Empty); +} + +// ------------------------------------------------------------ config validity + +TEST(MetricConfigTest, BucketIntervalIsBoundedAndDerivedOnce) { + EXPECT_EQ((MetricWindowConfig{100, 0, 5000}).bucketIntervalMs(), 10); + EXPECT_EQ((MetricWindowConfig{1000, 0, 5000}).bucketIntervalMs(), 100); + EXPECT_EQ((MetricWindowConfig{60000, 0, 90000}).bucketIntervalMs(), 100); +} + +TEST(MetricConfigTest, AcceptsAWorkableCombination) { + EXPECT_EQ(validate(MetricWindowConfig{1000, 2000, 5000}), ConfigError::None); +} + +TEST(MetricConfigTest, RejectsStaleShorterThanTheEvidenceItNeeds) { + // window=4s, sustained=4s, stale=5s produces its first zero at t=4 and goes + // stale at t=5 - four seconds short of ever firing. Each field on its own + // looks reasonable, which is exactly why the combined check has to exist. + const MetricWindowConfig cfg{4000, 4000, 5000}; + EXPECT_EQ(validate(cfg), ConfigError::StaleBeforeEvidence); + const std::string why = explain(cfg, ConfigError::StaleBeforeEvidence); + EXPECT_NE(why.find("8100"), std::string::npos) << why; // 4000+4000+100 +} + +TEST(MetricConfigTest, RejectsNonPositiveAndOversizedWindows) { + EXPECT_EQ(validate(MetricWindowConfig{0, 0, 100}), + ConfigError::RateWindowNotPositive); + EXPECT_EQ(validate(MetricWindowConfig{MetricWindowConfig::kMaxRateWindowMs + 1, + 0, 100000000}), + ConfigError::RateWindowTooLarge); + EXPECT_EQ(validate(MetricWindowConfig{1000, -1, 5000}), + ConfigError::SustainedNegative); + EXPECT_EQ(validate(MetricWindowConfig{1000, 0, 0}), + ConfigError::StaleAfterNotPositive); +} + +TEST(MetricConfigTest, StaleArithmeticDoesNotOverflow) { + constexpr int64_t kBig = 9223372036854775000LL; + const MetricWindowConfig cfg{1000, kBig, kBig}; + // Must decide, not wrap into an accidental "ok". + EXPECT_EQ(validate(cfg), ConfigError::StaleBeforeEvidence); +} + +// ---------------------------------------------------------------- rate source + +class MetricSourceTest : public ::testing::Test { + protected: + void SetUp() override { CounterRegistry::instance().resetForTesting(); } + void TearDown() override { CounterRegistry::instance().resetForTesting(); } + + MetricFeeds feeds; + + static MetricId parse(const char* text) { + const auto r = parseMetric(text); + EXPECT_TRUE(r.ok()) << text; + return r.id; + } + + // Drive the source forward to `now`, polling on every bucket boundary the + // way the collector loop would. + static gpufl::detail::MetricSample advance(MetricSource& src, int64_t from_ns, + int64_t to_ns, int64_t step_ns) { + gpufl::detail::MetricSample last; + for (int64_t t = from_ns; t <= to_ns; t += step_ns) last = src.poll(t); + return last; + } +}; + +TEST_F(MetricSourceTest, LaunchRateWarmsUpBeforeTheWindowIsFull) { + const MetricWindowConfig cfg{1000, 2000, 5000}; // 100ms buckets, 10 of them + MetricSource src(parse("kernel_launch_rate"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + // Half a window in: no verdict yet. A rule that fired here would be acting + // on a partial window, which reads as a lower rate than reality. + const auto mid = advance(src, 0, 500 * kMs, 10 * kMs); + EXPECT_EQ(mid.state, MetricState::WarmingUp); +} + +TEST_F(MetricSourceTest, LaunchRateWithNoLaunchesIsFreshZeroNotStale) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("kernel_launch_rate"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + const auto s = advance(src, 0, 1100 * kMs, 10 * kMs); + // The launch source exists from startup, so "no launches" is a measurement, + // not an absence. Reporting Stale here would be the opposite verdict. + EXPECT_EQ(s.state, MetricState::Fresh); + EXPECT_DOUBLE_EQ(s.value, 0.0); +} + +TEST_F(MetricSourceTest, EmptyBucketsStillPublishNewSequences) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("kernel_launch_rate"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + const auto a = advance(src, 0, 1100 * kMs, 10 * kMs); + const auto b = advance(src, 1110 * kMs, 1500 * kMs, 10 * kMs); + // Without this a genuine zero never accumulates as evidence and a total + // stall reads as "no new data" - the one case the feature exists for. + EXPECT_GT(b.sequence, a.sequence); + EXPECT_EQ(b.state, MetricState::Fresh); +} + +TEST_F(MetricSourceTest, LaunchRateReflectsObservedLaunches) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("kernel_launch_rate"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + // 500 launches spread over one window == 500/s. + for (int64_t t = 0; t < 1000 * kMs; t += 2 * kMs) { + feeds.noteKernelLaunch(t); + src.poll(t); + } + const auto s = advance(src, 1000 * kMs, 1010 * kMs, 10 * kMs); + EXPECT_EQ(s.state, MetricState::Fresh); + EXPECT_NEAR(s.value, 500.0, 60.0) << "rate=" << s.value; +} + +TEST_F(MetricSourceTest, PollingBetweenBucketsDoesNotAdvanceTheSequence) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("kernel_launch_rate"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + const auto warm = advance(src, 0, 1100 * kMs, 10 * kMs); + + const auto a = src.poll(1105 * kMs); + const auto b = src.poll(1110 * kMs); + const auto c = src.poll(1115 * kMs); + // The evaluator runs ~100x faster than a bucket closes. If every poll + // looked like new evidence, one reading would satisfy any sustained_ms. + EXPECT_EQ(a.sequence, b.sequence); + EXPECT_EQ(b.sequence, c.sequence); + EXPECT_GE(a.sequence, warm.sequence); +} + +TEST_F(MetricSourceTest, CustomCounterIsMissingUntilTheAppRegistersIt) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("custom.token_rate"), cfg, &feeds, ActiveCounterProvider()); + + const auto before = advance(src, 0, 1100 * kMs, 10 * kMs); + EXPECT_EQ(before.state, MetricState::Missing); + EXPECT_FALSE(src.customResolved()); + + // Asking about a counter must not create it - that would make "never + // registered" indistinguishable from "registered but idle". + EXPECT_EQ(CounterRegistry::instance().findCounter("token"), + CounterRegistry::kInvalidSlot); +} + +TEST_F(MetricSourceTest, CustomCounterResolvesLazilyAfterRegistration) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("custom.token_rate"), cfg, &feeds, ActiveCounterProvider()); + EXPECT_EQ(src.poll(0).state, MetricState::Missing); + + // An env rule is parsed during init(), long before application code reaches + // gpufl::counter(). Rejecting it at install time would kill every such rule. + auto tokens = gpufl::counter("token"); + const auto registered = advance(src, 10 * kMs, 1100 * kMs, 10 * kMs); + EXPECT_EQ(registered.state, MetricState::WarmingUp) + << "registered but never ticked is not Missing, and not Fresh 0 either"; + + for (int64_t t = 1100 * kMs; t < 2200 * kMs; t += 10 * kMs) { + tokens.add(10); + src.poll(t); + } + const auto s = src.poll(2200 * kMs); + EXPECT_EQ(s.state, MetricState::Fresh); + EXPECT_GT(s.value, 0.0); +} + +TEST_F(MetricSourceTest, CustomCounterGoesStaleWhenTicksStop) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("custom.stall_rate"), cfg, &feeds, ActiveCounterProvider()); + auto c = gpufl::counter("stall"); + + for (int64_t t = 0; t < 2000 * kMs; t += 10 * kMs) { + c.add(5); + src.poll(t); + } + ASSERT_EQ(src.poll(2000 * kMs).state, MetricState::Fresh); + + // Zeros first accumulate as fresh evidence... + const auto zeros = advance(src, 2010 * kMs, 4000 * kMs, 10 * kMs); + EXPECT_EQ(zeros.state, MetricState::Fresh); + EXPECT_DOUBLE_EQ(zeros.value, 0.0); + + // ...and only later does the source itself read as dead. Both halves matter: + // a rule must be able to fire on the stall before staleness hides it. + const auto dead = advance(src, 4010 * kMs, 9000 * kMs, 10 * kMs); + EXPECT_EQ(dead.state, MetricState::Stale); +} + +TEST_F(MetricSourceTest, PercentilePublishesNothingForAnEmptyWindow) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + const auto s = advance(src, 0, 1500 * kMs, 10 * kMs); + // 0 ms would read as instantaneous kernels rather than no kernels, and a + // rule watching for slow kernels would silently never fire. + EXPECT_NE(s.state, MetricState::Fresh); + EXPECT_DOUBLE_EQ(s.value, 0.0); +} + +TEST_F(MetricSourceTest, PercentileReportsTheMedianOverTheWindow) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + for (int64_t t = 0; t < 1200 * kMs; t += 10 * kMs) { + feeds.noteKernelDuration(t, 4.0); + src.poll(t); + } + const auto s = src.poll(1200 * kMs); + EXPECT_EQ(s.state, MetricState::Fresh); + EXPECT_NEAR(s.value, 4.0, 0.001); +} + +TEST_F(MetricSourceTest, GaugeAdvancesOnMeasurementsNotPolls) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("gpu[0].util_pct"), cfg, &feeds, ActiveCounterProvider()); + + EXPECT_EQ(src.poll(0).state, MetricState::WarmingUp); + + gpufl::DeviceSample sample; + sample.device_id = 0; + sample.gpu_util = 91; + feeds.noteDeviceSample(sample, 100 * kMs); + + const auto a = src.poll(110 * kMs); + EXPECT_EQ(a.state, MetricState::Fresh); + EXPECT_DOUBLE_EQ(a.value, 91.0); + + // NVML publishes every 100-500ms; the evaluator polls every ~1ms. A + // sequence that moved on polling would let one measurement satisfy any + // sustained_ms all by itself. + const auto b = src.poll(300 * kMs); + EXPECT_EQ(a.sequence, b.sequence); + + feeds.noteDeviceSample(sample, 400 * kMs); + EXPECT_GT(src.poll(410 * kMs).sequence, b.sequence); +} + +TEST_F(MetricSourceTest, GaugeGoesStaleWhenMeasurementsStop) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("gpu[0].util_pct"), cfg, &feeds, ActiveCounterProvider()); + + gpufl::DeviceSample sample; + sample.device_id = 0; + sample.gpu_util = 50; + feeds.noteDeviceSample(sample, 0); + + EXPECT_EQ(src.poll(1000 * kMs).state, MetricState::Fresh); + EXPECT_EQ(src.poll(9000 * kMs).state, MetricState::Stale); +} + +TEST_F(MetricSourceTest, MissingDeviceReadsAsWarmingUpNotZero) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("gpu[3].util_pct"), cfg, &feeds, ActiveCounterProvider()); + + gpufl::DeviceSample sample; + sample.device_id = 0; + sample.gpu_util = 77; + feeds.noteDeviceSample(sample, 0); + + // A rule on a GPU that does not exist must not read as 0% utilisation, + // which is a perfectly firable value for a "GPU went idle" rule. + EXPECT_EQ(src.poll(1000 * kMs).state, MetricState::WarmingUp); +} + +TEST_F(MetricSourceTest, EpochResetDiscardsEvidenceButKeepsSequenceMonotonic) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("kernel_launch_rate"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + for (int64_t t = 0; t < 1200 * kMs; t += 10 * kMs) { + feeds.noteKernelLaunch(t); + src.poll(t); + } + const auto before = src.poll(1200 * kMs); + ASSERT_EQ(before.state, MetricState::Fresh); + + src.resetEpoch(1200 * kMs); + const auto after = src.poll(1210 * kMs); + // Buckets filled while profiling was active describe a contaminated + // workload; letting them prove recovery is how a rule re-fires on its own + // overhead. + EXPECT_EQ(after.state, MetricState::WarmingUp); + // The sequence must not go backwards, or the evaluator would ignore real + // samples whose numbers it had already seen. + EXPECT_GE(after.sequence, before.sequence); +} + +TEST_F(MetricSourceTest, ALongCollectorStallDoesNotReplayFabricatedZeros) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("kernel_launch_rate"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + for (int64_t t = 0; t < 1200 * kMs; t += 10 * kMs) { + feeds.noteKernelLaunch(t); + src.poll(t); + } + ASSERT_EQ(src.poll(1200 * kMs).state, MetricState::Fresh); + + // The collector was blocked for a minute. Replaying 600 empty buckets would + // look like a measured run of zeros and could fire a stall rule that + // nothing actually observed. + const auto after = src.poll(61200 * kMs); + EXPECT_EQ(after.state, MetricState::WarmingUp); +} + +} // namespace From 805e6fc5dbc4bb9cd0d9bc53e38944419d7f7190 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Sun, 26 Jul 2026 20:14:03 -0700 Subject: [PATCH 06/28] test(metrics): name custom counters uniquely per test The two custom-counter tests passed alone and failed in the full suite on Linux. Counter slots are permanent by design, so a name shared with another test makes the result depend on execution order - and where a shared runtime is colocated with the test binary, resetting the local registry does not touch the one the source actually reads. The absence check now goes through the active provider for the same reason: asserting against the local registry would have passed without proving anything. --- tests/core/test_metric_registry.cpp | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/core/test_metric_registry.cpp b/tests/core/test_metric_registry.cpp index 7219ee9..240b1fa 100644 --- a/tests/core/test_metric_registry.cpp +++ b/tests/core/test_metric_registry.cpp @@ -229,7 +229,8 @@ TEST_F(MetricSourceTest, PollingBetweenBucketsDoesNotAdvanceTheSequence) { TEST_F(MetricSourceTest, CustomCounterIsMissingUntilTheAppRegistersIt) { const MetricWindowConfig cfg{1000, 2000, 5000}; - MetricSource src(parse("custom.token_rate"), cfg, &feeds, ActiveCounterProvider()); + MetricSource src(parse("custom.metric_absent_rate"), cfg, &feeds, + ActiveCounterProvider()); const auto before = advance(src, 0, 1100 * kMs, 10 * kMs); EXPECT_EQ(before.state, MetricState::Missing); @@ -237,18 +238,24 @@ TEST_F(MetricSourceTest, CustomCounterIsMissingUntilTheAppRegistersIt) { // Asking about a counter must not create it - that would make "never // registered" indistinguishable from "registered but idle". - EXPECT_EQ(CounterRegistry::instance().findCounter("token"), - CounterRegistry::kInvalidSlot); + // + // Checked through the ACTIVE provider, which is what the source consults. + // With a shared runtime present the local registry is a different registry + // entirely, and asserting against it would pass without proving anything. + EXPECT_EQ(ActiveCounterProvider()->lookup("metric_absent", + std::strlen("metric_absent")), + nullptr); } TEST_F(MetricSourceTest, CustomCounterResolvesLazilyAfterRegistration) { const MetricWindowConfig cfg{1000, 2000, 5000}; - MetricSource src(parse("custom.token_rate"), cfg, &feeds, ActiveCounterProvider()); + MetricSource src(parse("custom.metric_lazy_rate"), cfg, &feeds, + ActiveCounterProvider()); EXPECT_EQ(src.poll(0).state, MetricState::Missing); // An env rule is parsed during init(), long before application code reaches // gpufl::counter(). Rejecting it at install time would kill every such rule. - auto tokens = gpufl::counter("token"); + auto tokens = gpufl::counter("metric_lazy"); const auto registered = advance(src, 10 * kMs, 1100 * kMs, 10 * kMs); EXPECT_EQ(registered.state, MetricState::WarmingUp) << "registered but never ticked is not Missing, and not Fresh 0 either"; @@ -264,8 +271,11 @@ TEST_F(MetricSourceTest, CustomCounterResolvesLazilyAfterRegistration) { TEST_F(MetricSourceTest, CustomCounterGoesStaleWhenTicksStop) { const MetricWindowConfig cfg{1000, 2000, 5000}; - MetricSource src(parse("custom.stall_rate"), cfg, &feeds, ActiveCounterProvider()); - auto c = gpufl::counter("stall"); + MetricSource src(parse("custom.metric_stall_rate"), cfg, &feeds, + ActiveCounterProvider()); + // Unique to this test: slots are permanent by design, so a name shared with + // another test makes the result depend on execution order. + auto c = gpufl::counter("metric_stall"); for (int64_t t = 0; t < 2000 * kMs; t += 10 * kMs) { c.add(5); From 7e6f78ca5af7fe29cd7592a18575e74d7a348cbf Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Sun, 26 Jul 2026 20:31:53 -0700 Subject: [PATCH 07/28] feat(deep-window): rule evaluator with gates and validated config Turns a metric reading into a bounded window, without letting the rule fire on its own profiling overhead. Blackout and recovery stay distinct. Blackout is "a window is open, discard everything"; recovery is "the window closed, refill the clean epoch". Merged, contaminated samples would prove the workload had recovered and the rule would re-fire on the cost of the last window. Blackout covers any open window, manual or scheduled included, because contamination does not care who opened it. Windows carry a token so a manual window cannot spend the rule's budget, and so a request that the coordinator later turns down costs nothing. An open is serviced on a later beat, so the evaluator asks whether its request is still queued rather than treating "not open yet" as "never going to open". sustained_ms is a span between two observations, not an accumulation, and rearm is one predicate whose direction is validated - a rearm on the wrong side of the operator can never be reached, so the rule would fire once and then wait forever while looking healthy. Two gates, each with its own recorded reason. The engine gate matters most: without it a rule spends its whole budget opening windows that arm nothing. Custom counter rules are refused when the registry is not shared and more than one module is in play, since the target would tick one registry and this evaluator read another. An invalid rule never fails init(): config is parsed there, and failing hard would leave no session and nowhere to record the outcome. Fixed while testing: staleness was checked behind the repeat filter, so a source that died stopped advancing its sequence and was never seen to go stale - the rule sat in Pending on a reading nobody was taking. Nine tests were confirmed to fail with their own fix reverted. --- CMakeLists.txt | 1 + include/gpufl/core/deep_window.cpp | 63 +++ include/gpufl/core/deep_window.hpp | 28 ++ include/gpufl/core/deep_window_rule.cpp | 509 ++++++++++++++++++++++ include/gpufl/core/deep_window_rule.hpp | 270 ++++++++++++ include/gpufl/core/env_vars.hpp | 18 + tests/CMakeLists.txt | 1 + tests/core/test_deep_window_rule.cpp | 549 ++++++++++++++++++++++++ 8 files changed, 1439 insertions(+) create mode 100644 include/gpufl/core/deep_window_rule.cpp create mode 100644 include/gpufl/core/deep_window_rule.hpp create mode 100644 tests/core/test_deep_window_rule.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 5fe59e7..0daba4e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,6 +126,7 @@ target_sources(gpufl PRIVATE include/gpufl/core/counter_api.cpp include/gpufl/core/counter_provider.cpp include/gpufl/core/metric_id.cpp + include/gpufl/core/deep_window_rule.cpp include/gpufl/core/metric_registry.cpp include/gpufl/core/monitor_batch_manager.cpp include/gpufl/core/monitor_record_builders.cpp diff --git a/include/gpufl/core/deep_window.cpp b/include/gpufl/core/deep_window.cpp index 6b05f0c..f016e67 100644 --- a/include/gpufl/core/deep_window.cpp +++ b/include/gpufl/core/deep_window.cpp @@ -48,6 +48,21 @@ DeepWindowSpec g_pending_spec; // guarded by g_mu // When the last window closed, so a cooldown can be enforced. 0 = never. std::atomic g_last_close_ns{0}; +// Attribution for windows opened through a tagged request. +// +// A plain "a window opened" counter is not enough: a manual or scheduled +// window opening while a rule's request is outstanding would be counted +// against that rule's budget, and the budget is meant to bound what the RULE +// costs. Tokens start at 1 so 0 keeps its meaning of "not from a request". +std::atomic g_next_open_token{1}; +std::atomic g_last_opened_token{0}; +std::atomic g_opens_completed{0}; +uint64_t g_pending_token = 0; // guarded by g_mu +// Token of the request currently being serviced. Only TakePendingOpen_ sets +// it, and only for the duration of the Open() call it drives, so a direct +// Open() can never inherit someone else's attribution. +thread_local uint64_t g_claimed_token = 0; + int64_t g_opened_ns = 0; int64_t g_requested_duration_ms = 0; uint64_t g_requested_max_launches = 0; @@ -182,6 +197,12 @@ bool DeepWindow::Open(const DeepWindowSpec& spec) { return false; } + // Publish who this open belongs to, then clear the claim so a later + // direct Open() cannot inherit it. g_pending_token is set only by + // RequestOpenTagged and consumed exactly once, here. + g_last_opened_token.store(g_claimed_token, std::memory_order_release); + g_opens_completed.fetch_add(1, std::memory_order_acq_rel); + g_opened_ns = detail::GetTimestampNs(); g_name = spec.name.empty() ? "deep_window" : spec.name; g_requested_duration_ms = spec.max_duration_ms; @@ -321,6 +342,37 @@ void DeepWindow::RequestOpen(const DeepWindowSpec& spec) { ScheduleOpenAfter(0, spec); } +uint64_t DeepWindow::RequestOpenTagged(const DeepWindowSpec& spec) { + { + std::lock_guard lk(g_mu); + // Decided here so the caller learns now, rather than discovering later + // that a window it counted on never opened. + if (g_active.load(std::memory_order_relaxed)) return 0; + if (InCooldown(spec)) return 0; + } + const uint64_t token = g_next_open_token.fetch_add(1, std::memory_order_relaxed); + { + std::lock_guard lk(g_mu); + g_pending_token = token; + } + ScheduleOpenAfter(0, spec); + return token; +} + +uint64_t DeepWindow::LastOpenedToken() { + return g_last_opened_token.load(std::memory_order_acquire); +} + +uint64_t DeepWindow::PendingOpenToken() { + if (!g_open_requested.load(std::memory_order_acquire)) return 0; + std::lock_guard lk(g_mu); + return g_pending_token; +} + +uint64_t DeepWindow::OpensCompleted() { + return g_opens_completed.load(std::memory_order_acquire); +} + void DeepWindow::ScheduleOpenAfter(const int64_t delay_ms, const DeepWindowSpec& spec) { { @@ -346,9 +398,16 @@ void DeepWindow::TakePendingOpen_() { { std::lock_guard lk(g_mu); spec = g_pending_spec; + // Claim the token here, not in Open(): Open can still refuse, and a + // token left behind would keep reading as "queued" for the rest of the + // run, so a rule would wait forever for a window that was already + // turned down. + g_claimed_token = g_pending_token; + g_pending_token = 0; } // Outside the lock: Open takes it too. Open(spec); + g_claimed_token = 0; } void DeepWindow::OnLaunch() { @@ -414,6 +473,10 @@ void DeepWindow::ServicePending() { void DeepWindow::ResetForTesting() { std::lock_guard lk(g_mu); g_active.store(false, std::memory_order_release); + g_next_open_token.store(1, std::memory_order_relaxed); + g_last_opened_token.store(0, std::memory_order_relaxed); + g_opens_completed.store(0, std::memory_order_relaxed); + g_pending_token = 0; g_deadline_ns.store(0, std::memory_order_relaxed); g_launches_remaining.store(0, std::memory_order_relaxed); g_launches_covered.store(0, std::memory_order_relaxed); diff --git a/include/gpufl/core/deep_window.hpp b/include/gpufl/core/deep_window.hpp index e6313e4..a6cdda5 100644 --- a/include/gpufl/core/deep_window.hpp +++ b/include/gpufl/core/deep_window.hpp @@ -79,6 +79,34 @@ class DeepWindow { */ static void RequestOpen(const DeepWindowSpec& spec); + /** + * @brief Request an open and get back a token identifying it. + * + * For callers that must know whether the window they asked for actually + * happened - a rule with a window budget cannot count a window that never + * opened, and cannot let a manual one consume its budget either. + * + * Returns 0 when the request is refused outright (a window is already open, + * or the cooldown has not elapsed). A non-zero token matches + * LastOpenedToken() once, and only once, the requested window opens. + */ + static uint64_t RequestOpenTagged(const DeepWindowSpec& spec); + + /** @brief Token of the most recent open; 0 when it was not from a request. */ + static uint64_t LastOpenedToken(); + + /** + * @brief Token of the request still queued, or 0 when none is. + * + * An open is serviced on a later beat, so "has not opened yet" and "will + * never open" look identical without this. A caller that treated the first + * as the second would abandon a window that was about to open. + */ + static uint64_t PendingOpenToken(); + + /** @brief Monotonic count of windows that actually opened. */ + static uint64_t OpensCompleted(); + /** * @brief Same, but not before `delay_ms` have passed. * diff --git a/include/gpufl/core/deep_window_rule.cpp b/include/gpufl/core/deep_window_rule.cpp new file mode 100644 index 0000000..31bfc4e --- /dev/null +++ b/include/gpufl/core/deep_window_rule.cpp @@ -0,0 +1,509 @@ +#include "gpufl/core/deep_window_rule.hpp" + +#include +#include +#include +#include + +namespace gpufl::detail { +namespace { + +/** Largest window a rule may ask for. Property 4: nothing is unbounded. */ +constexpr int64_t kMaxWindowDurationMs = 60 * 1000; +constexpr uint64_t kMaxWindowLaunches = 5000000; +constexpr int kMaxWindowsHardLimit = 64; + +std::string trim(const std::string& s) { + const size_t b = s.find_first_not_of(" \t"); + if (b == std::string::npos) return {}; + const size_t e = s.find_last_not_of(" \t"); + return s.substr(b, e - b + 1); +} + +/** Parse "2s" / "500ms" / "2000" (bare = milliseconds). Negative on failure. */ +int64_t parseDurationMs(const std::string& text) { + const std::string s = trim(text); + if (s.empty()) return -1; + + size_t digits = 0; + while (digits < s.size() && s[digits] >= '0' && s[digits] <= '9') ++digits; + if (digits == 0) return -1; + + errno = 0; + const long long value = std::strtoll(s.substr(0, digits).c_str(), nullptr, 10); + if (errno != 0 || value < 0) return -1; + + const std::string unit = trim(s.substr(digits)); + if (unit.empty() || unit == "ms") return value; + if (unit == "s") { + if (value > kMaxWindowDurationMs) return -1; // also guards the *1000 + return value * 1000; + } + return -1; +} + +bool parseDouble(const std::string& text, double* out) { + const std::string s = trim(text); + if (s.empty()) return false; + char* end = nullptr; + errno = 0; + const double v = std::strtod(s.c_str(), &end); + if (errno != 0 || end == s.c_str() || *end != '\0') return false; + if (!std::isfinite(v)) return false; + *out = v; + return true; +} + +} // namespace + +const char* toString(const Comparison op) { + return op == Comparison::LessThan ? "<" : ">"; +} + +const char* toString(const RuleError e) { + switch (e) { + case RuleError::None: return "ok"; + case RuleError::BadMetric: return "bad_metric"; + case RuleError::BadTiming: return "bad_timing"; + case RuleError::ThresholdNotFinite: return "threshold_not_finite"; + case RuleError::RearmWrongSide: return "rearm_wrong_side"; + case RuleError::MaxWindowsOutOfRange: return "max_windows_out_of_range"; + case RuleError::WindowBoundsMissing: return "window_bounds_missing"; + case RuleError::WindowBoundsTooLarge: return "window_bounds_too_large"; + case RuleError::Unparsable: return "unparsable"; + case RuleError::DuplicateRule: return "duplicate_rule"; + } + return "unknown"; +} + +const char* toString(const RuleState s) { + switch (s) { + case RuleState::Inactive: return "inactive"; + case RuleState::WarmingUp: return "warming_up"; + case RuleState::Armed: return "armed"; + case RuleState::Pending: return "pending"; + case RuleState::Opening: return "opening"; + case RuleState::Blackout: return "blackout"; + case RuleState::Recovery: return "recovery"; + case RuleState::WaitingForRearm: return "waiting_for_rearm"; + } + return "unknown"; +} + +const char* toString(const RuleOutcome o) { + switch (o) { + case RuleOutcome::None: return "none"; + case RuleOutcome::NeverTrue: return "never_true"; + case RuleOutcome::Fired: return "fired"; + case RuleOutcome::Exhausted: return "exhausted"; + case RuleOutcome::Unsupported: return "unsupported"; + case RuleOutcome::InvalidConfig: return "invalid_config"; + } + return "unknown"; +} + +const char* toString(const RuleGate g) { + switch (g) { + case RuleGate::Ok: return "ok"; + case RuleGate::MetricUnavailable: return "metric_unavailable"; + case RuleGate::CountersNotShared: return "counters_not_shared"; + case RuleGate::NoDeepEngine: return "no_deep_engine"; + case RuleGate::WindowsUnsupported: return "windows_unsupported"; + } + return "unknown"; +} + +// ------------------------------------------------------------------- parsing + +RuleParseResult parseRuleExpression(const std::string& text) { + RuleParseResult out; + + const size_t op_at = text.find_first_of("<>"); + if (op_at == std::string::npos) { + out.error = RuleError::Unparsable; + out.detail = "expected '<' or '>' in \"" + text + "\""; + return out; + } + out.rule.op = text[op_at] == '<' ? Comparison::LessThan : Comparison::GreaterThan; + + const MetricParseResult metric = parseMetric(trim(text.substr(0, op_at))); + if (!metric.ok()) { + out.error = RuleError::BadMetric; + out.metric_error = metric.error; + out.detail = toString(metric.error); + return out; + } + out.rule.metric = metric.id; + + std::string rest = text.substr(op_at + 1); + // " for " is optional; without it the rule fires on the first + // fresh true reading. + int64_t sustained_ms = 0; + if (const size_t f = rest.find(" for "); f != std::string::npos) { + sustained_ms = parseDurationMs(rest.substr(f + 5)); + if (sustained_ms < 0) { + out.error = RuleError::Unparsable; + out.detail = "bad duration in \"" + text + "\""; + return out; + } + rest = rest.substr(0, f); + } + + if (!parseDouble(rest, &out.rule.threshold)) { + out.error = RuleError::Unparsable; + out.detail = "bad threshold in \"" + text + "\""; + return out; + } + + out.rule.timing.sustained_ms = sustained_ms; + // Plain "condition false" until an explicit hysteresis value is supplied. + out.rule.rearm_threshold = out.rule.threshold; + return out; +} + +RuleParseResult validateRule(const DeepWindowRule& rule) { + RuleParseResult out; + out.rule = rule; + + if (!std::isfinite(rule.threshold) || !std::isfinite(rule.rearm_threshold)) { + // NaN makes every comparison false, so the rule would sit armed forever + // while looking perfectly healthy. + out.error = RuleError::ThresholdNotFinite; + return out; + } + + // A rearm on the wrong side of the operator can never be reached, so the + // rule fires exactly once and then waits for a condition that cannot occur. + const bool rearm_ok = rule.op == Comparison::LessThan + ? rule.rearm_threshold >= rule.threshold + : rule.rearm_threshold <= rule.threshold; + if (!rearm_ok) { + out.error = RuleError::RearmWrongSide; + out.detail = std::string("rearm ") + toString(rule.op) + + " rule needs rearm on the other side of the threshold"; + return out; + } + + if (rule.max_windows < 1 || rule.max_windows > kMaxWindowsHardLimit) { + out.error = RuleError::MaxWindowsOutOfRange; + return out; + } + + if (rule.window.max_duration_ms <= 0 && rule.window.max_launches == 0) { + // A window with neither bound never closes on its own, which turns a + // bounded-cost feature into an always-on one. + out.error = RuleError::WindowBoundsMissing; + return out; + } + if (rule.window.max_duration_ms > kMaxWindowDurationMs || + rule.window.max_launches > kMaxWindowLaunches) { + out.error = RuleError::WindowBoundsTooLarge; + return out; + } + + if (const ConfigError e = validate(rule.timing); e != ConfigError::None) { + out.error = RuleError::BadTiming; + out.config_error = e; + out.detail = explain(rule.timing, e); + return out; + } + + return out; +} + +// ----------------------------------------------------------------- evaluator + +RuleEvaluator::Hooks RuleEvaluator::liveHooks() { + Hooks h; + h.request_open = [](void*, const DeepWindowSpec& spec) { + return DeepWindow::RequestOpenTagged(spec); + }; + h.window_active = [](void*) { return DeepWindow::Active(); }; + h.last_opened_token = [](void*) { return DeepWindow::LastOpenedToken(); }; + h.pending_open_token = [](void*) { return DeepWindow::PendingOpenToken(); }; + h.ctx = nullptr; + return h; +} + +RuleEvaluator::RuleEvaluator(DeepWindowRule rule, std::string rule_id, + const RuleCapabilities& caps, MetricSource* source, + Hooks hooks) + : rule_(std::move(rule)), + rule_id_(std::move(rule_id)), + source_(source), + hooks_(hooks) { + // Two independent gates, each with its own reason. A rule can be perfectly + // valid and still have nothing to watch, or nothing to arm. + RuleGate gate = RuleGate::Ok; + if (!caps.windows_supported) { + gate = RuleGate::WindowsUnsupported; + } else if (!caps.deep_engine_prepared) { + // An enum check on the configured engine is not enough: without this a + // rule spends its whole budget opening windows that arm nothing. + gate = RuleGate::NoDeepEngine; + } else if (rule_.metric.resolvesLazily() && caps.multi_module && + !caps.counters_shared) { + // The target ticks one registry and this evaluator reads another, so + // the counter can never be seen. Refusing here is the honest answer; + // the alternative is reporting a counter that is being ticked as + // Missing for the whole run. + gate = RuleGate::CountersNotShared; + } else if (!rule_.metric.resolvesLazily() && + rule_.metric.shape() == MetricShape::Gauge && + rule_.metric.device_index >= caps.device_count) { + // Built-in metrics are decided eagerly - unlike a custom counter, the + // answer cannot change later. + gate = RuleGate::MetricUnavailable; + } + + if (gate != RuleGate::Ok) { + state_ = RuleState::Inactive; + terminal_ = RuleOutcome::Unsupported; + reason_ = toString(gate); + } +} + +bool RuleEvaluator::conditionHolds(const double value) const { + return rule_.op == Comparison::LessThan ? value < rule_.threshold + : value > rule_.threshold; +} + +bool RuleEvaluator::rearmHolds(const double value) const { + // One predicate, not "false or threshold". With rearm == threshold this is + // exactly "condition false", which is the no-hysteresis default. + return rule_.op == Comparison::LessThan ? value >= rule_.rearm_threshold + : value <= rule_.rearm_threshold; +} + +void RuleEvaluator::toArmed() { + state_ = RuleState::Armed; + first_true_observed_ns_.reset(); + ++state_sequence_; +} + +void RuleEvaluator::enterBlackout(const int64_t) { + state_ = RuleState::Blackout; + first_true_observed_ns_.reset(); + ++state_sequence_; +} + +void RuleEvaluator::poll(const int64_t now_ns) { + if (state_ == RuleState::Inactive) return; + + const bool active = hooks_.window_active(hooks_.ctx); + + // 1. Confirm a requested open before anything else. The window we caused + // also puts us into blackout, so checking blackout first would lose the + // only chance to count it. + if (state_ == RuleState::Opening) { + if (pending_token_ != 0 && + hooks_.last_opened_token(hooks_.ctx) == pending_token_) { + pending_token_ = 0; + ++windows_opened_; + if (windows_opened_ >= static_cast(rule_.max_windows)) { + // Marked at the open that reaches the limit, not later: a run + // that crashes during this window still explains itself. + terminal_ = RuleOutcome::Exhausted; + } + enterBlackout(now_ns); + window_was_active_ = true; + return; + } + else if (hooks_.pending_open_token(hooks_.ctx) != pending_token_) { + // No longer queued and never opened: the coordinator turned it down + // when it got round to it. No budget is consumed - the budget + // bounds what the rule COST, and this cost nothing. + // + // Checked against the queue rather than "is a window open yet", + // because an open is serviced on a later beat and treating "not + // yet" as "never" would abandon a window about to open. + pending_token_ = 0; + reason_ = "open_request_dropped"; + toArmed(); + } + } + + // 2. Any open window contaminates, whoever opened it. Contamination does + // not care who asked. + if (active) { + if (state_ != RuleState::Blackout) enterBlackout(now_ns); + window_was_active_ = true; + return; + } + + // 3. A window just closed. Blackout and recovery are distinct: blackout is + // "discard everything", recovery is "refill the clean epoch". Merging + // them would let contaminated samples prove the workload recovered. + if (window_was_active_) { + window_was_active_ = false; + source_->resetEpoch(now_ns); + if (terminal_ == RuleOutcome::Exhausted) { + state_ = RuleState::Inactive; + ++state_sequence_; + return; + } + state_ = RuleState::Recovery; + ++state_sequence_; + } + + const MetricSample sample = source_->poll(now_ns); + last_metric_state_ = sample.state; + + // Staleness is checked BEFORE the repeat filter, deliberately. A source + // that dies stops advancing its sequence, so a rule that only looked at new + // samples would never notice - it would sit in Pending on a reading nobody + // is taking any more, which is the exact failure the two-timestamp design + // exists to prevent. + if (sample.state != MetricState::Fresh) { + // Not evidence the condition stopped holding, but not evidence it + // continued either, so the run is broken rather than completed. + if (state_ == RuleState::Pending) { + first_true_observed_ns_.reset(); + state_ = RuleState::Armed; + ++state_sequence_; + } + return; + } + + // Only a NEW fresh reading is evidence. The evaluator runs 100-2000x faster + // than a metric publishes, so counting repeats would let a single reading + // satisfy any sustained_ms on its own. + const bool is_new = !have_sequence_ || sample.sequence != last_sequence_; + if (!is_new) return; + last_sequence_ = sample.sequence; + have_sequence_ = true; + + ++samples_seen_; + last_value_ = sample.value; + last_observed_ns_ = sample.observed_ns; + + switch (state_) { + case RuleState::WarmingUp: + case RuleState::Recovery: + // A full window of clean data has now arrived, since a Fresh sample + // implies it. + state_ = state_ == RuleState::Recovery ? RuleState::WaitingForRearm + : RuleState::Armed; + ++state_sequence_; + if (state_ == RuleState::Armed) break; + [[fallthrough]]; + + case RuleState::WaitingForRearm: + if (rearmHolds(sample.value)) { + toArmed(); + } + break; + + case RuleState::Armed: + if (conditionHolds(sample.value)) { + first_true_observed_ns_ = sample.observed_ns; + state_ = RuleState::Pending; + ++state_sequence_; + // sustained_ms == 0 fires on this same reading, handled below. + if (rule_.timing.sustained_ms == 0) { + pending_token_ = hooks_.request_open(hooks_.ctx, rule_.window); + if (pending_token_ != 0) { + state_ = RuleState::Opening; + ++state_sequence_; + } else { + reason_ = "open_refused"; + toArmed(); + } + } + } + break; + + case RuleState::Pending: { + if (!conditionHolds(sample.value)) { + toArmed(); + break; + } + if (!first_true_observed_ns_.has_value()) { + first_true_observed_ns_ = sample.observed_ns; + break; + } + // A span between two observations, not an accumulation - so one + // stale reading re-read many times can never satisfy it. + const int64_t held_ns = sample.observed_ns - *first_true_observed_ns_; + if (held_ns < rule_.timing.sustained_ms * 1000000) break; + + pending_token_ = hooks_.request_open(hooks_.ctx, rule_.window); + if (pending_token_ != 0) { + state_ = RuleState::Opening; + ++state_sequence_; + } else { + // Refused by cooldown. Back to armed rather than holding + // pending, which would fire the instant the cooldown lapsed on + // evidence gathered long before. + reason_ = "open_refused"; + toArmed(); + } + break; + } + + case RuleState::Opening: + case RuleState::Blackout: + case RuleState::Inactive: + break; + } +} + +RuleSummary RuleEvaluator::snapshot(const int64_t now_ns) const { + RuleSummary s; + s.rule_id = rule_id_; + s.state = state_; + s.outcome = terminal_; + s.samples_seen = samples_seen_; + s.windows_opened = windows_opened_; + s.last_value = last_value_; + s.last_observed_ns = last_observed_ns_; + s.last_metric_state = last_metric_state_; + s.reason = reason_; + s.state_sequence = state_sequence_; + s.emitted_ns = now_ns; + return s; +} + +RuleSummary RuleEvaluator::finish(const int64_t now_ns) { + RuleSummary s = snapshot(now_ns); + + // Precedence, highest first. Several can apply at once and the most + // specific has to win, or a rule that was refused outright would report as + // one that simply never matched. + if (terminal_ == RuleOutcome::InvalidConfig || + terminal_ == RuleOutcome::Unsupported) { + s.outcome = terminal_; + } else if (terminal_ == RuleOutcome::Exhausted) { + s.outcome = RuleOutcome::Exhausted; + } else if (windows_opened_ > 0) { + s.outcome = RuleOutcome::Fired; + } else { + s.outcome = RuleOutcome::NeverTrue; + if (s.reason.empty()) { + // Distinguish "watched it and it never happened" from "never had + // anything to watch"; both leave windows_opened at 0. + s.reason = last_metric_state_ == MetricState::Missing + ? "custom_metric_never_registered" + : "condition_never_held"; + } + } + + s.state_sequence = ++state_sequence_; + return s; +} + +RuleSummary RuleEvaluator::refused(std::string rule_id, const RuleOutcome outcome, + std::string reason, const int64_t now_ns) { + RuleSummary s; + s.rule_id = std::move(rule_id); + // The terminal outcomes leave the evaluator standing in none of the running + // states, which is what `inactive` is for. + s.state = RuleState::Inactive; + s.outcome = outcome; + s.reason = std::move(reason); + s.emitted_ns = now_ns; + s.state_sequence = 1; + return s; +} + +} // namespace gpufl::detail diff --git a/include/gpufl/core/deep_window_rule.hpp b/include/gpufl/core/deep_window_rule.hpp new file mode 100644 index 0000000..4f9e81a --- /dev/null +++ b/include/gpufl/core/deep_window_rule.hpp @@ -0,0 +1,270 @@ +#pragma once + +#include +#include +#include +#include + +#include "gpufl/core/deep_window.hpp" +#include "gpufl/core/metric_id.hpp" +#include "gpufl/core/metric_registry.hpp" + +namespace gpufl::detail { + +enum class Comparison { LessThan, GreaterThan }; + +const char* toString(Comparison op); + +/** + * @brief A condition that opens a deep window. + * + * The predicate form already exists - it is the `if` a caller writes around + * gpufl::deepWindow(). A rule earns its place on two counts only: metrics + * gpufl has and the application does not, and thresholds that live in + * configuration rather than in code. + */ +struct DeepWindowRule { + MetricId metric; + Comparison op = Comparison::LessThan; + double threshold = 0.0; + /** + * Value the metric must recover past before the rule may fire again. + * + * Equal to `threshold` by default, which degenerates to plain "condition + * false" - the no-hysteresis case. A rule whose rearm sits on the wrong + * side of the operator can never rearm, so the direction is validated. + */ + double rearm_threshold = 0.0; + MetricWindowConfig timing; + int max_windows = 3; + DeepWindowSpec window; +}; + +/** + * @brief Why a rule was refused, or could not be honoured. + * + * Property 5 of the design: every reason a rule did not fire is recorded. A + * rejected rule that leaves no trace is indistinguishable from one that was + * simply never true, and the two call for opposite responses from the user. + */ +enum class RuleError { + None, + /// The metric name itself was refused; carries the parse reason. + BadMetric, + /// The timing combination cannot produce evidence; carries the config reason. + BadTiming, + ThresholdNotFinite, ///< NaN makes every comparison false, silently + RearmWrongSide, ///< LessThan needs rearm >= threshold, and vice versa + MaxWindowsOutOfRange, + WindowBoundsMissing, ///< a window with no bound at all never closes + WindowBoundsTooLarge, + Unparsable, ///< the --deep-when expression did not parse + DuplicateRule, ///< MVP allows exactly one +}; + +const char* toString(RuleError e); + +/** @brief Parse ` for `, e.g. "x<1000 for 2s". */ +struct RuleParseResult { + DeepWindowRule rule; + RuleError error = RuleError::None; + /// Set when error is BadMetric / BadTiming, so the reason is not lost. + MetricParseError metric_error = MetricParseError::None; + ConfigError config_error = ConfigError::None; + std::string detail; + + bool ok() const { return error == RuleError::None; } +}; + +/** + * @brief Parse the expression half of the config. + * + * Only the metric, operator, threshold and sustained duration come from the + * expression; everything else is a separate option. Splitting them keeps each + * refusal able to name the field it is about. + */ +RuleParseResult parseRuleExpression(const std::string& text); + +/** @brief Check a fully assembled rule. Returns RuleError::None when usable. */ +RuleParseResult validateRule(const DeepWindowRule& rule); + +// --------------------------------------------------------------------------- + +/** + * @brief Where the evaluator is standing. + * + * Never a verdict on the session - see RuleOutcome for that. Collapsing the + * two would make `armed` look like a conclusion, and force every reader to + * guess which kind of answer it was holding. + */ +enum class RuleState { + Inactive, ///< terminal: invalid, unsupported, or budget spent + WarmingUp, + Armed, + Pending, ///< condition true, waiting out sustained_ms + Opening, ///< an open was requested; waiting to see it happen + Blackout, ///< a window is open; everything observed is contaminated + Recovery, ///< window closed; refilling the clean epoch + WaitingForRearm, +}; + +const char* toString(RuleState s); + +/** @brief What the session concluded about the rule. */ +enum class RuleOutcome { + None, + NeverTrue, + Fired, + Exhausted, + Unsupported, + InvalidConfig, +}; + +const char* toString(RuleOutcome o); + +/** @brief Which gate refused a rule that was otherwise well-formed. */ +enum class RuleGate { + Ok, + /// The metric cannot be produced: no such device, sampler absent, or the + /// base mode does not generate it. + MetricUnavailable, + /// Custom counters are not shared across modules, so a counter the target + /// ticks is invisible to the evaluator that would read it. + CountersNotShared, + /// Windows are meaningless without an engine to arm inside them. An enum + /// check on the configured engine is necessary but not sufficient: without + /// this a rule burns its budget opening windows that arm nothing. + NoDeepEngine, + /// The base mode does not support bounded windows at all. + WindowsUnsupported, +}; + +const char* toString(RuleGate g); + +/** + * @brief Everything the session learned about one rule. + * + * `state` and `outcome` are separate on purpose. A crash mid-run leaves a + * meaningful state and no outcome; a clean run with a rule that never matched + * leaves `armed` and `never_true`. One field cannot carry both. + */ +struct RuleSummary { + std::string rule_id; + RuleState state = RuleState::Inactive; + RuleOutcome outcome = RuleOutcome::None; + uint64_t samples_seen = 0; + uint32_t windows_opened = 0; + /// Absent, not NaN: there is a real "no value yet", and NaN does not + /// survive the JSON and DB boundaries cleanly. + std::optional last_value; + std::optional last_observed_ns; + MetricState last_metric_state = MetricState::Missing; + std::string reason; + uint64_t state_sequence = 0; + int64_t emitted_ns = 0; +}; + +/** + * @brief The gates a rule must pass before it can open anything. + * + * Supplied by the caller rather than queried here so the evaluator stays + * testable without a GPU, and so the two gates are visibly independent - each + * has its own recorded failure reason. + */ +struct RuleCapabilities { + bool windows_supported = true; + bool deep_engine_prepared = true; + bool counters_shared = true; + /// More than one module holds a copy of gpufl - injection. Only then does + /// an unshared counter registry actually break anything. + bool multi_module = false; + int device_count = 1; +}; + +/** + * @brief Drives one rule from metric samples to a window request. + * + * Runs on the collector beat (~1ms) alongside serviceDeepWindow. + */ +class RuleEvaluator { +public: + /** @brief Injected so tests drive the coordinator without a GPU. */ + struct Hooks { + /// Ask for a window; returns a token, or 0 when refused outright. + uint64_t (*request_open)(void* ctx, const DeepWindowSpec&) = nullptr; + /// Is a window - any window, whoever opened it - currently open? + bool (*window_active)(void* ctx) = nullptr; + /// Token of the most recent open, so a manual window is not mistaken + /// for the one this rule asked for and charged to its budget. + uint64_t (*last_opened_token)(void* ctx) = nullptr; + /// Token of the request still queued, or 0. Without it "has not opened + /// yet" and "will never open" are indistinguishable, and the evaluator + /// abandons a window that was about to open. + uint64_t (*pending_open_token)(void* ctx) = nullptr; + void* ctx = nullptr; + }; + + /** @brief Hooks wired to the real DeepWindow coordinator. */ + static Hooks liveHooks(); + + RuleEvaluator(DeepWindowRule rule, std::string rule_id, + const RuleCapabilities& caps, MetricSource* source, Hooks hooks); + + /** @brief Advance the machine. Call every collector beat. */ + void poll(int64_t now_ns); + + /** @brief Finalise at shutdown and return what the session concluded. */ + RuleSummary finish(int64_t now_ns); + + /** @brief Current summary without concluding, for a mid-run emit. */ + RuleSummary snapshot(int64_t now_ns) const; + + RuleState state() const { return state_; } + uint32_t windowsOpened() const { return windows_opened_; } + + /** + * @brief Mark a rule that was refused before it could run. + * + * Kept separate from the constructor because an invalid rule must never + * fail init(): configuration is parsed during init, and a hard failure + * there would leave no session and no telemetry writer - nowhere to record + * the very outcome that has to be reported. + */ + static RuleSummary refused(std::string rule_id, RuleOutcome outcome, + std::string reason, int64_t now_ns); + +private: + bool conditionHolds(double value) const; + bool rearmHolds(double value) const; + void enterBlackout(int64_t now_ns); + void toArmed(); + + DeepWindowRule rule_; + std::string rule_id_; + MetricSource* source_ = nullptr; + Hooks hooks_; + + RuleState state_ = RuleState::WarmingUp; + RuleOutcome terminal_ = RuleOutcome::None; + std::string reason_; + + uint64_t samples_seen_ = 0; + uint64_t last_sequence_ = 0; + bool have_sequence_ = false; + uint32_t windows_opened_ = 0; + uint64_t state_sequence_ = 0; + + /// Start of the current unbroken run of true readings. Compared against a + /// later sample's timestamp, so sustained_ms is a span between two + /// observations rather than an accumulation that one stale reading could + /// satisfy on its own. + std::optional first_true_observed_ns_; + std::optional last_value_; + std::optional last_observed_ns_; + MetricState last_metric_state_ = MetricState::Missing; + + uint64_t pending_token_ = 0; + bool window_was_active_ = false; +}; + +} // namespace gpufl::detail diff --git a/include/gpufl/core/env_vars.hpp b/include/gpufl/core/env_vars.hpp index 9978b96..ce4fd9c 100644 --- a/include/gpufl/core/env_vars.hpp +++ b/include/gpufl/core/env_vars.hpp @@ -148,6 +148,24 @@ constexpr const char* kDeepWindowCooldownMs = "GPUFL_DEEP_WINDOW_COOLDOWN_MS"; // it from --deep-after. Unset = no scheduled window. constexpr const char* kDeepAfterMs = "GPUFL_DEEP_AFTER_MS"; +// ── Conditional deep window ───────────────────────────────────────────────── +// Open a window when a metric crosses a threshold, e.g. +// "custom.token_rate<1000 for 2s". Separate options rather than one grammar: +// the rule carries more fields than an expression can hold readably, and each +// one needs its own validation message when it is refused. +constexpr const char* kDeepWhen = "GPUFL_DEEP_WHEN"; +// Window the rate is measured over. Also sizes the warm-up. +constexpr const char* kDeepRateWindowMs = "GPUFL_DEEP_RATE_WINDOW_MS"; +// How long the source may go quiet before its readings stop counting as +// evidence. Validated against rate window + sustained, since a value shorter +// than those can never fire. +constexpr const char* kDeepStaleAfterMs = "GPUFL_DEEP_STALE_AFTER_MS"; +// Hysteresis: the value the metric must recover past before the rule may fire +// again. Defaults to the trigger threshold, which is plain "condition false". +constexpr const char* kDeepRearmAt = "GPUFL_DEEP_REARM_AT"; +// How many windows one rule may open in a session. +constexpr const char* kDeepMaxWindows = "GPUFL_DEEP_MAX_WINDOWS"; + // ── SASS metrics knobs ────────────────────────────────────────────────────── constexpr const char* kSassMetricsOnly = "GPUFL_SASS_METRICS_ONLY"; constexpr const char* kSassForceSafeActivity = "GPUFL_SASS_FORCE_SAFE_ACTIVITY"; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 645ae61..c5c31ae 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -20,6 +20,7 @@ set(GPUFL_TEST_SOURCES core/test_wire_contract.cpp core/test_counter_registry.cpp core/test_metric_registry.cpp + core/test_deep_window_rule.cpp core/test_monitor.cpp core/test_itanium_demangle.cpp core/test_sampler.cpp diff --git a/tests/core/test_deep_window_rule.cpp b/tests/core/test_deep_window_rule.cpp new file mode 100644 index 0000000..5a404f9 --- /dev/null +++ b/tests/core/test_deep_window_rule.cpp @@ -0,0 +1,549 @@ +#include + +#include +#include + +#include "gpufl.hpp" +#include "gpufl/core/counter_provider.hpp" +#include "gpufl/core/counter_registry.hpp" +#include "gpufl/core/deep_window_rule.hpp" +#include "gpufl/core/metric_registry.hpp" + +using namespace gpufl::detail; + +namespace { + +constexpr int64_t kMs = 1000000; + +// A stand-in window coordinator. Lets the state machine be driven without a +// GPU, and - more usefully - lets a test refuse an open on purpose, which is +// the branch that decides whether budget is consumed. +struct FakeCoordinator { + bool active = false; + bool refuse = false; + uint64_t next_token = 1; + uint64_t last_opened_token = 0; + uint64_t pending_token = 0; + int opens = 0; + + static uint64_t RequestOpen(void* ctx, const gpufl::DeepWindowSpec&) { + auto* self = static_cast(ctx); + if (self->refuse || self->active) return 0; + self->pending_token = self->next_token++; + return self->pending_token; + } + static bool Active(void* ctx) { + return static_cast(ctx)->active; + } + static uint64_t LastOpenedToken(void* ctx) { + return static_cast(ctx)->last_opened_token; + } + static uint64_t PendingOpenToken(void* ctx) { + return static_cast(ctx)->pending_token; + } + + RuleEvaluator::Hooks hooks() { + RuleEvaluator::Hooks h; + h.request_open = &RequestOpen; + h.window_active = &Active; + h.last_opened_token = &LastOpenedToken; + h.pending_open_token = &PendingOpenToken; + h.ctx = this; + return h; + } + + /** The coordinator refusing the queued request when it gets to it. */ + void dropRequest() { pending_token = 0; } + + /** The coordinator servicing a queued request. */ + void serviceOpen() { + if (pending_token == 0) return; + last_opened_token = pending_token; + pending_token = 0; + active = true; + ++opens; + } + /** A window opened by someone else - manual, or the scheduled trigger. */ + void openManual() { + last_opened_token = 0; + active = true; + ++opens; + } + void close() { active = false; } +}; + +DeepWindowRule makeRule(const char* expr) { + const auto parsed = parseRuleExpression(expr); + EXPECT_TRUE(parsed.ok()) << expr << " -> " << toString(parsed.error); + DeepWindowRule r = parsed.rule; + r.timing.rate_window_ms = 1000; + r.timing.stale_after_ms = 20000; + r.window.max_duration_ms = 500; + r.max_windows = 3; + return r; +} + +// ----------------------------------------------------------------- parsing + +TEST(RuleParseTest, ParsesMetricOperatorThresholdAndDuration) { + const auto r = parseRuleExpression("custom.token_rate<1000 for 2s"); + ASSERT_TRUE(r.ok()) << toString(r.error); + EXPECT_EQ(r.rule.metric.canonical, "custom.token_rate"); + EXPECT_EQ(r.rule.op, Comparison::LessThan); + EXPECT_DOUBLE_EQ(r.rule.threshold, 1000.0); + EXPECT_EQ(r.rule.timing.sustained_ms, 2000); + // No hysteresis unless asked for: rearm degenerates to "condition false". + EXPECT_DOUBLE_EQ(r.rule.rearm_threshold, 1000.0); +} + +TEST(RuleParseTest, AcceptsMillisecondsAndBareNumbers) { + EXPECT_EQ(parseRuleExpression("kernel_launch_rate>50 for 500ms").rule + .timing.sustained_ms, 500); + EXPECT_EQ(parseRuleExpression("kernel_launch_rate>50 for 750").rule + .timing.sustained_ms, 750); + EXPECT_EQ(parseRuleExpression("kernel_launch_rate>50").rule + .timing.sustained_ms, 0); +} + +TEST(RuleParseTest, RejectsGarbage) { + EXPECT_EQ(parseRuleExpression("kernel_launch_rate 50").error, + RuleError::Unparsable); + EXPECT_EQ(parseRuleExpression("kernel_launch_rate>abc").error, + RuleError::Unparsable); + EXPECT_EQ(parseRuleExpression("kernel_launch_rate>50 for 2 parsecs").error, + RuleError::Unparsable); + // The metric reason must survive, not be flattened into "unparsable". + const auto bad = parseRuleExpression("tokne_rate<5"); + EXPECT_EQ(bad.error, RuleError::BadMetric); + EXPECT_EQ(bad.metric_error, MetricParseError::MissingCustomPrefix); +} + +// -------------------------------------------------------------- validation + +TEST(RuleValidateTest, AcceptsAWorkableRule) { + EXPECT_EQ(validateRule(makeRule("custom.token_rate<1000 for 2s")).error, + RuleError::None); +} + +TEST(RuleValidateTest, RejectsRearmOnTheWrongSide) { + DeepWindowRule r = makeRule("custom.token_rate<1000 for 2s"); + r.rearm_threshold = 900; // below the threshold: can never be reached + // Otherwise the rule fires once and then waits forever for a condition + // that cannot occur, which looks identical to a healthy armed rule. + EXPECT_EQ(validateRule(r).error, RuleError::RearmWrongSide); + + DeepWindowRule g = makeRule("kernel_launch_rate>1000 for 2s"); + g.rearm_threshold = 1100; + EXPECT_EQ(validateRule(g).error, RuleError::RearmWrongSide); +} + +TEST(RuleValidateTest, RejectsNonFiniteThreshold) { + DeepWindowRule r = makeRule("custom.token_rate<1000 for 2s"); + r.threshold = std::nan(""); + r.rearm_threshold = r.threshold; + // Every comparison against NaN is false, so the rule would never fire and + // never report a problem. + EXPECT_EQ(validateRule(r).error, RuleError::ThresholdNotFinite); +} + +TEST(RuleValidateTest, RejectsBudgetOutOfRange) { + DeepWindowRule r = makeRule("custom.token_rate<1000 for 2s"); + r.max_windows = 0; + EXPECT_EQ(validateRule(r).error, RuleError::MaxWindowsOutOfRange); + r.max_windows = 100000; + EXPECT_EQ(validateRule(r).error, RuleError::MaxWindowsOutOfRange); +} + +TEST(RuleValidateTest, RejectsAWindowWithNoBound) { + DeepWindowRule r = makeRule("custom.token_rate<1000 for 2s"); + r.window.max_duration_ms = 0; + r.window.max_launches = 0; + // A window that never closes turns a bounded-cost feature into an + // always-on one. + EXPECT_EQ(validateRule(r).error, RuleError::WindowBoundsMissing); +} + +TEST(RuleValidateTest, RejectsTimingThatCanNeverFire) { + DeepWindowRule r = makeRule("custom.token_rate<1000 for 2s"); + r.timing.rate_window_ms = 4000; + r.timing.sustained_ms = 4000; + r.timing.stale_after_ms = 5000; + const auto v = validateRule(r); + EXPECT_EQ(v.error, RuleError::BadTiming); + EXPECT_EQ(v.config_error, ConfigError::StaleBeforeEvidence); + EXPECT_FALSE(v.detail.empty()) << "the arithmetic has to be in the message"; +} + +// ----------------------------------------------------------- state machine + +class RuleEvaluatorTest : public ::testing::Test { + protected: + void SetUp() override { CounterRegistry::instance().resetForTesting(); } + void TearDown() override { CounterRegistry::instance().resetForTesting(); } + + MetricFeeds feeds; + FakeCoordinator coord; + + // Drive the collector beat, feeding `launches` launches per bucket. + void run(RuleEvaluator& ev, MetricSource& src, int64_t from_ns, int64_t to_ns, + int launches_per_beat) { + for (int64_t t = from_ns; t <= to_ns; t += 10 * kMs) { + for (int i = 0; i < launches_per_beat; ++i) feeds.noteKernelLaunch(t); + ev.poll(t); + } + } +}; + +TEST_F(RuleEvaluatorTest, FiresOnlyAfterTheConditionIsSustained) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 2s"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + // Busy: 10 launches per 10ms beat == 1000/s, above the threshold. + run(ev, src, 0, 2000 * kMs, 10); + EXPECT_EQ(ev.state(), RuleState::Armed); + + // Goes quiet. The rule must not fire on the first true reading. + run(ev, src, 2010 * kMs, 4000 * kMs, 0); + EXPECT_EQ(ev.state(), RuleState::Pending) + << "fired before the condition was held for 2s"; + + run(ev, src, 4010 * kMs, 6000 * kMs, 0); + EXPECT_EQ(ev.state(), RuleState::Opening); + EXPECT_EQ(ev.windowsOpened(), 0u) << "budget consumed before a window opened"; +} + +TEST_F(RuleEvaluatorTest, BudgetCountsOnlyWindowsThatActuallyOpened) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + ASSERT_EQ(ev.state(), RuleState::Opening); + EXPECT_EQ(ev.windowsOpened(), 0u); + + coord.serviceOpen(); + ev.poll(4010 * kMs); + EXPECT_EQ(ev.state(), RuleState::Blackout); + EXPECT_EQ(ev.windowsOpened(), 1u); +} + +TEST_F(RuleEvaluatorTest, ADroppedRequestCostsNoBudget) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + ASSERT_EQ(ev.state(), RuleState::Opening); + + // The coordinator discards the request instead of opening. + coord.dropRequest(); + ev.poll(4010 * kMs); + EXPECT_EQ(ev.state(), RuleState::Armed); + EXPECT_EQ(ev.windowsOpened(), 0u) + << "budget bounds what the rule cost, and this cost nothing"; +} + +TEST_F(RuleEvaluatorTest, ARefusedOpenReturnsToArmedRatherThanHoldingPending) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + coord.refuse = true; // cooldown still running + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + EXPECT_EQ(ev.windowsOpened(), 0u); + + // The refusal must discard the evidence, not bank it. Accepting requests + // again is not enough on its own - the rule has to gather a fresh + // sustained run first, or a cooldown would just delay a fire that then + // lands on readings from long before. + coord.refuse = false; + ev.poll(4010 * kMs); + EXPECT_NE(ev.state(), RuleState::Opening) + << "fired on evidence gathered while the coordinator was refusing"; + + run(ev, src, 4020 * kMs, 5000 * kMs, 0); + EXPECT_EQ(ev.state(), RuleState::Opening) + << "never recovered after the refusal"; +} + +TEST_F(RuleEvaluatorTest, AManualWindowBlacksOutButCostsNoBudget) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 2s"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + ASSERT_EQ(ev.state(), RuleState::Armed); + + coord.openManual(); + ev.poll(1510 * kMs); + // Contamination does not care who opened the window. + EXPECT_EQ(ev.state(), RuleState::Blackout); + EXPECT_EQ(ev.windowsOpened(), 0u) + << "a manual window must not spend the rule's budget"; +} + +TEST_F(RuleEvaluatorTest, ContaminatedSamplesDoNotProveRecovery) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + coord.serviceOpen(); + ev.poll(4010 * kMs); + ASSERT_EQ(ev.state(), RuleState::Blackout); + + // Busy traffic while the window is open must not count as recovery. + run(ev, src, 4020 * kMs, 6000 * kMs, 10); + EXPECT_EQ(ev.state(), RuleState::Blackout); + + coord.close(); + ev.poll(6010 * kMs); + EXPECT_EQ(ev.state(), RuleState::Recovery) + << "the clean epoch only starts once the window has closed"; +} + +TEST_F(RuleEvaluatorTest, NoRefireUntilTheWorkloadRecovers) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + coord.serviceOpen(); + ev.poll(4010 * kMs); + coord.close(); + ev.poll(4020 * kMs); + + // Still quiet - the condition is still true. It must NOT reopen. + run(ev, src, 4030 * kMs, 9000 * kMs, 0); + EXPECT_EQ(ev.windowsOpened(), 1u) + << "refired while the condition never stopped holding"; + EXPECT_NE(ev.state(), RuleState::Blackout); + + // Recover, then degrade again: the second window is legitimate. + run(ev, src, 9010 * kMs, 11000 * kMs, 10); + run(ev, src, 11010 * kMs, 14000 * kMs, 0); + coord.serviceOpen(); + ev.poll(14010 * kMs); + EXPECT_EQ(ev.windowsOpened(), 2u); +} + +TEST_F(RuleEvaluatorTest, ExhaustionIsMarkedAtTheOpenThatReachesTheLimit) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + rule.max_windows = 1; + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + coord.serviceOpen(); + ev.poll(4010 * kMs); + ASSERT_EQ(ev.windowsOpened(), 1u); + + // Marked at the transition, not at shutdown, so a crashed run still + // explains itself. + EXPECT_EQ(ev.snapshot(4020 * kMs).outcome, RuleOutcome::Exhausted); + + coord.close(); + ev.poll(4030 * kMs); + // The evaluator has nowhere left to stand, so it goes inactive rather than + // back into recovery. + EXPECT_EQ(ev.state(), RuleState::Inactive); + + run(ev, src, 4040 * kMs, 10000 * kMs, 0); + EXPECT_EQ(ev.windowsOpened(), 1u); +} + +TEST_F(RuleEvaluatorTest, HysteresisNeedsARealRecoveryNotABrushPastTheThreshold) { + DeepWindowRule rule = makeRule("kernel_launch_rate<500 for 500ms"); + rule.rearm_threshold = 900; // must climb back above 900 to rearm + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + coord.serviceOpen(); + ev.poll(4010 * kMs); + coord.close(); + ev.poll(4020 * kMs); + + // ~600/s: above the trigger threshold but below the rearm threshold. + run(ev, src, 4030 * kMs, 7000 * kMs, 6); + EXPECT_EQ(ev.state(), RuleState::WaitingForRearm) + << "rearmed on a value that had not actually recovered"; + + run(ev, src, 7010 * kMs, 10000 * kMs, 10); + EXPECT_EQ(ev.state(), RuleState::Armed); +} + +TEST_F(RuleEvaluatorTest, StaleDataBreaksTheSustainedRun) { + // A gauge, not a rate. For a rate metric the validated timing GUARANTEES a + // stall fires before it goes stale, so a rate could never demonstrate this. + // A gauge stops for its own reasons - NVML no longer answering - while the + // measured quantity is still low, and that is the case worth guarding. + DeepWindowRule rule = makeRule("gpu[0].util_pct<10 for 2s"); + rule.timing.stale_after_ms = 3100; + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + + gpufl::DeviceSample idle; + idle.device_id = 0; + idle.gpu_util = 2; + for (int64_t t = 0; t <= 1000 * kMs; t += 100 * kMs) { + feeds.noteDeviceSample(idle, t); + ev.poll(t); + } + ASSERT_EQ(ev.state(), RuleState::Pending); + + // Measurements stop. Stale is not evidence the condition continued, so the + // run is broken rather than completed by readings nobody took. + for (int64_t t = 1010 * kMs; t <= 9000 * kMs; t += 10 * kMs) ev.poll(t); + EXPECT_EQ(ev.state(), RuleState::Armed); + EXPECT_EQ(ev.windowsOpened(), 0u); +} + +// ----------------------------------------------------------------- gates + +TEST_F(RuleEvaluatorTest, RuleIsRefusedWhenNoDeepEngineIsPrepared) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleCapabilities caps; + caps.deep_engine_prepared = false; + RuleEvaluator ev(rule, "r1", caps, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 4000 * kMs, 0); + EXPECT_EQ(ev.state(), RuleState::Inactive); + EXPECT_EQ(ev.windowsOpened(), 0u) + << "burned budget opening windows that would arm nothing"; + + const RuleSummary s = ev.finish(4000 * kMs); + EXPECT_EQ(s.outcome, RuleOutcome::Unsupported); + EXPECT_EQ(s.reason, "no_deep_engine"); +} + +TEST_F(RuleEvaluatorTest, CustomCounterRuleIsRefusedWhenCountersAreNotShared) { + DeepWindowRule rule = makeRule("custom.token_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleCapabilities caps; + caps.multi_module = true; // injection: target and evaluator differ + caps.counters_shared = false; + RuleEvaluator ev(rule, "r1", caps, &src, coord.hooks()); + + // The target would tick one registry and this evaluator read another, so + // the counter can never be seen. Saying so beats reporting a counter that + // is being ticked as Missing for the whole run. + EXPECT_EQ(ev.state(), RuleState::Inactive); + EXPECT_EQ(ev.finish(1000 * kMs).reason, "counters_not_shared"); +} + +TEST_F(RuleEvaluatorTest, UnsharedCountersAreFineInAnEmbeddedHost) { + DeepWindowRule rule = makeRule("custom.token_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleCapabilities caps; + caps.multi_module = false; // one copy of gpufl in the process + caps.counters_shared = false; + RuleEvaluator ev(rule, "r1", caps, &src, coord.hooks()); + EXPECT_NE(ev.state(), RuleState::Inactive); +} + +TEST_F(RuleEvaluatorTest, GaugeRuleOnAMissingDeviceIsRefusedEagerly) { + DeepWindowRule rule = makeRule("gpu[3].util_pct<10 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleCapabilities caps; + caps.device_count = 1; + RuleEvaluator ev(rule, "r1", caps, &src, coord.hooks()); + // A built-in metric's availability cannot change later, unlike a custom + // counter's, so it is decided now. + EXPECT_EQ(ev.state(), RuleState::Inactive); + EXPECT_EQ(ev.finish(0).reason, "metric_unavailable"); +} + +// --------------------------------------------------------------- summaries + +TEST_F(RuleEvaluatorTest, ARuleThatNeverMatchedStillReportsAnOutcome) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 4000 * kMs, 10); // always busy + const RuleSummary s = ev.finish(4000 * kMs); + // "log once" is invisible in the UI, and absence of a record cannot be + // read as evidence the rule never fired. + EXPECT_EQ(s.outcome, RuleOutcome::NeverTrue); + EXPECT_EQ(s.reason, "condition_never_held"); + EXPECT_GT(s.samples_seen, 0u); + EXPECT_TRUE(s.last_value.has_value()); +} + +TEST_F(RuleEvaluatorTest, StateAndOutcomeAreSeparateFields) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + const RuleSummary mid = ev.snapshot(1500 * kMs); + // `armed` is where the evaluator is standing, not a verdict on the session. + EXPECT_EQ(mid.state, RuleState::Armed); + EXPECT_EQ(mid.outcome, RuleOutcome::None); +} + +TEST_F(RuleEvaluatorTest, ANeverRegisteredCounterIsNamedInTheSummary) { + DeepWindowRule rule = makeRule("custom.never_ticked_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + + for (int64_t t = 0; t <= 4000 * kMs; t += 10 * kMs) ev.poll(t); + const RuleSummary s = ev.finish(4000 * kMs); + EXPECT_EQ(s.outcome, RuleOutcome::NeverTrue); + EXPECT_EQ(s.reason, "custom_metric_never_registered"); + EXPECT_EQ(s.last_metric_state, MetricState::Missing); +} + +TEST_F(RuleEvaluatorTest, StateSequenceIsMonotonic) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + uint64_t previous = 0; + for (int64_t t = 0; t <= 6000 * kMs; t += 10 * kMs) { + if (t < 1500 * kMs) feeds.noteKernelLaunch(t); + ev.poll(t); + const uint64_t seq = ev.snapshot(t).state_sequence; + // A late record must never overwrite a newer one at the backend, and + // that ordering is only as good as this number. + EXPECT_GE(seq, previous); + previous = seq; + } + EXPECT_GT(ev.finish(6000 * kMs).state_sequence, previous); +} + +TEST(RuleRefusedTest, AnInvalidRuleStillProducesAReportableSummary) { + // Configuration is parsed during init(); failing hard there would leave no + // session and no telemetry writer - nowhere to record this very outcome. + const RuleSummary s = RuleEvaluator::refused( + "r1", RuleOutcome::InvalidConfig, "rearm_wrong_side", 42); + EXPECT_EQ(s.state, RuleState::Inactive); + EXPECT_EQ(s.outcome, RuleOutcome::InvalidConfig); + EXPECT_EQ(s.reason, "rearm_wrong_side"); + EXPECT_GT(s.state_sequence, 0u); +} + +} // namespace From 438ed404aa938f053c9326eb045391a4df26f048 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Sun, 26 Jul 2026 20:32:55 -0700 Subject: [PATCH 08/28] fix(test): include for std::nan on gcc --- tests/core/test_deep_window_rule.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/core/test_deep_window_rule.cpp b/tests/core/test_deep_window_rule.cpp index 5a404f9..2539afe 100644 --- a/tests/core/test_deep_window_rule.cpp +++ b/tests/core/test_deep_window_rule.cpp @@ -1,5 +1,6 @@ #include +#include #include #include From d1030ea6d31807e5986c29ec9ed67b0193967fe9 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Sun, 26 Jul 2026 20:43:52 -0700 Subject: [PATCH 09/28] feat(deep-window): wire the rule in and record what it concluded The evaluator existed but nothing drove it. GPUFL_DEEP_WHEN now installs a rule at init, the collector beat advances it beside the window service, and the launch callback and sampler feed it. Windows carry the comparison that caused them. A bare observed value stops being readable the first time somebody edits the threshold, so the rule id, operator, threshold, rate window, sustained duration and first-true timestamp travel with the window. Manual windows carry no trigger at all rather than an all-zero one that reads like a real rule. Every rule emits a summary, including one that never fired. Absence has to keep meaning "the run ended unexpectedly" - if a rule that was never true also left nothing, the UI could not tell the two apart. An invalid rule is installed as refused rather than dropped, so it still reports at shutdown, and init always succeeds: config is parsed there, so failing hard would leave nowhere to record the refusal. The launch feed is opt-in per rule. A rule watching a custom counter or a GPU gauge never reads it, and every launch in the process would otherwise pay for a feed nobody consumes. Device count is deliberately not guessed. Devices are not enumerated until the sampler's first measurement, and assuming one would refuse a valid gpu[3] rule on a four-GPU host; a device that never reports is named in the summary instead. --- CMakeLists.txt | 1 + .../gpufl/backends/nvidia/cupti_backend.cpp | 15 + .../gpufl/backends/nvidia/cupti_backend.hpp | 5 + include/gpufl/core/deep_window.cpp | 3 + include/gpufl/core/deep_window.hpp | 6 + include/gpufl/core/deep_window_rule.cpp | 34 +- include/gpufl/core/deep_window_rule.hpp | 8 + include/gpufl/core/deep_window_rules.cpp | 300 ++++++++++++++++++ include/gpufl/core/deep_window_rules.hpp | 65 ++++ include/gpufl/core/events.hpp | 58 ++++ include/gpufl/core/gpufl.cpp | 5 + .../gpufl/core/model/deep_window_model.cpp | 45 +++ .../gpufl/core/model/deep_window_model.hpp | 15 + include/gpufl/core/monitor.cpp | 11 + include/gpufl/core/monitor_backend.hpp | 14 + include/gpufl/core/sampler.cpp | 7 + tests/CMakeLists.txt | 1 + tests/core/test_deep_window_rules_install.cpp | 119 +++++++ 18 files changed, 706 insertions(+), 6 deletions(-) create mode 100644 include/gpufl/core/deep_window_rules.cpp create mode 100644 include/gpufl/core/deep_window_rules.hpp create mode 100644 tests/core/test_deep_window_rules_install.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0daba4e..8d2899c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,6 +127,7 @@ target_sources(gpufl PRIVATE include/gpufl/core/counter_provider.cpp include/gpufl/core/metric_id.cpp include/gpufl/core/deep_window_rule.cpp + include/gpufl/core/deep_window_rules.cpp include/gpufl/core/metric_registry.cpp include/gpufl/core/monitor_batch_manager.cpp include/gpufl/core/monitor_record_builders.cpp diff --git a/include/gpufl/backends/nvidia/cupti_backend.cpp b/include/gpufl/backends/nvidia/cupti_backend.cpp index 07a3212..b621417 100644 --- a/include/gpufl/backends/nvidia/cupti_backend.cpp +++ b/include/gpufl/backends/nvidia/cupti_backend.cpp @@ -23,6 +23,7 @@ #include "gpufl/core/common.hpp" #include "gpufl/core/debug_logger.hpp" #include "gpufl/core/deep_window.hpp" +#include "gpufl/core/deep_window_rules.hpp" #include "gpufl/core/logger/logger.hpp" #include "gpufl/core/monitor.hpp" // Monitor::RequestSyntheticDrainAndWait #include "gpufl/core/model/perf_metric_model.hpp" @@ -746,6 +747,11 @@ void CuptiBackend::EngineLaunchTick() { // stop/collect. Closing before the engine tick also spares the engine a // beat it would only spend on a window that is already over. DeepWindow::OnLaunch(); + // The HOST launch rate a rule may watch. Gated on a rule wanting it, so a + // run without one pays a single relaxed atomic load per launch. + if (detail::DeepWindowRules::WantsLaunchFeed()) { + detail::DeepWindowRules::NoteKernelLaunch(detail::GetTimestampNs()); + } if (engine_) engine_->onLaunchTick(); } @@ -762,6 +768,15 @@ void CuptiBackend::DrainProfilingData() { if (rebound) cuCtxSetCurrent(prev); } +bool CuptiBackend::DeepEnginesPrepared() const { + // The REQUEST set, not the runtime state: under WindowOnly nothing is armed + // until a window opens, and this has to answer before one ever does. + const EngineRequestSet requested = + BuildEngineRequestSet(opts_.profiling_engine, combo_); + return requested.pc || requested.sass || requested.pm || requested.range || + requested.range_kernel; +} + std::vector CuptiBackend::ArmedEngineWireNames_() const { // Reuses the inspector the capability rows are built from, so a combo, a // Deep run and a single engine all narrow the same way: each path reports diff --git a/include/gpufl/backends/nvidia/cupti_backend.hpp b/include/gpufl/backends/nvidia/cupti_backend.hpp index 8441451..ddefc0b 100644 --- a/include/gpufl/backends/nvidia/cupti_backend.hpp +++ b/include/gpufl/backends/nvidia/cupti_backend.hpp @@ -247,6 +247,11 @@ class CuptiBackend : public IMonitorBackend { // or disarms it. std::vector ArmedEngineWireNames_() const; + // Capability gate for conditional windows: does this run have an engine a + // window could arm at all? A Trace-only run answers no, which is what stops + // a rule burning its budget on windows that collect nothing. + bool DeepEnginesPrepared() const override; + bool ShouldEnableNvtxMarkerActivityBeforeEngine_() const; bool ShouldEnableNvtxMarkerActivityForSelectedEngine_() const; static void EnableNvtxMarkerActivity_(const char* phase); diff --git a/include/gpufl/core/deep_window.cpp b/include/gpufl/core/deep_window.cpp index f016e67..1a1b4ba 100644 --- a/include/gpufl/core/deep_window.cpp +++ b/include/gpufl/core/deep_window.cpp @@ -63,6 +63,7 @@ uint64_t g_pending_token = 0; // guarded by g_mu // Open() can never inherit someone else's attribution. thread_local uint64_t g_claimed_token = 0; +DeepWindowTrigger g_trigger; // guarded by g_mu int64_t g_opened_ns = 0; int64_t g_requested_duration_ms = 0; uint64_t g_requested_max_launches = 0; @@ -204,6 +205,7 @@ bool DeepWindow::Open(const DeepWindowSpec& spec) { g_opens_completed.fetch_add(1, std::memory_order_acq_rel); g_opened_ns = detail::GetTimestampNs(); + g_trigger = spec.trigger; g_name = spec.name.empty() ? "deep_window" : spec.name; g_requested_duration_ms = spec.max_duration_ms; g_requested_max_launches = spec.max_launches; @@ -271,6 +273,7 @@ void DeepWindow::Close(const DeepWindowClose reason) { // the engine teardown, so a slow disarm doesn't shorten the quiet time. g_last_close_ns.store(end_ns, std::memory_order_relaxed); + ev.trigger = g_trigger; ev.pid = detail::GetPid(); ev.name = g_name; ev.close_reason = DeepWindowCloseName(reason); diff --git a/include/gpufl/core/deep_window.hpp b/include/gpufl/core/deep_window.hpp index a6cdda5..035a280 100644 --- a/include/gpufl/core/deep_window.hpp +++ b/include/gpufl/core/deep_window.hpp @@ -3,6 +3,8 @@ #include #include +#include "gpufl/core/events.hpp" + namespace gpufl { /** @@ -37,6 +39,10 @@ struct DeepWindowSpec { // here rather than in the caller's trigger. int64_t cooldown_ms = 0; std::string name = "deep_window"; + // What asked for this window, when a rule did. Travels with the spec so the + // window carries its own explanation rather than needing a lookup that + // would have to survive the window closing. + DeepWindowTrigger trigger; }; /** diff --git a/include/gpufl/core/deep_window_rule.cpp b/include/gpufl/core/deep_window_rule.cpp index 31bfc4e..eb6c122 100644 --- a/include/gpufl/core/deep_window_rule.cpp +++ b/include/gpufl/core/deep_window_rule.cpp @@ -248,7 +248,7 @@ RuleEvaluator::RuleEvaluator(DeepWindowRule rule, std::string rule_id, // the alternative is reporting a counter that is being ticked as // Missing for the whole run. gate = RuleGate::CountersNotShared; - } else if (!rule_.metric.resolvesLazily() && + } else if (caps.device_count > 0 && !rule_.metric.resolvesLazily() && rule_.metric.shape() == MetricShape::Gauge && rule_.metric.device_index >= caps.device_count) { // Built-in metrics are decided eagerly - unlike a custom counter, the @@ -263,6 +263,22 @@ RuleEvaluator::RuleEvaluator(DeepWindowRule rule, std::string rule_id, } } +DeepWindowSpec RuleEvaluator::specWithTrigger(const MetricSample& sample) const { + DeepWindowSpec spec = rule_.window; + spec.trigger.present = true; + spec.trigger.rule_id = rule_id_; + spec.trigger.metric = rule_.metric.canonical; + spec.trigger.op = toString(rule_.op); + spec.trigger.threshold = rule_.threshold; + spec.trigger.rearm_threshold = rule_.rearm_threshold; + spec.trigger.observed = sample.value; + spec.trigger.rate_window_ms = rule_.timing.rate_window_ms; + spec.trigger.sustained_ms = rule_.timing.sustained_ms; + spec.trigger.first_true_ns = first_true_observed_ns_.value_or(sample.observed_ns); + spec.trigger.fired_ns = sample.observed_ns; + return spec; +} + bool RuleEvaluator::conditionHolds(const double value) const { return rule_.op == Comparison::LessThan ? value < rule_.threshold : value > rule_.threshold; @@ -401,7 +417,7 @@ void RuleEvaluator::poll(const int64_t now_ns) { ++state_sequence_; // sustained_ms == 0 fires on this same reading, handled below. if (rule_.timing.sustained_ms == 0) { - pending_token_ = hooks_.request_open(hooks_.ctx, rule_.window); + pending_token_ = hooks_.request_open(hooks_.ctx, specWithTrigger(sample)); if (pending_token_ != 0) { state_ = RuleState::Opening; ++state_sequence_; @@ -427,7 +443,7 @@ void RuleEvaluator::poll(const int64_t now_ns) { const int64_t held_ns = sample.observed_ns - *first_true_observed_ns_; if (held_ns < rule_.timing.sustained_ms * 1000000) break; - pending_token_ = hooks_.request_open(hooks_.ctx, rule_.window); + pending_token_ = hooks_.request_open(hooks_.ctx, specWithTrigger(sample)); if (pending_token_ != 0) { state_ = RuleState::Opening; ++state_sequence_; @@ -482,9 +498,15 @@ RuleSummary RuleEvaluator::finish(const int64_t now_ns) { if (s.reason.empty()) { // Distinguish "watched it and it never happened" from "never had // anything to watch"; both leave windows_opened at 0. - s.reason = last_metric_state_ == MetricState::Missing - ? "custom_metric_never_registered" - : "condition_never_held"; + if (last_metric_state_ == MetricState::Missing) { + s.reason = "custom_metric_never_registered"; + } else if (samples_seen_ == 0) { + // Watched, but the source never produced a usable reading - + // a device that does not exist, or a sampler that never ran. + s.reason = "metric_source_never_reported"; + } else { + s.reason = "condition_never_held"; + } } } diff --git a/include/gpufl/core/deep_window_rule.hpp b/include/gpufl/core/deep_window_rule.hpp index 4f9e81a..95d2961 100644 --- a/include/gpufl/core/deep_window_rule.hpp +++ b/include/gpufl/core/deep_window_rule.hpp @@ -234,6 +234,14 @@ class RuleEvaluator { std::string reason, int64_t now_ns); private: + /** + * The window spec plus the comparison that produced it. + * + * Built at the request, not at the close: by the time the window closes the + * reading that caused it is long gone, and a bare observed value with no + * threshold beside it stops being readable the first time the rule changes. + */ + DeepWindowSpec specWithTrigger(const MetricSample& sample) const; bool conditionHolds(double value) const; bool rearmHolds(double value) const; void enterBlackout(int64_t now_ns); diff --git a/include/gpufl/core/deep_window_rules.cpp b/include/gpufl/core/deep_window_rules.cpp new file mode 100644 index 0000000..e3177d9 --- /dev/null +++ b/include/gpufl/core/deep_window_rules.cpp @@ -0,0 +1,300 @@ +#include "gpufl/core/deep_window_rules.hpp" + +#include +#include +#include +#include +#include + +#include "gpufl/core/common.hpp" +#include "gpufl/core/counter_provider.hpp" +#include "gpufl/core/debug_logger.hpp" +#include "gpufl/core/deep_window_rule.hpp" +#include "gpufl/core/env_vars.hpp" +#include "gpufl/core/events.hpp" +#include "gpufl/core/logger/logger.hpp" +#include "gpufl/core/metric_registry.hpp" +#include "gpufl/core/model/deep_window_model.hpp" +#include "gpufl.hpp" +#include "gpufl/core/monitor.hpp" +#include "gpufl/core/monitor_backend.hpp" +#include "gpufl/core/runtime.hpp" + +namespace gpufl::detail { +namespace { + +std::mutex g_mu; +bool g_installed = false; +bool g_finished = false; +std::string g_expression; +std::string g_rule_id; + +std::unique_ptr g_feeds; +std::unique_ptr g_source; +std::unique_ptr g_eval; +// Set when the rule was refused before it could run, so Finish() still has +// something to report. Holding the summary rather than an error string keeps +// the refused and the ran paths on one shape. +std::unique_ptr g_refused; + +// Read by the launch callback on every launch, so it must not take the lock. +std::atomic g_wants_launch_feed{false}; + +const char* EnvOrNull(const char* name) { + const char* v = std::getenv(name); + return (v && v[0] != '\0') ? v : nullptr; +} + +int64_t EnvIntOr(const char* name, const int64_t fallback) { + const char* v = EnvOrNull(name); + if (!v) return fallback; + char* end = nullptr; + const long long n = std::strtoll(v, &end, 10); + if (end == v || *end != '\0') { + GFL_LOG_ERROR(name, "='", v, "' is not an integer. Using ", fallback, "."); + return fallback; + } + return n; +} + +bool EnvDoubleOr(const char* name, double* out) { + const char* v = EnvOrNull(name); + if (!v) return false; + char* end = nullptr; + const double d = std::strtod(v, &end); + if (end == v || *end != '\0') { + GFL_LOG_ERROR(name, "='", v, "' is not a number. Ignoring."); + return false; + } + *out = d; + return true; +} + +/** + * Short, stable id for a rule. + * + * Hashed over the CANONICAL form, so two spellings of one rule do not produce + * two ids. An invalid rule has no canonical form, so it hashes its normalised + * raw input instead - a rejected rule still needs an id to be reported under. + */ +std::string RuleId(const std::string& canonical_input) { + uint64_t h = 1469598103934665603ull; // FNV-1a + for (const unsigned char c : canonical_input) { + h ^= c; + h *= 1099511628211ull; + } + std::ostringstream oss; + oss << std::hex << h; + return oss.str().substr(0, 12); +} + +std::string CanonicalConfig(const DeepWindowRule& r) { + std::ostringstream oss; + oss << "v1|" << r.metric.canonical << '|' << toString(r.op) << '|' + << r.threshold << '|' << r.rearm_threshold << '|' + << r.timing.rate_window_ms << '|' << r.timing.sustained_ms << '|' + << r.timing.stale_after_ms << '|' << r.max_windows << '|' + << r.window.max_duration_ms << '|' << r.window.max_launches; + return oss.str(); +} + +RuleCapabilities QueryCapabilities() { + RuleCapabilities caps; + // A Monitor-only run has no engine to arm inside a window at all. + caps.windows_supported = g_opts.profiling_engine != ProfilingEngine::Monitor; + + IMonitorBackend* backend = Monitor::GetBackend(); + caps.deep_engine_prepared = backend != nullptr && backend->DeepEnginesPrepared(); + + caps.counters_shared = CounterProvider::isShared(); + // Under injection the target and this evaluator are separate modules, which + // is the only situation where an unshared registry actually breaks a rule. + caps.multi_module = EnvOrNull(env::kCudaInjection64Path) != nullptr; + + // 0 = unknown. The device list is not enumerated until the sampler takes + // its first measurement, which is after init(), so a rule naming a device + // cannot be refused eagerly without guessing. Guessing 1 would refuse a + // valid gpu[3] rule on a four-GPU host - worse than deciding late. A device + // that never reports is named in the summary instead. + caps.device_count = 0; + return caps; +} + +void RefuseLocked(const RuleOutcome outcome, const std::string& reason) { + g_refused = std::make_unique(RuleEvaluator::refused( + g_rule_id, outcome, reason, detail::GetTimestampNs())); + g_installed = true; + GFL_LOG_ERROR("[DeepWindowRule] ", env::kDeepWhen, "='", g_expression, + "' disabled: ", reason, + ". Profiling continues; only the trigger is off."); +} + +} // namespace + +void DeepWindowRules::InstallFromEnv() { + const char* expr = EnvOrNull(env::kDeepWhen); + if (expr == nullptr) return; + + std::lock_guard lk(g_mu); + if (g_installed) { + // One rule for the MVP. Rejected loudly rather than silently ignored: + // a second rule that quietly does nothing is worse than an error. + GFL_LOG_ERROR("[DeepWindowRule] a rule is already installed; ignoring '", + expr, "'"); + return; + } + + g_expression = expr; + g_rule_id = RuleId("v1|raw|" + g_expression); + + RuleParseResult parsed = parseRuleExpression(g_expression); + if (!parsed.ok()) { + RefuseLocked(RuleOutcome::InvalidConfig, + parsed.detail.empty() ? toString(parsed.error) : parsed.detail); + return; + } + + DeepWindowRule rule = parsed.rule; + rule.timing.rate_window_ms = EnvIntOr(env::kDeepRateWindowMs, 1000); + rule.timing.stale_after_ms = EnvIntOr( + env::kDeepStaleAfterMs, + // Default derived from the other two so the out-of-the-box combination + // is one that CAN fire, rather than one the validator then rejects. + rule.timing.rate_window_ms + rule.timing.sustained_ms + + rule.timing.bucketIntervalMs() + 1000); + double rearm = 0.0; + if (EnvDoubleOr(env::kDeepRearmAt, &rearm)) rule.rearm_threshold = rearm; + rule.max_windows = static_cast(EnvIntOr(env::kDeepMaxWindows, 3)); + + rule.window.max_duration_ms = EnvIntOr(env::kDeepWindowMs, 0); + rule.window.max_launches = + static_cast(EnvIntOr(env::kDeepWindowMaxLaunches, 0)); + rule.window.cooldown_ms = EnvIntOr(env::kDeepWindowCooldownMs, 0); + rule.window.name = "deep_window"; + + const RuleParseResult checked = validateRule(rule); + if (!checked.ok()) { + RefuseLocked(RuleOutcome::InvalidConfig, + checked.detail.empty() ? toString(checked.error) + : checked.detail); + return; + } + + g_rule_id = RuleId(CanonicalConfig(rule)); + g_feeds = std::make_unique(); + g_feeds->seedStartup(detail::GetTimestampNs()); + g_source = std::make_unique(rule.metric, rule.timing, + g_feeds.get(), + ActiveCounterProvider()); + g_eval = std::make_unique(rule, g_rule_id, QueryCapabilities(), + g_source.get(), + RuleEvaluator::liveHooks()); + g_installed = true; + // The launch feed costs an atomic per launch, so only a rule that reads it + // turns it on. + g_wants_launch_feed.store( + rule.metric.kind == MetricKind::KernelLaunchRate || + rule.metric.kind == MetricKind::RecentKernelMs, + std::memory_order_release); + + GFL_LOG_DEBUG("[DeepWindowRule] installed id=", g_rule_id, " '", g_expression, + "' state=", toString(g_eval->state())); +} + +bool DeepWindowRules::Installed() { + std::lock_guard lk(g_mu); + return g_installed; +} + +bool DeepWindowRules::WantsLaunchFeed() { + return g_wants_launch_feed.load(std::memory_order_acquire); +} + +void DeepWindowRules::NoteKernelLaunch(const int64_t ts_ns) { + if (!WantsLaunchFeed()) return; + std::lock_guard lk(g_mu); + if (g_feeds) g_feeds->noteKernelLaunch(ts_ns); +} + +void DeepWindowRules::NoteKernelDuration(const int64_t ts_ns, const double ms) { + if (!WantsLaunchFeed()) return; + std::lock_guard lk(g_mu); + if (g_feeds) g_feeds->noteKernelDuration(ts_ns, ms); +} + +void DeepWindowRules::NoteDeviceSample(const DeviceSample& sample, + const int64_t ts_ns) { + std::lock_guard lk(g_mu); + if (g_feeds) g_feeds->noteDeviceSample(sample, ts_ns); +} + +void DeepWindowRules::Service() { + std::lock_guard lk(g_mu); + if (!g_eval) return; + g_eval->poll(detail::GetTimestampNs()); +} + +void DeepWindowRules::Finish() { + RuleSummary summary; + { + std::lock_guard lk(g_mu); + if (!g_installed || g_finished) return; + g_finished = true; + if (g_refused) { + summary = *g_refused; + } else if (g_eval) { + summary = g_eval->finish(detail::GetTimestampNs()); + } else { + return; + } + } + + const Runtime* rt = runtime(); + if (!rt || !rt->logger) { + // Nothing to write to. Logged rather than dropped silently, since this + // is the record that explains why no window ever appeared. + GFL_LOG_ERROR("[DeepWindowRule] no logger at shutdown; summary lost: ", + toString(summary.outcome), " ", summary.reason); + return; + } + + DeepWindowRuleSummaryEvent ev; + ev.pid = detail::GetPid(); + ev.app = rt->app_name; + ev.session_id = rt->session_id; + ev.rule_id = summary.rule_id; + ev.expression = g_expression; + ev.state = toString(summary.state); + ev.outcome = toString(summary.outcome); + ev.reason = summary.reason; + ev.metric_state = toString(summary.last_metric_state); + ev.samples_seen = summary.samples_seen; + ev.windows_opened = summary.windows_opened; + ev.has_last_value = summary.last_value.has_value(); + if (ev.has_last_value) { + ev.last_value = *summary.last_value; + ev.last_observed_ns = summary.last_observed_ns.value_or(0); + } + ev.state_sequence = summary.state_sequence; + ev.emitted_ns = summary.emitted_ns; + + rt->logger->write(model::DeepWindowRuleSummaryModel(ev)); + GFL_LOG_DEBUG("[DeepWindowRule] summary id=", ev.rule_id, + " outcome=", ev.outcome, " windows=", ev.windows_opened, + " reason=", ev.reason); +} + +void DeepWindowRules::ResetForTesting() { + std::lock_guard lk(g_mu); + g_installed = false; + g_finished = false; + g_expression.clear(); + g_rule_id.clear(); + g_eval.reset(); + g_source.reset(); + g_feeds.reset(); + g_refused.reset(); + g_wants_launch_feed.store(false, std::memory_order_release); +} + +} // namespace gpufl::detail diff --git a/include/gpufl/core/deep_window_rules.hpp b/include/gpufl/core/deep_window_rules.hpp new file mode 100644 index 0000000..243535e --- /dev/null +++ b/include/gpufl/core/deep_window_rules.hpp @@ -0,0 +1,65 @@ +#pragma once + +#include +#include + +namespace gpufl { +struct DeviceSample; +} + +namespace gpufl::detail { + +/** + * @brief Owns the one conditional rule for the session. + * + * A facade rather than an object the caller holds, because the feed points sit + * in the launch callback and the sampler while the evaluation sits on the + * collector - three call sites that have no way to pass an instance between + * them. + * + * One rule for the MVP. Multi-rule arbitration is deferred rather than guessed: + * two rules that both want a window need a policy for who wins, and inventing + * one before anybody has asked is how it ends up wrong. + */ +class DeepWindowRules { +public: + /** + * @brief Read GPUFL_DEEP_WHEN and friends. Called once from init(). + * + * NEVER fails init(), whatever the configuration says. Config is parsed + * during init, so a hard failure here would leave no session and no + * telemetry writer - nowhere to record the very outcome a rejected rule has + * to report. An invalid rule disables the trigger and nothing else; the + * profiling session runs normally. + */ + static void InstallFromEnv(); + + /** @brief Advance the rule. Called on the collector beat. */ + static void Service(); + + /** @brief A kernel launch was observed at the host launch API. */ + static void NoteKernelLaunch(int64_t ts_ns); + /** @brief A completed kernel's duration, for recent_kernel_ms. */ + static void NoteKernelDuration(int64_t ts_ns, double duration_ms); + /** @brief A successful device measurement. Never called on a poll. */ + static void NoteDeviceSample(const DeviceSample& sample, int64_t ts_ns); + + /** + * @brief Write the rule summary. Called once during shutdown. + * + * Emitted even when the rule never fired: "log once" is invisible in the + * UI, and no record at all is indistinguishable from a rule that was simply + * never true. + */ + static void Finish(); + + /** @brief True when a rule is installed - valid or refused. */ + static bool Installed(); + + /** @brief Cheap enough for the launch callback to ask every launch. */ + static bool WantsLaunchFeed(); + + static void ResetForTesting(); +}; + +} // namespace gpufl::detail diff --git a/include/gpufl/core/events.hpp b/include/gpufl/core/events.hpp index 922d393..d59f0d1 100644 --- a/include/gpufl/core/events.hpp +++ b/include/gpufl/core/events.hpp @@ -649,7 +649,32 @@ struct NvtxMarkerEvent { * `close_reason` is what tells the reader that was the deadline expiring * rather than the profiler failing. */ +/** + * What a rule observed at the moment it asked for a window. + * + * Carried on the window rather than looked up later. A bare `trigger_value=842` + * becomes unreadable the first time somebody edits the threshold, so the whole + * comparison travels with the window it caused. + * + * `present` is false for a window nobody triggered - manual, scheduled, or the + * launcher's --deep-after. + */ +struct DeepWindowTrigger { + bool present = false; + std::string rule_id; + std::string metric; // canonical name, device index included + std::string op; // "<" or ">" + double threshold = 0.0; + double rearm_threshold = 0.0; + double observed = 0.0; + int64_t rate_window_ms = 0; + int64_t sustained_ms = 0; + int64_t first_true_ns = 0; // start of the run of true readings + int64_t fired_ns = 0; +}; + struct DeepWindowEvent { + DeepWindowTrigger trigger; int pid = 0; std::string app; std::string session_id; @@ -669,6 +694,39 @@ struct DeepWindowEvent { uint64_t requested_max_launches = 0; }; +/** + * What the session concluded about a conditional rule. + * + * Emitted even when the rule never fired. A rule that leaves no record is + * indistinguishable from one that was never true, and from a run that crashed + * before it could report - three situations calling for different responses. + * + * `state` and `outcome` are separate: state is where the evaluator was + * standing, outcome is the verdict. One field cannot carry both without making + * `armed` look like a conclusion. + */ +struct DeepWindowRuleSummaryEvent { + int pid = 0; + std::string app; + std::string session_id; + std::string rule_id; + std::string expression; // the configured rule, as written + std::string state; + std::string outcome; + std::string reason; + std::string metric_state; + uint64_t samples_seen = 0; + uint32_t windows_opened = 0; + // Absent rather than a sentinel: there is a real "no value yet", and NaN + // does not survive the JSON and DB boundaries cleanly. + bool has_last_value = false; + double last_value = 0.0; + int64_t last_observed_ns = 0; + // Monotonic, so a redelivered or late record cannot overwrite a newer one. + uint64_t state_sequence = 0; + int64_t emitted_ns = 0; +}; + /** * One CUDA graph launch event captured by CUPTI's * CUPTI_ACTIVITY_KIND_GRAPH_TRACE stream. diff --git a/include/gpufl/core/gpufl.cpp b/include/gpufl/core/gpufl.cpp index 9236d46..6b60252 100644 --- a/include/gpufl/core/gpufl.cpp +++ b/include/gpufl/core/gpufl.cpp @@ -22,6 +22,7 @@ #include "gpufl/core/config_file_loader.hpp" #include "gpufl/core/debug_logger.hpp" #include "gpufl/core/deep_window.hpp" +#include "gpufl/core/deep_window_rules.hpp" #include "gpufl/core/events.hpp" #include "gpufl/core/logger/logger.hpp" #include "gpufl/core/remote_config.hpp" @@ -612,6 +613,10 @@ bool init(const InitOptions& opts) { // from here: it only records the request; the arm itself happens on the // app thread at the first launch past the delay. scheduleEnvDeepWindow(); + // Same reason as above: the rule needs a live backend to ask about deep + // engine capability, and a metric window that starts from a running + // runtime rather than from a half-built one. + detail::DeepWindowRules::InstallFromEnv(); GFL_LOG_DEBUG("Initialization complete!"); return true; diff --git a/include/gpufl/core/model/deep_window_model.cpp b/include/gpufl/core/model/deep_window_model.cpp index 28efc9b..3892cd5 100644 --- a/include/gpufl/core/model/deep_window_model.cpp +++ b/include/gpufl/core/model/deep_window_model.cpp @@ -15,6 +15,24 @@ std::string DeepWindowModel::buildJson() const { } engines << ']'; + // Omitted entirely when nothing triggered the window, so a manual window + // does not carry an all-zero rule that reads like a real one. + std::ostringstream trigger; + if (e_.trigger.present) { + trigger << ",\"trigger\":{" + << "\"rule_id\":\"" << jsonEscape(e_.trigger.rule_id) << "\"" + << ",\"metric\":\"" << jsonEscape(e_.trigger.metric) << "\"" + << ",\"op\":\"" << jsonEscape(e_.trigger.op) << "\"" + << ",\"threshold\":" << e_.trigger.threshold + << ",\"rearm_threshold\":" << e_.trigger.rearm_threshold + << ",\"observed\":" << e_.trigger.observed + << ",\"rate_window_ms\":" << e_.trigger.rate_window_ms + << ",\"sustained_ms\":" << e_.trigger.sustained_ms + << ",\"first_true_ns\":" << e_.trigger.first_true_ns + << ",\"fired_ns\":" << e_.trigger.fired_ns + << "}"; + } + std::ostringstream oss; oss << "{\"type\":\"deep_window_event\"" << ",\"pid\":" << e_.pid @@ -29,8 +47,35 @@ std::string DeepWindowModel::buildJson() const { << ",\"launches_covered\":" << e_.launches_covered << ",\"requested_duration_ms\":" << e_.requested_duration_ms << ",\"requested_max_launches\":" << e_.requested_max_launches + << trigger.str() << "}"; return oss.str(); } +std::string DeepWindowRuleSummaryModel::buildJson() const { + std::ostringstream oss; + oss << "{\"type\":\"deep_window_rule_summary\"" + << ",\"pid\":" << e_.pid + << ",\"app\":\"" << jsonEscape(e_.app) << "\"" + << ",\"session_id\":\"" << jsonEscape(e_.session_id) << "\"" + << ",\"rule_id\":\"" << jsonEscape(e_.rule_id) << "\"" + << ",\"expression\":\"" << jsonEscape(e_.expression) << "\"" + << ",\"state\":\"" << jsonEscape(e_.state) << "\"" + << ",\"outcome\":\"" << jsonEscape(e_.outcome) << "\"" + << ",\"reason\":\"" << jsonEscape(e_.reason) << "\"" + << ",\"metric_state\":\"" << jsonEscape(e_.metric_state) << "\"" + << ",\"samples_seen\":" << e_.samples_seen + << ",\"windows_opened\":" << e_.windows_opened + << ",\"state_sequence\":" << e_.state_sequence + << ",\"emitted_ns\":" << e_.emitted_ns; + // Written only when there is one. A null would have to be distinguished + // from 0 downstream, and 0 is a legitimate reading for every metric here. + if (e_.has_last_value) { + oss << ",\"last_value\":" << e_.last_value + << ",\"last_observed_ns\":" << e_.last_observed_ns; + } + oss << "}"; + return oss.str(); +} + } // namespace gpufl::model diff --git a/include/gpufl/core/model/deep_window_model.hpp b/include/gpufl/core/model/deep_window_model.hpp index cafd346..a7634b6 100644 --- a/include/gpufl/core/model/deep_window_model.hpp +++ b/include/gpufl/core/model/deep_window_model.hpp @@ -20,4 +20,19 @@ struct DeepWindowModel final : IJsonSerializable { const DeepWindowEvent& e_; }; +/** + * JSON serializer for the conditional-rule summary. Same channel as the window + * itself: the two are read together, and a summary that arrived on a different + * channel could be ingested after the windows it explains. + */ +struct DeepWindowRuleSummaryModel final : IJsonSerializable { + explicit DeepWindowRuleSummaryModel(const DeepWindowRuleSummaryEvent& e) + : e_(e) {} + std::string buildJson() const override; + Channel channel() const override { return Channel::Scope; } + + private: + const DeepWindowRuleSummaryEvent& e_; +}; + } // namespace gpufl::model diff --git a/include/gpufl/core/monitor.cpp b/include/gpufl/core/monitor.cpp index e03b598..a31e9ee 100644 --- a/include/gpufl/core/monitor.cpp +++ b/include/gpufl/core/monitor.cpp @@ -1,5 +1,7 @@ #include "gpufl/core/monitor.hpp" +#include "gpufl/core/deep_window_rules.hpp" + #include #include #include @@ -459,6 +461,9 @@ void CollectorLoop() { // decides how closely a deep window tracks its deadline, and it is a // lock-free check when no window is closing. if (g_state.adapter) g_state.adapter->serviceDeepWindow(); + // Immediately after, so a rule that decides to open is serviced on the + // very next beat rather than a full loop later. + detail::DeepWindowRules::Service(); if (!RecordProcessor::processNext()) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); @@ -522,6 +527,10 @@ void Monitor::Initialize(const MonitorOptions& opts) { void Monitor::Shutdown() { if (!g_state.initialized.exchange(false)) return; + // Before the logger goes away. A rule that never fired still owes an + // explanation, and this is the only place it can be written. + detail::DeepWindowRules::Finish(); + if (g_state.adapter) { g_state.adapter->stop(); g_state.adapter->shutdown(); @@ -549,6 +558,8 @@ void Monitor::Shutdown() { void Monitor::DrainAndFinalizeForExit() { if (!g_state.initialized.exchange(false)) return; + detail::DeepWindowRules::Finish(); + // The backend's PC sampling cycle thread stops issuing CUPTI reads as soon // as process-exit teardown is flagged (PcSamplingEngine::drainData), so it // sits idle here and can't fault mid-flush. We deliberately do NOT join it diff --git a/include/gpufl/core/monitor_backend.hpp b/include/gpufl/core/monitor_backend.hpp index e44a8d2..33ca0f0 100644 --- a/include/gpufl/core/monitor_backend.hpp +++ b/include/gpufl/core/monitor_backend.hpp @@ -152,6 +152,20 @@ class IMonitorBackend { */ virtual void ServiceDeepWindow() {} + /** + * @brief Is any engine that a window could arm actually prepared? + * + * The capability gate for conditional windows. Checking the configured + * engine enum is necessary but not sufficient: a Trace-only run resolves to + * a valid engine and still arms nothing inside a window, so a rule would + * spend its whole budget opening windows that collect no deep data. + * + * Answers about what was PREPARED, not what is currently armed - under + * WindowOnly nothing is armed until a window opens, which is precisely when + * the answer is needed. + */ + virtual bool DeepEnginesPrepared() const { return false; } + virtual void OnPerfScopeStart(const char* name) {} virtual void OnPerfScopeStop(const char* name) {} // Perf-scope counterparts of OnDeepWindowStart/Stop; see those. diff --git a/include/gpufl/core/sampler.cpp b/include/gpufl/core/sampler.cpp index 4900a57..85f9841 100644 --- a/include/gpufl/core/sampler.cpp +++ b/include/gpufl/core/sampler.cpp @@ -1,5 +1,7 @@ #include "gpufl/core/sampler.hpp" +#include "gpufl/core/deep_window_rules.hpp" + #include "gpufl/core/common.hpp" #include "gpufl/core/debug_logger.hpp" #include "gpufl/core/logger/logger.hpp" @@ -106,6 +108,11 @@ void Sampler::runLoop_() { const int64_t ts = detail::GetTimestampNs(); for (const DeviceSample& d : collector_->sampleAll()) { + // A rule reads gauges from here, not by polling: the timestamp has + // to be when the measurement was actually taken, or a metric could + // never be detected as having stopped. + detail::DeepWindowRules::NoteDeviceSample(d, ts); + DeviceMetricBatchRow row; row.ts_ns = ts; row.device_id = d.device_id; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c5c31ae..87a6b1f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -21,6 +21,7 @@ set(GPUFL_TEST_SOURCES core/test_counter_registry.cpp core/test_metric_registry.cpp core/test_deep_window_rule.cpp + core/test_deep_window_rules_install.cpp core/test_monitor.cpp core/test_itanium_demangle.cpp core/test_sampler.cpp diff --git a/tests/core/test_deep_window_rules_install.cpp b/tests/core/test_deep_window_rules_install.cpp new file mode 100644 index 0000000..cd2bcfb --- /dev/null +++ b/tests/core/test_deep_window_rules_install.cpp @@ -0,0 +1,119 @@ +#include + +#include +#include + +#include "gpufl/core/deep_window_rules.hpp" +#include "gpufl/core/env_vars.hpp" + +using gpufl::detail::DeepWindowRules; + +namespace { + +// Env is process-global, so every test here sets and clears the whole group. +// Leaving one behind would silently change the next test's rule. +void setEnv(const char* name, const char* value) { +#if defined(_WIN32) + _putenv_s(name, value ? value : ""); +#else + if (value) ::setenv(name, value, 1); else ::unsetenv(name); +#endif +} + +class RuleInstallTest : public ::testing::Test { + protected: + void SetUp() override { clearAll(); DeepWindowRules::ResetForTesting(); } + void TearDown() override { clearAll(); DeepWindowRules::ResetForTesting(); } + + static void clearAll() { + for (const char* n : {gpufl::env::kDeepWhen, gpufl::env::kDeepRateWindowMs, + gpufl::env::kDeepStaleAfterMs, gpufl::env::kDeepRearmAt, + gpufl::env::kDeepMaxWindows, gpufl::env::kDeepWindowMs, + gpufl::env::kDeepWindowMaxLaunches}) { + setEnv(n, nullptr); + } + } +}; + +TEST_F(RuleInstallTest, NoRuleWithoutTheEnvVar) { + DeepWindowRules::InstallFromEnv(); + EXPECT_FALSE(DeepWindowRules::Installed()); + EXPECT_FALSE(DeepWindowRules::WantsLaunchFeed()); +} + +TEST_F(RuleInstallTest, AValidRuleInstallsAndAsksForTheLaunchFeed) { + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + DeepWindowRules::InstallFromEnv(); + + EXPECT_TRUE(DeepWindowRules::Installed()); + // The feed costs an atomic per launch, so only a rule that reads it should + // switch it on. + EXPECT_TRUE(DeepWindowRules::WantsLaunchFeed()); +} + +TEST_F(RuleInstallTest, ACustomCounterRuleDoesNotPayForTheLaunchFeed) { + setEnv(gpufl::env::kDeepWhen, "custom.token_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + DeepWindowRules::InstallFromEnv(); + + EXPECT_TRUE(DeepWindowRules::Installed()); + EXPECT_FALSE(DeepWindowRules::WantsLaunchFeed()) + << "every launch would pay for a feed this rule never reads"; +} + +TEST_F(RuleInstallTest, AnInvalidRuleIsInstalledAsRefusedRatherThanIgnored) { + setEnv(gpufl::env::kDeepWhen, "tokne_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + DeepWindowRules::InstallFromEnv(); + + // Installed, so Finish() still has something to report. A rejected rule + // that vanishes looks exactly like one that was simply never true. + EXPECT_TRUE(DeepWindowRules::Installed()); + EXPECT_FALSE(DeepWindowRules::WantsLaunchFeed()); +} + +TEST_F(RuleInstallTest, AWindowWithNoBoundIsRefused) { + // Neither GPUFL_DEEP_WINDOW_MS nor MAX_LAUNCHES: the window would never + // close, which turns a bounded-cost feature into an always-on one. + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 2s"); + DeepWindowRules::InstallFromEnv(); + EXPECT_TRUE(DeepWindowRules::Installed()); + EXPECT_FALSE(DeepWindowRules::WantsLaunchFeed()); +} + +TEST_F(RuleInstallTest, TheDefaultStaleAfterIsOneThatCanActuallyFire) { + // Left unset, stale-after must be derived from the other two rather than + // defaulted to a constant the validator would then reject. + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 30s"); + setEnv(gpufl::env::kDeepRateWindowMs, "5000"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + DeepWindowRules::InstallFromEnv(); + + EXPECT_TRUE(DeepWindowRules::Installed()); + EXPECT_TRUE(DeepWindowRules::WantsLaunchFeed()) + << "a workable rule was refused by its own default stale-after"; +} + +TEST_F(RuleInstallTest, ServiceAndFinishAreSafeWithoutARule) { + // Both run unconditionally from the collector and from shutdown. + DeepWindowRules::Service(); + DeepWindowRules::Finish(); + DeepWindowRules::NoteKernelLaunch(1); + SUCCEED(); +} + +TEST_F(RuleInstallTest, ASecondRuleIsRefusedRatherThanSilentlyReplacing) { + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + DeepWindowRules::InstallFromEnv(); + ASSERT_TRUE(DeepWindowRules::WantsLaunchFeed()); + + setEnv(gpufl::env::kDeepWhen, "custom.token_rate<5 for 2s"); + DeepWindowRules::InstallFromEnv(); + // The first rule stands. Replacing it silently would make which rule ran + // depend on call order. + EXPECT_TRUE(DeepWindowRules::WantsLaunchFeed()); +} + +} // namespace From c42ace48fda3ca056c4a118931c746328bd13ff1 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Sun, 26 Jul 2026 22:10:30 -0700 Subject: [PATCH 10/28] fix(deep-window): review fixes across the counter and rule paths Counter add is lock-free again. A handle now IS the slot's address, so add is one relaxed atomic; it was going through a slot id, which had to be bounds-checked, which meant locking the registry - putting every ticking thread on one mutex and distorting the throughput a rule exists to measure. The header claimed this contract all along. The pending open is one record. Spec and owner token were separate globals, so an untagged request arriving after a tagged one replaced only the spec: a manual window then opened carrying a rule's token and was charged to that rule's budget, which is the exact mis-attribution the token was added to prevent. Custom metrics read since the session baseline. Slots are permanent, so a counter ticked by a previous session made a new session believe it had already moved, and a stall rule could arm on evidence this run never saw. The launch callback takes no lock at all. It held the rule mutex and then the feed mutex, on the application's launch path, contending with the collector's polling. Finish runs after the collector is joined, and freezes the evaluator first; it could previously write a summary and then let a window open that the summary does not mention. It also releases the session before attempting the write, so a shutdown with no logger no longer leaves the session claimed and the next init() silently rule-less. recent_kernel_ms is fed. Only launches were wired, so a rule watching kernel duration read nothing and ended every run as missing. Window detection uses the monotonic open counter. A launch-bounded window can open and close between two beats, and a boolean poll would miss it entirely - its contaminated samples then feeding the rule as clean. Malformed numeric options fail closed instead of silently defaulting, capability uses the resolved engine rather than the requested one, and the runtime installs unconditionally rather than only with the Python bindings - a launcher-only install left the injected evaluator and the target on separate fallback registries. The cross-module script now exits non-zero on a wrong value, resolves the library per platform, and runs under CTest. --- CMakeLists.txt | 11 +- .../backends/nvidia/kernel_launch_handler.cpp | 6 + include/gpufl/core/counter_api.cpp | 33 ++-- include/gpufl/core/counter_registry.cpp | 31 +--- include/gpufl/core/counter_registry.hpp | 73 ++++++--- include/gpufl/core/deep_window.cpp | 51 ++++--- include/gpufl/core/deep_window_rule.cpp | 15 +- include/gpufl/core/deep_window_rule.hpp | 7 + include/gpufl/core/deep_window_rules.cpp | 141 ++++++++++++++---- include/gpufl/core/metric_registry.cpp | 43 ++++-- include/gpufl/core/metric_registry.hpp | 12 +- include/gpufl/core/monitor.cpp | 15 +- runtime/counter_runtime.cpp | 50 +++---- scripts/counter_cross_module_check.py | 96 +++++++++--- tests/CMakeLists.txt | 19 +++ tests/core/test_counter_registry.cpp | 16 +- tests/core/test_deep_window_rule.cpp | 25 ++++ tests/core/test_deep_window_rules_install.cpp | 28 ++++ tests/core/test_metric_registry.cpp | 33 ++++ 19 files changed, 505 insertions(+), 200 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d2899c..4453834 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,16 @@ add_library(gpufl_counter_runtime SHARED include/gpufl/core/counter_registry.cpp ) target_include_directories(gpufl_counter_runtime PUBLIC include) + +# Installed unconditionally, NOT only with the Python bindings. A launcher-only +# install that omits it leaves the injected evaluator and the target application +# each falling back to their own in-process registry - the exact split this +# library exists to prevent, and one that shows up as a counter reading Missing +# forever rather than as a missing file. +install(TARGETS gpufl_counter_runtime + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} +) set_target_properties(gpufl_counter_runtime PROPERTIES CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN ON @@ -799,7 +809,6 @@ if(BUILD_PYTHON) $ COMMENT "Colocating gpufl_counter_runtime with _gpufl_client" ) - install(TARGETS gpufl_counter_runtime DESTINATION gpufl) # If CUDA is available, link it to the Python module if(GPUFL_HAS_CUDA) diff --git a/include/gpufl/backends/nvidia/kernel_launch_handler.cpp b/include/gpufl/backends/nvidia/kernel_launch_handler.cpp index b025d45..d559d3e 100644 --- a/include/gpufl/backends/nvidia/kernel_launch_handler.cpp +++ b/include/gpufl/backends/nvidia/kernel_launch_handler.cpp @@ -1,5 +1,6 @@ #include "gpufl/backends/nvidia/kernel_launch_handler.hpp" +#include "gpufl/core/deep_window_rules.hpp" #include "gpufl/core/env_vars.hpp" #include // std::min(initializer_list) - see occupancy calc below @@ -545,6 +546,11 @@ bool KernelLaunchHandler::handleActivityRecord(const CUpti_Activity* record, kernelName); out.cpu_start_ns = wallStartNs; out.duration_ns = static_cast(k->end - k->start); + // Feed recent_kernel_ms. This is the only place a COMPLETED kernel's + // duration exists - the launch callback sees a launch, not a result - so a + // rule watching kernel duration reads nothing at all without this. + detail::DeepWindowRules::NoteKernelDuration( + wallStartNs, static_cast(out.duration_ns) / 1e6); out.dyn_shared = k->dynamicSharedMemory; out.static_shared = k->staticSharedMemory; out.num_regs = k->registersPerThread; diff --git a/include/gpufl/core/counter_api.cpp b/include/gpufl/core/counter_api.cpp index 187091c..85f030b 100644 --- a/include/gpufl/core/counter_api.cpp +++ b/include/gpufl/core/counter_api.cpp @@ -17,39 +17,32 @@ namespace { // paths go through one code shape and cannot drift. using detail::CounterRegistry; +using Slot = CounterRegistry::Slot; + +Slot* AsSlot(gpufl_counter_handle h) { return static_cast(h); } + gpufl_counter_handle LocalRegister(const char* name, size_t len) { if (name == nullptr) return nullptr; - const auto slot = CounterRegistry::instance().registerCounter(std::string(name, len)); - if (slot == CounterRegistry::kInvalidSlot) return nullptr; - return reinterpret_cast(static_cast(slot) + 1u); + CounterRegistry& reg = CounterRegistry::instance(); + return reg.slotFor(reg.registerCounter(std::string(name, len))); } -bool LocalSlot(gpufl_counter_handle h, CounterRegistry::SlotId* out) { - if (h == nullptr) return false; - *out = static_cast(reinterpret_cast(h) - 1u); - return true; +gpufl_counter_handle LocalLookup(const char* name, size_t len) { + if (name == nullptr) return nullptr; + CounterRegistry& reg = CounterRegistry::instance(); + return reg.slotFor(reg.findCounter(std::string(name, len))); } void LocalAdd(gpufl_counter_handle h, uint64_t v) { - CounterRegistry::SlotId slot; - if (LocalSlot(h, &slot)) CounterRegistry::instance().addRaw(slot, v); + CounterRegistry::addRaw(AsSlot(h), v); } uint64_t LocalLoad(gpufl_counter_handle h) { - CounterRegistry::SlotId slot; - return LocalSlot(h, &slot) ? CounterRegistry::instance().rawValue(slot) : 0; + return CounterRegistry::rawValue(AsSlot(h)); } uint64_t LocalLoadSinceBaseline(gpufl_counter_handle h) { - CounterRegistry::SlotId slot; - return LocalSlot(h, &slot) ? CounterRegistry::instance().valueSinceBaseline(slot) : 0; -} - -gpufl_counter_handle LocalLookup(const char* name, size_t len) { - if (name == nullptr) return nullptr; - const auto slot = CounterRegistry::instance().findCounter(std::string(name, len)); - if (slot == CounterRegistry::kInvalidSlot) return nullptr; - return reinterpret_cast(static_cast(slot) + 1u); + return CounterRegistry::valueSinceBaseline(AsSlot(h)); } void LocalBeginSession() { CounterRegistry::instance().beginSession(); } diff --git a/include/gpufl/core/counter_registry.cpp b/include/gpufl/core/counter_registry.cpp index 6f4b416..3a456d1 100644 --- a/include/gpufl/core/counter_registry.cpp +++ b/include/gpufl/core/counter_registry.cpp @@ -72,37 +72,19 @@ CounterRegistry::SlotId CounterRegistry::registerCounter(const std::string& name slots_.back().name = name; // Registered mid-generation: baseline at the current value so ticks from a // previous session, or from before init(), do not land in this one. - slots_.back().baseline = slots_.back().value.load(std::memory_order_relaxed); + slots_.back().baseline.store( + slots_.back().value.load(std::memory_order_relaxed), + std::memory_order_relaxed); byName_.emplace(name, slot); return slot; } -std::atomic* CounterRegistry::valueSlot(SlotId slot) { +CounterRegistry::Slot* CounterRegistry::slotFor(const SlotId slot) { std::lock_guard lk(mu_); if (slot >= slots_.size()) return nullptr; // Safe to hand out and keep: a deque never relocates existing elements, and // slots are never erased outside resetForTesting(). - return &slots_[slot].value; -} - -void CounterRegistry::addRaw(SlotId slot, uint64_t value) { - std::lock_guard lk(mu_); - if (slot >= slots_.size()) return; - slots_[slot].value.fetch_add(value, std::memory_order_relaxed); -} - -uint64_t CounterRegistry::rawValue(SlotId slot) const { - std::lock_guard lk(mu_); - if (slot >= slots_.size()) return 0; - return slots_[slot].value.load(std::memory_order_relaxed); -} - -uint64_t CounterRegistry::valueSinceBaseline(SlotId slot) const { - std::lock_guard lk(mu_); - if (slot >= slots_.size()) return 0; - // Unsigned subtraction, deliberately: correct across a wrap, where a signed - // difference would go negative. - return slots_[slot].value.load(std::memory_order_relaxed) - slots_[slot].baseline; + return &slots_[slot]; } const std::string& CounterRegistry::name(SlotId slot) const { @@ -120,7 +102,8 @@ size_t CounterRegistry::counterCount() const { void CounterRegistry::beginSession() { std::lock_guard lk(mu_); for (auto& slot : slots_) { - slot.baseline = slot.value.load(std::memory_order_relaxed); + slot.baseline.store(slot.value.load(std::memory_order_relaxed), + std::memory_order_relaxed); } sessionActive_.store(true, std::memory_order_release); } diff --git a/include/gpufl/core/counter_registry.hpp b/include/gpufl/core/counter_registry.hpp index 67499b0..e5b3b1d 100644 --- a/include/gpufl/core/counter_registry.hpp +++ b/include/gpufl/core/counter_registry.hpp @@ -42,6 +42,26 @@ class CounterRegistry { using SlotId = uint32_t; static constexpr SlotId kInvalidSlot = 0xFFFFFFFFu; + /** + * @brief One counter's storage, addressed directly by a handle. + * + * Public because the hot path is a pointer to this, not an index. Going + * through an index means bounds-checking the container, and checking it + * means locking it - which would serialise every ticking thread on one + * mutex and distort the very throughput a rule is trying to measure. + * + * Never freed, so the pointer stays valid across shutdown()/init(); a deque + * also never relocates existing elements, so registration cannot move one + * out from under a caller that already holds it. + */ + struct Slot { + std::atomic value{0}; + /// Atomic so valueSinceBaseline stays lock-free too. Written only by + /// beginSession and by registration, both rare. + std::atomic baseline{0}; + std::string name; ///< guarded by mu_, never changes once set + }; + /** Bounds. A permanent table is exactly what must not grow without limit. */ static constexpr size_t kMaxCounters = 256; static constexpr size_t kMaxNameLength = 96; @@ -69,6 +89,15 @@ class CounterRegistry { */ SlotId registerCounter(const std::string& name); + /** + * @brief Stable address of a slot, or nullptr for kInvalidSlot. + * + * Taken once at registration and held. Everything on the hot path goes + * through this rather than through a SlotId, so no lock is needed and the + * container is never touched. + */ + Slot* slotFor(SlotId slot); + /** * @brief Find @p name WITHOUT creating it. kInvalidSlot if absent. * @@ -80,29 +109,33 @@ class CounterRegistry { SlotId findCounter(const std::string& name) const; /** - * @brief Stable address of a slot's value, or nullptr for kInvalidSlot. + * @brief Add to a slot. One relaxed atomic, no lock, no lookup. * - * Handed to the caller once, at registration, so add() never touches the - * container. Reading slots_.size() or indexing it without the mutex would - * race a concurrent registration even though a deque keeps references to - * existing elements valid. - */ - std::atomic* valueSlot(SlotId slot); - - /** - * @brief Add without validating. For the C ABI boundary only. + * The whole point of handing out a Slot*: a decode loop ticking a counter + * must not serialise on a registry mutex, or the profiler changes the + * throughput the rule is watching. * - * gpufl::Counter::add already rejects non-positive and absurd values before - * it gets here, and re-checking across the boundary would just duplicate a - * contract in two places that could then disagree. + * Validation happens at gpufl::Counter::add, before this is reached; + * re-checking here would duplicate a contract in two places that could + * then disagree. */ - void addRaw(SlotId slot, uint64_t value); + static void addRaw(Slot* slot, uint64_t value) { + if (slot != nullptr) slot->value.fetch_add(value, std::memory_order_relaxed); + } /** @brief Raw slot value, including anything added before this runtime. */ - uint64_t rawValue(SlotId slot) const; + static uint64_t rawValue(const Slot* slot) { + return slot == nullptr ? 0 : slot->value.load(std::memory_order_relaxed); + } /** @brief Value accrued since the current runtime took its baseline. */ - uint64_t valueSinceBaseline(SlotId slot) const; + static uint64_t valueSinceBaseline(const Slot* slot) { + if (slot == nullptr) return 0; + // Unsigned subtraction, deliberately: correct across a wrap, where a + // signed difference would go negative. + return slot->value.load(std::memory_order_relaxed) - + slot->baseline.load(std::memory_order_relaxed); + } const std::string& name(SlotId slot) const; size_t counterCount() const; @@ -125,17 +158,11 @@ class CounterRegistry { private: CounterRegistry() = default; - struct Slot { - std::atomic value{0}; - uint64_t baseline = 0; // guarded by mu_ - std::string name; // guarded by mu_, never changes once set - }; - static bool nameIsValid(const std::string& name); /** @brief Truncate an untrusted name so a rejection cannot bloat the log. */ static std::string forLog(const std::string& name); - // Guards registration and the name/baseline fields. Never taken by add(). + // Guards registration and the name map. Never taken by addRaw/rawValue. mutable std::mutex mu_; // Deque, not vector: a handle holds a Slot* and a vector would invalidate // it on growth. Slots are never erased. diff --git a/include/gpufl/core/deep_window.cpp b/include/gpufl/core/deep_window.cpp index 1a1b4ba..da95b44 100644 --- a/include/gpufl/core/deep_window.cpp +++ b/include/gpufl/core/deep_window.cpp @@ -39,11 +39,20 @@ std::atomic g_close_requested{false}; std::atomic g_close_reason{static_cast(DeepWindowClose::Deadline)}; // An open asked for by a thread that can't arm one itself. Checked lock-free -// on the launch beat; g_pending_spec is only read once this is set, so the +// on the launch beat; g_pending is only read once this is set, so the // hot path pays for it exactly when a trigger is waiting. std::atomic g_open_requested{false}; std::atomic g_pending_open_at_ns{0}; // 0 = at the next launch -DeepWindowSpec g_pending_spec; // guarded by g_mu + +// The queued request, as ONE record. Spec and owner token were separate fields +// once, and an untagged request arriving after a tagged one then replaced only +// the spec - so a manual window opened carrying a rule's token and was charged +// to that rule's budget. Replacing the record replaces both or neither. +struct PendingOpen { + DeepWindowSpec spec; + uint64_t owner_token = 0; // 0 = nobody is waiting on this one +}; +PendingOpen g_pending; // guarded by g_mu // When the last window closed, so a cooldown can be enforced. 0 = never. std::atomic g_last_close_ns{0}; @@ -57,7 +66,6 @@ std::atomic g_last_close_ns{0}; std::atomic g_next_open_token{1}; std::atomic g_last_opened_token{0}; std::atomic g_opens_completed{0}; -uint64_t g_pending_token = 0; // guarded by g_mu // Token of the request currently being serviced. Only TakePendingOpen_ sets // it, and only for the duration of the Open() call it drives, so a direct // Open() can never inherit someone else's attribution. @@ -199,7 +207,7 @@ bool DeepWindow::Open(const DeepWindowSpec& spec) { } // Publish who this open belongs to, then clear the claim so a later - // direct Open() cannot inherit it. g_pending_token is set only by + // direct Open() cannot inherit it. The claim is taken only by // RequestOpenTagged and consumed exactly once, here. g_last_opened_token.store(g_claimed_token, std::memory_order_release); g_opens_completed.fetch_add(1, std::memory_order_acq_rel); @@ -346,19 +354,24 @@ void DeepWindow::RequestOpen(const DeepWindowSpec& spec) { } uint64_t DeepWindow::RequestOpenTagged(const DeepWindowSpec& spec) { + uint64_t token = 0; { std::lock_guard lk(g_mu); // Decided here so the caller learns now, rather than discovering later // that a window it counted on never opened. if (g_active.load(std::memory_order_relaxed)) return 0; if (InCooldown(spec)) return 0; + + token = g_next_open_token.fetch_add(1, std::memory_order_relaxed); + // Spec and token published together, under one lock. Splitting them is + // how an untagged request could inherit a rule's attribution. + g_pending.spec = spec; + g_pending.owner_token = token; + g_pending_open_at_ns.store(0, std::memory_order_relaxed); } - const uint64_t token = g_next_open_token.fetch_add(1, std::memory_order_relaxed); - { - std::lock_guard lk(g_mu); - g_pending_token = token; - } - ScheduleOpenAfter(0, spec); + g_open_requested.store(true, std::memory_order_release); + GFL_LOG_DEBUG("[DeepWindow] tagged open requested token=", token, + " duration_ms=", spec.max_duration_ms); return token; } @@ -369,7 +382,7 @@ uint64_t DeepWindow::LastOpenedToken() { uint64_t DeepWindow::PendingOpenToken() { if (!g_open_requested.load(std::memory_order_acquire)) return 0; std::lock_guard lk(g_mu); - return g_pending_token; + return g_pending.owner_token; } uint64_t DeepWindow::OpensCompleted() { @@ -380,7 +393,11 @@ void DeepWindow::ScheduleOpenAfter(const int64_t delay_ms, const DeepWindowSpec& spec) { { std::lock_guard lk(g_mu); - g_pending_spec = spec; + g_pending.spec = spec; + // Untagged: nobody is counting this window against a budget. Cleared + // explicitly, so replacing a tagged request cannot leave its owner + // attached to somebody else's window. + g_pending.owner_token = 0; g_pending_open_at_ns.store( delay_ms > 0 ? detail::GetTimestampNs() + delay_ms * 1000000 : 0, std::memory_order_relaxed); @@ -400,13 +417,13 @@ void DeepWindow::TakePendingOpen_() { DeepWindowSpec spec; { std::lock_guard lk(g_mu); - spec = g_pending_spec; + spec = g_pending.spec; // Claim the token here, not in Open(): Open can still refuse, and a // token left behind would keep reading as "queued" for the rest of the // run, so a rule would wait forever for a window that was already // turned down. - g_claimed_token = g_pending_token; - g_pending_token = 0; + g_claimed_token = g_pending.owner_token; + g_pending.owner_token = 0; } // Outside the lock: Open takes it too. Open(spec); @@ -479,7 +496,7 @@ void DeepWindow::ResetForTesting() { g_next_open_token.store(1, std::memory_order_relaxed); g_last_opened_token.store(0, std::memory_order_relaxed); g_opens_completed.store(0, std::memory_order_relaxed); - g_pending_token = 0; + g_pending = PendingOpen{}; g_deadline_ns.store(0, std::memory_order_relaxed); g_launches_remaining.store(0, std::memory_order_relaxed); g_launches_covered.store(0, std::memory_order_relaxed); @@ -488,7 +505,7 @@ void DeepWindow::ResetForTesting() { std::memory_order_relaxed); g_open_requested.store(false, std::memory_order_relaxed); g_pending_open_at_ns.store(0, std::memory_order_relaxed); - g_pending_spec = DeepWindowSpec{}; + g_pending = PendingOpen{}; g_last_close_ns.store(0, std::memory_order_relaxed); g_opened_ns = 0; g_requested_duration_ms = 0; diff --git a/include/gpufl/core/deep_window_rule.cpp b/include/gpufl/core/deep_window_rule.cpp index eb6c122..a9c53e6 100644 --- a/include/gpufl/core/deep_window_rule.cpp +++ b/include/gpufl/core/deep_window_rule.cpp @@ -219,6 +219,7 @@ RuleEvaluator::Hooks RuleEvaluator::liveHooks() { return DeepWindow::RequestOpenTagged(spec); }; h.window_active = [](void*) { return DeepWindow::Active(); }; + h.opens_completed = [](void*) { return DeepWindow::OpensCompleted(); }; h.last_opened_token = [](void*) { return DeepWindow::LastOpenedToken(); }; h.pending_open_token = [](void*) { return DeepWindow::PendingOpenToken(); }; h.ctx = nullptr; @@ -308,6 +309,15 @@ void RuleEvaluator::poll(const int64_t now_ns) { const bool active = hooks_.window_active(hooks_.ctx); + // A window can open and close entirely between two beats - a launch-bounded + // one over a busy loop routinely does. The counter catches that; a boolean + // would not, and the samples taken during it would feed the rule as clean. + const uint64_t opens = hooks_.opens_completed != nullptr + ? hooks_.opens_completed(hooks_.ctx) : 0; + const bool missed_window = have_opens_ && opens != opens_seen_ && !active; + opens_seen_ = opens; + have_opens_ = true; + // 1. Confirm a requested open before anything else. The window we caused // also puts us into blackout, so checking blackout first would lose the // only chance to count it. @@ -347,10 +357,11 @@ void RuleEvaluator::poll(const int64_t now_ns) { return; } - // 3. A window just closed. Blackout and recovery are distinct: blackout is + // 3. A window just closed - either one we watched, or one that came and + // went between beats. Blackout and recovery are distinct: blackout is // "discard everything", recovery is "refill the clean epoch". Merging // them would let contaminated samples prove the workload recovered. - if (window_was_active_) { + if (window_was_active_ || missed_window) { window_was_active_ = false; source_->resetEpoch(now_ns); if (terminal_ == RuleOutcome::Exhausted) { diff --git a/include/gpufl/core/deep_window_rule.hpp b/include/gpufl/core/deep_window_rule.hpp index 95d2961..f84d415 100644 --- a/include/gpufl/core/deep_window_rule.hpp +++ b/include/gpufl/core/deep_window_rule.hpp @@ -194,6 +194,11 @@ class RuleEvaluator { uint64_t (*request_open)(void* ctx, const DeepWindowSpec&) = nullptr; /// Is a window - any window, whoever opened it - currently open? bool (*window_active)(void* ctx) = nullptr; + /// Monotonic count of windows that have opened. A short launch-bounded + /// window can open AND close between two beats, so polling a boolean + /// would miss it entirely and its contaminated samples would feed the + /// rule as if profiling had never been on. + uint64_t (*opens_completed)(void* ctx) = nullptr; /// Token of the most recent open, so a manual window is not mistaken /// for the one this rule asked for and charged to its budget. uint64_t (*last_opened_token)(void* ctx) = nullptr; @@ -273,6 +278,8 @@ class RuleEvaluator { uint64_t pending_token_ = 0; bool window_was_active_ = false; + uint64_t opens_seen_ = 0; + bool have_opens_ = false; }; } // namespace gpufl::detail diff --git a/include/gpufl/core/deep_window_rules.cpp b/include/gpufl/core/deep_window_rules.cpp index e3177d9..a348905 100644 --- a/include/gpufl/core/deep_window_rules.cpp +++ b/include/gpufl/core/deep_window_rules.cpp @@ -29,7 +29,19 @@ bool g_finished = false; std::string g_expression; std::string g_rule_id; -std::unique_ptr g_feeds; +/** + * Process-lifetime feeds, never destroyed. + * + * The launch callback writes here without taking any lock, so the object must + * not be able to disappear underneath it. Same reasoning as the counter slots: + * a lifetime that ends is a race the hot path would have to pay to defend + * against. Contents are cleared on install instead. + */ +MetricFeeds& Feeds() { + static MetricFeeds feeds; + return feeds; +} + std::unique_ptr g_source; std::unique_ptr g_eval; // Set when the rule was refused before it could run, so Finish() still has @@ -39,31 +51,42 @@ std::unique_ptr g_refused; // Read by the launch callback on every launch, so it must not take the lock. std::atomic g_wants_launch_feed{false}; +std::atomic g_wants_duration_feed{false}; +// Mirrors g_installed without the mutex, for the feed entry points. +std::atomic g_installed_relaxed{false}; const char* EnvOrNull(const char* name) { const char* v = std::getenv(name); return (v && v[0] != '\0') ? v : nullptr; } -int64_t EnvIntOr(const char* name, const int64_t fallback) { +/** + * Integer from env, or the fallback when unset. + * + * A malformed value is a HARD failure, reported through @p bad rather than + * quietly replaced by the default. Substituting silently means a typo in a + * threshold or a budget still opens real windows, under settings the user never + * asked for and cannot see. + */ +int64_t EnvIntOr(const char* name, const int64_t fallback, std::string* bad) { const char* v = EnvOrNull(name); if (!v) return fallback; char* end = nullptr; const long long n = std::strtoll(v, &end, 10); - if (end == v || *end != '\0') { - GFL_LOG_ERROR(name, "='", v, "' is not an integer. Using ", fallback, "."); + if (end == v || *end != 0) { + if (bad->empty()) *bad = std::string(name) + "='" + v + "' is not an integer"; return fallback; } return n; } -bool EnvDoubleOr(const char* name, double* out) { +bool EnvDoubleOr(const char* name, double* out, std::string* bad) { const char* v = EnvOrNull(name); if (!v) return false; char* end = nullptr; const double d = std::strtod(v, &end); - if (end == v || *end != '\0') { - GFL_LOG_ERROR(name, "='", v, "' is not a number. Ignoring."); + if (end == v || *end != 0) { + if (bad->empty()) *bad = std::string(name) + "='" + v + "' is not a number"; return false; } *out = d; @@ -101,7 +124,11 @@ std::string CanonicalConfig(const DeepWindowRule& r) { RuleCapabilities QueryCapabilities() { RuleCapabilities caps; // A Monitor-only run has no engine to arm inside a window at all. - caps.windows_supported = g_opts.profiling_engine != ProfilingEngine::Monitor; + // The RESOLVED engine, not the requested one: a request can be overridden + // at init, and gating on what was asked for would answer about a run that + // is not the one happening. + caps.windows_supported = + Monitor::ResolvedProfilingEngine() != ProfilingEngine::Monitor; IMonitorBackend* backend = Monitor::GetBackend(); caps.deep_engine_prepared = backend != nullptr && backend->DeepEnginesPrepared(); @@ -129,6 +156,24 @@ void RefuseLocked(const RuleOutcome outcome, const std::string& reason) { ". Profiling continues; only the trigger is off."); } +/** + * Drop the session so a later init() installs cleanly. + * + * An embedded host can shutdown() and init() again in one process, and a + * session left claimed makes the second run's rule look like a duplicate - + * that run then has no trigger at all and nothing says why. + */ +void ReleaseSession() { + std::lock_guard lk(g_mu); + g_installed = false; + g_finished = false; + g_eval.reset(); + g_source.reset(); + g_refused.reset(); + g_expression.clear(); + g_rule_id.clear(); +} + } // namespace void DeepWindowRules::InstallFromEnv() { @@ -154,24 +199,32 @@ void DeepWindowRules::InstallFromEnv() { return; } + std::string bad_env; DeepWindowRule rule = parsed.rule; - rule.timing.rate_window_ms = EnvIntOr(env::kDeepRateWindowMs, 1000); + rule.timing.rate_window_ms = EnvIntOr(env::kDeepRateWindowMs, 1000, &bad_env); rule.timing.stale_after_ms = EnvIntOr( env::kDeepStaleAfterMs, // Default derived from the other two so the out-of-the-box combination // is one that CAN fire, rather than one the validator then rejects. rule.timing.rate_window_ms + rule.timing.sustained_ms + - rule.timing.bucketIntervalMs() + 1000); + rule.timing.bucketIntervalMs() + 1000, &bad_env); double rearm = 0.0; - if (EnvDoubleOr(env::kDeepRearmAt, &rearm)) rule.rearm_threshold = rearm; - rule.max_windows = static_cast(EnvIntOr(env::kDeepMaxWindows, 3)); + if (EnvDoubleOr(env::kDeepRearmAt, &rearm, &bad_env)) rule.rearm_threshold = rearm; + rule.max_windows = static_cast(EnvIntOr(env::kDeepMaxWindows, 3, &bad_env)); - rule.window.max_duration_ms = EnvIntOr(env::kDeepWindowMs, 0); + rule.window.max_duration_ms = EnvIntOr(env::kDeepWindowMs, 0, &bad_env); rule.window.max_launches = - static_cast(EnvIntOr(env::kDeepWindowMaxLaunches, 0)); - rule.window.cooldown_ms = EnvIntOr(env::kDeepWindowCooldownMs, 0); + static_cast(EnvIntOr(env::kDeepWindowMaxLaunches, 0, &bad_env)); + rule.window.cooldown_ms = EnvIntOr(env::kDeepWindowCooldownMs, 0, &bad_env); rule.window.name = "deep_window"; + if (!bad_env.empty()) { + // Fail closed. A malformed number silently replaced by a default opens + // real windows under settings nobody chose. + RefuseLocked(RuleOutcome::InvalidConfig, bad_env); + return; + } + const RuleParseResult checked = validateRule(rule); if (!checked.ok()) { RefuseLocked(RuleOutcome::InvalidConfig, @@ -181,10 +234,12 @@ void DeepWindowRules::InstallFromEnv() { } g_rule_id = RuleId(CanonicalConfig(rule)); - g_feeds = std::make_unique(); - g_feeds->seedStartup(detail::GetTimestampNs()); + // Cleared rather than reallocated: the feeds outlive every session so the + // lock-free launch path never has to check whether they still exist. + Feeds().resetForTesting(); + Feeds().seedStartup(detail::GetTimestampNs()); g_source = std::make_unique(rule.metric, rule.timing, - g_feeds.get(), + &Feeds(), ActiveCounterProvider()); g_eval = std::make_unique(rule, g_rule_id, QueryCapabilities(), g_source.get(), @@ -192,10 +247,11 @@ void DeepWindowRules::InstallFromEnv() { g_installed = true; // The launch feed costs an atomic per launch, so only a rule that reads it // turns it on. - g_wants_launch_feed.store( - rule.metric.kind == MetricKind::KernelLaunchRate || - rule.metric.kind == MetricKind::RecentKernelMs, - std::memory_order_release); + g_wants_launch_feed.store(rule.metric.kind == MetricKind::KernelLaunchRate, + std::memory_order_release); + g_wants_duration_feed.store(rule.metric.kind == MetricKind::RecentKernelMs, + std::memory_order_release); + g_installed_relaxed.store(true, std::memory_order_release); GFL_LOG_DEBUG("[DeepWindowRule] installed id=", g_rule_id, " '", g_expression, "' state=", toString(g_eval->state())); @@ -211,35 +267,47 @@ bool DeepWindowRules::WantsLaunchFeed() { } void DeepWindowRules::NoteKernelLaunch(const int64_t ts_ns) { + // No lock at all: one atomic gate, then three relaxed atomics. This runs on + // the application's launch path, and taking g_mu here would queue it behind + // the collector's evaluation - changing the launch rate being measured. if (!WantsLaunchFeed()) return; - std::lock_guard lk(g_mu); - if (g_feeds) g_feeds->noteKernelLaunch(ts_ns); + Feeds().noteKernelLaunch(ts_ns); } void DeepWindowRules::NoteKernelDuration(const int64_t ts_ns, const double ms) { - if (!WantsLaunchFeed()) return; - std::lock_guard lk(g_mu); - if (g_feeds) g_feeds->noteKernelDuration(ts_ns, ms); + // Fed from activity processing, not from the per-launch path, so the + // feed's own lock is acceptable here. + if (!g_wants_duration_feed.load(std::memory_order_acquire)) return; + Feeds().noteKernelDuration(ts_ns, ms); } void DeepWindowRules::NoteDeviceSample(const DeviceSample& sample, const int64_t ts_ns) { - std::lock_guard lk(g_mu); - if (g_feeds) g_feeds->noteDeviceSample(sample, ts_ns); + if (!g_installed_relaxed.load(std::memory_order_acquire)) return; + Feeds().noteDeviceSample(sample, ts_ns); } void DeepWindowRules::Service() { std::lock_guard lk(g_mu); - if (!g_eval) return; + // Once the summary is written the evaluator must not run again: a later + // poll could open a window the recorded summary does not mention, and the + // session would report fewer windows than it actually took. + if (!g_eval || g_finished) return; g_eval->poll(detail::GetTimestampNs()); } void DeepWindowRules::Finish() { RuleSummary summary; + std::string expression; { std::lock_guard lk(g_mu); if (!g_installed || g_finished) return; g_finished = true; + // Set before producing the summary, so a collector beat that is already + // inside Service() cannot advance the evaluator past what is reported. + g_wants_launch_feed.store(false, std::memory_order_release); + g_wants_duration_feed.store(false, std::memory_order_release); + g_installed_relaxed.store(false, std::memory_order_release); if (g_refused) { summary = *g_refused; } else if (g_eval) { @@ -247,8 +315,15 @@ void DeepWindowRules::Finish() { } else { return; } + expression = g_expression; } + // Released BEFORE the write is attempted. Tying the release to a successful + // write meant that a shutdown with no logger left the session claimed + // forever, and a host that called init() again got no rule at all - a + // reporting failure silently turning into a functional one. + ReleaseSession(); + const Runtime* rt = runtime(); if (!rt || !rt->logger) { // Nothing to write to. Logged rather than dropped silently, since this @@ -263,7 +338,7 @@ void DeepWindowRules::Finish() { ev.app = rt->app_name; ev.session_id = rt->session_id; ev.rule_id = summary.rule_id; - ev.expression = g_expression; + ev.expression = expression; ev.state = toString(summary.state); ev.outcome = toString(summary.outcome); ev.reason = summary.reason; @@ -292,9 +367,11 @@ void DeepWindowRules::ResetForTesting() { g_rule_id.clear(); g_eval.reset(); g_source.reset(); - g_feeds.reset(); + Feeds().resetForTesting(); g_refused.reset(); g_wants_launch_feed.store(false, std::memory_order_release); + g_wants_duration_feed.store(false, std::memory_order_release); + g_installed_relaxed.store(false, std::memory_order_release); } } // namespace gpufl::detail diff --git a/include/gpufl/core/metric_registry.cpp b/include/gpufl/core/metric_registry.cpp index 5ce12c2..7e463d9 100644 --- a/include/gpufl/core/metric_registry.cpp +++ b/include/gpufl/core/metric_registry.cpp @@ -25,10 +25,10 @@ const char* toString(const MetricState s) { // ---------------------------------------------------------------- MetricFeeds void MetricFeeds::noteKernelLaunch(const int64_t ts_ns) { - std::lock_guard lk(mu_); - ++launch_.count; - launch_.last_event_ns = ts_ns; - launch_.seeded = true; + // Lock-free: this is the application's launch path. + launch_count_.fetch_add(1, std::memory_order_relaxed); + launch_last_ns_.store(ts_ns, std::memory_order_relaxed); + launch_seeded_.store(true, std::memory_order_release); } void MetricFeeds::noteKernelDuration(const int64_t ts_ns, const double duration_ms) { @@ -61,11 +61,13 @@ void MetricFeeds::noteDeviceSample(const DeviceSample& sample, const int64_t ts_ } void MetricFeeds::seedStartup(const int64_t ts_ns) { - std::lock_guard lk(mu_); - if (launch_.seeded) return; - launch_.last_event_ns = ts_ns; - launch_.seeded = true; - durations_.last_event_ns = ts_ns; + { + std::lock_guard lk(mu_); + durations_.last_event_ns = ts_ns; + } + if (launch_seeded_.load(std::memory_order_acquire)) return; + launch_last_ns_.store(ts_ns, std::memory_order_relaxed); + launch_seeded_.store(true, std::memory_order_release); } int MetricFeeds::deviceCount() const { @@ -74,8 +76,13 @@ int MetricFeeds::deviceCount() const { } MetricFeeds::LaunchFeed MetricFeeds::launchFeed() const { - std::lock_guard lk(mu_); - return launch_; + LaunchFeed out; + // Seeded first: it is released last on the write side, so observing it + // true means the count and timestamp beside it are already visible. + out.seeded = launch_seeded_.load(std::memory_order_acquire); + out.count = launch_count_.load(std::memory_order_relaxed); + out.last_event_ns = launch_last_ns_.load(std::memory_order_relaxed); + return out; } MetricFeeds::DurationFeed MetricFeeds::drainDurations() { @@ -109,9 +116,11 @@ MetricFeeds::GaugeFeed MetricFeeds::gaugeFeed(const MetricKind kind, void MetricFeeds::resetForTesting() { std::lock_guard lk(mu_); - launch_ = LaunchFeed{}; durations_ = DurationFeed{}; devices_.clear(); + launch_count_.store(0, std::memory_order_relaxed); + launch_last_ns_.store(0, std::memory_order_relaxed); + launch_seeded_.store(false, std::memory_order_release); } // --------------------------------------------------------------- MetricSource @@ -169,7 +178,12 @@ void MetricSource::closeBucket(const int64_t boundary_ns) { if (id_.kind == MetricKind::KernelLaunchRate) { total = feeds_->launchFeed().count; } else if (resolveCustomHandle()) { - total = counters_->load(handle_); + // Since the session baseline, not the raw lifetime total. A + // permanent slot keeps its value across shutdown()/init(), so a + // counter ticked by a PREVIOUS session would otherwise read as + // already-ticked here and a stall rule would arm on evidence + // this run never saw. + total = counters_->load_since_baseline(handle_); if (total > 0) first_tick_seen_ = true; } // Unsigned delta: correct across a wrap, which is why the counter is @@ -249,7 +263,8 @@ MetricSample MetricSource::poll(const int64_t now_ns) { current_.sequence = total_closes_; return current_; } - if (!first_tick_seen_ && counters_->load(handle_) > 0) { + if (!first_tick_seen_ && + counters_->load_since_baseline(handle_) > 0) { first_tick_seen_ = true; // Seed the source clock at the moment the counter is first // seen to move; ticks before this point have no timestamp. diff --git a/include/gpufl/core/metric_registry.hpp b/include/gpufl/core/metric_registry.hpp index 0580b4b..d7c0720 100644 --- a/include/gpufl/core/metric_registry.hpp +++ b/include/gpufl/core/metric_registry.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -126,8 +127,17 @@ class MetricFeeds { void resetForTesting(); private: + // The launch feed is atomics, not mutex-guarded state. It is written from + // the CUDA launch callback on every launch, and a lock there would put the + // application's launch path behind the collector's polling - changing the + // launch rate the rule is trying to measure. + std::atomic launch_count_{0}; + std::atomic launch_last_ns_{0}; + std::atomic launch_seeded_{false}; + + // Durations and gauges are fed from the activity/sampler threads, not from + // the per-launch path, so a lock is fine here. mutable std::mutex mu_; - LaunchFeed launch_; DurationFeed durations_; struct DeviceGauges { GaugeFeed util; diff --git a/include/gpufl/core/monitor.cpp b/include/gpufl/core/monitor.cpp index a31e9ee..2a59597 100644 --- a/include/gpufl/core/monitor.cpp +++ b/include/gpufl/core/monitor.cpp @@ -527,10 +527,6 @@ void Monitor::Initialize(const MonitorOptions& opts) { void Monitor::Shutdown() { if (!g_state.initialized.exchange(false)) return; - // Before the logger goes away. A rule that never fired still owes an - // explanation, and this is the only place it can be written. - detail::DeepWindowRules::Finish(); - if (g_state.adapter) { g_state.adapter->stop(); g_state.adapter->shutdown(); @@ -539,6 +535,11 @@ void Monitor::Shutdown() { g_state.collectorRunning.store(false); if (g_state.collectorThread.joinable()) g_state.collectorThread.join(); + // AFTER the collector has stopped, and before the logger goes away. Writing + // the summary while the collector still runs would let the evaluator open a + // window the recorded summary never mentions. + detail::DeepWindowRules::Finish(); + while (RecordProcessor::processNext()) {} if (Runtime* rt = runtime(); rt && rt->logger) { drainSyntheticKernels(rt); @@ -558,8 +559,6 @@ void Monitor::Shutdown() { void Monitor::DrainAndFinalizeForExit() { if (!g_state.initialized.exchange(false)) return; - detail::DeepWindowRules::Finish(); - // The backend's PC sampling cycle thread stops issuing CUPTI reads as soon // as process-exit teardown is flagged (PcSamplingEngine::drainData), so it // sits idle here and can't fault mid-flush. We deliberately do NOT join it @@ -570,6 +569,10 @@ void Monitor::DrainAndFinalizeForExit() { g_state.collectorRunning.store(false); if (g_state.collectorThread.joinable()) g_state.collectorThread.join(); + // Same ordering as Shutdown: the collector is stopped first, so nothing can + // advance the rule past what this summary reports. + detail::DeepWindowRules::Finish(); + while (RecordProcessor::processNext()) {} if (Runtime* rt = runtime(); rt && rt->logger) { drainSyntheticKernels(rt); diff --git a/runtime/counter_runtime.cpp b/runtime/counter_runtime.cpp index c647e74..55d44cf 100644 --- a/runtime/counter_runtime.cpp +++ b/runtime/counter_runtime.cpp @@ -16,50 +16,38 @@ namespace { using gpufl::detail::CounterRegistry; -// Handles are slot ids biased by one, cast to a pointer, rather than addresses -// into the registry. An id survives anything the container does, and NULL then -// falls out naturally as "invalid" without needing a sentinel address. -gpufl_counter_handle ToHandle(const CounterRegistry::SlotId slot) { - if (slot == CounterRegistry::kInvalidSlot) return nullptr; - return reinterpret_cast( - static_cast(slot) + 1u); -} +// A handle IS the slot's address. Not a slot id: an id has to be +// bounds-checked against the container, checking it means locking the +// container, and that would put every ticking thread on one mutex - distorting +// the throughput a rule exists to measure. Slots are never freed and a deque +// never relocates them, so the address stays good for the life of the process. +using Slot = CounterRegistry::Slot; -bool FromHandle(const gpufl_counter_handle handle, CounterRegistry::SlotId* out) { - if (handle == nullptr) return false; - *out = static_cast( - reinterpret_cast(handle) - 1u); - return true; -} +Slot* AsSlot(const gpufl_counter_handle h) { return static_cast(h); } gpufl_counter_handle RegisterCounter(const char* name, const size_t name_length) { if (name == nullptr) return nullptr; - return ToHandle(CounterRegistry::instance().registerCounter( - std::string(name, name_length))); + CounterRegistry& reg = CounterRegistry::instance(); + return reg.slotFor(reg.registerCounter(std::string(name, name_length))); +} + +gpufl_counter_handle Lookup(const char* name, const size_t name_length) { + if (name == nullptr) return nullptr; + CounterRegistry& reg = CounterRegistry::instance(); + return reg.slotFor(reg.findCounter(std::string(name, name_length))); } +// Lock-free from here down: one relaxed atomic each. void Add(const gpufl_counter_handle handle, const uint64_t value) { - CounterRegistry::SlotId slot; - if (!FromHandle(handle, &slot)) return; - CounterRegistry::instance().addRaw(slot, value); + CounterRegistry::addRaw(AsSlot(handle), value); } uint64_t Load(const gpufl_counter_handle handle) { - CounterRegistry::SlotId slot; - if (!FromHandle(handle, &slot)) return 0; - return CounterRegistry::instance().rawValue(slot); + return CounterRegistry::rawValue(AsSlot(handle)); } uint64_t LoadSinceBaseline(const gpufl_counter_handle handle) { - CounterRegistry::SlotId slot; - if (!FromHandle(handle, &slot)) return 0; - return CounterRegistry::instance().valueSinceBaseline(slot); -} - -gpufl_counter_handle Lookup(const char* name, const size_t name_length) { - if (name == nullptr) return nullptr; - return ToHandle(CounterRegistry::instance().findCounter( - std::string(name, name_length))); + return CounterRegistry::valueSinceBaseline(AsSlot(handle)); } void BeginSession() { CounterRegistry::instance().beginSession(); } diff --git a/scripts/counter_cross_module_check.py b/scripts/counter_cross_module_check.py index a49aeea..2ac6c58 100644 --- a/scripts/counter_cross_module_check.py +++ b/scripts/counter_cross_module_check.py @@ -1,8 +1,8 @@ # Proves the counter registry is shared across module boundaries. # -# gpufl is a static library, so the Python extension and gpufl_inject.dll each -# hold their own copy of it. Without the shared runtime they would also hold -# their own counter registries, and a target ticking a counter would be +# gpufl is a static library, so the Python extension and the injection library +# each hold their own copy of it. Without the shared runtime they would also +# hold their own counter registries, and a target ticking a counter would be # invisible to the injected evaluator - the case counters exist for. # # Run standalone, or under the launcher: @@ -11,33 +11,87 @@ # python scripts/counter_cross_module_check.py # gpufl trace -o out --passes Trace -- python scripts/counter_cross_module_check.py # -# Expected: "value_via_abi=42". A private registry in the extension reads 0. -import ctypes, os, sys, pathlib -sys.path.insert(0, str(pathlib.Path(os.environ["GPUFL_REPO"]) / "python")) -import gpufl +# Exits non-zero on any failure, so it is usable as a CTest case rather than +# something a human has to read the output of. A check nobody runs, or one that +# prints a wrong answer and still succeeds, is not evidence of anything. +import ctypes +import os +import pathlib +import sys + +EXPECTED = 42 + +repo = os.environ.get("GPUFL_REPO") +if not repo: + print("XMOD: set GPUFL_REPO to the repository root", file=sys.stderr) + sys.exit(2) +sys.path.insert(0, str(pathlib.Path(repo) / "python")) +import gpufl # noqa: E402 tokens = gpufl.counter("xmod_token") tokens.add(41) tokens.add(1) -# Read the slot back through the runtime DLL directly - a third binder, using -# only the C ABI. If the extension had its own registry this would read 0. -lib = ctypes.CDLL("gpufl_counter_runtime.dll") + +def runtime_names(): + """Platform library names, most specific first.""" + if sys.platform == "win32": + return ["gpufl_counter_runtime.dll"] + if sys.platform == "darwin": + return ["libgpufl_counter_runtime.dylib"] + return ["libgpufl_counter_runtime.so"] + + +lib = None +for name in runtime_names(): + try: + lib = ctypes.CDLL(name) + break + except OSError: + continue +if lib is None: + # Not a pass. The whole point is that this library is reachable; if it is + # not, the two modules are already on separate registries. + print("XMOD: shared runtime not loadable (%s)" % ", ".join(runtime_names()), + file=sys.stderr) + sys.exit(1) + lib.gpufl_get_counter_provider_v1.restype = ctypes.c_void_p p = lib.gpufl_get_counter_provider_v1() if not p: - print("XMOD: no provider"); sys.exit(1) + print("XMOD: no provider", file=sys.stderr) + sys.exit(1) + class Provider(ctypes.Structure): - _fields_ = [("abi_version", ctypes.c_uint32), ("struct_size", ctypes.c_uint32), - ("register_counter", ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t)), - ("add", ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_uint64)), - ("load", ctypes.CFUNCTYPE(ctypes.c_uint64, ctypes.c_void_p)), - ("load_since_baseline", ctypes.CFUNCTYPE(ctypes.c_uint64, ctypes.c_void_p)), - ("begin_session", ctypes.CFUNCTYPE(None)), - ("end_session", ctypes.CFUNCTYPE(None)), - ("session_active", ctypes.CFUNCTYPE(ctypes.c_int))] + _fields_ = [ + ("abi_version", ctypes.c_uint32), + ("struct_size", ctypes.c_uint32), + ("register_counter", ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t)), + ("add", ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_uint64)), + ("load", ctypes.CFUNCTYPE(ctypes.c_uint64, ctypes.c_void_p)), + ("load_since_baseline", ctypes.CFUNCTYPE(ctypes.c_uint64, ctypes.c_void_p)), + ("begin_session", ctypes.CFUNCTYPE(None)), + ("end_session", ctypes.CFUNCTYPE(None)), + ("session_active", ctypes.CFUNCTYPE(ctypes.c_int)), + ("lookup", ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t)), + ] + + prov = ctypes.cast(p, ctypes.POINTER(Provider)).contents name = b"xmod_token" -h = prov.register_counter(name, len(name)) -print("XMOD: abi=%d value_via_abi=%d" % (prov.abi_version, prov.load(h))) + +# lookup, not register: if the extension had its own registry, the counter +# would be absent here rather than present with a value of 0. Those are +# different failures and the message should say which one happened. +h = prov.lookup(name, len(name)) +if not h: + print("XMOD: counter absent from the shared runtime - the extension is on " + "its own registry", file=sys.stderr) + sys.exit(1) + +value = prov.load(h) +print("XMOD: abi=%d value_via_abi=%d" % (prov.abi_version, value)) +if value != EXPECTED: + print("XMOD: expected %d, got %d" % (EXPECTED, value), file=sys.stderr) + sys.exit(1) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 87a6b1f..13ea730 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -245,3 +245,22 @@ else() DISCOVERY_MODE POST_BUILD ) endif() + +# Cross-module counter sharing. Cannot be proven from inside one executable - +# it needs the Python extension AND the shared runtime as separate modules - so +# it runs as its own process rather than as a gtest case. Skipped rather than +# failed when the bindings are not built, since most configurations do not +# build them. +if(BUILD_PYTHON AND TARGET gpufl_counter_runtime) + find_package(Python3 COMPONENTS Interpreter QUIET) + if(Python3_Interpreter_FOUND) + add_test(NAME counter_cross_module + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_SOURCE_DIR}/scripts/counter_cross_module_check.py) + set_tests_properties(counter_cross_module PROPERTIES + ENVIRONMENT "GPUFL_REPO=${CMAKE_SOURCE_DIR}" + # The runtime has to be findable by name; the build tree puts it + # beside the extension. + WORKING_DIRECTORY $) + endif() +endif() diff --git a/tests/core/test_counter_registry.cpp b/tests/core/test_counter_registry.cpp index e4851ff..271b1fd 100644 --- a/tests/core/test_counter_registry.cpp +++ b/tests/core/test_counter_registry.cpp @@ -252,17 +252,17 @@ TEST_F(CounterRegistryTest, AddAboveThePerCallBoundIsRefused) { TEST_F(CounterRegistryTest, DeltaIsCorrectAcrossAWrap) { // Rates are unsigned deltas precisely so a wrap needs no saturation or CAS - // loop. Driven against the in-module registry: reaching the wrap needs the - // slot's address, which the C ABI deliberately does not hand out. - const auto slot = reg().registerCounter("wrap_probe"); - auto* raw = reg().valueSlot(slot); - ASSERT_NE(raw, nullptr); + // loop. Driven against the in-module registry, since reaching the wrap + // means writing the raw value rather than adding to it. + auto* slot = reg().slotFor(reg().registerCounter("wrap_probe")); + ASSERT_NE(slot, nullptr); - raw->store(std::numeric_limits::max() - 5, std::memory_order_relaxed); + slot->value.store(std::numeric_limits::max() - 5, + std::memory_order_relaxed); reg().beginSession(); - reg().addRaw(slot, 10); // wraps past zero + CounterRegistry::addRaw(slot, 10); // wraps past zero - EXPECT_EQ(reg().valueSinceBaseline(slot), 10u) + EXPECT_EQ(CounterRegistry::valueSinceBaseline(slot), 10u) << "unsigned subtraction must stay correct across a wrap"; } diff --git a/tests/core/test_deep_window_rule.cpp b/tests/core/test_deep_window_rule.cpp index 2539afe..e093528 100644 --- a/tests/core/test_deep_window_rule.cpp +++ b/tests/core/test_deep_window_rule.cpp @@ -39,6 +39,9 @@ struct FakeCoordinator { static uint64_t LastOpenedToken(void* ctx) { return static_cast(ctx)->last_opened_token; } + static uint64_t OpensCompleted(void* ctx) { + return static_cast(static_cast(ctx)->opens); + } static uint64_t PendingOpenToken(void* ctx) { return static_cast(ctx)->pending_token; } @@ -49,6 +52,7 @@ struct FakeCoordinator { h.window_active = &Active; h.last_opened_token = &LastOpenedToken; h.pending_open_token = &PendingOpenToken; + h.opens_completed = &OpensCompleted; h.ctx = this; return h; } @@ -547,4 +551,25 @@ TEST(RuleRefusedTest, AnInvalidRuleStillProducesAReportableSummary) { EXPECT_GT(s.state_sequence, 0u); } +TEST_F(RuleEvaluatorTest, AWindowThatOpensAndClosesBetweenBeatsIsNotMissed) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + ASSERT_EQ(ev.state(), RuleState::Armed); + + // A launch-bounded window over a busy loop can open and close inside one + // collector beat. Polling a boolean would see nothing and let the samples + // taken during it count as clean. + coord.openManual(); + coord.close(); + + ev.poll(1510 * kMs); + EXPECT_EQ(ev.state(), RuleState::Recovery) + << "a whole window came and went unnoticed"; +} + + } // namespace diff --git a/tests/core/test_deep_window_rules_install.cpp b/tests/core/test_deep_window_rules_install.cpp index cd2bcfb..492c858 100644 --- a/tests/core/test_deep_window_rules_install.cpp +++ b/tests/core/test_deep_window_rules_install.cpp @@ -116,4 +116,32 @@ TEST_F(RuleInstallTest, ASecondRuleIsRefusedRatherThanSilentlyReplacing) { EXPECT_TRUE(DeepWindowRules::WantsLaunchFeed()); } +TEST_F(RuleInstallTest, AMalformedNumericOptionRefusesRatherThanDefaulting) { + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + setEnv(gpufl::env::kDeepMaxWindows, "three"); + DeepWindowRules::InstallFromEnv(); + + // Silently substituting the default would open real windows under a budget + // the user never chose and cannot see. + EXPECT_TRUE(DeepWindowRules::Installed()); + EXPECT_FALSE(DeepWindowRules::WantsLaunchFeed()) + << "a typo'd option was quietly replaced by a default"; +} + +TEST_F(RuleInstallTest, ARuleCanBeInstalledAgainAfterFinish) { + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + DeepWindowRules::InstallFromEnv(); + ASSERT_TRUE(DeepWindowRules::Installed()); + + // An embedded host may shutdown() and init() again in one process. Without + // releasing the session, the second run's rule is refused as a duplicate + // and that run silently has no trigger at all. + DeepWindowRules::Finish(); + DeepWindowRules::InstallFromEnv(); + EXPECT_TRUE(DeepWindowRules::WantsLaunchFeed()) + << "the second session was left with no rule"; +} + } // namespace diff --git a/tests/core/test_metric_registry.cpp b/tests/core/test_metric_registry.cpp index 240b1fa..40fd935 100644 --- a/tests/core/test_metric_registry.cpp +++ b/tests/core/test_metric_registry.cpp @@ -413,4 +413,37 @@ TEST_F(MetricSourceTest, ALongCollectorStallDoesNotReplayFabricatedZeros) { EXPECT_EQ(after.state, MetricState::WarmingUp); } +TEST_F(MetricSourceTest, CustomCounterIgnoresTicksFromAnEarlierSession) { + // Slots are permanent by design, so a counter ticked by a previous session + // still holds that value. Reading the raw total would make the new session + // believe the counter had already moved, arm on evidence it never saw, and + // fire a stall rule on a workload that had not started. + auto c = gpufl::counter("metric_prev_session"); + c.add(5000); // "previous session" traffic + + ActiveCounterProvider()->begin_session(); // this session's baseline + + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("custom.metric_prev_session_rate"), cfg, &feeds, + ActiveCounterProvider()); + + const auto s = advance(src, 0, 3000 * kMs, 10 * kMs); + EXPECT_EQ(s.state, MetricState::WarmingUp) + << "counted a previous session's ticks as this session's first tick"; + ActiveCounterProvider()->end_session(); +} + +TEST_F(MetricSourceTest, LaunchFeedIsVisibleWithoutTakingALock) { + // The launch path writes atomics only. This does not prove the absence of a + // lock, but it does pin the visibility contract the atomics have to keep: + // seeded is released last, so observing it means count and timestamp are + // already visible. + MetricFeeds f; + f.noteKernelLaunch(1234); + const auto feed = f.launchFeed(); + EXPECT_TRUE(feed.seeded); + EXPECT_EQ(feed.count, 1u); + EXPECT_EQ(feed.last_event_ns, 1234); +} + } // namespace From 115dd107e61869d9cf20e02361ba42f8c9bf56e1 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Sun, 26 Jul 2026 22:14:10 -0700 Subject: [PATCH 11/28] feat(deep-window): emit terminal rule outcomes at the transition exhausted and unsupported are conclusions a run reaches long before it ends. Holding them until shutdown meant a process that crashed afterwards explained nothing, and the session read as one whose rule simply never fired. The shutdown summary still follows and always carries a higher state_sequence, so the backend's strictly-greater upsert keeps the final row and treats a redelivery of either as a no-op. --- include/gpufl/core/deep_window_rule.cpp | 10 ++ include/gpufl/core/deep_window_rule.hpp | 12 +++ include/gpufl/core/deep_window_rules.cpp | 115 +++++++++++++++-------- tests/core/test_deep_window_rule.cpp | 54 +++++++++++ 4 files changed, 151 insertions(+), 40 deletions(-) diff --git a/include/gpufl/core/deep_window_rule.cpp b/include/gpufl/core/deep_window_rule.cpp index a9c53e6..aac21f2 100644 --- a/include/gpufl/core/deep_window_rule.cpp +++ b/include/gpufl/core/deep_window_rule.cpp @@ -475,6 +475,16 @@ void RuleEvaluator::poll(const int64_t now_ns) { } } +bool RuleEvaluator::takeTerminalToEmit() { + if (terminal_emitted_) return false; + if (terminal_ != RuleOutcome::Exhausted && + terminal_ != RuleOutcome::Unsupported) { + return false; + } + terminal_emitted_ = true; + return true; +} + RuleSummary RuleEvaluator::snapshot(const int64_t now_ns) const { RuleSummary s; s.rule_id = rule_id_; diff --git a/include/gpufl/core/deep_window_rule.hpp b/include/gpufl/core/deep_window_rule.hpp index f84d415..68d4e8d 100644 --- a/include/gpufl/core/deep_window_rule.hpp +++ b/include/gpufl/core/deep_window_rule.hpp @@ -224,6 +224,17 @@ class RuleEvaluator { /** @brief Current summary without concluding, for a mid-run emit. */ RuleSummary snapshot(int64_t now_ns) const; + /** + * @brief True once, when a terminal outcome is first reached. + * + * `exhausted` and `unsupported` are conclusions the run can reach long + * before it ends. Holding them until shutdown means a process that crashes + * afterwards explains nothing, and the session looks like one where the + * rule simply never fired. Reported here so the caller can write it at the + * transition; the shutdown summary still follows, with a higher sequence. + */ + bool takeTerminalToEmit(); + RuleState state() const { return state_; } uint32_t windowsOpened() const { return windows_opened_; } @@ -280,6 +291,7 @@ class RuleEvaluator { bool window_was_active_ = false; uint64_t opens_seen_ = 0; bool have_opens_ = false; + bool terminal_emitted_ = false; }; } // namespace gpufl::detail diff --git a/include/gpufl/core/deep_window_rules.cpp b/include/gpufl/core/deep_window_rules.cpp index a348905..0e33b25 100644 --- a/include/gpufl/core/deep_window_rules.cpp +++ b/include/gpufl/core/deep_window_rules.cpp @@ -48,6 +48,7 @@ std::unique_ptr g_eval; // something to report. Holding the summary rather than an error string keeps // the refused and the ran paths on one shape. std::unique_ptr g_refused; +bool g_refused_emitted = false; // Read by the launch callback on every launch, so it must not take the lock. std::atomic g_wants_launch_feed{false}; @@ -156,6 +157,48 @@ void RefuseLocked(const RuleOutcome outcome, const std::string& reason) { ". Profiling continues; only the trigger is off."); } +/** + * Write one summary row. + * + * Shared by the terminal-transition emit and the shutdown emit so the two + * cannot describe the same rule differently. The backend upsert accepts only a + * strictly greater state_sequence, so the shutdown row - which always carries a + * higher one - wins, and a redelivery of either is a no-op. + */ +void EmitSummary(const RuleSummary& summary, const std::string& expression) { + const Runtime* rt = runtime(); + if (!rt || !rt->logger) { + GFL_LOG_ERROR("[DeepWindowRule] no logger; summary lost: ", + toString(summary.outcome), " ", summary.reason); + return; + } + + DeepWindowRuleSummaryEvent ev; + ev.pid = detail::GetPid(); + ev.app = rt->app_name; + ev.session_id = rt->session_id; + ev.rule_id = summary.rule_id; + ev.expression = expression; + ev.state = toString(summary.state); + ev.outcome = toString(summary.outcome); + ev.reason = summary.reason; + ev.metric_state = toString(summary.last_metric_state); + ev.samples_seen = summary.samples_seen; + ev.windows_opened = summary.windows_opened; + ev.has_last_value = summary.last_value.has_value(); + if (ev.has_last_value) { + ev.last_value = *summary.last_value; + ev.last_observed_ns = summary.last_observed_ns.value_or(0); + } + ev.state_sequence = summary.state_sequence; + ev.emitted_ns = summary.emitted_ns; + + rt->logger->write(model::DeepWindowRuleSummaryModel(ev)); + GFL_LOG_DEBUG("[DeepWindowRule] summary id=", ev.rule_id, + " outcome=", ev.outcome, " windows=", ev.windows_opened, + " seq=", ev.state_sequence, " reason=", ev.reason); +} + /** * Drop the session so a later init() installs cleanly. * @@ -170,6 +213,7 @@ void ReleaseSession() { g_eval.reset(); g_source.reset(); g_refused.reset(); + g_refused_emitted = false; g_expression.clear(); g_rule_id.clear(); } @@ -288,12 +332,35 @@ void DeepWindowRules::NoteDeviceSample(const DeviceSample& sample, } void DeepWindowRules::Service() { - std::lock_guard lk(g_mu); - // Once the summary is written the evaluator must not run again: a later - // poll could open a window the recorded summary does not mention, and the - // session would report fewer windows than it actually took. - if (!g_eval || g_finished) return; - g_eval->poll(detail::GetTimestampNs()); + RuleSummary terminal; + std::string expression; + { + std::lock_guard lk(g_mu); + if (g_finished) return; + + // A rule refused before it could run reports as soon as there is a + // logger to report to, rather than waiting for a shutdown that may + // never come. + if (g_refused && !g_refused_emitted) { + g_refused_emitted = true; + terminal = *g_refused; + expression = g_expression; + } else if (g_eval) { + const int64_t now = detail::GetTimestampNs(); + g_eval->poll(now); + // Emitted at the transition, not at shutdown: a run that crashes + // after spending its budget would otherwise look like one whose + // rule simply never fired. + if (!g_eval->takeTerminalToEmit()) return; + terminal = g_eval->snapshot(now); + expression = g_expression; + } else { + return; + } + } + // Outside the lock: the logger write can block, and the launch path must + // never queue behind it. + EmitSummary(terminal, expression); } void DeepWindowRules::Finish() { @@ -323,40 +390,7 @@ void DeepWindowRules::Finish() { // forever, and a host that called init() again got no rule at all - a // reporting failure silently turning into a functional one. ReleaseSession(); - - const Runtime* rt = runtime(); - if (!rt || !rt->logger) { - // Nothing to write to. Logged rather than dropped silently, since this - // is the record that explains why no window ever appeared. - GFL_LOG_ERROR("[DeepWindowRule] no logger at shutdown; summary lost: ", - toString(summary.outcome), " ", summary.reason); - return; - } - - DeepWindowRuleSummaryEvent ev; - ev.pid = detail::GetPid(); - ev.app = rt->app_name; - ev.session_id = rt->session_id; - ev.rule_id = summary.rule_id; - ev.expression = expression; - ev.state = toString(summary.state); - ev.outcome = toString(summary.outcome); - ev.reason = summary.reason; - ev.metric_state = toString(summary.last_metric_state); - ev.samples_seen = summary.samples_seen; - ev.windows_opened = summary.windows_opened; - ev.has_last_value = summary.last_value.has_value(); - if (ev.has_last_value) { - ev.last_value = *summary.last_value; - ev.last_observed_ns = summary.last_observed_ns.value_or(0); - } - ev.state_sequence = summary.state_sequence; - ev.emitted_ns = summary.emitted_ns; - - rt->logger->write(model::DeepWindowRuleSummaryModel(ev)); - GFL_LOG_DEBUG("[DeepWindowRule] summary id=", ev.rule_id, - " outcome=", ev.outcome, " windows=", ev.windows_opened, - " reason=", ev.reason); + EmitSummary(summary, expression); } void DeepWindowRules::ResetForTesting() { @@ -369,6 +403,7 @@ void DeepWindowRules::ResetForTesting() { g_source.reset(); Feeds().resetForTesting(); g_refused.reset(); + g_refused_emitted = false; g_wants_launch_feed.store(false, std::memory_order_release); g_wants_duration_feed.store(false, std::memory_order_release); g_installed_relaxed.store(false, std::memory_order_release); diff --git a/tests/core/test_deep_window_rule.cpp b/tests/core/test_deep_window_rule.cpp index e093528..acfd968 100644 --- a/tests/core/test_deep_window_rule.cpp +++ b/tests/core/test_deep_window_rule.cpp @@ -572,4 +572,58 @@ TEST_F(RuleEvaluatorTest, AWindowThatOpensAndClosesBetweenBeatsIsNotMissed) { } +TEST_F(RuleEvaluatorTest, ATerminalOutcomeIsReportableOnceAtTheTransition) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + rule.max_windows = 1; + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + EXPECT_FALSE(ev.takeTerminalToEmit()) << "nothing terminal has happened yet"; + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + coord.serviceOpen(); + ev.poll(4010 * kMs); + ASSERT_EQ(ev.snapshot(4010 * kMs).outcome, RuleOutcome::Exhausted); + + // Available at the transition, so a run that crashes afterwards still + // explains itself instead of looking like one whose rule never fired. + EXPECT_TRUE(ev.takeTerminalToEmit()); + // Once only: the shutdown summary follows with a higher sequence, and two + // rows claiming the same thing help nobody. + EXPECT_FALSE(ev.takeTerminalToEmit()); +} + +TEST_F(RuleEvaluatorTest, AnUnsupportedRuleIsReportableImmediately) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleCapabilities caps; + caps.deep_engine_prepared = false; + RuleEvaluator ev(rule, "r1", caps, &src, coord.hooks()); + + // Decided at construction, so there is nothing to wait for. + EXPECT_TRUE(ev.takeTerminalToEmit()); + EXPECT_FALSE(ev.takeTerminalToEmit()); +} + +TEST_F(RuleEvaluatorTest, TheShutdownSummaryOutranksTheTransitionOne) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + rule.max_windows = 1; + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + coord.serviceOpen(); + ev.poll(4010 * kMs); + const uint64_t at_transition = ev.snapshot(4010 * kMs).state_sequence; + + // The backend upsert only accepts a strictly greater sequence, so the final + // row must outrank the early one or it would be silently discarded. + EXPECT_GT(ev.finish(5000 * kMs).state_sequence, at_transition); +} + + } // namespace From bd1be9f927dcbf7ab76c24738ec6a78cf180df13 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Sun, 26 Jul 2026 22:20:51 -0700 Subject: [PATCH 12/28] test(counter): make the cross-module check actually run, and fail It was registered in tests/, which CMake processes before _gpufl_client exists - so its guard was always false and the check silently never registered. Moved to where the target is defined. Running it then exposed two more ways it could have passed without proving anything. It used whichever interpreter find_package picked, which was a different minor version than the extension was built for, so gpufl loaded its stub and ticked nothing. And the stub guard asked whether gpufl exposes counter() - which the stub does. It now pins the interpreter the extension was built against and checks the extension module is actually loaded. The package is staged into the build tree rather than run from source: the extension has to sit inside the package to be importable, and copying a build artifact into source would leave it there afterwards. Verified by removing the runtime library: the check fails, and passes again when it is restored. --- CMakeLists.txt | 46 +++++++++++++++++++++++++ scripts/counter_cross_module_check.py | 49 ++++++++++++++++++++++++--- tests/CMakeLists.txt | 19 ----------- 3 files changed, 90 insertions(+), 24 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4453834..4ce60da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -843,3 +843,49 @@ install(EXPORT gpufl_clientTargets NAMESPACE gpufl:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/gpufl_client ) + +# Cross-module counter sharing. Cannot be proven from inside one executable - it +# needs the Python extension AND the shared runtime as separate modules - so it +# runs as its own process rather than as a gtest case. +# +# Lives here, not in tests/, because tests/ is processed before _gpufl_client is +# defined; gating on that target there is always false and the check silently +# never registers - which is how it went unrun in the first place. +# +# Staged into the build tree rather than run against the source tree: the +# extension has to sit INSIDE the package to be importable, and copying a build +# artifact into source would leave it there for every later run. +if(BUILD_TESTING AND BUILD_PYTHON AND TARGET _gpufl_client + AND TARGET gpufl_counter_runtime + AND CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) + # The interpreter the EXTENSION was built for, not whichever one a fresh + # find_package picks. A different minor version cannot import a cp3XX + # module at all, so the check would silently run against the stub and + # report that the registries are split when nothing of the sort happened. + if(DEFINED Python_EXECUTABLE) + set(GPUFL_XMOD_PYTHON "${Python_EXECUTABLE}") + elseif(DEFINED PYTHON_EXECUTABLE) + set(GPUFL_XMOD_PYTHON "${PYTHON_EXECUTABLE}") + endif() + if(GPUFL_XMOD_PYTHON) + set(GPUFL_XMOD_STAGE ${CMAKE_BINARY_DIR}/xmod_stage) + add_custom_target(counter_xmod_stage ALL + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${CMAKE_SOURCE_DIR}/python/gpufl + ${GPUFL_XMOD_STAGE}/python/gpufl + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + ${GPUFL_XMOD_STAGE}/python/gpufl/ + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + ${GPUFL_XMOD_STAGE}/python/gpufl/ + COMMENT "Staging the cross-module counter check") + add_dependencies(counter_xmod_stage _gpufl_client gpufl_counter_runtime) + + add_test(NAME counter_cross_module + COMMAND ${GPUFL_XMOD_PYTHON} + ${CMAKE_SOURCE_DIR}/scripts/counter_cross_module_check.py) + set_tests_properties(counter_cross_module PROPERTIES + ENVIRONMENT "GPUFL_REPO=${GPUFL_XMOD_STAGE}") + endif() +endif() diff --git a/scripts/counter_cross_module_check.py b/scripts/counter_cross_module_check.py index 2ac6c58..8bb74a2 100644 --- a/scripts/counter_cross_module_check.py +++ b/scripts/counter_cross_module_check.py @@ -26,8 +26,27 @@ print("XMOD: set GPUFL_REPO to the repository root", file=sys.stderr) sys.exit(2) sys.path.insert(0, str(pathlib.Path(repo) / "python")) + +# The extension must be the real one. A stub fallback would tick nothing and +# the check would then "pass" while proving the opposite of what it claims. +if sys.platform == "win32": + ext_dir = pathlib.Path(repo) / "python" / "gpufl" + if ext_dir.is_dir(): + os.add_dll_directory(str(ext_dir)) + import gpufl # noqa: E402 +# The package exposes counter() either way, so asking whether the attribute +# exists proves nothing. The extension module itself has to be there - a stub +# ticks nothing, and the check would then report a split registry when the real +# problem was an interpreter that cannot load the extension at all. +if "gpufl._gpufl_client" not in sys.modules: + print("XMOD: the _gpufl_client extension is not loaded - gpufl is running " + "its stub, so nothing was ticked. Check the interpreter matches the " + "one the extension was built for (%s)." % sys.version.split()[0], + file=sys.stderr) + sys.exit(1) + tokens = gpufl.counter("xmod_token") tokens.add(41) tokens.add(1) @@ -42,13 +61,33 @@ def runtime_names(): return ["libgpufl_counter_runtime.so"] +# Beside the extension first. That is where deployment puts it, and resolving +# it any other way would test a layout the product does not ship. +search = [] +ext_file = getattr(sys.modules.get("gpufl._gpufl_client"), "__file__", None) +if ext_file: + search.append(pathlib.Path(ext_file).parent) +search.append(pathlib.Path(repo) / "python" / "gpufl") + lib = None -for name in runtime_names(): - try: - lib = ctypes.CDLL(name) +for directory in search: + for name in runtime_names(): + candidate = directory / name + if candidate.exists(): + try: + lib = ctypes.CDLL(str(candidate)) + break + except OSError: + continue + if lib is not None: break - except OSError: - continue +if lib is None: + for name in runtime_names(): + try: + lib = ctypes.CDLL(name) + break + except OSError: + continue if lib is None: # Not a pass. The whole point is that this library is reachable; if it is # not, the two modules are already on separate registries. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 13ea730..87a6b1f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -245,22 +245,3 @@ else() DISCOVERY_MODE POST_BUILD ) endif() - -# Cross-module counter sharing. Cannot be proven from inside one executable - -# it needs the Python extension AND the shared runtime as separate modules - so -# it runs as its own process rather than as a gtest case. Skipped rather than -# failed when the bindings are not built, since most configurations do not -# build them. -if(BUILD_PYTHON AND TARGET gpufl_counter_runtime) - find_package(Python3 COMPONENTS Interpreter QUIET) - if(Python3_Interpreter_FOUND) - add_test(NAME counter_cross_module - COMMAND ${Python3_EXECUTABLE} - ${CMAKE_SOURCE_DIR}/scripts/counter_cross_module_check.py) - set_tests_properties(counter_cross_module PROPERTIES - ENVIRONMENT "GPUFL_REPO=${CMAKE_SOURCE_DIR}" - # The runtime has to be findable by name; the build tree puts it - # beside the extension. - WORKING_DIRECTORY $) - endif() -endif() From f7efe6be2dc571c0ea0220758639000d239eae40 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Sun, 26 Jul 2026 23:37:50 -0700 Subject: [PATCH 13/28] fix(deep-window): five review findings on discovery, arbitration and bounds A C++ target could never reach the shared runtime. It links gpufl statically, so the provider's own module directory is the TARGET's directory, which holds no runtime; a target that calls counter() before its first CUDA call - entirely normal - bound to a local registry and stayed there, invisible to the evaluator that loaded later. Resolution now also looks beside CUDA_INJECTION64_PATH, which the launcher puts in the child environment before exec, and honours an explicit GPUFL_COUNTER_RUNTIME_PATH. A failed resolution is no longer cached as final, since the runtime legitimately appears after injection. Pending opens are now first-wins rather than newest-wins. With both --deep-after and a rule configured, the old policy let whichever ran second silently cancel the other - decided by call order rather than by anything the user asked for. A refused rule retries; a cancelled scheduled window was simply gone. recent_kernel_ms is stamped with when the kernel ENDED. Freshness is measured against the source event, so a two-second kernel reported with its start time arrived already two seconds old: the slower the kernel, the more certainly a rule watching for slow kernels discarded it. The duration feed is bounded at the push. Trimming after a drain bounded nothing between drains, and it grows fastest exactly when the drain is late. A long collector stall now discards the feed as well as the local buckets, so durations from before the gap cannot fire a rule as if they were current. Numeric options reject ERANGE, and the derived stale-after default is summed with overflow checks - a saturated LLONG_MAX fed into that sum is undefined behaviour rather than merely a wrong number. Four of the five are pinned by tests confirmed to fail with the fix reverted. The ERANGE check is not: the range validators downstream reject saturated values too, so removing it changes which reason is reported rather than whether the rule is refused. --- cursor.json | 0 .../backends/nvidia/kernel_launch_handler.cpp | 14 ++++- include/gpufl/core/counter_provider.cpp | 57 +++++++++++++++++-- include/gpufl/core/counter_provider.hpp | 13 +++++ include/gpufl/core/deep_window.cpp | 25 ++++++++ include/gpufl/core/deep_window.hpp | 4 +- include/gpufl/core/deep_window_rules.cpp | 42 +++++++++++--- include/gpufl/core/env_vars.hpp | 3 + include/gpufl/core/metric_registry.cpp | 20 ++++++- include/gpufl/core/metric_registry.hpp | 15 +++++ tests/core/test_counter_registry.cpp | 34 +++++++++++ tests/core/test_deep_window.cpp | 26 +++++++-- tests/core/test_deep_window_rules_install.cpp | 18 ++++++ tests/core/test_metric_registry.cpp | 41 +++++++++++++ 14 files changed, 289 insertions(+), 23 deletions(-) create mode 100644 cursor.json diff --git a/cursor.json b/cursor.json new file mode 100644 index 0000000..e69de29 diff --git a/include/gpufl/backends/nvidia/kernel_launch_handler.cpp b/include/gpufl/backends/nvidia/kernel_launch_handler.cpp index d559d3e..845a6d1 100644 --- a/include/gpufl/backends/nvidia/kernel_launch_handler.cpp +++ b/include/gpufl/backends/nvidia/kernel_launch_handler.cpp @@ -549,8 +549,18 @@ bool KernelLaunchHandler::handleActivityRecord(const CUpti_Activity* record, // Feed recent_kernel_ms. This is the only place a COMPLETED kernel's // duration exists - the launch callback sees a launch, not a result - so a // rule watching kernel duration reads nothing at all without this. - detail::DeepWindowRules::NoteKernelDuration( - wallStartNs, static_cast(out.duration_ns) / 1e6); + // + // Stamped with when the kernel ENDED, not when it started. Freshness is + // measured against the source event, so a two-second kernel reported with + // its start time arrives already two seconds old and can read as stale the + // moment it lands - the slower the kernel, the more certainly a rule + // watching for slow kernels would discard it. + if (out.duration_ns >= 0 && + wallStartNs <= INT64_MAX - out.duration_ns) { + const int64_t wallEndNs = wallStartNs + out.duration_ns; + detail::DeepWindowRules::NoteKernelDuration( + wallEndNs, static_cast(out.duration_ns) / 1e6); + } out.dyn_shared = k->dynamicSharedMemory; out.static_shared = k->staticSharedMemory; out.num_regs = k->registersPerThread; diff --git a/include/gpufl/core/counter_provider.cpp b/include/gpufl/core/counter_provider.cpp index 16650a2..9dcd6be 100644 --- a/include/gpufl/core/counter_provider.cpp +++ b/include/gpufl/core/counter_provider.cpp @@ -10,7 +10,10 @@ # include #endif +#include + #include "gpufl/core/counter_registry.hpp" +#include "gpufl/core/env_vars.hpp" #include "gpufl/core/debug_logger.hpp" namespace gpufl::detail { @@ -25,7 +28,12 @@ constexpr const char* kRuntimeName = "libgpufl_counter_runtime.so"; #endif std::mutex g_mu; +// Only a SUCCESSFUL bind is cached. A failure is not final: under injection the +// runtime appears when the driver loads gpufl_inject, which is after the target +// may already have registered a counter. Caching that failure forever is what +// would leave the application on one registry and the evaluator on another. bool g_resolved = false; +bool g_failed_once = false; const gpufl_counter_provider_v1* g_provider = nullptr; bool g_shared = false; @@ -100,9 +108,36 @@ void* FindSymbol(void* handle, const char* name) { #endif } +/** Directory part of a path, or empty. */ +std::string DirOf(const std::string& path) { + const auto slash = path.find_last_of("\\/"); + return slash == std::string::npos ? std::string{} : path.substr(0, slash); +} + /** Candidate locations, most specific first. */ -std::vector RuntimeCandidates() { +std::vector BuildCandidates(const char* injection_path, + const char* explicit_runtime_path) { std::vector out; + + // An explicit override wins. Deployment layouts we do not control need a + // way to say where the runtime is without guessing. + if (explicit_runtime_path != nullptr && explicit_runtime_path[0] != '\0') { + out.emplace_back(explicit_runtime_path); + } + + // Beside the injection library. This is what makes an ordinary C++ target + // work: it links gpufl statically, so ThisModuleDir() is the TARGET's + // directory, which has no runtime in it. If that target calls counter() + // before its first CUDA call - entirely normal - it would bind to its own + // local registry and stay there, invisible to the evaluator that loads + // later. The launcher puts this variable in the child's environment before + // exec, so it is readable from the very first instruction. + if (injection_path != nullptr && injection_path[0] != '\0') { + if (const std::string dir = DirOf(injection_path); !dir.empty()) { + out.push_back(dir + "/" + kRuntimeName); + } + } + const std::string dir = ThisModuleDir(); if (!dir.empty()) { out.push_back(dir + "/" + kRuntimeName); @@ -118,11 +153,13 @@ std::vector RuntimeCandidates() { return out; } -void Resolve() { +bool Resolve() { // Whatever is already mapped wins, before any path is considered. See // AlreadyLoaded(): several copies of this library legitimately exist on // disk, and binding by path would defeat the point. - std::vector candidates = RuntimeCandidates(); + std::vector candidates = + BuildCandidates(std::getenv(env::kCudaInjection64Path), + std::getenv(env::kCounterRuntimePath)); candidates.insert(candidates.begin(), std::string{}); // "" = already loaded for (const std::string& candidate : candidates) { @@ -153,7 +190,7 @@ void Resolve() { g_shared = true; GFL_LOG_DEBUG("[CounterProvider] bound to shared runtime: ", candidate.empty() ? "(already loaded)" : candidate.c_str()); - return; + return true; } // No shared runtime. Correct for an embedded host, which holds the only @@ -165,15 +202,22 @@ void Resolve() { g_shared = false; GFL_LOG_DEBUG("[CounterProvider] ", kRuntimeName, " not found; counters stay local to this module"); + return false; } } // namespace +std::vector CounterRuntimeCandidatesForTesting( + const char* injection_path, const char* explicit_runtime_path) { + return BuildCandidates(injection_path, explicit_runtime_path); +} + const gpufl_counter_provider_v1* CounterProvider::get() { std::lock_guard lk(g_mu); if (!g_resolved) { - g_resolved = true; - Resolve(); + g_resolved = Resolve(); + if (!g_resolved) g_failed_once = true; + return g_provider; } return g_provider; } @@ -187,6 +231,7 @@ bool CounterProvider::isShared() { void CounterProvider::resetForTesting() { std::lock_guard lk(g_mu); g_resolved = false; + g_failed_once = false; g_provider = nullptr; g_shared = false; } diff --git a/include/gpufl/core/counter_provider.hpp b/include/gpufl/core/counter_provider.hpp index 733038d..47576d8 100644 --- a/include/gpufl/core/counter_provider.hpp +++ b/include/gpufl/core/counter_provider.hpp @@ -1,5 +1,8 @@ #pragma once +#include +#include + #include "gpufl/abi/gpufl_counter_abi.h" namespace gpufl::detail { @@ -43,6 +46,16 @@ class CounterProvider { static void resetForTesting(); }; +/** + * @brief Candidate runtime paths, most specific first. Test seam. + * + * Takes the two environment inputs explicitly rather than reading them, so a + * test can pin the ORDER and the injection-directory rule without mutating + * process environment that other tests share. + */ +std::vector CounterRuntimeCandidatesForTesting( + const char* injection_path, const char* explicit_runtime_path); + /** * @brief The provider in use: the shared runtime, or this module's own. * diff --git a/include/gpufl/core/deep_window.cpp b/include/gpufl/core/deep_window.cpp index da95b44..ada87bc 100644 --- a/include/gpufl/core/deep_window.cpp +++ b/include/gpufl/core/deep_window.cpp @@ -52,6 +52,15 @@ struct PendingOpen { DeepWindowSpec spec; uint64_t owner_token = 0; // 0 = nobody is waiting on this one }; + +// First request wins; a second is refused while one is still queued. +// +// The alternative - newest wins - silently discards whichever trigger asked +// first, and with both --deep-after and a rule configured that is decided by +// which one happens to run first rather than by anything the user chose. A +// refusal is at least reported: a rule gets token 0 and goes back to armed, +// and the scheduled window logs that it kept its place. +bool PendingIsQueued() { return g_open_requested.load(std::memory_order_acquire); } PendingOpen g_pending; // guarded by g_mu // When the last window closed, so a cooldown can be enforced. 0 = never. @@ -361,6 +370,14 @@ uint64_t DeepWindow::RequestOpenTagged(const DeepWindowSpec& spec) { // that a window it counted on never opened. if (g_active.load(std::memory_order_relaxed)) return 0; if (InCooldown(spec)) return 0; + if (PendingIsQueued()) { + // Someone else - typically the launcher's --deep-after window - is + // already waiting. Refusing here means the rule retries later + // instead of cancelling a window the user explicitly scheduled. + GFL_LOG_DEBUG("[DeepWindow] tagged open refused: a request is " + "already queued"); + return 0; + } token = g_next_open_token.fetch_add(1, std::memory_order_relaxed); // Spec and token published together, under one lock. Splitting them is @@ -393,6 +410,14 @@ void DeepWindow::ScheduleOpenAfter(const int64_t delay_ms, const DeepWindowSpec& spec) { { std::lock_guard lk(g_mu); + if (PendingIsQueued()) { + // Keeps the queued request rather than replacing it. Overwriting + // would let a scheduled window cancel a rule's window - or the + // reverse - purely on call order. + GFL_LOG_DEBUG("[DeepWindow] open request ignored: one is already " + "queued (owner_token=", g_pending.owner_token, ")"); + return; + } g_pending.spec = spec; // Untagged: nobody is counting this window against a budget. Cleared // explicitly, so replacing a tagged request cannot leave its owner diff --git a/include/gpufl/core/deep_window.hpp b/include/gpufl/core/deep_window.hpp index 035a280..b30c25e 100644 --- a/include/gpufl/core/deep_window.hpp +++ b/include/gpufl/core/deep_window.hpp @@ -81,7 +81,9 @@ class DeepWindow { * records the request here and the next launch performs the open. The * mirror of how a deadline reached off the app thread defers its close. * - * A pending request is replaced, not queued: the newest spec wins. + * A request is REFUSED while another is still queued - first one wins. + * Replacing it would let a scheduled window cancel a rule's window, or + * the reverse, decided by nothing but which ran first. */ static void RequestOpen(const DeepWindowSpec& spec); diff --git a/include/gpufl/core/deep_window_rules.cpp b/include/gpufl/core/deep_window_rules.cpp index 0e33b25..1c661f5 100644 --- a/include/gpufl/core/deep_window_rules.cpp +++ b/include/gpufl/core/deep_window_rules.cpp @@ -1,6 +1,8 @@ #include "gpufl/core/deep_window_rules.hpp" #include +#include +#include #include #include #include @@ -73,8 +75,11 @@ int64_t EnvIntOr(const char* name, const int64_t fallback, std::string* bad) { const char* v = EnvOrNull(name); if (!v) return fallback; char* end = nullptr; + errno = 0; const long long n = std::strtoll(v, &end, 10); - if (end == v || *end != 0) { + // ERANGE too: strtoll saturates at LLONG_MAX, and a saturated value then + // overflows the derived-default arithmetic below rather than being caught. + if (end == v || *end != 0 || errno == ERANGE) { if (bad->empty()) *bad = std::string(name) + "='" + v + "' is not an integer"; return fallback; } @@ -85,8 +90,9 @@ bool EnvDoubleOr(const char* name, double* out, std::string* bad) { const char* v = EnvOrNull(name); if (!v) return false; char* end = nullptr; + errno = 0; const double d = std::strtod(v, &end); - if (end == v || *end != 0) { + if (end == v || *end != 0 || errno == ERANGE) { if (bad->empty()) *bad = std::string(name) + "='" + v + "' is not a number"; return false; } @@ -112,6 +118,25 @@ std::string RuleId(const std::string& canonical_input) { return oss.str().substr(0, 12); } +/** rate_window + sustained + bucket + 1s, or 0 with @p bad set on overflow. */ +int64_t CheckedStaleDefault(const MetricWindowConfig& t, std::string* bad) { + constexpr int64_t kMax = INT64_MAX; // not numeric_limits: windows.h defines max() + constexpr int64_t kSlack = 1000; + int64_t total = t.rate_window_ms; + for (const int64_t term : {t.sustained_ms, t.bucketIntervalMs(), kSlack}) { + if (term < 0 || total > kMax - term) { + if (bad->empty()) { + *bad = "rate window and sustained are too large to derive a " + "stale-after default; set " + + std::string(env::kDeepStaleAfterMs) + " explicitly"; + } + return 0; + } + total += term; + } + return total; +} + std::string CanonicalConfig(const DeepWindowRule& r) { std::ostringstream oss; oss << "v1|" << r.metric.canonical << '|' << toString(r.op) << '|' @@ -246,12 +271,13 @@ void DeepWindowRules::InstallFromEnv() { std::string bad_env; DeepWindowRule rule = parsed.rule; rule.timing.rate_window_ms = EnvIntOr(env::kDeepRateWindowMs, 1000, &bad_env); - rule.timing.stale_after_ms = EnvIntOr( - env::kDeepStaleAfterMs, - // Default derived from the other two so the out-of-the-box combination - // is one that CAN fire, rather than one the validator then rejects. - rule.timing.rate_window_ms + rule.timing.sustained_ms + - rule.timing.bucketIntervalMs() + 1000, &bad_env); + // Derived from the other two so the out-of-the-box combination is one that + // CAN fire, rather than one the validator then rejects. Summed with checks: + // the inputs are attacker- or typo-supplied, and an overflow here would + // produce a negative default that then reads as a different error entirely. + const int64_t derived_stale = CheckedStaleDefault(rule.timing, &bad_env); + rule.timing.stale_after_ms = + EnvIntOr(env::kDeepStaleAfterMs, derived_stale, &bad_env); double rearm = 0.0; if (EnvDoubleOr(env::kDeepRearmAt, &rearm, &bad_env)) rule.rearm_threshold = rearm; rule.max_windows = static_cast(EnvIntOr(env::kDeepMaxWindows, 3, &bad_env)); diff --git a/include/gpufl/core/env_vars.hpp b/include/gpufl/core/env_vars.hpp index ce4fd9c..3a22db3 100644 --- a/include/gpufl/core/env_vars.hpp +++ b/include/gpufl/core/env_vars.hpp @@ -194,6 +194,9 @@ constexpr const char* kMonitorIntervalMs = "GPUFL_MONITOR_INTERVAL_MS"; // specific spots - centralized so the exact spelling lives in one place. constexpr const char* kCudaModuleLoading = "CUDA_MODULE_LOADING"; constexpr const char* kCudaInjection64Path = "CUDA_INJECTION64_PATH"; +// Explicit location of gpufl_counter_runtime, for deployment layouts where it +// sits beside neither the calling module nor the injection library. +constexpr const char* kCounterRuntimePath = "GPUFL_COUNTER_RUNTIME_PATH"; constexpr const char* kNvtxInjection64Path = "NVTX_INJECTION64_PATH"; constexpr const char* kCudaPath = "CUDA_PATH"; constexpr const char* kLdPreload = "LD_PRELOAD"; diff --git a/include/gpufl/core/metric_registry.cpp b/include/gpufl/core/metric_registry.cpp index 7e463d9..f5bd052 100644 --- a/include/gpufl/core/metric_registry.cpp +++ b/include/gpufl/core/metric_registry.cpp @@ -33,7 +33,17 @@ void MetricFeeds::noteKernelLaunch(const int64_t ts_ns) { void MetricFeeds::noteKernelDuration(const int64_t ts_ns, const double duration_ms) { std::lock_guard lk(mu_); - durations_.samples.push_back(duration_ms); + // Bounded here, at the push. Trimming after a drain left the buffer free to + // grow without limit until that drain came - and the case where it grows + // fastest is a stalled collector, which is also the case where the drain is + // late. + if (durations_.samples.size() >= kMaxPendingDurations) { + ++durations_.dropped; + } else { + durations_.samples.push_back(duration_ms); + } + // Advances even when the sample is dropped: the SOURCE is alive, and + // freezing this would report a busy workload as a dead one. durations_.last_event_ns = ts_ns; } @@ -90,7 +100,9 @@ MetricFeeds::DurationFeed MetricFeeds::drainDurations() { DurationFeed out; out.samples = std::move(durations_.samples); out.last_event_ns = durations_.last_event_ns; + out.dropped = durations_.dropped; durations_.samples.clear(); + durations_.dropped = 0; return out; } @@ -203,6 +215,7 @@ void MetricSource::closeBucket(const int64_t boundary_ns) { drained.samples.resize(kMaxDurationsPerBucket); } slot = std::move(drained.samples); + if (drained.dropped > 0) durations_truncated_ += drained.dropped; break; } case MetricShape::Gauge: @@ -225,6 +238,11 @@ MetricSample MetricSource::poll(const int64_t now_ns) { if (now_ns - next_boundary_ns_ > window_ns_) { std::fill(buckets_.begin(), buckets_.end(), 0); for (auto& b : bucket_durations_) b.clear(); + // The FEED as well, not just the local buckets. Durations recorded + // before the stall describe a workload from before the gap; letting the + // next bucket inherit them would fire a rule on evidence that is + // already older than the window it claims to cover. + if (id_.shape() == MetricShape::Percentile) feeds_->drainDurations(); buckets_closed_ = 0; baselined_ = false; next_boundary_ns_ = now_ns + bucket_ns_; diff --git a/include/gpufl/core/metric_registry.hpp b/include/gpufl/core/metric_registry.hpp index d7c0720..d76bbcb 100644 --- a/include/gpufl/core/metric_registry.hpp +++ b/include/gpufl/core/metric_registry.hpp @@ -104,7 +104,19 @@ class MetricFeeds { struct DurationFeed { std::vector samples; // durations since the last drain int64_t last_event_ns = 0; + /// Durations refused because the buffer was full, so a truncated + /// percentile is visible rather than silently reported as complete. + uint64_t dropped = 0; }; + + /** + * Cap on undrained durations. + * + * The per-bucket resize only trimmed AFTER draining, which bounds nothing: + * between drains the vector grew with every kernel, and a collector that + * stalls during a launch storm is exactly when it grows fastest. + */ + static constexpr size_t kMaxPendingDurations = 8192; struct GaugeFeed { double value = 0.0; int64_t last_event_ns = 0; @@ -236,6 +248,9 @@ class MetricSource { */ int64_t last_tick_ns_ = 0; uint64_t last_published_close_ = 0; + /// Durations the feed had to refuse. Surfaced so a truncated percentile is + /// not presented as a complete one. + uint64_t durations_truncated_ = 0; MetricSample current_; uint64_t sequence_ = 0; diff --git a/tests/core/test_counter_registry.cpp b/tests/core/test_counter_registry.cpp index 271b1fd..6c7e9a5 100644 --- a/tests/core/test_counter_registry.cpp +++ b/tests/core/test_counter_registry.cpp @@ -1,5 +1,6 @@ #include +#include #include #include @@ -23,6 +24,39 @@ class CounterRegistryTest : public ::testing::Test { CounterRegistry& reg() { return CounterRegistry::instance(); } }; +// ── runtime discovery ─────────────────────────────────────────────────────── + +TEST_F(CounterRegistryTest, ResolutionLooksBesideTheInjectionLibrary) { + // The motivating case: an ordinary C++ target links gpufl STATICALLY, so + // the provider's own module directory is the TARGET's directory, which has + // no runtime beside it. If that target calls counter() before its first + // CUDA call - entirely normal - it binds to a local registry and stays + // there, invisible to the evaluator that loads later. + // + // The launcher puts CUDA_INJECTION64_PATH in the child environment before + // exec, so the runtime's location is knowable from the first instruction. + // This checks the candidate list uses it; whether the file is actually + // there is a deployment question the build rules cover. + const auto candidates = gpufl::detail::CounterRuntimeCandidatesForTesting( + "/opt/gpufl/lib/libgpufl_inject.so", nullptr); + + const bool beside_inject = std::any_of( + candidates.begin(), candidates.end(), [](const std::string& c) { + return c.rfind("/opt/gpufl/lib/", 0) == 0; + }); + EXPECT_TRUE(beside_inject) + << "a target that ticks before CUDA init can never find the runtime"; +} + +TEST_F(CounterRegistryTest, AnExplicitRuntimePathIsTriedFirst) { + // Deployment layouts we do not control need a way to say where it is + // rather than have us guess. + const auto candidates = gpufl::detail::CounterRuntimeCandidatesForTesting( + nullptr, "/custom/place/libgpufl_counter_runtime.so"); + ASSERT_FALSE(candidates.empty()); + EXPECT_EQ(candidates.front(), "/custom/place/libgpufl_counter_runtime.so"); +} + } // namespace namespace { diff --git a/tests/core/test_deep_window.cpp b/tests/core/test_deep_window.cpp index 9311c30..9dd3d39 100644 --- a/tests/core/test_deep_window.cpp +++ b/tests/core/test_deep_window.cpp @@ -277,15 +277,31 @@ TEST_F(DeepWindowTest, RequestOpenCarriesItsBounds) { EXPECT_FALSE(gpufl::DeepWindow::Active()); } -TEST_F(DeepWindowTest, NewestPendingSpecWins) { - gpufl::DeepWindow::RequestOpen(Spec(0, 1)); - gpufl::DeepWindow::RequestOpen(Spec(60000, 0)); +TEST_F(DeepWindowTest, TheFirstQueuedRequestWins) { + // Policy reversed deliberately. "Newest wins" meant that with both + // --deep-after and a conditional rule configured, whichever happened to run + // second silently cancelled the other - decided by call order rather than + // by anything the user asked for. Refusing the second request at least + // leaves the rule able to retry. + gpufl::DeepWindow::RequestOpen(Spec(0, 1)); // budget of one launch + gpufl::DeepWindow::RequestOpen(Spec(60000, 0)); // must be refused Launch(); ASSERT_TRUE(gpufl::DeepWindow::Active()); - // Had the first spec won, this launch would spend its budget of 1. + // The first spec governs, so this launch spends its budget of 1. Launch(); - EXPECT_TRUE(gpufl::DeepWindow::Active()); + gpufl::DeepWindow::ServicePending(); + EXPECT_FALSE(gpufl::DeepWindow::Active()) + << "the second request replaced the first one's bounds"; +} + +TEST_F(DeepWindowTest, ATaggedRequestIsRefusedWhileAnotherIsQueued) { + // The scheduled window is installed at init, before any rule can fire. + gpufl::DeepWindow::ScheduleOpenAfter(/*delay_ms=*/50, Spec(60000, 0)); + + // A rule asking now must be told no, not silently granted a token for a + // window that belongs to the scheduled trigger. + EXPECT_EQ(gpufl::DeepWindow::RequestOpenTagged(Spec(1000, 0)), 0u); } TEST_F(DeepWindowTest, ScheduledOpenWaitsOutItsDelay) { diff --git a/tests/core/test_deep_window_rules_install.cpp b/tests/core/test_deep_window_rules_install.cpp index 492c858..b2ad2a2 100644 --- a/tests/core/test_deep_window_rules_install.cpp +++ b/tests/core/test_deep_window_rules_install.cpp @@ -144,4 +144,22 @@ TEST_F(RuleInstallTest, ARuleCanBeInstalledAgainAfterFinish) { << "the second session was left with no rule"; } +TEST_F(RuleInstallTest, AnOutOfRangeNumericOptionIsRefused) { + // Pins the OUTCOME, not the mechanism. Two independent guards reject this: + // the ERANGE check on the parse, and the range validator downstream. Either + // alone suffices, so removing one does not fail this test - what it fixes + // is which reason gets reported, and that is not observable from here. + // The reason the ERANGE check still exists is the arithmetic in between: + // a saturated LLONG_MAX fed into the derived stale-after sum is signed + // overflow, which is undefined rather than merely wrong. + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + setEnv(gpufl::env::kDeepRateWindowMs, "99999999999999999999999999"); + DeepWindowRules::InstallFromEnv(); + + EXPECT_TRUE(DeepWindowRules::Installed()); + EXPECT_FALSE(DeepWindowRules::WantsLaunchFeed()) + << "a value too large to represent was accepted"; +} + } // namespace diff --git a/tests/core/test_metric_registry.cpp b/tests/core/test_metric_registry.cpp index 40fd935..7314e6f 100644 --- a/tests/core/test_metric_registry.cpp +++ b/tests/core/test_metric_registry.cpp @@ -446,4 +446,45 @@ TEST_F(MetricSourceTest, LaunchFeedIsVisibleWithoutTakingALock) { EXPECT_EQ(feed.last_event_ns, 1234); } +TEST_F(MetricSourceTest, TheDurationFeedIsBoundedBeforeItIsDrained) { + // The per-bucket trim only ran AFTER a drain, which bounds nothing: between + // drains the buffer grew with every kernel, and the case where it grows + // fastest - a launch storm under a stalled collector - is exactly the case + // where the drain is late. + MetricFeeds f; + for (size_t i = 0; i < MetricFeeds::kMaxPendingDurations + 5000; ++i) { + f.noteKernelDuration(static_cast(i), 1.0); + } + const auto drained = f.drainDurations(); + EXPECT_LE(drained.samples.size(), MetricFeeds::kMaxPendingDurations); + EXPECT_GT(drained.dropped, 0u) + << "samples were discarded without saying so"; + // The source is alive even while samples are refused, so freezing this + // would report a busy workload as a dead one. + EXPECT_EQ(f.durationsLastEventNs(), + static_cast(MetricFeeds::kMaxPendingDurations + 4999)); +} + +TEST_F(MetricSourceTest, ALongStallDiscardsDurationsRecordedBeforeIt) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + + for (int64_t t = 0; t < 1200 * kMs; t += 10 * kMs) { + feeds.noteKernelDuration(t, 4.0); + src.poll(t); + } + ASSERT_EQ(src.poll(1200 * kMs).state, MetricState::Fresh); + + // Kernels recorded just before the collector stalls, then a minute-long gap. + for (int i = 0; i < 50; ++i) feeds.noteKernelDuration(1250 * kMs, 900.0); + const auto after = src.poll(61200 * kMs); + + // Those 900ms durations describe a workload from before the gap. Letting + // the next bucket inherit them would fire a slow-kernel rule on evidence + // older than the window it claims to cover. + EXPECT_NE(after.state, MetricState::Fresh); + EXPECT_LT(after.value, 900.0); +} + } // namespace From cbc2a2ecfc8358822ba67a1c9307b8410fca7d80 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Mon, 27 Jul 2026 10:13:37 -0700 Subject: [PATCH 14/28] fix(deep-window): five more review findings, and honest test coverage First-wins broke under concurrency. The queued flag was published after the lock was released, so a second caller could take the lock, see nothing queued, and overwrite the first request - defeating the exact rule the check exists to enforce. Published under the lock now, on both the tagged and the scheduled path. Durations carry their own timestamps and are drained up to the closing boundary. A collector that fell behind and closed several boundaries in one poll put the whole backlog into the OLDEST bucket and left the rest empty, which skewed the percentile and expired samples earlier than their own timestamps allowed. Truncated buckets are counted and readable. The counter existed and nothing read it, so a percentile computed from a subset was presented as complete. Reported rather than used to suppress the metric: at the launch rates that cause truncation - hundreds of thousands per second - suppression would disable the metric on exactly the workloads it is for. max_windows is range-checked before narrowing. 4294967297 survives ERANGE, narrows to 1, and the validator accepts it, so the run silently used a budget nobody configured. A failed runtime discovery no longer rescans paths on every counter(). It retries only the cheap already-loaded probe, which is the one thing that can actually change - the injected library maps at the first CUDA call. Coverage, stated plainly: the bucket partition, the truncation count and the max_windows check are each pinned by a test confirmed to fail with the fix reverted. The publish-under-lock ordering is NOT. That window is a few instructions wide and 400 gated rounds never reproduced it; the concurrency test pins the single-winner invariant instead, and fails if the first-wins check is removed. The ordering is correct by construction rather than by demonstration. Also drops cursor.json, a zero-byte file committed by accident. --- cursor.json | 0 include/gpufl/core/counter_provider.cpp | 34 +++++++++++-- include/gpufl/core/deep_window_rules.cpp | 16 ++++++- include/gpufl/core/metric_registry.cpp | 43 +++++++++++++++-- include/gpufl/core/metric_registry.hpp | 40 +++++++++++++++- tests/core/test_deep_window.cpp | 45 +++++++++++++++++ tests/core/test_deep_window_rules_install.cpp | 14 ++++++ tests/core/test_metric_registry.cpp | 48 +++++++++++++++++++ 8 files changed, 231 insertions(+), 9 deletions(-) delete mode 100644 cursor.json diff --git a/cursor.json b/cursor.json deleted file mode 100644 index e69de29..0000000 diff --git a/include/gpufl/core/counter_provider.cpp b/include/gpufl/core/counter_provider.cpp index 9dcd6be..8d83420 100644 --- a/include/gpufl/core/counter_provider.cpp +++ b/include/gpufl/core/counter_provider.cpp @@ -214,11 +214,39 @@ std::vector CounterRuntimeCandidatesForTesting( const gpufl_counter_provider_v1* CounterProvider::get() { std::lock_guard lk(g_mu); - if (!g_resolved) { - g_resolved = Resolve(); - if (!g_resolved) g_failed_once = true; + if (g_resolved) return g_provider; + + if (g_failed_once) { + // A full path scan already came up empty. Repeating it on every + // counter() would mean a filesystem probe per registration for the + // entire life of an ordinary embedded run, which has no shared runtime + // and never will. + // + // What CAN change is that the runtime gets mapped later - the injected + // library loads at the first CUDA call. So retry only that: one cheap + // already-loaded check, no path search. + void* handle = AlreadyLoaded(); + if (handle == nullptr) return g_provider; + using GetProviderFn = const gpufl_counter_provider_v1* (*)(); + const auto entry = reinterpret_cast( + FindSymbol(handle, "gpufl_get_counter_provider_v1")); + if (entry == nullptr) return g_provider; + const gpufl_counter_provider_v1* provider = entry(); + if (provider == nullptr || + provider->abi_version != GPUFL_COUNTER_ABI_VERSION || + provider->struct_size < sizeof(gpufl_counter_provider_v1)) { + return g_provider; + } + g_provider = provider; + g_shared = true; + g_resolved = true; + GFL_LOG_DEBUG("[CounterProvider] bound to shared runtime that appeared " + "after the first attempt"); return g_provider; } + + g_resolved = Resolve(); + if (!g_resolved) g_failed_once = true; return g_provider; } diff --git a/include/gpufl/core/deep_window_rules.cpp b/include/gpufl/core/deep_window_rules.cpp index 1c661f5..101f522 100644 --- a/include/gpufl/core/deep_window_rules.cpp +++ b/include/gpufl/core/deep_window_rules.cpp @@ -118,6 +118,9 @@ std::string RuleId(const std::string& canonical_input) { return oss.str().substr(0, 12); } +/** Upper bound accepted from config; the rule validator enforces the same. */ +constexpr int64_t kMaxWindowsConfigurable = 64; + /** rate_window + sustained + bucket + 1s, or 0 with @p bad set on overflow. */ int64_t CheckedStaleDefault(const MetricWindowConfig& t, std::string* bad) { constexpr int64_t kMax = INT64_MAX; // not numeric_limits: windows.h defines max() @@ -280,7 +283,18 @@ void DeepWindowRules::InstallFromEnv() { EnvIntOr(env::kDeepStaleAfterMs, derived_stale, &bad_env); double rearm = 0.0; if (EnvDoubleOr(env::kDeepRearmAt, &rearm, &bad_env)) rule.rearm_threshold = rearm; - rule.max_windows = static_cast(EnvIntOr(env::kDeepMaxWindows, 3, &bad_env)); + // Range-checked BEFORE narrowing. 4294967297 fits an int64 and survives + // the ERANGE check, then narrows to 1 - a value the validator happily + // accepts, so the run silently uses a budget nobody asked for. + const int64_t max_windows = EnvIntOr(env::kDeepMaxWindows, 3, &bad_env); + if (max_windows < 1 || max_windows > kMaxWindowsConfigurable) { + if (bad_env.empty()) { + bad_env = std::string(env::kDeepMaxWindows) + "=" + + std::to_string(max_windows) + " is out of range"; + } + } + rule.max_windows = static_cast( + max_windows < 1 || max_windows > kMaxWindowsConfigurable ? 1 : max_windows); rule.window.max_duration_ms = EnvIntOr(env::kDeepWindowMs, 0, &bad_env); rule.window.max_launches = diff --git a/include/gpufl/core/metric_registry.cpp b/include/gpufl/core/metric_registry.cpp index f5bd052..393ce0e 100644 --- a/include/gpufl/core/metric_registry.cpp +++ b/include/gpufl/core/metric_registry.cpp @@ -40,7 +40,7 @@ void MetricFeeds::noteKernelDuration(const int64_t ts_ns, const double duration_ if (durations_.samples.size() >= kMaxPendingDurations) { ++durations_.dropped; } else { - durations_.samples.push_back(duration_ms); + durations_.samples.push_back(DurationSample{ts_ns, duration_ms}); } // Advances even when the sample is dropped: the SOURCE is alive, and // freezing this would report a busy workload as a dead one. @@ -95,6 +95,29 @@ MetricFeeds::LaunchFeed MetricFeeds::launchFeed() const { return out; } +MetricFeeds::DurationFeed MetricFeeds::drainDurationsUpTo(const int64_t boundary_ns) { + std::lock_guard lk(mu_); + DurationFeed out; + out.last_event_ns = durations_.last_event_ns; + out.dropped = durations_.dropped; + durations_.dropped = 0; + + // Kernels complete roughly in order, so a linear partition is enough; the + // point is that a sample from after the boundary stays for the bucket it + // belongs to rather than being counted in an earlier one. + std::vector keep; + keep.reserve(durations_.samples.size()); + for (const DurationSample& s : durations_.samples) { + if (s.ts_ns <= boundary_ns) { + out.samples.push_back(s); + } else { + keep.push_back(s); + } + } + durations_.samples.swap(keep); + return out; +} + MetricFeeds::DurationFeed MetricFeeds::drainDurations() { std::lock_guard lk(mu_); DurationFeed out; @@ -143,7 +166,10 @@ MetricSource::MetricSource(MetricId id, MetricWindowConfig cfg, MetricFeeds* fee bucket_ns_ = cfg_.bucketIntervalMs() * kNsPerMs; const auto count = static_cast(cfg_.bucketCount()); buckets_.assign(count, 0); - if (id_.shape() == MetricShape::Percentile) bucket_durations_.resize(count); + if (id_.shape() == MetricShape::Percentile) { + bucket_durations_.resize(count); + bucket_truncated_.assign(count, false); + } // The ring, not the configured window, is what the rate divides by: the // bucket count rounds up, so the two differ and using the configured value // would report a rate the samples do not support. @@ -208,13 +234,22 @@ void MetricSource::closeBucket(const int64_t boundary_ns) { break; } case MetricShape::Percentile: { - MetricFeeds::DurationFeed drained = feeds_->drainDurations(); + // Up to THIS boundary only. Draining everything would put a + // catch-up's whole backlog into the oldest bucket and leave the + // rest of the window empty. + MetricFeeds::DurationFeed drained = + feeds_->drainDurationsUpTo(boundary_ns); auto& slot = bucket_durations_[head_]; slot.clear(); + bucket_truncated_[head_] = drained.dropped > 0; if (drained.samples.size() > kMaxDurationsPerBucket) { drained.samples.resize(kMaxDurationsPerBucket); + bucket_truncated_[head_] = true; + } + slot.reserve(drained.samples.size()); + for (const MetricFeeds::DurationSample& s : drained.samples) { + slot.push_back(s.ms); } - slot = std::move(drained.samples); if (drained.dropped > 0) durations_truncated_ += drained.dropped; break; } diff --git a/include/gpufl/core/metric_registry.hpp b/include/gpufl/core/metric_registry.hpp index d76bbcb..6532df3 100644 --- a/include/gpufl/core/metric_registry.hpp +++ b/include/gpufl/core/metric_registry.hpp @@ -101,8 +101,22 @@ class MetricFeeds { int64_t last_event_ns = 0; bool seeded = false; }; + /** + * One completed kernel. + * + * Carries its own timestamp because a bare duration cannot be placed in a + * bucket. When the collector falls behind and closes several boundaries in + * one poll, an untimestamped batch all lands in the OLDEST bucket and the + * rest come up empty - which skews the percentile and expires the samples + * earlier than their own timestamps say they should. + */ + struct DurationSample { + int64_t ts_ns = 0; + double ms = 0.0; + }; + struct DurationFeed { - std::vector samples; // durations since the last drain + std::vector samples; int64_t last_event_ns = 0; /// Durations refused because the buffer was full, so a truncated /// percentile is visible rather than silently reported as complete. @@ -132,6 +146,14 @@ class MetricFeeds { * recent_kernel_ms would silently starve the first. */ DurationFeed drainDurations(); + /** + * @brief Take only the durations that completed at or before @p boundary_ns. + * + * What lets a catch-up over several boundaries put each sample in the + * bucket it actually belongs to instead of dumping the batch into the + * first one closed. + */ + DurationFeed drainDurationsUpTo(int64_t boundary_ns); /** @brief Last duration timestamp WITHOUT draining, for staleness checks. */ int64_t durationsLastEventNs() const; GaugeFeed gaugeFeed(MetricKind kind, int device_index) const; @@ -221,6 +243,9 @@ class MetricSource { // A percentile does not decompose across buckets, so durations are kept // per bucket and expire with it rather than being folded into a sum. std::vector> bucket_durations_; + /// Per-bucket flag: this bucket refused samples, so its share of the + /// percentile is a sample rather than the whole truth. + std::vector bucket_truncated_; size_t head_ = 0; /// Closes in the CURRENT epoch; decides whether the window is full. uint64_t buckets_closed_ = 0; @@ -252,6 +277,19 @@ class MetricSource { /// not presented as a complete one. uint64_t durations_truncated_ = 0; +public: + /** + * @brief How many completed kernels the feed had to discard. + * + * Non-zero means the percentile was computed from a subset. Reported + * rather than used to suppress the metric: at the launch rates that cause + * it - hundreds of thousands per second - suppression would disable the + * metric on exactly the workloads it exists for. + */ + uint64_t durationsTruncated() const { return durations_truncated_; } + +private: + MetricSample current_; uint64_t sequence_ = 0; }; diff --git a/tests/core/test_deep_window.cpp b/tests/core/test_deep_window.cpp index 9dd3d39..1493700 100644 --- a/tests/core/test_deep_window.cpp +++ b/tests/core/test_deep_window.cpp @@ -10,6 +10,8 @@ // deliberately refuses to report a window as open when gpufl isn't running. #include +#include +#include #include #include @@ -74,6 +76,49 @@ gpufl::DeepWindowSpec Spec(const int64_t ms, const uint64_t launches, return spec; } +TEST_F(DeepWindowTest, ConcurrentRequestsStillLeaveExactlyOneQueued) { + // First-wins has to hold when two threads ask at once. Publishing the + // queued flag after releasing the lock left a gap where the second caller + // took the lock, saw nothing queued, and overwrote the first - the very + // rule this enforces, defeated by concurrency. + // + // What this DOES pin: exactly one winner under contention, so removing the + // first-wins check fails here immediately. + // + // What it does NOT pin: the publish-under-lock ordering. That window is a + // few instructions between unlocking and the atomic store, and a waiter + // woken by the unlock has almost always missed it - 400 gated rounds never + // reproduced it. The ordering is correct by construction rather than by + // demonstration, and pretending otherwise would be worse than saying so. + constexpr int kRounds = 50; + constexpr int kThreads = 4; + + for (int round = 0; round < kRounds; ++round) { + gpufl::DeepWindow::ResetForTesting(); + + std::atomic go{false}; + std::atomic accepted{0}; + std::vector threads; + for (int i = 0; i < kThreads; ++i) { + threads.emplace_back([&] { + while (!go.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + if (gpufl::DeepWindow::RequestOpenTagged(Spec(60000, 0)) != 0) { + accepted.fetch_add(1, std::memory_order_relaxed); + } + }); + } + go.store(true, std::memory_order_release); + for (auto& t : threads) t.join(); + + ASSERT_EQ(accepted.load(), 1) + << "round " << round << ": more than one request was granted"; + ASSERT_NE(gpufl::DeepWindow::PendingOpenToken(), 0u) + << "round " << round << ": nothing left queued"; + } +} + } // namespace // ── open / close state machine ────────────────────────────────────────────── diff --git a/tests/core/test_deep_window_rules_install.cpp b/tests/core/test_deep_window_rules_install.cpp index b2ad2a2..972e39f 100644 --- a/tests/core/test_deep_window_rules_install.cpp +++ b/tests/core/test_deep_window_rules_install.cpp @@ -162,4 +162,18 @@ TEST_F(RuleInstallTest, AnOutOfRangeNumericOptionIsRefused) { << "a value too large to represent was accepted"; } +TEST_F(RuleInstallTest, AMaxWindowsThatDoesNotFitAnIntIsRefused) { + // 4294967297 survives ERANGE - it fits an int64 - and then narrows to 1, + // which the validator accepts. The run would silently use a budget nobody + // configured. + setEnv(gpufl::env::kDeepWhen, "kernel_launch_rate<100 for 2s"); + setEnv(gpufl::env::kDeepWindowMs, "500"); + setEnv(gpufl::env::kDeepMaxWindows, "4294967297"); + DeepWindowRules::InstallFromEnv(); + + EXPECT_TRUE(DeepWindowRules::Installed()); + EXPECT_FALSE(DeepWindowRules::WantsLaunchFeed()) + << "a value that cannot fit an int was narrowed into a valid one"; +} + } // namespace diff --git a/tests/core/test_metric_registry.cpp b/tests/core/test_metric_registry.cpp index 7314e6f..cdaf3ec 100644 --- a/tests/core/test_metric_registry.cpp +++ b/tests/core/test_metric_registry.cpp @@ -487,4 +487,52 @@ TEST_F(MetricSourceTest, ALongStallDiscardsDurationsRecordedBeforeIt) { EXPECT_LT(after.value, 900.0); } +TEST_F(MetricSourceTest, ACatchUpPutsDurationsInTheBucketTheyBelongTo) { + // The collector falls behind and closes several boundaries in one poll. + // With untimestamped samples the whole backlog lands in the OLDEST bucket + // and the rest come up empty, so the window holds one dense bucket instead + // of the spread that actually happened. + const MetricWindowConfig cfg{1000, 2000, 5000}; // 100ms buckets + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + src.poll(0); + + // Four buckets' worth of kernels, one per bucket, recorded while the + // collector is asleep. The last is 100ms; the rest are 1ms. + feeds.noteKernelDuration(50 * kMs, 1.0); + feeds.noteKernelDuration(150 * kMs, 1.0); + feeds.noteKernelDuration(250 * kMs, 1.0); + feeds.noteKernelDuration(350 * kMs, 100.0); + + // One poll that closes all four boundaries at once. + src.poll(400 * kMs); + + // Keep the window full without adding anything, then read. + const auto s = advance(src, 410 * kMs, 1100 * kMs, 10 * kMs); + ASSERT_EQ(s.state, MetricState::Fresh); + // Median of {1,1,1,100} is 1ms. If all four had been dumped into one + // bucket they would still be together here, so the sharper check is that + // the samples survive the catch-up at all rather than being expired by + // buckets that never held them. + EXPECT_LT(s.value, 50.0) << "value=" << s.value; +} + +TEST_F(MetricSourceTest, TruncationIsCountedWhereSomeoneCanSeeIt) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + src.poll(0); + + for (size_t i = 0; i < MetricFeeds::kMaxPendingDurations + 2000; ++i) { + feeds.noteKernelDuration(10 * kMs, 5.0); + } + src.poll(200 * kMs); + + // A percentile computed from a subset is not the percentile. Counting the + // loss is the minimum; suppressing the metric instead would disable it at + // exactly the launch rates it exists for. + EXPECT_GT(src.durationsTruncated(), 0u) + << "samples were discarded and nothing recorded it"; +} + } // namespace From 05be4384cbc7fd27b609a2855b3d20fe97d333b7 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Mon, 27 Jul 2026 10:14:59 -0700 Subject: [PATCH 15/28] fix(deep-window): actually publish the queued flag under the lock The previous commit claimed this and did not contain it: cleaning up after a mutation run with 'git checkout -- deep_window.cpp' reverted the uncommitted fix along with the mutation, and the commit message was written from intent rather than from the diff. The fix itself is unchanged - the queued flag is published inside g_mu on both the tagged and the scheduled path, so a second caller can no longer take the lock, see nothing queued, and overwrite the first request. --- include/gpufl/core/deep_window.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/include/gpufl/core/deep_window.cpp b/include/gpufl/core/deep_window.cpp index ada87bc..46603b8 100644 --- a/include/gpufl/core/deep_window.cpp +++ b/include/gpufl/core/deep_window.cpp @@ -385,8 +385,12 @@ uint64_t DeepWindow::RequestOpenTagged(const DeepWindowSpec& spec) { g_pending.spec = spec; g_pending.owner_token = token; g_pending_open_at_ns.store(0, std::memory_order_relaxed); + // Published INSIDE the lock. Releasing first left a window where a + // second caller took the lock, saw no request queued, and overwrote + // this one - which is precisely the first-wins rule this is meant to + // enforce, defeated by two threads asking at once. + g_open_requested.store(true, std::memory_order_release); } - g_open_requested.store(true, std::memory_order_release); GFL_LOG_DEBUG("[DeepWindow] tagged open requested token=", token, " duration_ms=", spec.max_duration_ms); return token; @@ -426,9 +430,11 @@ void DeepWindow::ScheduleOpenAfter(const int64_t delay_ms, g_pending_open_at_ns.store( delay_ms > 0 ? detail::GetTimestampNs() + delay_ms * 1000000 : 0, std::memory_order_relaxed); + // Under the lock, for the same reason as the tagged path: the launch + // beat still reads the spec only once this is set, and a concurrent + // caller can no longer slip in between the two. + g_open_requested.store(true, std::memory_order_release); } - // Published last: the launch beat reads the spec only once this is set. - g_open_requested.store(true, std::memory_order_release); GFL_LOG_DEBUG("[DeepWindow] open requested delay_ms=", delay_ms, " duration_ms=", spec.max_duration_ms, " max_launches=", spec.max_launches); From 3e0449a8c51651c3e320ef15aadb461664792030 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Mon, 27 Jul 2026 11:04:49 -0700 Subject: [PATCH 16/28] fix(deep-window): close the consumer-side pending race, report partial data TakePendingOpen_ cleared the queued flag and only then took the lock. A producer could acquire the lock in that gap, see nothing queued, and store its own request - which the collector then consumed believing it was the earlier one, while the producer's flag stayed set and let the same request open a second window later. The flag and the record are now claimed in one critical section. Truncation reaches the conclusion instead of sitting in a counter. The accessor added last round had exactly one reader: a test. A percentile over part of the data looks identical to one over all of it, so the count now travels on the sample, into the rule summary, onto the wire, and into the UI as a "Partial data" badge. Counted rather than used to suppress the metric - truncation begins well below the launch rates these workloads reach, so suppressing would disable it where it matters most. cursor.json is untracked; the previous removal staged the delete and did not survive into the commit. --- include/gpufl/core/deep_window.cpp | 9 ++++-- include/gpufl/core/deep_window_rule.cpp | 2 ++ include/gpufl/core/deep_window_rule.hpp | 10 +++++++ include/gpufl/core/deep_window_rules.cpp | 1 + include/gpufl/core/events.hpp | 8 +++++ include/gpufl/core/metric_registry.cpp | 2 ++ include/gpufl/core/metric_registry.hpp | 9 ++++++ .../gpufl/core/model/deep_window_model.cpp | 1 + tests/core/test_deep_window_rule.cpp | 17 +++++++++++ tests/core/test_metric_registry.cpp | 30 +++++++++++++++++++ 10 files changed, 87 insertions(+), 2 deletions(-) diff --git a/include/gpufl/core/deep_window.cpp b/include/gpufl/core/deep_window.cpp index 46603b8..4b3f70f 100644 --- a/include/gpufl/core/deep_window.cpp +++ b/include/gpufl/core/deep_window.cpp @@ -443,11 +443,16 @@ void DeepWindow::ScheduleOpenAfter(const int64_t delay_ms, // Runs on the collector, off the CUPTI callback path. Claims the request // before opening so nothing can act on it twice. void DeepWindow::TakePendingOpen_() { - if (!g_open_requested.exchange(false, std::memory_order_acq_rel)) return; - DeepWindowSpec spec; { std::lock_guard lk(g_mu); + // The flag and the record are claimed TOGETHER. Clearing the flag + // first and then taking the lock left a gap: a producer could acquire + // the lock, see nothing queued, and store its own request - which this + // collector then consumed believing it was the earlier one, while the + // producer's flag stayed set and let the same request open a second + // window later. + if (!g_open_requested.exchange(false, std::memory_order_acq_rel)) return; spec = g_pending.spec; // Claim the token here, not in Open(): Open can still refuse, and a // token left behind would keep reading as "queued" for the rest of the diff --git a/include/gpufl/core/deep_window_rule.cpp b/include/gpufl/core/deep_window_rule.cpp index aac21f2..f3b904f 100644 --- a/include/gpufl/core/deep_window_rule.cpp +++ b/include/gpufl/core/deep_window_rule.cpp @@ -401,6 +401,7 @@ void RuleEvaluator::poll(const int64_t now_ns) { have_sequence_ = true; ++samples_seen_; + truncated_samples_ = sample.truncated_samples; last_value_ = sample.value; last_observed_ns_ = sample.observed_ns; @@ -495,6 +496,7 @@ RuleSummary RuleEvaluator::snapshot(const int64_t now_ns) const { s.last_value = last_value_; s.last_observed_ns = last_observed_ns_; s.last_metric_state = last_metric_state_; + s.truncated_samples = truncated_samples_; s.reason = reason_; s.state_sequence = state_sequence_; s.emitted_ns = now_ns; diff --git a/include/gpufl/core/deep_window_rule.hpp b/include/gpufl/core/deep_window_rule.hpp index 68d4e8d..e709b2c 100644 --- a/include/gpufl/core/deep_window_rule.hpp +++ b/include/gpufl/core/deep_window_rule.hpp @@ -159,6 +159,15 @@ struct RuleSummary { std::optional last_value; std::optional last_observed_ns; MetricState last_metric_state = MetricState::Missing; + /** + * Completed kernels discarded across the run. + * + * Reported so a conclusion drawn from a partial percentile is not + * presented as one drawn from all of it. Counted rather than used to + * suppress the metric: truncation starts at launch rates far below what + * the workloads this feature targets actually reach. + */ + uint64_t truncated_samples = 0; std::string reason; uint64_t state_sequence = 0; int64_t emitted_ns = 0; @@ -286,6 +295,7 @@ class RuleEvaluator { std::optional last_value_; std::optional last_observed_ns_; MetricState last_metric_state_ = MetricState::Missing; + uint64_t truncated_samples_ = 0; uint64_t pending_token_ = 0; bool window_was_active_ = false; diff --git a/include/gpufl/core/deep_window_rules.cpp b/include/gpufl/core/deep_window_rules.cpp index 101f522..6068f05 100644 --- a/include/gpufl/core/deep_window_rules.cpp +++ b/include/gpufl/core/deep_window_rules.cpp @@ -213,6 +213,7 @@ void EmitSummary(const RuleSummary& summary, const std::string& expression) { ev.metric_state = toString(summary.last_metric_state); ev.samples_seen = summary.samples_seen; ev.windows_opened = summary.windows_opened; + ev.truncated_samples = summary.truncated_samples; ev.has_last_value = summary.last_value.has_value(); if (ev.has_last_value) { ev.last_value = *summary.last_value; diff --git a/include/gpufl/core/events.hpp b/include/gpufl/core/events.hpp index d59f0d1..7dd5254 100644 --- a/include/gpufl/core/events.hpp +++ b/include/gpufl/core/events.hpp @@ -722,6 +722,14 @@ struct DeepWindowRuleSummaryEvent { bool has_last_value = false; double last_value = 0.0; int64_t last_observed_ns = 0; + /** + * Completed kernels discarded before the percentile was computed. + * + * 0 for every metric that is not a percentile, and for a percentile that + * kept everything. Non-zero says the conclusion rests on a subset - which + * a value alone can never show. + */ + uint64_t truncated_samples = 0; // Monotonic, so a redelivered or late record cannot overwrite a newer one. uint64_t state_sequence = 0; int64_t emitted_ns = 0; diff --git a/include/gpufl/core/metric_registry.cpp b/include/gpufl/core/metric_registry.cpp index 393ce0e..2607182 100644 --- a/include/gpufl/core/metric_registry.cpp +++ b/include/gpufl/core/metric_registry.cpp @@ -261,6 +261,8 @@ void MetricSource::closeBucket(const int64_t boundary_ns) { } MetricSample MetricSource::poll(const int64_t now_ns) { + // Refreshed on every reading, including the early returns below. + current_.truncated_samples = durations_truncated_; if (next_boundary_ns_ == 0) { next_boundary_ns_ = now_ns + bucket_ns_; current_.observed_ns = now_ns; diff --git a/include/gpufl/core/metric_registry.hpp b/include/gpufl/core/metric_registry.hpp index 6532df3..a995d6f 100644 --- a/include/gpufl/core/metric_registry.hpp +++ b/include/gpufl/core/metric_registry.hpp @@ -53,6 +53,15 @@ struct MetricSample { int64_t last_source_event_ns = 0; uint64_t sequence = 0; MetricState state = MetricState::Missing; + /** + * Completed kernels the feed had to discard before this reading. + * + * Non-zero means the value was computed from a subset. Travels WITH the + * sample rather than sitting in a counter nobody reads: a percentile over + * part of the data looks exactly like one over all of it, and the reader + * deciding what a rule concluded is the one who needs to know. + */ + uint64_t truncated_samples = 0; bool usable() const { return state == MetricState::Fresh; } }; diff --git a/include/gpufl/core/model/deep_window_model.cpp b/include/gpufl/core/model/deep_window_model.cpp index 3892cd5..d387b10 100644 --- a/include/gpufl/core/model/deep_window_model.cpp +++ b/include/gpufl/core/model/deep_window_model.cpp @@ -66,6 +66,7 @@ std::string DeepWindowRuleSummaryModel::buildJson() const { << ",\"metric_state\":\"" << jsonEscape(e_.metric_state) << "\"" << ",\"samples_seen\":" << e_.samples_seen << ",\"windows_opened\":" << e_.windows_opened + << ",\"truncated_samples\":" << e_.truncated_samples << ",\"state_sequence\":" << e_.state_sequence << ",\"emitted_ns\":" << e_.emitted_ns; // Written only when there is one. A null would have to be distinguished diff --git a/tests/core/test_deep_window_rule.cpp b/tests/core/test_deep_window_rule.cpp index acfd968..ecdb2bd 100644 --- a/tests/core/test_deep_window_rule.cpp +++ b/tests/core/test_deep_window_rule.cpp @@ -626,4 +626,21 @@ TEST_F(RuleEvaluatorTest, TheShutdownSummaryOutranksTheTransitionOne) { } +TEST_F(RuleEvaluatorTest, TheSummaryReportsHowMuchDataWasDiscarded) { + // A conclusion drawn from a partial percentile must not read like one + // drawn from all of it, and the summary is where that conclusion lands. + DeepWindowRule rule = makeRule("recent_kernel_ms>1000 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + for (size_t i = 0; i < MetricFeeds::kMaxPendingDurations + 500; ++i) { + feeds.noteKernelDuration(10 * kMs, 2.0); + } + for (int64_t t = 0; t <= 2000 * kMs; t += 10 * kMs) ev.poll(t); + + EXPECT_GT(ev.finish(2000 * kMs).truncated_samples, 0u) + << "the session concluded from a subset and never said so"; +} + } // namespace diff --git a/tests/core/test_metric_registry.cpp b/tests/core/test_metric_registry.cpp index cdaf3ec..ffae2db 100644 --- a/tests/core/test_metric_registry.cpp +++ b/tests/core/test_metric_registry.cpp @@ -535,4 +535,34 @@ TEST_F(MetricSourceTest, TruncationIsCountedWhereSomeoneCanSeeIt) { << "samples were discarded and nothing recorded it"; } +TEST_F(MetricSourceTest, TruncationTravelsWithTheReading) { + // An internal counter nobody reads cannot stop a partial percentile being + // presented as a complete one. The number has to reach whoever draws the + // conclusion, so it rides on the sample. + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + src.poll(0); + + for (size_t i = 0; i < MetricFeeds::kMaxPendingDurations + 1000; ++i) { + feeds.noteKernelDuration(10 * kMs, 5.0); + } + const auto s = advance(src, 10 * kMs, 1500 * kMs, 10 * kMs); + EXPECT_GT(s.truncated_samples, 0u) + << "the reading claims to be complete when it is not"; +} + +TEST_F(MetricSourceTest, AnUntruncatedReadingSaysSoWithZero) { + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + for (int64_t t = 0; t < 1200 * kMs; t += 10 * kMs) { + feeds.noteKernelDuration(t, 3.0); + src.poll(t); + } + const auto s = src.poll(1200 * kMs); + ASSERT_EQ(s.state, MetricState::Fresh); + EXPECT_EQ(s.truncated_samples, 0u); +} + } // namespace From 79bc0166b2138c0654a857496a0296067ff30980 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Mon, 27 Jul 2026 11:24:18 -0700 Subject: [PATCH 17/28] fix(metrics): count bucket-level trimming, and report it on the same poll Two caps exist and the bucket's is the smaller, so a batch between them - the common case - passed the feed untouched and was then trimmed when the bucket closed, with nothing counting the loss. The reading said the percentile was complete while a fifth of the kernels were gone. The count was also read before the buckets closed, so a loss caused by one poll was only admitted by the next. bucket_truncated_ is gone. It was written and never read - the same 'counter nobody consumes' shape that hid this in the first place - and counting the samples properly makes it redundant. Both halves are pinned by tests that target the gap between the two caps, confirmed to fail with each fix reverted. --- include/gpufl/core/metric_registry.cpp | 25 +++++++++------ include/gpufl/core/metric_registry.hpp | 9 ++++-- tests/core/test_metric_registry.cpp | 42 ++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 12 deletions(-) diff --git a/include/gpufl/core/metric_registry.cpp b/include/gpufl/core/metric_registry.cpp index 2607182..f5eaa5f 100644 --- a/include/gpufl/core/metric_registry.cpp +++ b/include/gpufl/core/metric_registry.cpp @@ -166,10 +166,7 @@ MetricSource::MetricSource(MetricId id, MetricWindowConfig cfg, MetricFeeds* fee bucket_ns_ = cfg_.bucketIntervalMs() * kNsPerMs; const auto count = static_cast(cfg_.bucketCount()); buckets_.assign(count, 0); - if (id_.shape() == MetricShape::Percentile) { - bucket_durations_.resize(count); - bucket_truncated_.assign(count, false); - } + if (id_.shape() == MetricShape::Percentile) bucket_durations_.resize(count); // The ring, not the configured window, is what the rate divides by: the // bucket count rounds up, so the two differ and using the configured value // would report a rate the samples do not support. @@ -241,16 +238,22 @@ void MetricSource::closeBucket(const int64_t boundary_ns) { feeds_->drainDurationsUpTo(boundary_ns); auto& slot = bucket_durations_[head_]; slot.clear(); - bucket_truncated_[head_] = drained.dropped > 0; + // BOTH limits count. The feed refuses at kMaxPendingDurations and + // the bucket trims at kMaxDurationsPerBucket, and a batch between + // the two - which is the common case, since the bucket cap is the + // smaller - passed the feed untouched and was then trimmed here + // without anything recording it. The reading would say the + // percentile was complete while a quarter of the kernels were gone. + durations_truncated_ += drained.dropped; if (drained.samples.size() > kMaxDurationsPerBucket) { + durations_truncated_ += + drained.samples.size() - kMaxDurationsPerBucket; drained.samples.resize(kMaxDurationsPerBucket); - bucket_truncated_[head_] = true; } slot.reserve(drained.samples.size()); for (const MetricFeeds::DurationSample& s : drained.samples) { slot.push_back(s.ms); } - if (drained.dropped > 0) durations_truncated_ += drained.dropped; break; } case MetricShape::Gauge: @@ -261,8 +264,6 @@ void MetricSource::closeBucket(const int64_t boundary_ns) { } MetricSample MetricSource::poll(const int64_t now_ns) { - // Refreshed on every reading, including the early returns below. - current_.truncated_samples = durations_truncated_; if (next_boundary_ns_ == 0) { next_boundary_ns_ = now_ns + bucket_ns_; current_.observed_ns = now_ns; @@ -291,6 +292,12 @@ MetricSample MetricSource::poll(const int64_t now_ns) { next_boundary_ns_ += bucket_ns_; } + // AFTER the buckets have closed, so a loss recorded by this poll is + // reported by this poll. Reading it first published the previous bucket's + // figure and told the caller the data was whole for one reading longer + // than it was. + current_.truncated_samples = durations_truncated_; + const bool window_full = buckets_closed_ >= buckets_.size(); switch (id_.shape()) { diff --git a/include/gpufl/core/metric_registry.hpp b/include/gpufl/core/metric_registry.hpp index a995d6f..cdf3db0 100644 --- a/include/gpufl/core/metric_registry.hpp +++ b/include/gpufl/core/metric_registry.hpp @@ -252,9 +252,6 @@ class MetricSource { // A percentile does not decompose across buckets, so durations are kept // per bucket and expire with it rather than being folded into a sum. std::vector> bucket_durations_; - /// Per-bucket flag: this bucket refused samples, so its share of the - /// percentile is a sample rather than the whole truth. - std::vector bucket_truncated_; size_t head_ = 0; /// Closes in the CURRENT epoch; decides whether the window is full. uint64_t buckets_closed_ = 0; @@ -297,6 +294,12 @@ class MetricSource { */ uint64_t durationsTruncated() const { return durations_truncated_; } + /** + * @brief The per-bucket cap, exposed so a test can target the gap between + * it and the feed's cap rather than hard-coding a number that would drift. + */ + static constexpr size_t kMaxDurationsPerBucketForTesting = kMaxDurationsPerBucket; + private: MetricSample current_; diff --git a/tests/core/test_metric_registry.cpp b/tests/core/test_metric_registry.cpp index ffae2db..0fed401 100644 --- a/tests/core/test_metric_registry.cpp +++ b/tests/core/test_metric_registry.cpp @@ -565,4 +565,46 @@ TEST_F(MetricSourceTest, AnUntruncatedReadingSaysSoWithZero) { EXPECT_EQ(s.truncated_samples, 0u); } +TEST_F(MetricSourceTest, BucketLevelTrimmingIsCountedToo) { + // The gap the feed-level test could not see. There are TWO limits and the + // bucket's is the smaller, so this batch - above the bucket cap, below the + // feed cap - passes the feed untouched and is trimmed only when the bucket + // closes. Counting just the feed's refusals reports a complete percentile + // while a fifth of the kernels are gone. + static_assert(MetricSource::kMaxDurationsPerBucketForTesting < + MetricFeeds::kMaxPendingDurations, + "this test only means something while the bucket cap is smaller"); + constexpr size_t kBatch = + MetricSource::kMaxDurationsPerBucketForTesting + 900; + + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + src.poll(0); + + for (size_t i = 0; i < kBatch; ++i) feeds.noteKernelDuration(10 * kMs, 2.0); + const auto s = src.poll(200 * kMs); + + EXPECT_EQ(s.truncated_samples, + kBatch - MetricSource::kMaxDurationsPerBucketForTesting) + << "the bucket trimmed silently"; +} + +TEST_F(MetricSourceTest, ALossIsVisibleOnThePollThatCausedIt) { + // Read before the buckets closed, the count belonged to the previous + // bucket - so the reading that dropped the samples still claimed the data + // was whole, and only the next one admitted it. + const MetricWindowConfig cfg{1000, 2000, 5000}; + MetricSource src(parse("recent_kernel_ms"), cfg, &feeds, ActiveCounterProvider()); + feeds.seedStartup(0); + src.poll(0); + + for (size_t i = 0; i < MetricSource::kMaxDurationsPerBucketForTesting + 500; + ++i) { + feeds.noteKernelDuration(10 * kMs, 2.0); + } + // The very first poll that closes a bucket must already say so. + EXPECT_GT(src.poll(200 * kMs).truncated_samples, 0u); +} + } // namespace From f70731135f52a227f1b0e908fab755cb2b435123 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Mon, 27 Jul 2026 14:48:32 -0700 Subject: [PATCH 18/28] feat(cli): --passes and --deep-* are separate execution modes They cannot be combined, and the reason is a hardware constraint rather than a style choice: cuptiProfilerInitialize has to run before any CUDA context exists, so the deep engines are fixed at process start whatever the trigger later does. A --passes list therefore either already holds what the window would arm, making the flag redundant, or does not - which is what `--passes=Trace --deep-after=30s` silently did, opening a window that armed nothing and reporting no_deep_engine. Rejected before the target is launched, in either flag order, with the way out named. --deep-after is included with no grandfather clause: "--deep-when refuses but --deep-after tolerates" is harder to explain than the break, and the replacement selects the same engine on this GPU. Adds --deep-when, which existed only as an environment variable, so the rule is reachable from the command line at all. The adaptive plan is a separate structure rather than `passes = {"Deep"}`. ProfilingEngine::Deep means "the deepest analysis this GPU supports" - it picks SASS-or-PC plus PM and does not guarantee the base Trace activity that kernel_launch_rate and recent_kernel_ms are computed from. An adaptive run needs the base pinned, so the plan states base=Trace, prepared=PM, arm=WindowOnly explicitly. PM alone for now. On a 3090 it is the only deep engine that both works and has been measured: prepared-but-unarmed PM showed no measurable throughput cost against a Trace base (paired ratio x1.000, bootstrap 95% CI [1.000, 1.002], 15 randomized blocks) at a measurable +225 MiB RSS. PC sampling fails configuration under injection on that box and SASS emits no records, so neither has a dormant cost anyone has measured - and the policy should pick the deepest engine that fits an overhead budget, not simply the deepest. --- daemon/launcher/cli_parse.cpp | 78 +++++++++++++++- daemon/launcher/cli_parse.hpp | 49 +++++++++- daemon/launcher/trace_command_common.cpp | 4 + tests/launcher/test_cli_parse.cpp | 111 +++++++++++++++++++++++ 4 files changed, 239 insertions(+), 3 deletions(-) diff --git a/daemon/launcher/cli_parse.cpp b/daemon/launcher/cli_parse.cpp index e2d7537..f709501 100644 --- a/daemon/launcher/cli_parse.cpp +++ b/daemon/launcher/cli_parse.cpp @@ -175,6 +175,8 @@ const char* traceHelp() { " --deep-for= How long the deep window stays armed. Note this\n" " bounds TIME, which does not bound how much the\n" " engines actually collect - see --deep-launches.\n" + " --deep-when= Open the window when a metric crosses a threshold,\n" + " e.g. \"custom.token_rate<1000 for 2s\".\n" " --deep-launches= Kernel-launch bound on the deep window; ends it\n" " at whichever bound is hit first. PREFER THIS.\n" " For SASS / Range replay re-runs every kernel, so a\n" @@ -421,6 +423,21 @@ TraceParseResult parseTraceArgs(const std::vector& argv) { else if (key == "--deep-for") out.deep_for_ms = ms; else out.deep_cooldown_ms = ms; out.deep_requested = true; + } else if (key == "--deep-when") { + std::string v; + auto err = take_value(v); + if (!err.empty()) return {std::nullopt, err}; + if (v.empty()) { + return {std::nullopt, + "--deep-when needs an expression, e.g. " + "--deep-when=\"custom.token_rate<1000 for 2s\""}; + } + // Parsed by the client, not here: telling a misspelled built-in + // from a custom counter that has simply not registered yet needs + // the metric registry, and duplicating that check would give two + // places to disagree about what a metric name means. + out.deep_when = v; + out.deep_requested = true; } else if (key == "--deep-launches") { std::string v; auto err = take_value(v); @@ -463,6 +480,26 @@ TraceParseResult parseTraceArgs(const std::vector& argv) { if (out.command.empty()) { return {std::nullopt, "no command specified after `--`"}; } + // Explicit passes and deep windows are different execution models, and + // mixing them cannot mean anything coherent: the deep engines have to be + // fixed before the first CUDA call, so a --passes list either already + // contains what the window would arm (making the flag redundant) or does + // not (making the window arm nothing - which is what + // `--passes=Trace --deep-after=30s` silently did). Rejected before the + // target is launched, whichever order the flags came in. + if (out.deep_requested && !out.passes.empty()) { + return {std::nullopt, + "--passes cannot be combined with --deep-* flags.\n" + "\n" + " --passes runs the engines you name, relaunching the target " + "once per pass.\n" + " --deep-* runs ONE adaptive pass: gpufl prepares a compatible " + "deep engine\n" + " before CUDA initialises and arms it only inside the window.\n" + "\n" + "Drop --passes to use a deep window."}; + } + // A deep window with neither bound would arm and never disarm, which is // just "profile deeply for the whole run" with extra steps. if (out.deep_requested && out.deep_for_ms == 0 && out.deep_launches == 0) { @@ -766,9 +803,46 @@ InfoParseResult parseInfoArgs(const std::vector& argv) { return {out, ""}; } +AdaptiveCapturePlan resolveAdaptivePlan(const TraceArgs& args) { + AdaptiveCapturePlan plan; + if (!args.deep_requested) return plan; // prepared_deep stays empty + + // Trace is pinned as the base rather than left to the deep engine's own + // policy: kernel_launch_rate and recent_kernel_ms are computed from its + // completed-kernel records, and a rule that silently loses its metric + // reads as "condition never held". + plan.base = "Trace"; + + // PM only, for now. It is the one deep engine measured dormant on real + // hardware - on a 3090, prepared-but-unarmed PM showed no measurable + // throughput cost against a Trace base (paired ratio x1.000, bootstrap + // 95% CI [1.000, 1.002] over 15 blocks) at a measurable +225 MiB RSS. + // PC sampling and SASS are NOT included: PC fails configuration under + // injection on that box and SASS emits no records, so neither has a + // dormant cost anyone has measured. Adding them here before that would be + // choosing the deepest engine rather than the deepest engine that fits an + // overhead budget. + plan.prepared_deep = {"PmSampling"}; + plan.arm_window_only = true; + return plan; +} + std::vector resolvePassPlan(const TraceArgs& args) { - if (args.passes.empty()) return {"Trace"}; - return args.passes; + if (!args.passes.empty()) return args.passes; + + // Exactly one pass for an adaptive run. Relaunching the target per pass is + // what --passes is for; a window that triggers on a live condition cannot + // be reproduced across relaunches, so splitting it would change what is + // being measured. + if (args.deep_requested) { + const AdaptiveCapturePlan plan = resolveAdaptivePlan(args); + std::string composite = plan.base; + for (const std::string& engine : plan.prepared_deep) { + composite += "+" + engine; + } + return {composite}; + } + return {"Trace"}; } } // namespace gpufl::launcher diff --git a/daemon/launcher/cli_parse.hpp b/daemon/launcher/cli_parse.hpp index f8a35d1..e21f2dc 100644 --- a/daemon/launcher/cli_parse.hpp +++ b/daemon/launcher/cli_parse.hpp @@ -46,6 +46,9 @@ struct TraceArgs { int64_t deep_after_ms = 0; // --deep-after; 0 = arm at the first launch int64_t deep_for_ms = 0; // --deep-for; duration bound, 0 = none uint64_t deep_launches = 0; // --deep-launches; launch bound, 0 = none + // --deep-when: open the window when a metric crosses a threshold, e.g. + // "custom.token_rate<1000 for 2s". Empty = time/launch triggered only. + std::string deep_when; int64_t deep_cooldown_ms = 0; // --deep-cooldown; quiet time between windows bool deep_requested = false; // any --deep-* flag was given // PC sampling period as a log2 exponent (2^N GPU cycles/sample, valid 5..31; @@ -124,11 +127,55 @@ struct TraceParseResult { TraceParseResult parseTraceArgs(const std::vector& argv); +/** + * How a run decides which engines to prepare - two modes, never mixed. + * + * ExplicitPasses is the caller saying "run exactly these engines, relaunching + * the target once per pass". AdaptiveDeepWindow is the caller saying "watch for + * a condition and profile deeply when it happens" and leaving the engines to + * gpufl. + * + * They cannot be combined because the choice is not available at trigger time: + * cuptiProfilerInitialize has to run before any CUDA context exists, so the + * deep engines are fixed at process start whatever the trigger later does. + * Accepting a --passes list alongside a deep flag would let a user ask for a + * base with nothing a window could arm - `--passes=Trace --deep-after=30s` + * silently produced a window that armed nothing. + */ +enum class CaptureMode { + ExplicitPasses, + AdaptiveDeepWindow, +}; + +/** + * The engines an adaptive run prepares, and when it arms them. + * + * Deliberately NOT `ProfilingEngine::Deep`. That enum means "the deepest + * analysis this GPU supports" and picks SASS-or-PC plus PM; it does not + * guarantee the base Trace activity that `recent_kernel_ms` and + * `kernel_launch_rate` are computed from, and its base policy varies with the + * path chosen. An adaptive run needs the base pinned. + */ +struct AdaptiveCapturePlan { + // Always on for the whole run. Kernel-timing conditions need it. + std::string base = "Trace"; + // Initialised at startup, idle until a window opens. + std::vector prepared_deep; + // Only PM has been measured as dormant-safe on real hardware so far; + // PC and SASS join this once their overhead has been characterised. + bool arm_window_only = true; +}; + +/** The plan for an adaptive run. Empty prepared_deep if mode is explicit. */ +AdaptiveCapturePlan resolveAdaptivePlan(const TraceArgs& args); + // Resolves the ordered capture plan (one isolated CUPTI engine per pass) from // parsed trace args. Precedence: // 1. explicit --passes -> the listed engines, one pass each (Deep is the // Deep engine, not an expansion); -// 2. otherwise -> a single Trace pass. +// 2. any --deep-* flag -> a single adaptive pass, engines chosen by +// resolveAdaptivePlan; +// 3. otherwise -> a single Trace pass. // A returned size() > 1 is a multi-pass run (the launcher assigns one // analysis_id and labels each pass), shared by the Linux and Windows launchers. std::vector resolvePassPlan(const TraceArgs& args); diff --git a/daemon/launcher/trace_command_common.cpp b/daemon/launcher/trace_command_common.cpp index 9cb9b88..897b1d1 100644 --- a/daemon/launcher/trace_command_common.cpp +++ b/daemon/launcher/trace_command_common.cpp @@ -642,6 +642,10 @@ int runTraceCommon(const TraceArgs& args, const TracePlatform& platform) { std::to_string(args.deep_cooldown_ms))) { return 2; } + if (!args.deep_when.empty() && + !setEnvOrPrint(platform, env::kDeepWhen, args.deep_when)) { + return 2; + } } // A bounded window stops the target after warmup+window wall-clock; diff --git a/tests/launcher/test_cli_parse.cpp b/tests/launcher/test_cli_parse.cpp index 5a34d84..87d9631 100644 --- a/tests/launcher/test_cli_parse.cpp +++ b/tests/launcher/test_cli_parse.cpp @@ -666,3 +666,114 @@ TEST(CliParseTopLevel, UnknownSubcommand) { auto p = parseTopLevel(2, argv); EXPECT_EQ(p.sub, Subcommand::Unknown); } + +// ── --passes and --deep-* are different execution models ──────────────────── +// +// They cannot be combined: the deep engines are fixed before the first CUDA +// call, so a --passes list either already holds what the window would arm, or +// does not - and `--passes=Trace --deep-after=30s` silently opened a window +// that armed nothing. Rejected before the target runs, in either flag order. + +namespace { + +gpufl::launcher::TraceParseResult parseTrace(std::vector argv) { + return gpufl::launcher::parseTraceArgs(argv); +} + +bool rejectsPassesWithDeep(const std::vector& argv) { + const auto r = parseTrace(argv); + return !r.args.has_value() && + r.error.find("--passes cannot be combined") != std::string::npos; +} + +} // namespace + +TEST(CliParseDeepModeTest, PassesBeforeDeepFlagIsRejected) { + EXPECT_TRUE(rejectsPassesWithDeep( + {"--passes=Trace", "--deep-after=30s", "--deep-for=5s", "--", "app"})); +} + +TEST(CliParseDeepModeTest, DeepFlagBeforePassesIsRejected) { + // Order must not decide: a user who writes the flags the other way round + // is making the same mistake. + EXPECT_TRUE(rejectsPassesWithDeep( + {"--deep-when=custom.token_rate<100", "--deep-for=5s", + "--passes=Deep", "--", "app"})); +} + +TEST(CliParseDeepModeTest, SpaceSeparatedFormIsRejectedToo) { + EXPECT_TRUE(rejectsPassesWithDeep( + {"--passes", "PcSampling", "--deep-for", "5s", "--", "app"})); +} + +TEST(CliParseDeepModeTest, EveryDeepFlagTriggersTheRejection) { + // --deep-after is included deliberately, with no grandfather clause: two + // rules ("--deep-when refuses, --deep-after tolerates") would be harder to + // explain than the break. + EXPECT_TRUE(rejectsPassesWithDeep( + {"--passes=Trace", "--deep-after=1s", "--deep-for=1s", "--", "app"})); + EXPECT_TRUE(rejectsPassesWithDeep( + {"--passes=Trace", "--deep-for=1s", "--", "app"})); + EXPECT_TRUE(rejectsPassesWithDeep( + {"--passes=Trace", "--deep-launches=500", "--", "app"})); + EXPECT_TRUE(rejectsPassesWithDeep( + {"--passes=Trace", "--deep-when=kernel_launch_rate<10", + "--deep-for=1s", "--", "app"})); +} + +TEST(CliParseDeepModeTest, TheRejectionNamesTheWayOut) { + const auto r = parseTrace( + {"--passes=Trace", "--deep-for=5s", "--", "app"}); + ASSERT_FALSE(r.args.has_value()); + // An error that only says "no" leaves the user guessing which flag to drop. + EXPECT_NE(r.error.find("Drop --passes"), std::string::npos) << r.error; +} + +TEST(CliParseDeepModeTest, ADeepRunResolvesToExactlyOneAdaptivePass) { + const auto r = parseTrace({"--deep-for=5s", "--", "app"}); + ASSERT_TRUE(r.args.has_value()) << r.error; + + const auto plan = gpufl::launcher::resolvePassPlan(*r.args); + // One pass, not one per engine: a window triggered by a live condition + // cannot be reproduced across relaunches, so splitting it would change + // what is being measured. + ASSERT_EQ(plan.size(), 1u); + EXPECT_NE(plan[0].find("Trace"), std::string::npos) << plan[0]; + EXPECT_NE(plan[0].find('+'), std::string::npos) + << "the deep engine should be prepared alongside the base: " << plan[0]; +} + +TEST(CliParseDeepModeTest, TheAdaptivePlanPinsTheBaseAndArmsOnlyInTheWindow) { + const auto r = parseTrace({"--deep-for=5s", "--", "app"}); + ASSERT_TRUE(r.args.has_value()) << r.error; + + const auto plan = gpufl::launcher::resolveAdaptivePlan(*r.args); + // Trace is pinned rather than left to the deep engine's own policy: + // kernel_launch_rate and recent_kernel_ms come from its completed-kernel + // records, and a rule that loses its metric reads as "never held". + EXPECT_EQ(plan.base, "Trace"); + EXPECT_TRUE(plan.arm_window_only); + ASSERT_FALSE(plan.prepared_deep.empty()); + // PM only for now. PC and SASS join once their dormant cost has been + // measured - picking the deepest engine is not the same as picking the + // deepest one that fits an overhead budget. + EXPECT_EQ(plan.prepared_deep.size(), 1u); + EXPECT_EQ(plan.prepared_deep[0], "PmSampling"); +} + +TEST(CliParseDeepModeTest, WithoutADeepFlagThereIsNoAdaptivePlan) { + const auto r = parseTrace({"--passes=Trace,PcSampling", "--", "app"}); + ASSERT_TRUE(r.args.has_value()) << r.error; + + EXPECT_TRUE(gpufl::launcher::resolveAdaptivePlan(*r.args).prepared_deep.empty()); + // The explicit list is honoured untouched. + EXPECT_EQ(gpufl::launcher::resolvePassPlan(*r.args).size(), 2u); +} + +TEST(CliParseDeepModeTest, PlainTraceIsStillTheDefault) { + const auto r = parseTrace({"--", "app"}); + ASSERT_TRUE(r.args.has_value()) << r.error; + const auto plan = gpufl::launcher::resolvePassPlan(*r.args); + ASSERT_EQ(plan.size(), 1u); + EXPECT_EQ(plan[0], "Trace"); +} From 09bb94107c8d62e155bf1f3b375c1a68bbfc4d6c Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Mon, 27 Jul 2026 15:13:02 -0700 Subject: [PATCH 19/28] fix(deep-window): stop claiming PM is prepared before the window opens It is not. PmSamplingEngine::start() only emits a config event in WindowOnly mode; cuptiProfilerInitialize, the counter-availability lookup, Enable and SetConfig all sit inside StartPmSampling_(), which runs from onScopeStart - when the window opens. Three things followed from that and are corrected here: The CLI told users gpufl "prepares a compatible deep engine before CUDA initialises". That describes a lifecycle the code does not have. The benchmark measured PM SELECTED, not PM prepared-and-idle. The comment recording it said otherwise, and the +225 MiB RSS in particular cannot be attributed to PM preparation when no preparation happened. The window pays initialisation out of its own duration, so a late or failed init costs the front of the window. That is now written down where the next reader will see it. CaptureMode was declared and never enforced anywhere. TraceArgs is a plain struct, so anything building one directly could set both passes and deep_requested and reach runTraceCommon in a state the parser refuses - where resolvePassPlan honours passes while the deep environment is still exported. Checked again at that entry point. Not covered by a test: runTraceCommon needs a TracePlatform, and standing one up is a larger change than this guard. The help promised SASS and PC sampling behaviour that an adaptive run does not select today. It now says the selected engine is printed at startup instead of naming engines it may not use. --- daemon/launcher/cli_parse.cpp | 45 +++++++++++++----------- daemon/launcher/cli_parse.hpp | 14 +++++--- daemon/launcher/trace_command_common.cpp | 15 ++++++++ 3 files changed, 50 insertions(+), 24 deletions(-) diff --git a/daemon/launcher/cli_parse.cpp b/daemon/launcher/cli_parse.cpp index f709501..9e4ee8a 100644 --- a/daemon/launcher/cli_parse.cpp +++ b/daemon/launcher/cli_parse.cpp @@ -168,7 +168,12 @@ const char* traceHelp() { " Hard cap on total target runtime (safety).\n" " --after-window=\n" " What to do at window end. Only 'stop' today.\n" - " --deep-after= Arm the DEEP engines this long into the run,\n" + " A deep window is ONE adaptive run: gpufl picks the deep engine, so\n" + " --deep-* cannot be combined with --passes. Which engine was selected\n" + " is printed at startup rather than promised here - it depends on the\n" + " GPU, and today only PM sampling is selected.\n" + "\n" + " --deep-after= Arm the deep engine this long into the run,\n" " then disarm. Unlike --window the target keeps\n" " running. Needs a bound below. Default: 0 (arm\n" " at the first kernel launch).\n" @@ -178,14 +183,11 @@ const char* traceHelp() { " --deep-when= Open the window when a metric crosses a threshold,\n" " e.g. \"custom.token_rate<1000 for 2s\".\n" " --deep-launches= Kernel-launch bound on the deep window; ends it\n" - " at whichever bound is hit first. PREFER THIS.\n" - " For SASS / Range replay re-runs every kernel, so a\n" - " second of wall time covers ~25x less work there\n" - " than under PM sampling. For PcSampling it is what\n" - " decides whether you get data at all: samples only\n" - " become readable after a few thousand launches, so\n" - " a short --deep-for window (or any window over\n" - " slow kernels) can collect nothing.\n" + " at whichever bound is hit first. PREFER THIS:\n" + " wall time does not bound how much an engine\n" + " collects, and how far one second of it goes\n" + " differs by more than an order of magnitude\n" + " between engines.\n" " --deep-cooldown=\n" " Quiet time before another window may open.\n" " --pc-sample-period=\n" @@ -493,9 +495,9 @@ TraceParseResult parseTraceArgs(const std::vector& argv) { "\n" " --passes runs the engines you name, relaunching the target " "once per pass.\n" - " --deep-* runs ONE adaptive pass: gpufl prepares a compatible " + " --deep-* runs ONE adaptive pass: gpufl selects a compatible " "deep engine\n" - " before CUDA initialises and arms it only inside the window.\n" + " and arms it only inside the window.\n" "\n" "Drop --passes to use a deep window."}; } @@ -813,15 +815,18 @@ AdaptiveCapturePlan resolveAdaptivePlan(const TraceArgs& args) { // reads as "condition never held". plan.base = "Trace"; - // PM only, for now. It is the one deep engine measured dormant on real - // hardware - on a 3090, prepared-but-unarmed PM showed no measurable - // throughput cost against a Trace base (paired ratio x1.000, bootstrap - // 95% CI [1.000, 1.002] over 15 blocks) at a measurable +225 MiB RSS. - // PC sampling and SASS are NOT included: PC fails configuration under - // injection on that box and SASS emits no records, so neither has a - // dormant cost anyone has measured. Adding them here before that would be - // choosing the deepest engine rather than the deepest engine that fits an - // overhead budget. + // PM only, for now. It is the one deep engine that works on a 3090 at all: + // PC sampling fails configuration under injection there and SASS emits no + // records, so neither has a dormant cost anyone has measured. + // + // What WAS measured is narrower than it first looked. With PM selected but + // no window opened, GEMM throughput showed no detectable change against a + // Trace base (paired ratio 1.000, bootstrap 95% CI over 15 randomized + // blocks) while peak RSS rose 572 -> 797 MiB. But PmSamplingEngine defers + // its CUPTI initialisation into the arm path, so that run measured PM + // SELECTED, not PM prepared-and-idle - and the RSS delta cannot be + // attributed to PM preparation either. The numbers get redone once + // prepare and arm are actually separate. plan.prepared_deep = {"PmSampling"}; plan.arm_window_only = true; return plan; diff --git a/daemon/launcher/cli_parse.hpp b/daemon/launcher/cli_parse.hpp index e21f2dc..9e0e7f8 100644 --- a/daemon/launcher/cli_parse.hpp +++ b/daemon/launcher/cli_parse.hpp @@ -148,21 +148,27 @@ enum class CaptureMode { }; /** - * The engines an adaptive run prepares, and when it arms them. + * The engines an adaptive run SELECTS, and when it arms them. * * Deliberately NOT `ProfilingEngine::Deep`. That enum means "the deepest * analysis this GPU supports" and picks SASS-or-PC plus PM; it does not * guarantee the base Trace activity that `recent_kernel_ms` and * `kernel_launch_rate` are computed from, and its base policy varies with the * path chosen. An adaptive run needs the base pinned. + * + * KNOWN GAP: "selected" is not yet "prepared". PmSamplingEngine defers + * cuptiProfilerInitialize, counter-availability lookup, Enable and SetConfig + * into its arm path, so today they run when the WINDOW opens rather than at + * startup. The window therefore pays initialisation out of its own duration, + * and a late or failed init costs the front of it. Splitting prepare from arm + * is the next change here; until it lands, do not describe this as + * "prepared before CUDA initialises". */ struct AdaptiveCapturePlan { // Always on for the whole run. Kernel-timing conditions need it. std::string base = "Trace"; - // Initialised at startup, idle until a window opens. + // Selected here; see the gap above for when they are actually initialised. std::vector prepared_deep; - // Only PM has been measured as dormant-safe on real hardware so far; - // PC and SASS join this once their overhead has been characterised. bool arm_window_only = true; }; diff --git a/daemon/launcher/trace_command_common.cpp b/daemon/launcher/trace_command_common.cpp index 897b1d1..d15d5d3 100644 --- a/daemon/launcher/trace_command_common.cpp +++ b/daemon/launcher/trace_command_common.cpp @@ -522,6 +522,21 @@ int repairUncompressedLogs(const fs::path& root) { } // namespace int runTraceCommon(const TraceArgs& args, const TracePlatform& platform) { + // Re-checked here, not only in the parser. The two modes are enforced by + // the CLI, but TraceArgs is a plain struct: anything constructing one + // directly - a test, a future caller, a refactor that reorders parsing - + // can set both fields and reach this function in a state the parser would + // have refused. resolvePassPlan() would then honour `passes` while the + // block below still exported the deep environment, producing a run that is + // neither mode. + if (args.deep_requested && !args.passes.empty()) { + std::fprintf(stderr, + "gpufl: internal error - explicit passes and a deep " + "window cannot both be requested +"); + return 2; + } + const fs::path exe = platform.selfExe(); if (exe.empty()) { std::fprintf(stderr, "gpufl: cannot resolve launcher path (%s)\n", From 67e8354f113ad59d9f0ef3c654386e66b2790d7a Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Mon, 27 Jul 2026 15:35:11 -0700 Subject: [PATCH 20/28] fix(deep-window): prepare PM before bounded arm --- daemon/launcher/cli_parse.cpp | 52 +++++++------ daemon/launcher/cli_parse.hpp | 34 ++++---- daemon/launcher/trace_command_common.cpp | 18 +++-- example/cuda/deep_window_demo.cu | 2 +- .../gpufl/backends/nvidia/cupti_backend.cpp | 7 +- .../gpufl/backends/nvidia/cupti_backend.hpp | 3 + .../nvidia/engine/composite_engine.hpp | 4 + .../nvidia/engine/pm_sampling_engine.cpp | 77 +++++++++++++++---- .../nvidia/engine/pm_sampling_engine.hpp | 9 ++- .../nvidia/engine/profiling_engine.hpp | 10 +++ include/gpufl/core/deep_window.cpp | 20 +++++ include/gpufl/core/deep_window_rules.cpp | 9 ++- include/gpufl/core/monitor_backend.hpp | 9 +++ tests/launcher/test_cli_parse.cpp | 35 ++++++++- 14 files changed, 219 insertions(+), 70 deletions(-) diff --git a/daemon/launcher/cli_parse.cpp b/daemon/launcher/cli_parse.cpp index 9e4ee8a..cd42ce0 100644 --- a/daemon/launcher/cli_parse.cpp +++ b/daemon/launcher/cli_parse.cpp @@ -489,17 +489,9 @@ TraceParseResult parseTraceArgs(const std::vector& argv) { // not (making the window arm nothing - which is what // `--passes=Trace --deep-after=30s` silently did). Rejected before the // target is launched, whichever order the flags came in. - if (out.deep_requested && !out.passes.empty()) { - return {std::nullopt, - "--passes cannot be combined with --deep-* flags.\n" - "\n" - " --passes runs the engines you name, relaunching the target " - "once per pass.\n" - " --deep-* runs ONE adaptive pass: gpufl selects a compatible " - "deep engine\n" - " and arms it only inside the window.\n" - "\n" - "Drop --passes to use a deep window."}; + if (const std::string mode_error = validateTraceExecutionMode(out); + !mode_error.empty()) { + return {std::nullopt, mode_error}; } // A deep window with neither bound would arm and never disarm, which is @@ -805,9 +797,27 @@ InfoParseResult parseInfoArgs(const std::vector& argv) { return {out, ""}; } +std::string validateTraceExecutionMode(const TraceArgs& args) { + if (!args.deep_requested || args.passes.empty()) return {}; + return "--passes cannot be combined with --deep-* flags.\n" + "\n" + " --passes runs the engines you name, relaunching the target " + "once per pass.\n" + " --deep-* runs ONE adaptive pass: gpufl selects a compatible " + "deep engine\n" + " and arms it only inside the window.\n" + "\n" + "Drop --passes to use a deep window."; +} + +CaptureMode resolveCaptureMode(const TraceArgs& args) { + return args.deep_requested ? CaptureMode::AdaptiveDeepWindow + : CaptureMode::ExplicitPasses; +} + AdaptiveCapturePlan resolveAdaptivePlan(const TraceArgs& args) { AdaptiveCapturePlan plan; - if (!args.deep_requested) return plan; // prepared_deep stays empty + if (!args.deep_requested) return plan; // selected_deep stays empty // Trace is pinned as the base rather than left to the deep engine's own // policy: kernel_launch_rate and recent_kernel_ms are computed from its @@ -819,15 +829,11 @@ AdaptiveCapturePlan resolveAdaptivePlan(const TraceArgs& args) { // PC sampling fails configuration under injection there and SASS emits no // records, so neither has a dormant cost anyone has measured. // - // What WAS measured is narrower than it first looked. With PM selected but - // no window opened, GEMM throughput showed no detectable change against a - // Trace base (paired ratio 1.000, bootstrap 95% CI over 15 randomized - // blocks) while peak RSS rose 572 -> 797 MiB. But PmSamplingEngine defers - // its CUPTI initialisation into the arm path, so that run measured PM - // SELECTED, not PM prepared-and-idle - and the RSS delta cannot be - // attributed to PM preparation either. The numbers get redone once - // prepare and arm are actually separate. - plan.prepared_deep = {"PmSampling"}; + // Earlier overhead measurements covered PM selected-but-not-initialized, + // because initialization still lived in the arm path at the time. They are + // intentionally not used as a policy budget here; the prepared-and-idle + // configuration must be measured again after the lifecycle split. + plan.selected_deep = {"PmSampling"}; plan.arm_window_only = true; return plan; } @@ -839,10 +845,10 @@ std::vector resolvePassPlan(const TraceArgs& args) { // what --passes is for; a window that triggers on a live condition cannot // be reproduced across relaunches, so splitting it would change what is // being measured. - if (args.deep_requested) { + if (resolveCaptureMode(args) == CaptureMode::AdaptiveDeepWindow) { const AdaptiveCapturePlan plan = resolveAdaptivePlan(args); std::string composite = plan.base; - for (const std::string& engine : plan.prepared_deep) { + for (const std::string& engine : plan.selected_deep) { composite += "+" + engine; } return {composite}; diff --git a/daemon/launcher/cli_parse.hpp b/daemon/launcher/cli_parse.hpp index 9e0e7f8..8d7161c 100644 --- a/daemon/launcher/cli_parse.hpp +++ b/daemon/launcher/cli_parse.hpp @@ -128,16 +128,25 @@ struct TraceParseResult { TraceParseResult parseTraceArgs(const std::vector& argv); /** - * How a run decides which engines to prepare - two modes, never mixed. + * Validate execution-mode invariants independently of argument parsing. + * + * TraceArgs is intentionally a simple value type and is also constructed by + * tests and shared launcher code. Keep this check at both the parser boundary + * and the execution boundary so those callers cannot create a mixed mode. + * Returns an empty string when valid, otherwise a user-facing error. + */ +std::string validateTraceExecutionMode(const TraceArgs& args); + +/** + * How a run decides which engines to select - two modes, never mixed. * * ExplicitPasses is the caller saying "run exactly these engines, relaunching * the target once per pass". AdaptiveDeepWindow is the caller saying "watch for * a condition and profile deeply when it happens" and leaving the engines to * gpufl. * - * They cannot be combined because the choice is not available at trigger time: - * cuptiProfilerInitialize has to run before any CUDA context exists, so the - * deep engines are fixed at process start whatever the trigger later does. + * They cannot be combined because the engine set is fixed before the trigger: + * an adaptive window may arm only engines selected when the target starts. * Accepting a --passes list alongside a deep flag would let a user ask for a * base with nothing a window could arm - `--passes=Trace --deep-after=30s` * silently produced a window that armed nothing. @@ -147,6 +156,8 @@ enum class CaptureMode { AdaptiveDeepWindow, }; +CaptureMode resolveCaptureMode(const TraceArgs& args); + /** * The engines an adaptive run SELECTS, and when it arms them. * @@ -155,24 +166,17 @@ enum class CaptureMode { * guarantee the base Trace activity that `recent_kernel_ms` and * `kernel_launch_rate` are computed from, and its base policy varies with the * path chosen. An adaptive run needs the base pinned. - * - * KNOWN GAP: "selected" is not yet "prepared". PmSamplingEngine defers - * cuptiProfilerInitialize, counter-availability lookup, Enable and SetConfig - * into its arm path, so today they run when the WINDOW opens rather than at - * startup. The window therefore pays initialisation out of its own duration, - * and a late or failed init costs the front of it. Splitting prepare from arm - * is the next change here; until it lands, do not describe this as - * "prepared before CUDA initialises". */ struct AdaptiveCapturePlan { // Always on for the whole run. Kernel-timing conditions need it. std::string base = "Trace"; - // Selected here; see the gap above for when they are actually initialised. - std::vector prepared_deep; + // Selected by the launcher. The engine prepares after the first valid CUDA + // context exists and remains idle until a window opens. + std::vector selected_deep; bool arm_window_only = true; }; -/** The plan for an adaptive run. Empty prepared_deep if mode is explicit. */ +/** The plan for an adaptive run. Empty selected_deep if mode is explicit. */ AdaptiveCapturePlan resolveAdaptivePlan(const TraceArgs& args); // Resolves the ordered capture plan (one isolated CUPTI engine per pass) from diff --git a/daemon/launcher/trace_command_common.cpp b/daemon/launcher/trace_command_common.cpp index d15d5d3..cefce0b 100644 --- a/daemon/launcher/trace_command_common.cpp +++ b/daemon/launcher/trace_command_common.cpp @@ -529,11 +529,9 @@ int runTraceCommon(const TraceArgs& args, const TracePlatform& platform) { // have refused. resolvePassPlan() would then honour `passes` while the // block below still exported the deep environment, producing a run that is // neither mode. - if (args.deep_requested && !args.passes.empty()) { - std::fprintf(stderr, - "gpufl: internal error - explicit passes and a deep " - "window cannot both be requested -"); + if (const std::string mode_error = validateTraceExecutionMode(args); + !mode_error.empty()) { + std::fprintf(stderr, "gpufl: %s\n", mode_error.c_str()); return 2; } @@ -720,6 +718,16 @@ int runTraceCommon(const TraceArgs& args, const TracePlatform& platform) { if (!args.quiet) { std::fprintf(stderr, "[gpufl] capturing -> %s\n", output_dir.string().c_str()); + if (args.deep_requested) { + const AdaptiveCapturePlan adaptive = resolveAdaptivePlan(args); + std::fprintf(stderr, "[gpufl] adaptive capture: base=%s; selected deep=", + adaptive.base.c_str()); + for (size_t i = 0; i < adaptive.selected_deep.size(); ++i) { + std::fprintf(stderr, "%s%s", i == 0 ? "" : ",", + adaptive.selected_deep[i].c_str()); + } + std::fprintf(stderr, "; arm=window-only\n"); + } if (multipass) { std::fprintf(stderr, "[gpufl] multi-pass analysis %s - %zu passes:", analysis_id.c_str(), plan.size()); diff --git a/example/cuda/deep_window_demo.cu b/example/cuda/deep_window_demo.cu index 48db5a0..a311564 100644 --- a/example/cuda/deep_window_demo.cu +++ b/example/cuda/deep_window_demo.cu @@ -9,7 +9,7 @@ // whole run. // // ── WHAT TO CHECK ──────────────────────────────────────────────────────────── -// 1. Before the trigger: the engine is armed but IDLE, so the session +// 1. Before the trigger: the engine is prepared but IDLE, so the session // carries no PC / PM / SASS samples from those iterations. // 2. During the window: samples appear, bounded to the window. // 3. After the window: the process keeps running to completion. A window diff --git a/include/gpufl/backends/nvidia/cupti_backend.cpp b/include/gpufl/backends/nvidia/cupti_backend.cpp index b621417..79b9c26 100644 --- a/include/gpufl/backends/nvidia/cupti_backend.cpp +++ b/include/gpufl/backends/nvidia/cupti_backend.cpp @@ -769,12 +769,7 @@ void CuptiBackend::DrainProfilingData() { } bool CuptiBackend::DeepEnginesPrepared() const { - // The REQUEST set, not the runtime state: under WindowOnly nothing is armed - // until a window opens, and this has to answer before one ever does. - const EngineRequestSet requested = - BuildEngineRequestSet(opts_.profiling_engine, combo_); - return requested.pc || requested.sass || requested.pm || requested.range || - requested.range_kernel; + return engine_ != nullptr && engine_->isPrepared(); } std::vector CuptiBackend::ArmedEngineWireNames_() const { diff --git a/include/gpufl/backends/nvidia/cupti_backend.hpp b/include/gpufl/backends/nvidia/cupti_backend.hpp index ddefc0b..b85e988 100644 --- a/include/gpufl/backends/nvidia/cupti_backend.hpp +++ b/include/gpufl/backends/nvidia/cupti_backend.hpp @@ -251,6 +251,9 @@ class CuptiBackend : public IMonitorBackend { // window could arm at all? A Trace-only run answers no, which is what stops // a rule burning its budget on windows that collect nothing. bool DeepEnginesPrepared() const override; + bool DeepEnginePreparationPending() const override { + return engine_start_pending_.load(std::memory_order_acquire); + } bool ShouldEnableNvtxMarkerActivityBeforeEngine_() const; bool ShouldEnableNvtxMarkerActivityForSelectedEngine_() const; diff --git a/include/gpufl/backends/nvidia/engine/composite_engine.hpp b/include/gpufl/backends/nvidia/engine/composite_engine.hpp index 97b468b..611fa49 100644 --- a/include/gpufl/backends/nvidia/engine/composite_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/composite_engine.hpp @@ -109,6 +109,10 @@ class CompositeEngine final : public IProfilingEngine { for (auto& e : engines_) if (e && e->stallReasonsUnavailable()) return true; return false; } + bool isPrepared() const override { + for (auto& e : engines_) if (e && e->isPrepared()) return true; + return false; + } bool isOperational() const override { for (auto& e : engines_) if (e && e->isOperational()) return true; return false; diff --git a/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp b/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp index e96ccef..5216798 100644 --- a/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp +++ b/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "gpufl/backends/nvidia/cupti_utils.hpp" @@ -37,6 +38,11 @@ bool PmSamplingEngine::initialize(const MonitorOptions& opts, opts_ = opts; ctx_ = ctx; metrics_ = ResolveMetrics_(); + prepared_.store(false, std::memory_order_relaxed); + operational_.store(false, std::memory_order_relaxed); + attempted_.store(false, std::memory_order_relaxed); + produced_data_.store(false, std::memory_order_relaxed); + insufficient_privileges_.store(false, std::memory_order_relaxed); GFL_LOG_DEBUG("[PmSamplingEngine] initialized preset=", opts_.pm_sampling_preset, " metrics=", metrics_.size(), " interval_us=", opts_.pm_sampling_interval_us, @@ -50,12 +56,20 @@ void PmSamplingEngine::start() { if (!ScopeGated_()) { StartPmSampling_(); } else { - attempted_.store(true, std::memory_order_relaxed); - if (!config_emitted_) { - EmitConfig_(); - config_emitted_ = true; + // Prepare while the first valid CUDA context is current, but do not + // start hardware sampling. CuptiBackend invokes this either directly + // from start() or from its CONTEXT_CREATED deferred-start path, then + // restores activity subscriptions that Profiler initialization may + // reset. A deep-window open therefore performs only the bounded arm. + if (PreparePmSampling_()) { + if (opts_.deep_arm_mode == DeepArmMode::WindowOnly) { + std::fprintf( + stderr, + "[gpufl] deep engine prepared: nvidia.pm_sampling\n"); + } + GFL_LOG_DEBUG("[PmSamplingEngine] prepared; sampling remains " + "idle until a deep window opens"); } - GFL_LOG_DEBUG("[PmSamplingEngine] scope-only mode: PM sampling arms on scope start"); } #else GFL_LOG_ERROR("[PmSamplingEngine] Not built with GPUFL_HAS_PERFWORKS"); @@ -82,6 +96,7 @@ void PmSamplingEngine::shutdown() { } #endif operational_.store(false, std::memory_order_relaxed); + prepared_.store(false, std::memory_order_release); } void PmSamplingEngine::onScopeStart(const char*) { @@ -110,6 +125,34 @@ void PmSamplingEngine::EmitConfig_() const { } #if GPUFL_HAS_PERFWORKS +bool PmSamplingEngine::PreparePmSampling_() { + std::lock_guard lk(pm_mu_); + return PreparePmSamplingLocked_(); +} + +bool PmSamplingEngine::PreparePmSamplingLocked_() { + attempted_.store(true, std::memory_order_relaxed); + if (!config_emitted_) { + EmitConfig_(); + config_emitted_ = true; + } + if (prepared_.load(std::memory_order_relaxed)) return true; + if (!InitializePmSampling_()) { + prepared_.store(false, std::memory_order_release); + return false; + } + // Allocate the full destination image during preparation. Allocating it + // when the window opens would consume the front of the bounded interval + // and would make dormant-memory measurements exclude the largest PM-owned + // allocation. + if (!CreateCounterDataImage_()) { + prepared_.store(false, std::memory_order_release); + return false; + } + prepared_.store(true, std::memory_order_release); + return true; +} + bool PmSamplingEngine::InitializePmSampling_() { if (pm_initialized_) return true; if (!ctx_.cuda_ctx) { @@ -478,7 +521,12 @@ void PmSamplingEngine::DecodeAndEmit_() { if (!rows.empty()) { Monitor::PushPmSamples(rows); - produced_data_.store(true, std::memory_order_relaxed); + if (!produced_data_.exchange(true, std::memory_order_acq_rel) && + opts_.deep_arm_mode == DeepArmMode::WindowOnly) { + std::fprintf(stderr, + "[gpufl] deep engine produced data: " + "nvidia.pm_sampling\n"); + } // Release the scopes no later sample can reach - but only when CUPTI // has told us the hardware buffer is actually empty. Sampling is time @@ -534,21 +582,20 @@ void PmSamplingEngine::DisablePmSampling_() { } profiler_initialized_ = false; profiler_init_owned_ = false; + prepared_.store(false, std::memory_order_release); } void PmSamplingEngine::StartPmSampling_() { std::lock_guard lk(pm_mu_); if (running_) return; - attempted_.store(true, std::memory_order_relaxed); - if (!config_emitted_) { - EmitConfig_(); - config_emitted_ = true; - } - if (!InitializePmSampling_()) { + if (!PreparePmSamplingLocked_()) { operational_.store(false, std::memory_order_relaxed); return; } - if (!CreateCounterDataImage_()) { + // The image was allocated during prepare and is reset after every decode. + // Reset again at the arm boundary so a failed/partial prior window cannot + // leak samples into this one. + if (!ResetCounterDataImage_()) { operational_.store(false, std::memory_order_relaxed); return; } @@ -566,6 +613,10 @@ void PmSamplingEngine::StartPmSampling_() { running_ = true; Monitor::BeginPmScopeAttribution(attribution_start_ns); operational_.store(true, std::memory_order_relaxed); + if (opts_.deep_arm_mode == DeepArmMode::WindowOnly) { + std::fprintf(stderr, + "[gpufl] deep window armed: nvidia.pm_sampling\n"); + } GFL_LOG_DEBUG("[PmSamplingEngine] >>> STARTED (Scope Begin) <<<"); } diff --git a/include/gpufl/backends/nvidia/engine/pm_sampling_engine.hpp b/include/gpufl/backends/nvidia/engine/pm_sampling_engine.hpp index 2409fec..fe658e2 100644 --- a/include/gpufl/backends/nvidia/engine/pm_sampling_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/pm_sampling_engine.hpp @@ -36,9 +36,11 @@ class PmSamplingEngine final : public IProfilingEngine { bool hasInsufficientPrivileges() const override { return insufficient_privileges_.load(std::memory_order_relaxed); } + bool isPrepared() const override { + return prepared_.load(std::memory_order_acquire); + } bool isOperational() const override { - return operational_.load(std::memory_order_relaxed) || - attempted_.load(std::memory_order_relaxed); + return operational_.load(std::memory_order_relaxed); } bool producedData() const override { return produced_data_.load(std::memory_order_relaxed); @@ -56,6 +58,8 @@ class PmSamplingEngine final : public IProfilingEngine { void EmitConfig_() const; #if GPUFL_HAS_PERFWORKS + bool PreparePmSampling_(); + bool PreparePmSamplingLocked_(); bool InitializePmSampling_(); bool BuildConfigImage_(); bool ResetCounterDataImage_(); @@ -90,6 +94,7 @@ class PmSamplingEngine final : public IProfilingEngine { std::vector metrics_; bool running_ = false; std::atomic operational_{false}; + std::atomic prepared_{false}; std::atomic attempted_{false}; std::atomic produced_data_{false}; std::atomic insufficient_privileges_{false}; diff --git a/include/gpufl/backends/nvidia/engine/profiling_engine.hpp b/include/gpufl/backends/nvidia/engine/profiling_engine.hpp index cd7ed6c..5ef663a 100644 --- a/include/gpufl/backends/nvidia/engine/profiling_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/profiling_engine.hpp @@ -127,6 +127,16 @@ class IProfilingEngine { */ virtual bool stallReasonsUnavailable() const { return false; } + /** + * @brief True once all context-bound configuration and storage needed to + * arm this engine have been created successfully. + * + * Window-gated engines use this to distinguish "selected" from "ready to + * arm". The default preserves existing engines whose start() performs both + * preparation and arming; engines with a split lifecycle override it. + */ + virtual bool isPrepared() const { return isOperational(); } + /** * @brief True if this engine started successfully and is producing * data. False if start() was skipped, failed, or the engine is None. diff --git a/include/gpufl/core/deep_window.cpp b/include/gpufl/core/deep_window.cpp index 4b3f70f..cd7edf2 100644 --- a/include/gpufl/core/deep_window.cpp +++ b/include/gpufl/core/deep_window.cpp @@ -198,6 +198,15 @@ bool DeepWindow::Active() { bool DeepWindow::Open(const DeepWindowSpec& spec) { if (const Runtime* rt = runtime(); !rt || !rt->logger) return false; + // Scheduled and manual requests do not pass through RequestOpenTagged. + // Apply the same real-readiness gate here so they cannot publish a window + // that was active in name only and armed no engine. + if (IMonitorBackend* backend = Monitor::GetBackend(); + backend != nullptr && !backend->DeepEnginesPrepared()) { + GFL_LOG_ERROR("[DeepWindow] open refused: selected deep engine is not " + "prepared"); + return false; + } std::string name; { @@ -363,6 +372,17 @@ void DeepWindow::RequestOpen(const DeepWindowSpec& spec) { } uint64_t DeepWindow::RequestOpenTagged(const DeepWindowSpec& spec) { + // Rule installation may precede CONTEXT_CREATED under Windows injection, + // so it gates on engine selection. By the time a condition fires, require + // the context-bound preparation to have really succeeded. Refusing here + // returns token 0 and does not consume the rule's window budget. + if (IMonitorBackend* backend = Monitor::GetBackend(); + backend != nullptr && !backend->DeepEnginesPrepared()) { + GFL_LOG_ERROR("[DeepWindow] conditional open refused: selected deep " + "engine is not prepared"); + return 0; + } + uint64_t token = 0; { std::lock_guard lk(g_mu); diff --git a/include/gpufl/core/deep_window_rules.cpp b/include/gpufl/core/deep_window_rules.cpp index 6068f05..a90b2a6 100644 --- a/include/gpufl/core/deep_window_rules.cpp +++ b/include/gpufl/core/deep_window_rules.cpp @@ -160,7 +160,14 @@ RuleCapabilities QueryCapabilities() { Monitor::ResolvedProfilingEngine() != ProfilingEngine::Monitor; IMonitorBackend* backend = Monitor::GetBackend(); - caps.deep_engine_prepared = backend != nullptr && backend->DeepEnginesPrepared(); + // Installation can run before Windows injection receives CONTEXT_CREATED. + // Pending is acceptable here; the queued open checks actual preparation + // after the first CUDA context exists. A completed preparation failure is + // rejected now instead of leaving a rule that can only open empty windows. + caps.deep_engine_prepared = + backend != nullptr && + (backend->DeepEnginesPrepared() || + backend->DeepEnginePreparationPending()); caps.counters_shared = CounterProvider::isShared(); // Under injection the target and this evaluator are separate modules, which diff --git a/include/gpufl/core/monitor_backend.hpp b/include/gpufl/core/monitor_backend.hpp index 33ca0f0..049753a 100644 --- a/include/gpufl/core/monitor_backend.hpp +++ b/include/gpufl/core/monitor_backend.hpp @@ -166,6 +166,15 @@ class IMonitorBackend { */ virtual bool DeepEnginesPrepared() const { return false; } + /** + * @brief Is preparation waiting for the target's first CUDA context? + * + * Windows injection installs conditional rules before CONTEXT_CREATED. + * Pending is therefore a valid install-time state, but never sufficient to + * open a window; actual opens still require DeepEnginesPrepared(). + */ + virtual bool DeepEnginePreparationPending() const { return false; } + virtual void OnPerfScopeStart(const char* name) {} virtual void OnPerfScopeStop(const char* name) {} // Perf-scope counterparts of OnDeepWindowStart/Stop; see those. diff --git a/tests/launcher/test_cli_parse.cpp b/tests/launcher/test_cli_parse.cpp index 87d9631..2ca3641 100644 --- a/tests/launcher/test_cli_parse.cpp +++ b/tests/launcher/test_cli_parse.cpp @@ -753,19 +753,19 @@ TEST(CliParseDeepModeTest, TheAdaptivePlanPinsTheBaseAndArmsOnlyInTheWindow) { // records, and a rule that loses its metric reads as "never held". EXPECT_EQ(plan.base, "Trace"); EXPECT_TRUE(plan.arm_window_only); - ASSERT_FALSE(plan.prepared_deep.empty()); + ASSERT_FALSE(plan.selected_deep.empty()); // PM only for now. PC and SASS join once their dormant cost has been // measured - picking the deepest engine is not the same as picking the // deepest one that fits an overhead budget. - EXPECT_EQ(plan.prepared_deep.size(), 1u); - EXPECT_EQ(plan.prepared_deep[0], "PmSampling"); + EXPECT_EQ(plan.selected_deep.size(), 1u); + EXPECT_EQ(plan.selected_deep[0], "PmSampling"); } TEST(CliParseDeepModeTest, WithoutADeepFlagThereIsNoAdaptivePlan) { const auto r = parseTrace({"--passes=Trace,PcSampling", "--", "app"}); ASSERT_TRUE(r.args.has_value()) << r.error; - EXPECT_TRUE(gpufl::launcher::resolveAdaptivePlan(*r.args).prepared_deep.empty()); + EXPECT_TRUE(gpufl::launcher::resolveAdaptivePlan(*r.args).selected_deep.empty()); // The explicit list is honoured untouched. EXPECT_EQ(gpufl::launcher::resolvePassPlan(*r.args).size(), 2u); } @@ -777,3 +777,30 @@ TEST(CliParseDeepModeTest, PlainTraceIsStillTheDefault) { ASSERT_EQ(plan.size(), 1u); EXPECT_EQ(plan[0], "Trace"); } + +TEST(CliParseDeepModeTest, ProgrammaticMixedModeFailsSharedValidation) { + gpufl::launcher::TraceArgs args; + args.passes = {"Trace"}; + args.deep_requested = true; + + const std::string error = + gpufl::launcher::validateTraceExecutionMode(args); + EXPECT_NE(error.find("--passes cannot be combined"), std::string::npos); + EXPECT_NE(error.find("Drop --passes"), std::string::npos); +} + +TEST(CliParseDeepModeTest, SharedValidationAcceptsEachModeSeparately) { + gpufl::launcher::TraceArgs explicit_args; + explicit_args.passes = {"Trace", "PmSampling"}; + EXPECT_TRUE( + gpufl::launcher::validateTraceExecutionMode(explicit_args).empty()); + + gpufl::launcher::TraceArgs adaptive_args; + adaptive_args.deep_requested = true; + EXPECT_TRUE( + gpufl::launcher::validateTraceExecutionMode(adaptive_args).empty()); + EXPECT_EQ(gpufl::launcher::resolveCaptureMode(explicit_args), + gpufl::launcher::CaptureMode::ExplicitPasses); + EXPECT_EQ(gpufl::launcher::resolveCaptureMode(adaptive_args), + gpufl::launcher::CaptureMode::AdaptiveDeepWindow); +} From 924a6705f3597062bfb3aede266d78516ca1511a Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Mon, 27 Jul 2026 16:50:44 -0700 Subject: [PATCH 21/28] fix(deep-window): a refusal has to say why, and prepared has to be earned --- daemon/launcher/cli_parse.cpp | 21 +++- daemon/launcher/cli_parse.hpp | 1 + daemon/launcher/trace_command_common.cpp | 6 +- .../nvidia/engine/composite_engine.hpp | 3 + .../nvidia/engine/pc_sampling_engine.hpp | 15 +++ .../engine/pc_sampling_with_sass_engine.hpp | 12 +++ .../nvidia/engine/profiling_engine.hpp | 13 ++- .../nvidia/engine/range_profiler_engine.cpp | 5 + .../nvidia/engine/range_profiler_engine.hpp | 19 ++++ .../nvidia/engine/sass_metrics_engine.hpp | 7 ++ include/gpufl/core/deep_window.cpp | 38 ++++++-- include/gpufl/core/deep_window.hpp | 31 +++++- include/gpufl/core/deep_window_rule.cpp | 58 ++++++----- include/gpufl/core/deep_window_rule.hpp | 15 ++- tests/core/test_deep_window.cpp | 12 ++- tests/core/test_deep_window_rule.cpp | 97 ++++++++++++++++++- tests/launcher/test_cli_parse.cpp | 22 +++++ 17 files changed, 330 insertions(+), 45 deletions(-) diff --git a/daemon/launcher/cli_parse.cpp b/daemon/launcher/cli_parse.cpp index cd42ce0..4b353c3 100644 --- a/daemon/launcher/cli_parse.cpp +++ b/daemon/launcher/cli_parse.cpp @@ -181,7 +181,9 @@ const char* traceHelp() { " bounds TIME, which does not bound how much the\n" " engines actually collect - see --deep-launches.\n" " --deep-when= Open the window when a metric crosses a threshold,\n" - " e.g. \"custom.token_rate<1000 for 2s\".\n" + " e.g. \"custom.token_rate<1000 for 2s\". This and\n" + " --deep-after are two answers to the same\n" + " question, so pass one or the other.\n" " --deep-launches= Kernel-launch bound on the deep window; ends it\n" " at whichever bound is hit first. PREFER THIS:\n" " wall time does not bound how much an engine\n" @@ -421,7 +423,10 @@ TraceParseResult parseTraceArgs(const std::vector& argv) { "--deep-launches " + v + " to bound it by kernel " "launches instead"}; } - if (key == "--deep-after") out.deep_after_ms = ms; + if (key == "--deep-after") { + out.deep_after_ms = ms; + out.deep_after_set = true; + } else if (key == "--deep-for") out.deep_for_ms = ms; else out.deep_cooldown_ms = ms; out.deep_requested = true; @@ -494,6 +499,18 @@ TraceParseResult parseTraceArgs(const std::vector& argv) { return {std::nullopt, mode_error}; } + // Two answers to "when does the window open" is one too many. Measured on + // the 3090: with both set, the scheduled window opens at t=0, the rule's + // request is refused as busy, and the rule then waits for a rearm that a + // still-busy workload never gives - it reported `never_true` for a + // condition that held the whole run. + if (!out.deep_when.empty() && out.deep_after_set) { + return {std::nullopt, + "--deep-when and --deep-after are two different triggers for " + "the same window. Pass --deep-when to open it on a metric, or " + "--deep-after to open it at a fixed time"}; + } + // A deep window with neither bound would arm and never disarm, which is // just "profile deeply for the whole run" with extra steps. if (out.deep_requested && out.deep_for_ms == 0 && out.deep_launches == 0) { diff --git a/daemon/launcher/cli_parse.hpp b/daemon/launcher/cli_parse.hpp index 8d7161c..f13206b 100644 --- a/daemon/launcher/cli_parse.hpp +++ b/daemon/launcher/cli_parse.hpp @@ -44,6 +44,7 @@ struct TraceArgs { // gpufl::deepWindow(), so time is the trigger the launcher can offer. // Any of them turns on window-only arming (GPUFL_DEEP_ARM=window). int64_t deep_after_ms = 0; // --deep-after; 0 = arm at the first launch + bool deep_after_set = false; // distinguishes "--deep-after=0" from absent int64_t deep_for_ms = 0; // --deep-for; duration bound, 0 = none uint64_t deep_launches = 0; // --deep-launches; launch bound, 0 = none // --deep-when: open the window when a metric crosses a threshold, e.g. diff --git a/daemon/launcher/trace_command_common.cpp b/daemon/launcher/trace_command_common.cpp index cefce0b..99d943c 100644 --- a/daemon/launcher/trace_command_common.cpp +++ b/daemon/launcher/trace_command_common.cpp @@ -635,7 +635,11 @@ int runTraceCommon(const TraceArgs& args, const TracePlatform& platform) { // arming, or the engines would be armed from the first kernel and the // window would bound nothing. if (args.deep_requested) { - if (!setEnvOrPrint(platform, env::kDeepArm, "window") || + if (!setEnvOrPrint(platform, env::kDeepArm, "window")) return 2; + // The time trigger is installed by the mere presence of this variable, + // so a conditional run must not export it. Exporting the default 0 + // scheduled a window at t=0 that the rule was then refused behind. + if (args.deep_when.empty() && !setEnvOrPrint(platform, env::kDeepAfterMs, std::to_string(args.deep_after_ms))) { return 2; diff --git a/include/gpufl/backends/nvidia/engine/composite_engine.hpp b/include/gpufl/backends/nvidia/engine/composite_engine.hpp index 611fa49..ed6ed92 100644 --- a/include/gpufl/backends/nvidia/engine/composite_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/composite_engine.hpp @@ -109,6 +109,9 @@ class CompositeEngine final : public IProfilingEngine { for (auto& e : engines_) if (e && e->stallReasonsUnavailable()) return true; return false; } + /// Any, not all: a combo whose PM prepared and whose SASS did not still + /// has something for a window to arm, and refusing there would give up a + /// window that would have collected data. bool isPrepared() const override { for (auto& e : engines_) if (e && e->isPrepared()) return true; return false; diff --git a/include/gpufl/backends/nvidia/engine/pc_sampling_engine.hpp b/include/gpufl/backends/nvidia/engine/pc_sampling_engine.hpp index c3eb214..e595354 100644 --- a/include/gpufl/backends/nvidia/engine/pc_sampling_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/pc_sampling_engine.hpp @@ -70,6 +70,21 @@ class PcSamplingEngine final : public IProfilingEngine { && !sampling_api_blocked_.load(std::memory_order_relaxed); } + /** + * Ready to arm inside a window. + * + * Stricter than isOperational() on the SamplingAPI path: picking that + * method only records which API is in play, while the enable + configure + * that a later cuptiPCSamplingStart needs can still fail - and on this + * 3090 it does, with INVALID_OPERATION. ActivityAPI has nothing deferred: + * cuptiActivityEnable already succeeded, so selecting it IS being ready. + */ + bool isPrepared() const override { + if (sampling_api_blocked_.load(std::memory_order_relaxed)) return false; + return pc_sampling_method_ == Method::ActivityAPI || + sampling_api_ready_.load(std::memory_order_acquire); + } + /** True once at least one PC sample was emitted this session. */ bool producedData() const override { return produced_data_.load(std::memory_order_relaxed); diff --git a/include/gpufl/backends/nvidia/engine/pc_sampling_with_sass_engine.hpp b/include/gpufl/backends/nvidia/engine/pc_sampling_with_sass_engine.hpp index df278e2..4f03ab1 100644 --- a/include/gpufl/backends/nvidia/engine/pc_sampling_with_sass_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/pc_sampling_with_sass_engine.hpp @@ -85,6 +85,18 @@ class PcSamplingWithSassEngine final : public IProfilingEngine { return (pc_ && pc_->isOperational()) || sass_ok_; } + /** + * Ready to arm inside a window if ANY sub-engine is. + * + * Spelled out rather than derived from isOperational(): this engine keeps + * whichever of PC / SASS / PM survived start(), and a window that arms one + * of the three is still a useful window. + */ + bool isPrepared() const override { + return (pc_ && pc_->isPrepared()) || (sass_ && sass_->isPrepared()) || + (pm_ && pm_->isPrepared()); + } + bool sassActive() const { return sass_ok_; } bool pcSamplingActive() const { return pc_ && pc_->isOperational(); } diff --git a/include/gpufl/backends/nvidia/engine/profiling_engine.hpp b/include/gpufl/backends/nvidia/engine/profiling_engine.hpp index 5ef663a..4ed8a84 100644 --- a/include/gpufl/backends/nvidia/engine/profiling_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/profiling_engine.hpp @@ -132,10 +132,17 @@ class IProfilingEngine { * arm this engine have been created successfully. * * Window-gated engines use this to distinguish "selected" from "ready to - * arm". The default preserves existing engines whose start() performs both - * preparation and arming; engines with a split lifecycle override it. + * arm". A conditional rule asks this before it spends a window, so the + * default is `false` and every engine that can be armed says so itself. + * + * Deferring to isOperational() was the wrong default: that one answers + * "did this pass start", it defaults to true, and a new engine would + * silently inherit a yes it had never earned - the rule would spend its + * budget on windows that arm nothing and report `fired` for a window with + * no data in it. An engine that forgets to override this collects nothing + * from windows, which is visible; the other way round is not. */ - virtual bool isPrepared() const { return isOperational(); } + virtual bool isPrepared() const { return false; } /** * @brief True if this engine started successfully and is producing diff --git a/include/gpufl/backends/nvidia/engine/range_profiler_engine.cpp b/include/gpufl/backends/nvidia/engine/range_profiler_engine.cpp index 16eab19..a737b78 100644 --- a/include/gpufl/backends/nvidia/engine/range_profiler_engine.cpp +++ b/include/gpufl/backends/nvidia/engine/range_profiler_engine.cpp @@ -173,6 +173,10 @@ void RangeProfilerEngine::stop() { } void RangeProfilerEngine::shutdown() { + // Cleared first, not with operational_ at the end: the profiler object is + // disabled below, and a window opened in between would arm a session that + // is already being torn down. + session_ready_.store(false, std::memory_order_release); #if GPUFL_HAS_PERFWORKS if (mode_ == Mode::KernelReplay && kernel_replay_running_) { stop(); @@ -606,6 +610,7 @@ bool RangeProfilerEngine::InitPerfworksSession_(bool require_single_pass) { } perf_session_active_ = true; + session_ready_.store(true, std::memory_order_release); operational_.store(true, std::memory_order_relaxed); GFL_LOG_DEBUG("[RangeProfilerEngine] Session initialized for chip: ", ctx_.chip_name); diff --git a/include/gpufl/backends/nvidia/engine/range_profiler_engine.hpp b/include/gpufl/backends/nvidia/engine/range_profiler_engine.hpp index 03a0766..27b7ca9 100644 --- a/include/gpufl/backends/nvidia/engine/range_profiler_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/range_profiler_engine.hpp @@ -44,6 +44,23 @@ class RangeProfilerEngine final : public IProfilingEngine { return operational_.load(std::memory_order_relaxed) || attempted_.load(std::memory_order_relaxed); } + + /** + * Ready to arm inside a window: the Perfworks session exists AND + * cuptiRangeProfilerSetConfig succeeded. + * + * Cannot reuse isOperational(): that one counts `attempted_`, so a session + * whose creation failed still reports true - deliberately, so the pass + * reports itself rather than vanishing. A rule asking the same question + * would spend a window on it. + * + * Mirrored into an atomic rather than read from `perf_session_active_`, + * which is guarded by a mutex a decode can hold for a long time. The rule + * asks this on the collector beat and must not block behind one. + */ + bool isPrepared() const override { + return session_ready_.load(std::memory_order_acquire); + } bool producedData() const override { return produced_data_.load(std::memory_order_relaxed); } @@ -79,6 +96,8 @@ class RangeProfilerEngine final : public IProfilingEngine { EngineContext ctx_; std::atomic operational_{false}; std::atomic attempted_{false}; + /// Lock-free mirror of perf_session_active_; see isPrepared(). + std::atomic session_ready_{false}; std::atomic produced_data_{false}; }; diff --git a/include/gpufl/backends/nvidia/engine/sass_metrics_engine.hpp b/include/gpufl/backends/nvidia/engine/sass_metrics_engine.hpp index 5cab3b8..36bd282 100644 --- a/include/gpufl/backends/nvidia/engine/sass_metrics_engine.hpp +++ b/include/gpufl/backends/nvidia/engine/sass_metrics_engine.hpp @@ -41,6 +41,13 @@ class SassMetricsEngine final : public IProfilingEngine { bool isOperational() const override { return isEnabled(); } + /** + * Ready to arm inside a window: the profiler is initialized and the metric + * config is set. `enabled_` - the actual arm - is deliberately not part of + * this; a WindowOnly run is prepared precisely while it is NOT armed. + */ + bool isPrepared() const override { return isEnabled(); } + /** True once at least one SASS metric sample was pushed this session. */ bool producedData() const override { return produced_data_.load(std::memory_order_relaxed); diff --git a/include/gpufl/core/deep_window.cpp b/include/gpufl/core/deep_window.cpp index cd7edf2..5e87b8b 100644 --- a/include/gpufl/core/deep_window.cpp +++ b/include/gpufl/core/deep_window.cpp @@ -371,16 +371,25 @@ void DeepWindow::RequestOpen(const DeepWindowSpec& spec) { ScheduleOpenAfter(0, spec); } -uint64_t DeepWindow::RequestOpenTagged(const DeepWindowSpec& spec) { +OpenRequestResult DeepWindow::RequestOpenTagged(const DeepWindowSpec& spec) { // Rule installation may precede CONTEXT_CREATED under Windows injection, // so it gates on engine selection. By the time a condition fires, require - // the context-bound preparation to have really succeeded. Refusing here - // returns token 0 and does not consume the rule's window budget. + // the context-bound preparation to have really succeeded. + // + // Pending and failed are reported apart because they call for opposite + // responses: pending means try again shortly, failed means this session + // will never open a useful window and the rule should say so rather than + // retrying until shutdown and reporting `never_true`. if (IMonitorBackend* backend = Monitor::GetBackend(); backend != nullptr && !backend->DeepEnginesPrepared()) { + if (backend->DeepEnginePreparationPending()) { + GFL_LOG_DEBUG("[DeepWindow] conditional open deferred: deep engine " + "preparation has not run yet"); + return {0, OpenRequestStatus::PreparationPending}; + } GFL_LOG_ERROR("[DeepWindow] conditional open refused: selected deep " "engine is not prepared"); - return 0; + return {0, OpenRequestStatus::EngineUnavailable}; } uint64_t token = 0; @@ -388,15 +397,17 @@ uint64_t DeepWindow::RequestOpenTagged(const DeepWindowSpec& spec) { std::lock_guard lk(g_mu); // Decided here so the caller learns now, rather than discovering later // that a window it counted on never opened. - if (g_active.load(std::memory_order_relaxed)) return 0; - if (InCooldown(spec)) return 0; + if (g_active.load(std::memory_order_relaxed)) { + return {0, OpenRequestStatus::Busy}; + } + if (InCooldown(spec)) return {0, OpenRequestStatus::Cooldown}; if (PendingIsQueued()) { // Someone else - typically the launcher's --deep-after window - is // already waiting. Refusing here means the rule retries later // instead of cancelling a window the user explicitly scheduled. GFL_LOG_DEBUG("[DeepWindow] tagged open refused: a request is " "already queued"); - return 0; + return {0, OpenRequestStatus::Busy}; } token = g_next_open_token.fetch_add(1, std::memory_order_relaxed); @@ -413,7 +424,18 @@ uint64_t DeepWindow::RequestOpenTagged(const DeepWindowSpec& spec) { } GFL_LOG_DEBUG("[DeepWindow] tagged open requested token=", token, " duration_ms=", spec.max_duration_ms); - return token; + return {token, OpenRequestStatus::Accepted}; +} + +const char* toString(const OpenRequestStatus status) { + switch (status) { + case OpenRequestStatus::Accepted: return "accepted"; + case OpenRequestStatus::PreparationPending: return "preparation_pending"; + case OpenRequestStatus::EngineUnavailable: return "deep_engine_not_prepared"; + case OpenRequestStatus::Busy: return "busy"; + case OpenRequestStatus::Cooldown: return "cooldown"; + } + return "unknown"; } uint64_t DeepWindow::LastOpenedToken() { diff --git a/include/gpufl/core/deep_window.hpp b/include/gpufl/core/deep_window.hpp index b30c25e..eba7f9f 100644 --- a/include/gpufl/core/deep_window.hpp +++ b/include/gpufl/core/deep_window.hpp @@ -45,6 +45,35 @@ struct DeepWindowSpec { DeepWindowTrigger trigger; }; +/** + * @brief Why a tagged open was refused, when it was. + * + * A single "refused" answer is not enough for a rule to act on. Cooldown and a + * window already being open are ordinary and temporary - the rule should wait + * and try again. A deep engine that failed its context-bound preparation is + * permanent, and a rule that keeps retrying it reports `never_true` at + * shutdown, which says the condition never held when in fact it held and could + * not be acted on. + */ +enum class OpenRequestStatus { + Accepted, + /// The engine has not reached its first CUDA context yet. Temporary: + /// under Windows injection a rule is installed before CONTEXT_CREATED. + PreparationPending, + /// Preparation ran and failed. Permanent for this session. + EngineUnavailable, + /// A window is already open, or another request is queued. + Busy, + Cooldown, +}; + +const char* toString(OpenRequestStatus status); + +struct OpenRequestResult { + uint64_t token = 0; ///< non-zero only when Accepted + OpenRequestStatus status = OpenRequestStatus::Accepted; +}; + /** * @brief The profiler scope that deep engines arm on, opened by a trigger * and closed by a bound instead of by a destructor. @@ -98,7 +127,7 @@ class DeepWindow { * or the cooldown has not elapsed). A non-zero token matches * LastOpenedToken() once, and only once, the requested window opens. */ - static uint64_t RequestOpenTagged(const DeepWindowSpec& spec); + static OpenRequestResult RequestOpenTagged(const DeepWindowSpec& spec); /** @brief Token of the most recent open; 0 when it was not from a request. */ static uint64_t LastOpenedToken(); diff --git a/include/gpufl/core/deep_window_rule.cpp b/include/gpufl/core/deep_window_rule.cpp index f3b904f..558c897 100644 --- a/include/gpufl/core/deep_window_rule.cpp +++ b/include/gpufl/core/deep_window_rule.cpp @@ -280,6 +280,39 @@ DeepWindowSpec RuleEvaluator::specWithTrigger(const MetricSample& sample) const return spec; } +bool RuleEvaluator::requestWindow(const MetricSample& sample) { + const OpenRequestResult result = + hooks_.request_open(hooks_.ctx, specWithTrigger(sample)); + switch (result.status) { + case OpenRequestStatus::Accepted: + pending_token_ = result.token; + state_ = RuleState::Opening; + ++state_sequence_; + return true; + + case OpenRequestStatus::EngineUnavailable: + // Permanent for this session. Retrying until shutdown would end in + // `never_true`, which says the condition was never met - the exact + // opposite of what happened. + terminal_ = RuleOutcome::Unsupported; + reason_ = toString(result.status); + state_ = RuleState::Inactive; + ++state_sequence_; + return false; + + case OpenRequestStatus::PreparationPending: + case OpenRequestStatus::Busy: + case OpenRequestStatus::Cooldown: + // Ordinary and temporary. Back to armed rather than holding + // Pending, which would fire the instant the refusal lifted on + // evidence gathered long before. + reason_ = toString(result.status); + toArmed(); + return false; + } + return false; +} + bool RuleEvaluator::conditionHolds(const double value) const { return rule_.op == Comparison::LessThan ? value < rule_.threshold : value > rule_.threshold; @@ -427,17 +460,8 @@ void RuleEvaluator::poll(const int64_t now_ns) { first_true_observed_ns_ = sample.observed_ns; state_ = RuleState::Pending; ++state_sequence_; - // sustained_ms == 0 fires on this same reading, handled below. - if (rule_.timing.sustained_ms == 0) { - pending_token_ = hooks_.request_open(hooks_.ctx, specWithTrigger(sample)); - if (pending_token_ != 0) { - state_ = RuleState::Opening; - ++state_sequence_; - } else { - reason_ = "open_refused"; - toArmed(); - } - } + // sustained_ms == 0 fires on this same reading. + if (rule_.timing.sustained_ms == 0) requestWindow(sample); } break; @@ -455,17 +479,7 @@ void RuleEvaluator::poll(const int64_t now_ns) { const int64_t held_ns = sample.observed_ns - *first_true_observed_ns_; if (held_ns < rule_.timing.sustained_ms * 1000000) break; - pending_token_ = hooks_.request_open(hooks_.ctx, specWithTrigger(sample)); - if (pending_token_ != 0) { - state_ = RuleState::Opening; - ++state_sequence_; - } else { - // Refused by cooldown. Back to armed rather than holding - // pending, which would fire the instant the cooldown lapsed on - // evidence gathered long before. - reason_ = "open_refused"; - toArmed(); - } + requestWindow(sample); break; } diff --git a/include/gpufl/core/deep_window_rule.hpp b/include/gpufl/core/deep_window_rule.hpp index e709b2c..8037c31 100644 --- a/include/gpufl/core/deep_window_rule.hpp +++ b/include/gpufl/core/deep_window_rule.hpp @@ -199,8 +199,11 @@ class RuleEvaluator { public: /** @brief Injected so tests drive the coordinator without a GPU. */ struct Hooks { - /// Ask for a window; returns a token, or 0 when refused outright. - uint64_t (*request_open)(void* ctx, const DeepWindowSpec&) = nullptr; + /// Ask for a window. The REASON matters, not just the refusal: + /// cooldown and busy are temporary and worth retrying, a failed + /// engine preparation is permanent and retrying it until shutdown + /// makes the summary claim the condition never held. + OpenRequestResult (*request_open)(void* ctx, const DeepWindowSpec&) = nullptr; /// Is a window - any window, whoever opened it - currently open? bool (*window_active)(void* ctx) = nullptr; /// Monotonic count of windows that have opened. A short launch-bounded @@ -267,6 +270,14 @@ class RuleEvaluator { * threshold beside it stops being readable the first time the rule changes. */ DeepWindowSpec specWithTrigger(const MetricSample& sample) const; + /** + * Ask for a window and act on the answer. + * + * One place, so the two call sites - sustained_ms == 0 and the + * held-long-enough path - cannot reach different conclusions about what a + * refusal means. Returns true only when a window was actually requested. + */ + bool requestWindow(const MetricSample& sample); bool conditionHolds(double value) const; bool rearmHolds(double value) const; void enterBlackout(int64_t now_ns); diff --git a/tests/core/test_deep_window.cpp b/tests/core/test_deep_window.cpp index 1493700..b01d8d0 100644 --- a/tests/core/test_deep_window.cpp +++ b/tests/core/test_deep_window.cpp @@ -104,7 +104,8 @@ TEST_F(DeepWindowTest, ConcurrentRequestsStillLeaveExactlyOneQueued) { while (!go.load(std::memory_order_acquire)) { std::this_thread::yield(); } - if (gpufl::DeepWindow::RequestOpenTagged(Spec(60000, 0)) != 0) { + if (gpufl::DeepWindow::RequestOpenTagged(Spec(60000, 0)).status == + gpufl::OpenRequestStatus::Accepted) { accepted.fetch_add(1, std::memory_order_relaxed); } }); @@ -345,8 +346,13 @@ TEST_F(DeepWindowTest, ATaggedRequestIsRefusedWhileAnotherIsQueued) { gpufl::DeepWindow::ScheduleOpenAfter(/*delay_ms=*/50, Spec(60000, 0)); // A rule asking now must be told no, not silently granted a token for a - // window that belongs to the scheduled trigger. - EXPECT_EQ(gpufl::DeepWindow::RequestOpenTagged(Spec(1000, 0)), 0u); + // window that belongs to the scheduled trigger. `busy` and not one of the + // engine statuses: the rule has to keep retrying, since the scheduled + // window will release the queue. + const gpufl::OpenRequestResult r = + gpufl::DeepWindow::RequestOpenTagged(Spec(1000, 0)); + EXPECT_EQ(r.token, 0u); + EXPECT_EQ(r.status, gpufl::OpenRequestStatus::Busy); } TEST_F(DeepWindowTest, ScheduledOpenWaitsOutItsDelay) { diff --git a/tests/core/test_deep_window_rule.cpp b/tests/core/test_deep_window_rule.cpp index ecdb2bd..ac7bd82 100644 --- a/tests/core/test_deep_window_rule.cpp +++ b/tests/core/test_deep_window_rule.cpp @@ -27,11 +27,16 @@ struct FakeCoordinator { uint64_t pending_token = 0; int opens = 0; - static uint64_t RequestOpen(void* ctx, const gpufl::DeepWindowSpec&) { + /// What the coordinator should answer next. Set per test. + gpufl::OpenRequestStatus refusal = gpufl::OpenRequestStatus::Cooldown; + + static gpufl::OpenRequestResult RequestOpen(void* ctx, + const gpufl::DeepWindowSpec&) { auto* self = static_cast(ctx); - if (self->refuse || self->active) return 0; + if (self->refuse) return {0, self->refusal}; + if (self->active) return {0, gpufl::OpenRequestStatus::Busy}; self->pending_token = self->next_token++; - return self->pending_token; + return {self->pending_token, gpufl::OpenRequestStatus::Accepted}; } static bool Active(void* ctx) { return static_cast(ctx)->active; @@ -643,4 +648,90 @@ TEST_F(RuleEvaluatorTest, TheSummaryReportsHowMuchDataWasDiscarded) { << "the session concluded from a subset and never said so"; } +// ── a refusal has to say WHY ──────────────────────────────────────────────── +// +// One undifferentiated "refused" made every refusal look temporary, so a rule +// whose deep engine had permanently failed retried until shutdown and then +// reported `never_true` - which says the condition was never met, the exact +// opposite of what happened. + +TEST_F(RuleEvaluatorTest, AFailedEnginePreparationEndsTheRuleAsUnsupported) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + coord.refuse = true; + coord.refusal = gpufl::OpenRequestStatus::EngineUnavailable; + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + + EXPECT_EQ(ev.state(), RuleState::Inactive); + const RuleSummary s = ev.finish(4000 * kMs); + EXPECT_EQ(s.outcome, RuleOutcome::Unsupported) + << "a condition that held but could not be acted on is not never_true"; + EXPECT_EQ(s.reason, "deep_engine_not_prepared"); + EXPECT_EQ(s.windows_opened, 0u); +} + +TEST_F(RuleEvaluatorTest, PreparationPendingIsRetriedRatherThanConcluded) { + // Windows injection installs a rule before CONTEXT_CREATED, so "not ready + // yet" is an ordinary startup state. Treating it as failure would kill + // every rule on that platform. + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + coord.refuse = true; + coord.refusal = gpufl::OpenRequestStatus::PreparationPending; + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + ASSERT_NE(ev.state(), RuleState::Inactive) << "gave up on a pending engine"; + + // Preparation completes; the rule must still be able to fire. + coord.refuse = false; + run(ev, src, 4010 * kMs, 5200 * kMs, 0); + EXPECT_EQ(ev.state(), RuleState::Opening); +} + +TEST_F(RuleEvaluatorTest, ACooldownRefusalIsNotAPermanentFailure) { + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + coord.refuse = true; + coord.refusal = gpufl::OpenRequestStatus::Cooldown; + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + + EXPECT_NE(ev.state(), RuleState::Inactive); + // Recording an ordinary quiet period as a permanent failure would tell the + // user their engine is broken when the rule is simply waiting its turn. + EXPECT_NE(ev.snapshot(4000 * kMs).outcome, RuleOutcome::Unsupported); +} + +TEST_F(RuleEvaluatorTest, ABusyRefusalStillFiresOnceTheWindowIsFree) { + // The scheduled --deep-after window holds the queue; the rule waits. + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + coord.refuse = true; + coord.refusal = gpufl::OpenRequestStatus::Busy; + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + ASSERT_NE(ev.state(), RuleState::Inactive); + + coord.refuse = false; + run(ev, src, 4010 * kMs, 5200 * kMs, 0); + EXPECT_EQ(ev.state(), RuleState::Opening); + + coord.serviceOpen(); + ev.poll(5210 * kMs); + EXPECT_EQ(ev.windowsOpened(), 1u); +} + } // namespace diff --git a/tests/launcher/test_cli_parse.cpp b/tests/launcher/test_cli_parse.cpp index 2ca3641..a73a5a2 100644 --- a/tests/launcher/test_cli_parse.cpp +++ b/tests/launcher/test_cli_parse.cpp @@ -729,6 +729,28 @@ TEST(CliParseDeepModeTest, TheRejectionNamesTheWayOut) { EXPECT_NE(r.error.find("Drop --passes"), std::string::npos) << r.error; } +TEST(CliParseDeepModeTest, TheTwoWindowTriggersAreMutuallyExclusive) { + // Both set is not "either may open it". Measured on the 3090: the + // scheduled window opens at t=0, the rule is refused behind it, and the + // summary then reports `never_true` for a condition that held all run. + const auto r = parseTrace({"--deep-when=kernel_launch_rate<10", + "--deep-after=1s", "--deep-for=2s", "--", "app"}); + ASSERT_FALSE(r.args.has_value()); + EXPECT_NE(r.error.find("--deep-when"), std::string::npos) << r.error; + EXPECT_NE(r.error.find("--deep-after"), std::string::npos) << r.error; +} + +TEST(CliParseDeepModeTest, AConditionalRunDoesNotCarryATimeTrigger) { + const auto r = parseTrace({"--deep-when=kernel_launch_rate<10", + "--deep-for=2s", "--", "app"}); + ASSERT_TRUE(r.args.has_value()) << r.error; + // deep_after_ms still holds its default 0; what must not happen is the + // launcher treating that default as a request. GPUFL_DEEP_AFTER_MS installs + // the time trigger by being present at all, so the flag being unset has to + // survive as far as the environment. + EXPECT_FALSE(r.args->deep_after_set); +} + TEST(CliParseDeepModeTest, ADeepRunResolvesToExactlyOneAdaptivePass) { const auto r = parseTrace({"--deep-for=5s", "--", "app"}); ASSERT_TRUE(r.args.has_value()) << r.error; From c40a62f86c86fc39c5bcdfd43da2ca3eed15b1b8 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Mon, 27 Jul 2026 18:12:13 -0700 Subject: [PATCH 22/28] fix(deep-window): a trigger the run did not ask for, and a verdict it did not earn --- daemon/launcher/CMakeLists.txt | 1 + daemon/launcher/deep_window_env.cpp | 84 ++++++++++++ daemon/launcher/trace_command.cpp | 9 ++ daemon/launcher/trace_command_common.cpp | 45 +----- daemon/launcher/trace_command_common.hpp | 30 ++++ daemon/launcher/trace_command_win.cpp | 12 ++ include/gpufl/core/deep_window.cpp | 1 + include/gpufl/core/deep_window.hpp | 15 +- include/gpufl/core/deep_window_rule.cpp | 34 +++++ include/gpufl/core/deep_window_rule.hpp | 13 ++ tests/CMakeLists.txt | 2 + tests/core/test_deep_window_rule.cpp | 71 ++++++++++ tests/launcher/test_deep_window_env.cpp | 166 +++++++++++++++++++++++ 13 files changed, 438 insertions(+), 45 deletions(-) create mode 100644 daemon/launcher/deep_window_env.cpp create mode 100644 tests/launcher/test_deep_window_env.cpp diff --git a/daemon/launcher/CMakeLists.txt b/daemon/launcher/CMakeLists.txt index cbf9059..ace2f50 100644 --- a/daemon/launcher/CMakeLists.txt +++ b/daemon/launcher/CMakeLists.txt @@ -24,6 +24,7 @@ add_executable(gpufl_launcher cli_parse.cpp info_command.cpp trace_command_common.cpp + deep_window_env.cpp ${GPUFL_LAUNCHER_TRACE_IMPL} monitor_command.cpp ../monitor/monitor_runner.cpp diff --git a/daemon/launcher/deep_window_env.cpp b/daemon/launcher/deep_window_env.cpp new file mode 100644 index 0000000..c5bdea2 --- /dev/null +++ b/daemon/launcher/deep_window_env.cpp @@ -0,0 +1,84 @@ +// Publishing the deep-window options into the target's environment. +// +// Its own translation unit so a test can link it without the rest of the trace +// command, which pulls in zlib, the agent launcher and the uploader. What is +// being tested here is a decision, not a launch. + +#include +#include + +#include "cli_parse.hpp" +#include "gpufl/core/env_vars.hpp" +#include "trace_command_common.hpp" + +namespace gpufl::launcher { + +bool setEnvOrPrint(const TracePlatform& platform, const char* key, + const std::string& value) { + std::string error; + if (platform.setEnv(key, value, error)) return true; + std::fprintf(stderr, "gpufl: %s\n", error.c_str()); + return false; +} + +bool unsetEnvOrPrint(const TracePlatform& platform, const char* key) { + std::string error; + if (platform.unsetEnv(key, error)) return true; + std::fprintf(stderr, "gpufl: %s\n", error.c_str()); + return false; +} + +// --deep-*: bound how long the DEEP engines stay armed inside a target that +// keeps running. Distinct from --window, which bounds the target's lifetime. +// Asking for a deep window implies window-only arming, or the engines would be +// armed from the first kernel and the window would bound nothing. +bool applyDeepWindowEnv(const TraceArgs& args, const TracePlatform& platform) { + if (!args.deep_requested) { + // Not this run's business. A `--passes` run sets neither trigger and + // scrubs neither: configuring a window purely through the environment + // is the supported way to reach an engine the adaptive plan does not + // select yet. + return true; + } + if (!setEnvOrPrint(platform, env::kDeepArm, "window")) return false; + + // Both trigger variables install by EXISTING, whatever their value, so the + // run has to say which one it owns and REMOVE the other. Merely not + // setting one leaves whatever the parent shell had: a --deep-when run + // under an exported GPUFL_DEEP_AFTER_MS opened a scheduled window at t=0 + // and the rule spent the run refused behind it, and a --deep-after run + // under an exported GPUFL_DEEP_WHEN installed a rule nobody asked for. The + // CLI rejects the two flags together; this is the same rule applied to the + // environment the target inherits. + if (!args.deep_when.empty()) { + if (!unsetEnvOrPrint(platform, env::kDeepAfterMs)) return false; + if (!setEnvOrPrint(platform, env::kDeepWhen, args.deep_when)) { + return false; + } + } else { + if (!unsetEnvOrPrint(platform, env::kDeepWhen)) return false; + if (!setEnvOrPrint(platform, env::kDeepAfterMs, + std::to_string(args.deep_after_ms))) { + return false; + } + } + + if (args.deep_for_ms > 0 && + !setEnvOrPrint(platform, env::kDeepWindowMs, + std::to_string(args.deep_for_ms))) { + return false; + } + if (args.deep_launches > 0 && + !setEnvOrPrint(platform, env::kDeepWindowMaxLaunches, + std::to_string(args.deep_launches))) { + return false; + } + if (args.deep_cooldown_ms > 0 && + !setEnvOrPrint(platform, env::kDeepWindowCooldownMs, + std::to_string(args.deep_cooldown_ms))) { + return false; + } + return true; +} + +} // namespace gpufl::launcher diff --git a/daemon/launcher/trace_command.cpp b/daemon/launcher/trace_command.cpp index fc9260f..11243f2 100644 --- a/daemon/launcher/trace_command.cpp +++ b/daemon/launcher/trace_command.cpp @@ -78,6 +78,15 @@ class PosixTracePlatform final : public TracePlatform { return false; } + bool unsetEnv(const char* key, std::string& error) const override { + // unsetenv succeeds when the name is not set, which is the outcome + // being asked for either way. + if (::unsetenv(key) == 0) return true; + error = "unsetenv " + std::string(key) + " failed: " + + std::strerror(errno); + return false; + } + bool prepareInjectionEnv(const fs::path& inject_lib, std::string& error) const override { std::string ld_preload = inject_lib.string(); diff --git a/daemon/launcher/trace_command_common.cpp b/daemon/launcher/trace_command_common.cpp index 99d943c..afa9756 100644 --- a/daemon/launcher/trace_command_common.cpp +++ b/daemon/launcher/trace_command_common.cpp @@ -39,15 +39,6 @@ fs::path findInjectLib(const TracePlatform& platform, const fs::path& exe) { return {}; } -bool setEnvOrPrint(const TracePlatform& platform, - const char* key, - const std::string& value) { - std::string error; - if (platform.setEnv(key, value, error)) return true; - std::fprintf(stderr, "gpufl: %s\n", error.c_str()); - return false; -} - // A '+'-joined pass token ("Trace+PcSampling") runs those engines together in // one process via GPUFL_ENGINE_COMBO. Returns the comma-joined combo for a // composite token, or "" for a single-engine token. @@ -629,41 +620,7 @@ int runTraceCommon(const TraceArgs& args, const TracePlatform& platform) { return 2; } - // --deep-*: bound how long the DEEP engines stay armed inside a target - // that keeps running. Distinct from --window above, which bounds the - // target's lifetime. Asking for a deep window implies window-only - // arming, or the engines would be armed from the first kernel and the - // window would bound nothing. - if (args.deep_requested) { - if (!setEnvOrPrint(platform, env::kDeepArm, "window")) return 2; - // The time trigger is installed by the mere presence of this variable, - // so a conditional run must not export it. Exporting the default 0 - // scheduled a window at t=0 that the rule was then refused behind. - if (args.deep_when.empty() && - !setEnvOrPrint(platform, env::kDeepAfterMs, - std::to_string(args.deep_after_ms))) { - return 2; - } - if (args.deep_for_ms > 0 && - !setEnvOrPrint(platform, env::kDeepWindowMs, - std::to_string(args.deep_for_ms))) { - return 2; - } - if (args.deep_launches > 0 && - !setEnvOrPrint(platform, env::kDeepWindowMaxLaunches, - std::to_string(args.deep_launches))) { - return 2; - } - if (args.deep_cooldown_ms > 0 && - !setEnvOrPrint(platform, env::kDeepWindowCooldownMs, - std::to_string(args.deep_cooldown_ms))) { - return 2; - } - if (!args.deep_when.empty() && - !setEnvOrPrint(platform, env::kDeepWhen, args.deep_when)) { - return 2; - } - } + if (!applyDeepWindowEnv(args, platform)) return 2; // A bounded window stops the target after warmup+window wall-clock; // run_ms == 0 keeps the historical "run until the target exits" behavior. diff --git a/daemon/launcher/trace_command_common.hpp b/daemon/launcher/trace_command_common.hpp index 51c1c5f..9654dcc 100644 --- a/daemon/launcher/trace_command_common.hpp +++ b/daemon/launcher/trace_command_common.hpp @@ -45,6 +45,17 @@ class TracePlatform { virtual bool setEnv(const char* key, const std::string& value, std::string& error) const = 0; + /** + * Remove a variable from the environment the target will inherit. + * + * Needed because some GPUFL variables are triggers that install by + * EXISTING, not by their value: not setting one is not the same as the + * target not seeing one. Not-present is the only way to say "off", so a + * mode that owns a trigger has to be able to take the other one away. + * + * Removing a variable that is not set is success, not an error. + */ + virtual bool unsetEnv(const char* key, std::string& error) const = 0; virtual bool prepareInjectionEnv(const fs::path& inject_lib, std::string& error) const = 0; virtual TraceProcessResult runProcess( @@ -52,6 +63,25 @@ class TracePlatform { const RunOptions& opts) const = 0; }; +/** @brief platform.setEnv, printing the platform's reason on failure. */ +bool setEnvOrPrint(const TracePlatform& platform, const char* key, + const std::string& value); +/** @brief platform.unsetEnv, printing the platform's reason on failure. */ +bool unsetEnvOrPrint(const TracePlatform& platform, const char* key); + +/** + * @brief Publish the deep-window options into the target's environment. + * + * Separate from runTraceCommon so the trigger-ownership rule can be tested + * without launching a process: which variable a mode sets is only half of it, + * and the half that bit - which one it REMOVES - is invisible in any test that + * only inspects what was set. + * + * @return false when the platform refused an environment change; the reason is + * already on stderr. + */ +bool applyDeepWindowEnv(const TraceArgs& args, const TracePlatform& platform); + int runTraceCommon(const TraceArgs& args, const TracePlatform& platform); } // namespace gpufl::launcher diff --git a/daemon/launcher/trace_command_win.cpp b/daemon/launcher/trace_command_win.cpp index 6514156..720f2c9 100644 --- a/daemon/launcher/trace_command_win.cpp +++ b/daemon/launcher/trace_command_win.cpp @@ -134,6 +134,18 @@ class WindowsTracePlatform final : public TracePlatform { return false; } + bool unsetEnv(const char* key, std::string& error) const override { + // A null value deletes the variable. Deleting one that was never set + // reports ERROR_ENVVAR_NOT_FOUND, which is the state being asked for. + if (SetEnvironmentVariableA(key, nullptr)) return true; + const DWORD err = GetLastError(); + if (err == ERROR_ENVVAR_NOT_FOUND) return true; + error = "SetEnvironmentVariable(" + std::string(key) + + ", null) failed (err=" + + std::to_string(static_cast(err)) + ")"; + return false; + } + bool prepareInjectionEnv(const fs::path& inject_lib, std::string& error) const override { std::string path = inject_lib.parent_path().string(); diff --git a/include/gpufl/core/deep_window.cpp b/include/gpufl/core/deep_window.cpp index 5e87b8b..cd0ca4c 100644 --- a/include/gpufl/core/deep_window.cpp +++ b/include/gpufl/core/deep_window.cpp @@ -434,6 +434,7 @@ const char* toString(const OpenRequestStatus status) { case OpenRequestStatus::EngineUnavailable: return "deep_engine_not_prepared"; case OpenRequestStatus::Busy: return "busy"; case OpenRequestStatus::Cooldown: return "cooldown"; + case OpenRequestStatus::InvalidResult: return "invalid_open_result"; } return "unknown"; } diff --git a/include/gpufl/core/deep_window.hpp b/include/gpufl/core/deep_window.hpp index eba7f9f..b452362 100644 --- a/include/gpufl/core/deep_window.hpp +++ b/include/gpufl/core/deep_window.hpp @@ -65,13 +65,26 @@ enum class OpenRequestStatus { /// A window is already open, or another request is queued. Busy, Cooldown, + /// The coordinator answered with something that cannot be acted on - + /// accepted but no token. Its own value rather than a silent fallback: it + /// means a coordinator is broken, and reporting it as cooldown would hide + /// that behind a state the user would read as normal. + InvalidResult, }; const char* toString(OpenRequestStatus status); struct OpenRequestResult { uint64_t token = 0; ///< non-zero only when Accepted - OpenRequestStatus status = OpenRequestStatus::Accepted; + /// Defaults to a refusal, not to Accepted. A default-constructed result is + /// one nobody filled in, and taking that as "yes, token 0" left the + /// evaluator waiting in Opening for a window that was never asked for. + OpenRequestStatus status = OpenRequestStatus::InvalidResult; + + /// True only when this result can actually be waited on. + bool accepted() const { + return status == OpenRequestStatus::Accepted && token != 0; + } }; /** diff --git a/include/gpufl/core/deep_window_rule.cpp b/include/gpufl/core/deep_window_rule.cpp index 558c897..91ca2da 100644 --- a/include/gpufl/core/deep_window_rule.cpp +++ b/include/gpufl/core/deep_window_rule.cpp @@ -5,6 +5,8 @@ #include #include +#include "gpufl/core/debug_logger.hpp" + namespace gpufl::detail { namespace { @@ -94,6 +96,7 @@ const char* toString(const RuleOutcome o) { switch (o) { case RuleOutcome::None: return "none"; case RuleOutcome::NeverTrue: return "never_true"; + case RuleOutcome::Blocked: return "blocked"; case RuleOutcome::Fired: return "fired"; case RuleOutcome::Exhausted: return "exhausted"; case RuleOutcome::Unsupported: return "unsupported"; @@ -283,6 +286,27 @@ DeepWindowSpec RuleEvaluator::specWithTrigger(const MetricSample& sample) const bool RuleEvaluator::requestWindow(const MetricSample& sample) { const OpenRequestResult result = hooks_.request_open(hooks_.ctx, specWithTrigger(sample)); + + // Reaching here at all means the condition held long enough to act on. + // Never cleared: what it answers at shutdown is "did this rule ever get as + // far as asking", which decides whether `never_true` is an honest verdict. + // Set before the answer is known on purpose - an accepted request whose + // window is later dropped also asked and also got nothing. + open_was_attempted_ = true; + + // Accepted with no token is not a yes. Waiting on token 0 leaves the + // evaluator in Opening for the rest of the run, watching a request that + // was never queued - a silent stall rather than a reported fault. + if (result.status == OpenRequestStatus::Accepted && !result.accepted()) { + GFL_LOG_ERROR("[DeepWindowRule] window coordinator accepted a request " + "without a token; treating the rule as unsupported"); + terminal_ = RuleOutcome::Unsupported; + reason_ = toString(OpenRequestStatus::InvalidResult); + state_ = RuleState::Inactive; + ++state_sequence_; + return false; + } + switch (result.status) { case OpenRequestStatus::Accepted: pending_token_ = result.token; @@ -291,6 +315,7 @@ bool RuleEvaluator::requestWindow(const MetricSample& sample) { return true; case OpenRequestStatus::EngineUnavailable: + case OpenRequestStatus::InvalidResult: // Permanent for this session. Retrying until shutdown would end in // `never_true`, which says the condition was never met - the exact // opposite of what happened. @@ -530,6 +555,15 @@ RuleSummary RuleEvaluator::finish(const int64_t now_ns) { s.outcome = RuleOutcome::Exhausted; } else if (windows_opened_ > 0) { s.outcome = RuleOutcome::Fired; + } else if (open_was_attempted_) { + // The condition held long enough to ask, and every ask was turned + // down - by a manual window holding the coordinator, by a cooldown + // longer than the run had left, or by preparation that never + // completed. `never_true` would say the condition never occurred, + // which is the opposite, and would send the user to look at their + // threshold instead of at what was blocking the window. + s.outcome = RuleOutcome::Blocked; + if (s.reason.empty()) s.reason = "open_refused"; } else { s.outcome = RuleOutcome::NeverTrue; if (s.reason.empty()) { diff --git a/include/gpufl/core/deep_window_rule.hpp b/include/gpufl/core/deep_window_rule.hpp index 8037c31..0a18634 100644 --- a/include/gpufl/core/deep_window_rule.hpp +++ b/include/gpufl/core/deep_window_rule.hpp @@ -114,6 +114,16 @@ const char* toString(RuleState s); enum class RuleOutcome { None, NeverTrue, + /** + * The condition held and a window was asked for, but none ever opened. + * + * Separate from NeverTrue because the two send the user to opposite + * places: never_true says look at your threshold, blocked says look at + * what is holding the window - a manual window, a cooldown longer than + * the run, an engine still preparing. Both leave windows_opened at 0, so + * one field cannot carry them. + */ + Blocked, Fired, Exhausted, Unsupported, @@ -308,6 +318,9 @@ class RuleEvaluator { MetricState last_metric_state_ = MetricState::Missing; uint64_t truncated_samples_ = 0; + /// Whether this rule ever got as far as asking for a window. Decides + /// `blocked` against `never_true` at shutdown; never cleared. + bool open_was_attempted_ = false; uint64_t pending_token_ = 0; bool window_was_active_ = false; uint64_t opens_seen_ = 0; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 87a6b1f..ec43fa2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -31,8 +31,10 @@ set(GPUFL_TEST_SOURCES # need to depend on the launcher target (which is Linux-only). launcher/test_cli_parse.cpp launcher/test_info_command.cpp + launcher/test_deep_window_env.cpp ${CMAKE_SOURCE_DIR}/daemon/launcher/cli_parse.cpp ${CMAKE_SOURCE_DIR}/daemon/launcher/info_command.cpp + ${CMAKE_SOURCE_DIR}/daemon/launcher/deep_window_env.cpp ) if(GPUFL_ENABLE_NVIDIA AND GPUFL_HAS_CUPTI) diff --git a/tests/core/test_deep_window_rule.cpp b/tests/core/test_deep_window_rule.cpp index ac7bd82..9f585d5 100644 --- a/tests/core/test_deep_window_rule.cpp +++ b/tests/core/test_deep_window_rule.cpp @@ -30,9 +30,15 @@ struct FakeCoordinator { /// What the coordinator should answer next. Set per test. gpufl::OpenRequestStatus refusal = gpufl::OpenRequestStatus::Cooldown; + /// A broken coordinator: says yes and hands back nothing to wait on. + bool accept_without_token = false; + static gpufl::OpenRequestResult RequestOpen(void* ctx, const gpufl::DeepWindowSpec&) { auto* self = static_cast(ctx); + if (self->accept_without_token) { + return {0, gpufl::OpenRequestStatus::Accepted}; + } if (self->refuse) return {0, self->refusal}; if (self->active) return {0, gpufl::OpenRequestStatus::Busy}; self->pending_token = self->next_token++; @@ -734,4 +740,69 @@ TEST_F(RuleEvaluatorTest, ABusyRefusalStillFiresOnceTheWindowIsFree) { EXPECT_EQ(ev.windowsOpened(), 1u); } +// ── refused to the end is not "never true" ────────────────────────────────── + +TEST_F(RuleEvaluatorTest, ARunRefusedToTheEndReportsBlockedNotNeverTrue) { + // The refusal is temporary and correctly retried, but the run ends before + // it lifts - a manual window held for the rest of the session, or a + // cooldown longer than the time left. The condition held throughout. + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + coord.refuse = true; + coord.refusal = gpufl::OpenRequestStatus::Cooldown; + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + + const RuleSummary s = ev.finish(4000 * kMs); + EXPECT_EQ(s.outcome, RuleOutcome::Blocked) + << "never_true sends the user to their threshold; the threshold was fine"; + EXPECT_EQ(s.reason, "cooldown") << "and it has to name what was in the way"; + EXPECT_EQ(s.windows_opened, 0u); +} + +TEST_F(RuleEvaluatorTest, AConditionThatNeverHeldIsStillNeverTrue) { + // The other side of the same line: nothing was ever asked for, so + // `blocked` would be inventing an obstacle that did not exist. + DeepWindowRule rule = makeRule("kernel_launch_rate>10000 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + + run(ev, src, 0, 4000 * kMs, 1); + + const RuleSummary s = ev.finish(4000 * kMs); + EXPECT_EQ(s.outcome, RuleOutcome::NeverTrue); + EXPECT_EQ(s.reason, "condition_never_held"); +} + +TEST_F(RuleEvaluatorTest, AnAcceptedRequestWithNoTokenIsAFaultNotAWait) { + // A coordinator that answers "accepted, token 0" has a bug. Waiting on it + // parks the evaluator in Opening for the rest of the run, watching a + // request that was never queued - no window, no error, no explanation. + DeepWindowRule rule = makeRule("kernel_launch_rate<100 for 500ms"); + MetricSource src(rule.metric, rule.timing, &feeds, ActiveCounterProvider()); + RuleEvaluator ev(rule, "r1", RuleCapabilities{}, &src, coord.hooks()); + feeds.seedStartup(0); + coord.accept_without_token = true; + + run(ev, src, 0, 1500 * kMs, 10); + run(ev, src, 1510 * kMs, 4000 * kMs, 0); + + EXPECT_NE(ev.state(), RuleState::Opening) << "stalled on a phantom request"; + const RuleSummary s = ev.finish(4000 * kMs); + EXPECT_EQ(s.outcome, RuleOutcome::Unsupported); + EXPECT_EQ(s.reason, "invalid_open_result"); +} + +TEST_F(RuleEvaluatorTest, ADefaultConstructedResultIsARefusal) { + // Nobody writes {} on purpose; a hook that forgets a return path produces + // it. Defaulting the status to Accepted made that silently mean "yes". + const gpufl::OpenRequestResult unset; + EXPECT_FALSE(unset.accepted()); + EXPECT_NE(unset.status, gpufl::OpenRequestStatus::Accepted); +} + } // namespace diff --git a/tests/launcher/test_deep_window_env.cpp b/tests/launcher/test_deep_window_env.cpp new file mode 100644 index 0000000..5415ea3 --- /dev/null +++ b/tests/launcher/test_deep_window_env.cpp @@ -0,0 +1,166 @@ +#include + +#include +#include +#include + +#include "cli_parse.hpp" +#include "gpufl/core/env_vars.hpp" +#include "trace_command_common.hpp" + +using gpufl::launcher::TraceArgs; +using gpufl::launcher::TracePlatform; +using gpufl::launcher::applyDeepWindowEnv; +namespace env = gpufl::env; + +namespace { + +// Records what the launcher did to the environment instead of doing it. The +// point of the recording is the REMOVALS: a test that only inspects what was +// set cannot tell "we left it alone" from "we took it away", and those two +// differ by whether a trigger the parent shell exported survives. +class RecordingPlatform final : public TracePlatform { + public: + /// The environment the target would inherit. Seed it to model a parent + /// shell that already had a GPUFL variable exported. + std::map env; + std::vector removed; + + bool has(const char* key) const { return env.count(key) != 0; } + std::string get(const char* key) const { + const auto it = env.find(key); + return it == env.end() ? std::string() : it->second; + } + + bool setEnv(const char* key, const std::string& value, + std::string&) const override { + const_cast(this)->env[key] = value; + return true; + } + bool unsetEnv(const char* key, std::string&) const override { + auto* self = const_cast(this); + self->env.erase(key); + self->removed.emplace_back(key); + return true; + } + + // Not reached: applyDeepWindowEnv only touches the environment. + const char* platformName() const override { return "recording"; } + const char* injectLibraryName() const override { return "none"; } + gpufl::launcher::fs::path selfExe() const override { return {}; } + std::vector injectLibCandidates( + const gpufl::launcher::fs::path&) const override { return {}; } + gpufl::launcher::fs::path defaultOutputDir( + const std::string&) const override { return {}; } + std::string defaultAppName(const std::string&) const override { return {}; } + bool prepareInjectionEnv(const gpufl::launcher::fs::path&, + std::string&) const override { return true; } + gpufl::launcher::TraceProcessResult runProcess( + const std::vector&, + const gpufl::launcher::RunOptions&) const override { return {}; } +}; + +TraceArgs conditionalRun() { + TraceArgs a; + a.deep_requested = true; + a.deep_when = "kernel_launch_rate<100 for 500ms"; + a.deep_for_ms = 2000; + return a; +} + +TraceArgs scheduledRun() { + TraceArgs a; + a.deep_requested = true; + a.deep_after_ms = 3000; + a.deep_after_set = true; + a.deep_for_ms = 2000; + return a; +} + +// ── trigger ownership ─────────────────────────────────────────────────────── +// +// GPUFL_DEEP_AFTER_MS and GPUFL_DEEP_WHEN install their triggers by EXISTING, +// whatever their value. Not setting one is therefore not the same as the +// target not seeing one, and every test here is about that difference. + +TEST(DeepWindowEnvTest, AConditionalRunRemovesAnInheritedTimeTrigger) { + RecordingPlatform p; + p.env[env::kDeepAfterMs] = "0"; // exported by the parent shell + + ASSERT_TRUE(applyDeepWindowEnv(conditionalRun(), p)); + + // Left in place, this opens a window at t=0 that the rule is then refused + // behind - and the rule reports never_true for a condition that held. + EXPECT_FALSE(p.has(env::kDeepAfterMs)); + EXPECT_EQ(p.get(env::kDeepWhen), "kernel_launch_rate<100 for 500ms"); +} + +TEST(DeepWindowEnvTest, AScheduledRunRemovesAnInheritedRule) { + RecordingPlatform p; + p.env[env::kDeepWhen] = "custom.token_rate<10 for 1s"; + + ASSERT_TRUE(applyDeepWindowEnv(scheduledRun(), p)); + + // Otherwise --deep-after silently installs a rule nobody asked for, which + // then competes with the window the user did ask for. + EXPECT_FALSE(p.has(env::kDeepWhen)); + EXPECT_EQ(p.get(env::kDeepAfterMs), "3000"); +} + +TEST(DeepWindowEnvTest, EachModeRemovesTheOtherTriggerEvenWhenUnset) { + // The removal is unconditional, not "only if we saw one". The launcher + // cannot see the parent environment of a target it has not spawned yet on + // every platform, so it does not try to decide. + RecordingPlatform cond; + ASSERT_TRUE(applyDeepWindowEnv(conditionalRun(), cond)); + EXPECT_NE(std::find(cond.removed.begin(), cond.removed.end(), + env::kDeepAfterMs), cond.removed.end()); + + RecordingPlatform sched; + ASSERT_TRUE(applyDeepWindowEnv(scheduledRun(), sched)); + EXPECT_NE(std::find(sched.removed.begin(), sched.removed.end(), + env::kDeepWhen), sched.removed.end()); +} + +TEST(DeepWindowEnvTest, ARunWithNoDeepFlagsTouchesNeitherTrigger) { + // `--passes PcSampling` with GPUFL_DEEP_WHEN exported is the supported way + // to reach an engine the adaptive plan does not select yet. Scrubbing here + // would take that away. + RecordingPlatform p; + p.env[env::kDeepWhen] = "custom.token_rate<10 for 1s"; + p.env[env::kDeepAfterMs] = "5000"; + + TraceArgs plain; // deep_requested stays false + ASSERT_TRUE(applyDeepWindowEnv(plain, p)); + + EXPECT_TRUE(p.removed.empty()); + EXPECT_EQ(p.get(env::kDeepWhen), "custom.token_rate<10 for 1s"); + EXPECT_EQ(p.get(env::kDeepAfterMs), "5000"); + EXPECT_FALSE(p.has(env::kDeepArm)) << "window-only arming was not asked for"; +} + +TEST(DeepWindowEnvTest, TheWindowBoundsAndArmModeStillTravel) { + RecordingPlatform p; + TraceArgs a = conditionalRun(); + a.deep_launches = 500; + a.deep_cooldown_ms = 1500; + + ASSERT_TRUE(applyDeepWindowEnv(a, p)); + + EXPECT_EQ(p.get(env::kDeepArm), "window"); + EXPECT_EQ(p.get(env::kDeepWindowMs), "2000"); + EXPECT_EQ(p.get(env::kDeepWindowMaxLaunches), "500"); + EXPECT_EQ(p.get(env::kDeepWindowCooldownMs), "1500"); +} + +TEST(DeepWindowEnvTest, AnUnsetBoundIsNotPublishedAsZero) { + // 0 means "no bound" to the client, and publishing it would override a + // bound the parent environment had legitimately set. + RecordingPlatform p; + ASSERT_TRUE(applyDeepWindowEnv(conditionalRun(), p)); + + EXPECT_FALSE(p.has(env::kDeepWindowMaxLaunches)); + EXPECT_FALSE(p.has(env::kDeepWindowCooldownMs)); +} + +} // namespace From a18810b698acfb731abb7741ae44daa3a980abba Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Mon, 27 Jul 2026 21:16:45 -0700 Subject: [PATCH 23/28] fix(upload): require a successful agent exit --- daemon/launcher/agent_launcher.cpp | 25 ++++++++++---- daemon/launcher/agent_launcher.hpp | 11 ++++++- daemon/launcher/trace_command_common.cpp | 8 ++++- daemon/launcher/upload_command.cpp | 14 ++++++-- tests/CMakeLists.txt | 2 ++ tests/launcher/test_agent_launcher.cpp | 42 ++++++++++++++++++++++++ 6 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 tests/launcher/test_agent_launcher.cpp diff --git a/daemon/launcher/agent_launcher.cpp b/daemon/launcher/agent_launcher.cpp index d9198c7..42cc67e 100644 --- a/daemon/launcher/agent_launcher.cpp +++ b/daemon/launcher/agent_launcher.cpp @@ -1,5 +1,6 @@ #include "agent_launcher.hpp" +#include #include #include #include @@ -170,36 +171,46 @@ bool AgentProcess::start(const std::vector& command, #endif } -bool AgentProcess::waitForExit(int timeoutMs) { - if (!running_) return true; +AgentWaitResult AgentProcess::waitForExit(int timeoutMs) { + if (!running_) return {true, 0}; #ifdef _WIN32 auto process = process_; const DWORD r = WaitForSingleObject( process, timeoutMs < 0 ? INFINITE : static_cast(timeoutMs)); if (r == WAIT_OBJECT_0) { + DWORD exit_code = 0; + const bool have_exit_code = GetExitCodeProcess(process, &exit_code) != 0; CloseHandle(thread_); CloseHandle(process); process_ = nullptr; thread_ = nullptr; running_ = false; - return true; + return {true, have_exit_code ? static_cast(exit_code) : -1}; } - return false; // timeout/failure - caller falls back to stop() + return {}; // timeout/failure - caller falls back to stop() #else constexpr int step_ms = 100; int waited = 0; int status = 0; while (timeoutMs < 0 || waited < timeoutMs) { const pid_t rc = ::waitpid(pid_, &status, WNOHANG); - if (rc == pid_ || rc < 0) { // exited, or already reaped / gone + if (rc == pid_) { + running_ = false; + pid_ = -1; + if (WIFEXITED(status)) return {true, WEXITSTATUS(status)}; + if (WIFSIGNALED(status)) return {true, 128 + WTERMSIG(status)}; + return {true, -1}; + } + if (rc < 0) { + if (errno == EINTR) continue; running_ = false; pid_ = -1; - return true; + return {true, -1}; } usleep(step_ms * 1000); waited += step_ms; } - return false; + return {}; #endif } diff --git a/daemon/launcher/agent_launcher.hpp b/daemon/launcher/agent_launcher.hpp index 5406444..cbdc968 100644 --- a/daemon/launcher/agent_launcher.hpp +++ b/daemon/launcher/agent_launcher.hpp @@ -26,6 +26,15 @@ struct AgentLaunchPlan { std::string description; }; +struct AgentWaitResult { + bool exited = false; + // Valid when exited=true. A negative value means the platform could not + // recover a trustworthy child status and must never be treated as success. + int exit_code = -1; + + bool succeeded() const { return exited && exit_code == 0; } +}; + class AgentProcess { public: AgentProcess() = default; @@ -35,7 +44,7 @@ class AgentProcess { bool start(const std::vector& command, std::string& error); void stop(); - bool waitForExit(int timeoutMs); + AgentWaitResult waitForExit(int timeoutMs); bool isRunning() const { return running_; } private: diff --git a/daemon/launcher/trace_command_common.cpp b/daemon/launcher/trace_command_common.cpp index afa9756..0872c06 100644 --- a/daemon/launcher/trace_command_common.cpp +++ b/daemon/launcher/trace_command_common.cpp @@ -804,7 +804,8 @@ int runTraceCommon(const TraceArgs& args, const TracePlatform& platform) { // hard-kill the agent mid-upload and drop its late windows. const int cap = args.agent_drain_ms; GFL_LOG_DEBUG("waiting up to ", cap / 1000.0, "s for agent to finish uploading"); - if (agent.waitForExit(cap)) { + const AgentWaitResult wait = agent.waitForExit(cap); + if (wait.succeeded()) { GFL_LOG_DEBUG("agent finished uploading"); // Clean drain: every window was 202-accepted, so no more chunks are // coming. Signal upload-complete from here (outside the agent's racing @@ -814,12 +815,17 @@ int runTraceCommon(const TraceArgs& args, const TracePlatform& platform) { config.api_key = resolveOption(args.api_key, env::kApiKey); config.api_path = args.api_version.empty() ? "" : "/api/" + args.api_version; signalSessionsComplete(output_dir, config, args.quiet); + } else if (wait.exited) { + GFL_LOG_ERROR("agent exited before completing the upload (exit code ", + wait.exit_code, "); session-complete was not sent"); + if (overall_rc == 0) overall_rc = 4; } else { if (!args.quiet) { GFL_LOG_ERROR("agent still running after ", cap / 1000.0, "s cap - stopping (late windows may need a post-hoc `gpufl upload`) "); } agent.stop(); + if (overall_rc == 0) overall_rc = 4; } } diff --git a/daemon/launcher/upload_command.cpp b/daemon/launcher/upload_command.cpp index 9efb75a..c507571 100644 --- a/daemon/launcher/upload_command.cpp +++ b/daemon/launcher/upload_command.cpp @@ -121,9 +121,17 @@ int runUpload(const UploadArgs& args) { return 3; } const int cap_ms = args.timeout_s > 0 ? args.timeout_s * 1000 : 300000; - if (agent.waitForExit(cap_ms)) { - if (!args.quiet) std::printf("Upload complete.\n"); - return 0; + const AgentWaitResult wait = agent.waitForExit(cap_ms); + if (wait.exited) { + if (wait.succeeded()) { + if (!args.quiet) std::printf("Upload complete.\n"); + return 0; + } + std::fprintf(stderr, + "gpufl upload: agent exited before completing the upload " + "(exit code %d)\n", + wait.exit_code); + return 4; } agent.stop(); std::fprintf(stderr, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ec43fa2..3ec7bc7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -32,9 +32,11 @@ set(GPUFL_TEST_SOURCES launcher/test_cli_parse.cpp launcher/test_info_command.cpp launcher/test_deep_window_env.cpp + launcher/test_agent_launcher.cpp ${CMAKE_SOURCE_DIR}/daemon/launcher/cli_parse.cpp ${CMAKE_SOURCE_DIR}/daemon/launcher/info_command.cpp ${CMAKE_SOURCE_DIR}/daemon/launcher/deep_window_env.cpp + ${CMAKE_SOURCE_DIR}/daemon/launcher/agent_launcher.cpp ) if(GPUFL_ENABLE_NVIDIA AND GPUFL_HAS_CUPTI) diff --git a/tests/launcher/test_agent_launcher.cpp b/tests/launcher/test_agent_launcher.cpp new file mode 100644 index 0000000..a777764 --- /dev/null +++ b/tests/launcher/test_agent_launcher.cpp @@ -0,0 +1,42 @@ +#include + +#include +#include + +#include "agent_launcher.hpp" + +namespace { + +std::vector exitCommand(const int code) { +#ifdef _WIN32 + return {"cmd.exe", "/d", "/s", "/c", "exit " + std::to_string(code)}; +#else + return {"/bin/sh", "-c", "exit " + std::to_string(code)}; +#endif +} + +} // namespace + +TEST(AgentProcessTest, AZeroExitIsACompletedUpload) { + gpufl::launcher::AgentProcess process; + std::string error; + ASSERT_TRUE(process.start(exitCommand(0), error)) << error; + + const auto result = process.waitForExit(5000); + + EXPECT_TRUE(result.exited); + EXPECT_EQ(result.exit_code, 0); + EXPECT_TRUE(result.succeeded()); +} + +TEST(AgentProcessTest, ANonzeroExitIsNotACompletedUpload) { + gpufl::launcher::AgentProcess process; + std::string error; + ASSERT_TRUE(process.start(exitCommand(7), error)) << error; + + const auto result = process.waitForExit(5000); + + EXPECT_TRUE(result.exited); + EXPECT_EQ(result.exit_code, 7); + EXPECT_FALSE(result.succeeded()); +} From 021ff7a375e2648f224530585c28b497dd1a7f4d Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Mon, 27 Jul 2026 22:04:39 -0700 Subject: [PATCH 24/28] replace GFL_LOG_ERROR in the upload related call with fprint --- daemon/launcher/trace_command_common.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/daemon/launcher/trace_command_common.cpp b/daemon/launcher/trace_command_common.cpp index 0872c06..9540cd4 100644 --- a/daemon/launcher/trace_command_common.cpp +++ b/daemon/launcher/trace_command_common.cpp @@ -816,13 +816,19 @@ int runTraceCommon(const TraceArgs& args, const TracePlatform& platform) { config.api_path = args.api_version.empty() ? "" : "/api/" + args.api_version; signalSessionsComplete(output_dir, config, args.quiet); } else if (wait.exited) { - GFL_LOG_ERROR("agent exited before completing the upload (exit code ", - wait.exit_code, "); session-complete was not sent"); + std::fprintf( + stderr, + "gpufl trace --upload: agent exited before completing the upload " + "(exit code %d); session-complete was not sent\n", + wait.exit_code); if (overall_rc == 0) overall_rc = 4; } else { if (!args.quiet) { - GFL_LOG_ERROR("agent still running after ", cap / 1000.0, - "s cap - stopping (late windows may need a post-hoc `gpufl upload`) "); + std::fprintf( + stderr, + "gpufl trace --upload: agent still running after %.1fs - " + "stopping; late windows may require a post-hoc `gpufl upload`\n", + cap / 1000.0); } agent.stop(); if (overall_rc == 0) overall_rc = 4; From 80164d7ebba73ae4f33ba241dd4b2ec77e650a85 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Tue, 28 Jul 2026 14:43:34 -0700 Subject: [PATCH 25/28] feat(deep-window): add counter telemetry and safe CUDA boundaries --- CMakeLists.txt | 146 +++++- daemon/launcher/trace_command_common.cpp | 5 +- .../nvidia/capture_capability_resolver.cpp | 6 +- .../nvidia/cupti_capture_capabilities.cpp | 3 +- .../nvidia/cupti_engine_selection.cpp | 2 +- .../nvidia/engine/pm_sampling_engine.cpp | 12 +- include/gpufl/core/debug_logger.hpp | 19 + include/gpufl/core/deep_window_rule.cpp | 4 + include/gpufl/core/deep_window_rule.hpp | 8 + include/gpufl/core/deep_window_rules.cpp | 50 ++ include/gpufl/core/deep_window_rules.hpp | 11 + include/gpufl/core/events.hpp | 46 ++ include/gpufl/core/metric_registry.cpp | 27 ++ include/gpufl/core/metric_registry.hpp | 19 + .../gpufl/core/model/deep_window_model.cpp | 22 + .../gpufl/core/model/deep_window_model.hpp | 13 + include/gpufl/core/monitor.cpp | 8 + include/gpufl/core/nvtx_counters.cpp | 381 +++++++++++++++ include/gpufl/core/nvtx_counters.hpp | 226 +++++++++ include/gpufl/inject/cuda_interpose_linux.cpp | 290 +++++++++++ include/gpufl/inject/inject_entry.cpp | 209 +++++++- include/gpufl/upload/upload_logs.cpp | 12 +- scripts/counter_workload_bench.cu | 70 +++ scripts/counter_workload_bench.py | 224 +++++++++ scripts/verify_early_memcpy_capture.py | 86 ++++ tests/CMakeLists.txt | 14 + tests/common/early_memcpy_target.cu | 69 +++ tests/core/test_debug_logger.cpp | 42 ++ tests/core/test_nvtx_counters.cpp | 453 ++++++++++++++++++ tests/package/consumer/CMakeLists.txt | 13 + tests/package/consumer/main.cpp | 13 + tests/package/counters_consumer_check.cmake | 102 ++++ 32 files changed, 2576 insertions(+), 29 deletions(-) create mode 100644 include/gpufl/core/nvtx_counters.cpp create mode 100644 include/gpufl/core/nvtx_counters.hpp create mode 100644 include/gpufl/inject/cuda_interpose_linux.cpp create mode 100644 scripts/counter_workload_bench.cu create mode 100644 scripts/counter_workload_bench.py create mode 100644 scripts/verify_early_memcpy_capture.py create mode 100644 tests/common/early_memcpy_target.cu create mode 100644 tests/core/test_debug_logger.cpp create mode 100644 tests/core/test_nvtx_counters.cpp create mode 100644 tests/package/consumer/CMakeLists.txt create mode 100644 tests/package/consumer/main.cpp create mode 100644 tests/package/counters_consumer_check.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 4ce60da..cc8226b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,6 +76,46 @@ set_target_properties(gpufl_counter_runtime PROPERTIES VISIBILITY_INLINES_HIDDEN ON ) +# Counters WITHOUT the profiler. What an application links to tick counters a +# rule can watch (`gpufl::counter("tokens").add(n)`), while the profiling +# itself arrives from outside via `gpufl trace`. Linking gpufl::gpufl for two +# lines of instrumentation reads as "I am embedding the SDK" - and the target +# name is most of the answer to that confusion; nothing here starts a session, +# and none of the SDK's dependencies (CUPTI, NVML, zlib, OpenSSL, the +# uploader) are in it. +# +# `gpufl` layers ON TOP of this rather than compiling the same sources again: +# two archives both defining the registry is a duplicate-symbol error for +# anyone who links both. +add_library(gpufl_counters STATIC + include/gpufl/core/counter_registry.cpp + include/gpufl/core/counter_api.cpp + include/gpufl/core/counter_provider.cpp + include/gpufl/core/debug_logger.cpp +) +add_library(gpufl::counters ALIAS gpufl_counters) +# The ALIAS above only exists in this build tree. The INSTALLED name comes +# from EXPORT_NAME plus the export namespace, and without this the package +# would ship the target as gpufl::gpufl_counters while every document says +# gpufl::counters. +set_target_properties(gpufl_counters PROPERTIES EXPORT_NAME counters) +target_include_directories(gpufl_counters + PUBLIC + $ + $ +) +target_compile_features(gpufl_counters INTERFACE cxx_std_17) +# The provider dlopens the shared runtime and the registry uses std::mutex; +# consumers must not have to know either. Threads::Threads rather than a raw +# `pthread`: the raw name would be exported verbatim into the targets file and +# bypass whatever the consumer's toolchain says threading means there. +find_package(Threads REQUIRED) +target_link_libraries(gpufl_counters PUBLIC Threads::Threads) +if(UNIX) + target_link_libraries(gpufl_counters PUBLIC ${CMAKE_DL_LIBS}) +endif() +set_target_properties(gpufl_counters PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_include_directories(gpufl PUBLIC $ @@ -104,6 +144,11 @@ target_compile_definitions(gpufl PUBLIC # Enable PIC for static library (required when linking into shared libraries like Python modules) set_target_properties(gpufl PROPERTIES POSITION_INDEPENDENT_CODE ON) +# PUBLIC: consumers of the full SDK get the counter layer through this link, +# and the counter sources live in gpufl_counters ONLY - see that target for +# why they must not be compiled into both archives. +target_link_libraries(gpufl PUBLIC gpufl_counters) + target_sources(gpufl PRIVATE include/gpufl/core/dictionary_manager.cpp include/gpufl/core/sass_compressor.cpp @@ -132,9 +177,7 @@ target_sources(gpufl PRIVATE include/gpufl/core/runtime.cpp include/gpufl/core/backend_factory.cpp include/gpufl/core/monitor_adapter.cpp - include/gpufl/core/counter_registry.cpp - include/gpufl/core/counter_api.cpp - include/gpufl/core/counter_provider.cpp + include/gpufl/core/nvtx_counters.cpp include/gpufl/core/metric_id.cpp include/gpufl/core/deep_window_rule.cpp include/gpufl/core/deep_window_rules.cpp @@ -144,7 +187,6 @@ target_sources(gpufl PRIVATE include/gpufl/core/monitor.cpp include/gpufl/core/gpufl.cpp include/gpufl/core/common.cpp - include/gpufl/core/debug_logger.cpp include/gpufl/core/stack_trace.cpp include/gpufl/core/itanium_demangle.cpp include/gpufl/core/scope_registry.cpp @@ -179,7 +221,15 @@ else() GIT_REPOSITORY https://github.com/madler/zlib.git GIT_TAG v1.3.1 ) + # zlib's own install rules cache ABSOLUTE destinations at configure time + # (INSTALL_LIB_DIR = ${CMAKE_INSTALL_PREFIX}/lib), so they ignore + # `cmake --install --prefix` and try to write into Program Files - which is + # how the install-tree consumer test failed before this. Skip them all; the + # archive is installed below with a relative destination that the prefix + # override can move. + set(SKIP_INSTALL_ALL ON) FetchContent_MakeAvailable(zlib) + unset(SKIP_INSTALL_ALL) # zlib.h lives in the source dir; zconf.h is generated in the binary dir. # Add both privately to gpufl - consumers never include zlib headers directly. target_link_libraries(gpufl PRIVATE zlibstatic) @@ -220,6 +270,12 @@ FetchContent_Declare( # cpp-httplib's CMakeLists defines build flags for its own tests / examples. # Turn them off so we only build the header-only interface target. set(HTTPLIB_COMPILE OFF CACHE BOOL "" FORCE) +# HTTPLIB_INSTALL stays ON, deliberately: gpufl's link interface records +# httplib::httplib, and install(EXPORT gpufl_clientTargets) refuses to +# generate unless that target is in SOME export set - httplib's own is what +# satisfies the check. Turning it off fails the whole configure. The price is +# cpp-httplib's headers and package files landing in the install prefix; the +# fix that removes them is reworking the SDK export, not this switch. FetchContent_MakeAvailable(httplib) find_package(OpenSSL QUIET) @@ -667,6 +723,16 @@ if(BUILD_GPUFL_INJECT AND ((UNIX AND NOT APPLE) OR WIN32)) add_library(gpufl_inject SHARED include/gpufl/inject/inject_entry.cpp ) + if(UNIX AND NOT APPLE AND GPUFL_HAS_CUDA) + # Compile CUDA boundary wrappers against the toolkit's official ABI. + # inject_entry.cpp keeps its header-free fallback for no-CUDA builds. + target_sources(gpufl_inject PRIVATE + include/gpufl/inject/cuda_interpose_linux.cpp + ) + target_compile_definitions(gpufl_inject PRIVATE + GPUFL_TYPED_CUDA_INTERPOSE=1 + ) + endif() target_link_libraries(gpufl_inject PRIVATE gpufl::gpufl) # The injected DLL is loaded by the CUDA driver into the profiled target, @@ -774,6 +840,10 @@ endfunction() if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR AND BUILD_TESTING) enable_testing() + # Keep googletest's install rules out of the package: the install-tree + # consumer test installs this build into a scratch prefix, and gtest + # headers and archives are not something gpufl_client ships. + set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) add_subdirectory(tests) endif() @@ -828,6 +898,28 @@ include(GNUInstallDirs) # Install header files install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT counters +) + +# Two export sets, deliberately. The generated targets file REFUSES to load +# when any imported target it references is absent, and the SDK records its +# private dependencies (httplib, zlib, OpenSSL, CUDA) in its link interface - +# so shipping counters in the SDK's set made `find_package(gpufl_client)` fail +# for a counters-only consumer with "httplib::httplib is missing". Measured, +# not theoretical. The counters set references nothing but system dl/pthread +# and loads anywhere; the SDK set loads only for consumers that opt in AND +# have its dependencies. +install(TARGETS gpufl_counters + EXPORT gpufl_countersTargets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT counters + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) +install(EXPORT gpufl_countersTargets + FILE gpufl_countersTargets.cmake + NAMESPACE gpufl:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/gpufl_client + COMPONENT counters ) install(TARGETS gpufl @@ -844,6 +936,27 @@ install(EXPORT gpufl_clientTargets DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/gpufl_client ) +# What makes `find_package(gpufl_client)` actually resolve: CMake looks for +# gpufl_clientConfig.cmake, and a bare Targets file is not one. Version file +# alongside so `find_package(gpufl_client 1.2)` can hold a floor. +include(CMakePackageConfigHelpers) +configure_package_config_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/gpufl_clientConfig.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/gpufl_clientConfig.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/gpufl_client +) +write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/gpufl_clientConfigVersion.cmake + VERSION ${PROJECT_VERSION} + COMPATIBILITY SameMajorVersion +) +install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/gpufl_clientConfig.cmake + ${CMAKE_CURRENT_BINARY_DIR}/gpufl_clientConfigVersion.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/gpufl_client + COMPONENT counters +) + # Cross-module counter sharing. Cannot be proven from inside one executable - it # needs the Python extension AND the shared runtime as separate modules - so it # runs as its own process rather than as a gtest case. @@ -889,3 +1002,28 @@ if(BUILD_TESTING AND BUILD_PYTHON AND TARGET _gpufl_client ENVIRONMENT "GPUFL_REPO=${GPUFL_XMOD_STAGE}") endif() endif() + +# The install-tree consumer, as CTest rather than a manual run: install into a +# scratch prefix, configure the fixture consumer against it with find_package, +# build it, run it. This is the only test that can see EXPORT_NAME, the Config +# file install, and the counters export set's isolation from the SDK's +# dependencies - the unit suite links the build tree and passes no matter what +# the package ships. +if(BUILD_TESTING) + enable_testing() + add_test(NAME counters_package_consumer + COMMAND ${CMAKE_COMMAND} + -DGPUFL_BINARY_DIR=${CMAKE_BINARY_DIR} + -DGPUFL_SOURCE_DIR=${CMAKE_SOURCE_DIR} + -DGPUFL_CONFIG=$ + -DGPUFL_GENERATOR=${CMAKE_GENERATOR} + -DGPUFL_PLATFORM=${CMAKE_GENERATOR_PLATFORM} + -DGPUFL_TOOLSET=${CMAKE_GENERATOR_TOOLSET} + -P ${CMAKE_SOURCE_DIR}/tests/package/counters_consumer_check.cmake) + # Self-contained: the script builds gpufl_counters itself and installs + # ONLY the counters component, so this test does not depend on - and + # cannot be broken by - the rest of the project's install rules. A Debug + # run without gpufl_counter_runtime built proved why that isolation + # matters. + set_tests_properties(counters_package_consumer PROPERTIES TIMEOUT 300) +endif() diff --git a/daemon/launcher/trace_command_common.cpp b/daemon/launcher/trace_command_common.cpp index 9540cd4..8f05dff 100644 --- a/daemon/launcher/trace_command_common.cpp +++ b/daemon/launcher/trace_command_common.cpp @@ -435,7 +435,10 @@ void signalSessionsComplete(const fs::path& output_dir, if (res.second) { GFL_LOG_DEBUG("signalled upload-complete for session ", res.first); } else if (!quiet) { - GFL_LOG_ERROR("failed to signal upload-complete for session ", res.first); + std::fprintf(stderr, + "gpufl trace --upload: failed to signal " + "upload-complete for session %s\n", + res.first.c_str()); } } } diff --git a/include/gpufl/backends/nvidia/capture_capability_resolver.cpp b/include/gpufl/backends/nvidia/capture_capability_resolver.cpp index 7aad0a8..1db8849 100644 --- a/include/gpufl/backends/nvidia/capture_capability_resolver.cpp +++ b/include/gpufl/backends/nvidia/capture_capability_resolver.cpp @@ -36,7 +36,7 @@ std::vector BuildCaptureCapabilityWarnings( // for a deep window those differ by a lot, and quoting the wrong one // contradicts the advice. warnings.push_back( - "[gpufl] PC sampling collected 0 stall samples - too few kernel " + "PC sampling collected 0 stall samples - too few kernel " "launches were sampled. PC sampling accumulates per kernel and " "needs a few thousand launches before any samples are readable. " "Sample more launches - note that a longer wall-clock window is " @@ -47,14 +47,14 @@ std::vector BuildCaptureCapabilityWarnings( if (input.requests.sass && input.engine_state.sass.active && !input.engine_state.sass.has_data) { warnings.push_back( - "[gpufl] SASS metrics collected 0 instruction samples - the " + "SASS metrics collected 0 instruction samples - the " "profiled kernels were too short. Run more iterations / a " "longer-running kernel to collect instruction-level data."); } if (input.requests.pm && input.engine_state.pm.active && !input.engine_state.pm.has_data) { warnings.push_back( - "[gpufl] PM sampling collected 0 hardware samples - the profiled " + "PM sampling collected 0 hardware samples - the profiled " "workload was too short for the sampling interval."); } return warnings; diff --git a/include/gpufl/backends/nvidia/cupti_capture_capabilities.cpp b/include/gpufl/backends/nvidia/cupti_capture_capabilities.cpp index 9a467ae..a3361e8 100644 --- a/include/gpufl/backends/nvidia/cupti_capture_capabilities.cpp +++ b/include/gpufl/backends/nvidia/cupti_capture_capabilities.cpp @@ -1,7 +1,6 @@ #include "gpufl/backends/nvidia/cupti_backend.hpp" #include -#include #include #include "gpufl/backends/nvidia/capture_capability_resolver.hpp" @@ -83,7 +82,7 @@ void CuptiBackend::EmitCaptureCapabilities_() const { // run otherwise gives no local hint that a too-short workload starved the // sampler. Point at the remedies. for (const std::string& warning : BuildCaptureCapabilityWarnings(input)) { - std::fprintf(stderr, "%s\n", warning.c_str()); + GFL_LOG_WARN(warning); } CaptureCapabilitiesEvent evt = BuildCaptureCapabilitiesEvent(input); diff --git a/include/gpufl/backends/nvidia/cupti_engine_selection.cpp b/include/gpufl/backends/nvidia/cupti_engine_selection.cpp index 1fc1a22..5acbc19 100644 --- a/include/gpufl/backends/nvidia/cupti_engine_selection.cpp +++ b/include/gpufl/backends/nvidia/cupti_engine_selection.cpp @@ -215,7 +215,7 @@ EngineRuntimeState InspectEngineRuntimeState(const IProfilingEngine* engine, if (comboActive) { // Keep the matrix log close to the single place that inspects // runtime engine state. - GFL_LOG_ERROR("[Composite][matrix] ", sub->name(), " armed=", + GFL_LOG_DEBUG("[Composite][matrix] ", sub->name(), " armed=", armed ? "yes" : "no", " produced=", produced ? "yes" : "no"); } diff --git a/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp b/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp index 5216798..0737115 100644 --- a/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp +++ b/include/gpufl/backends/nvidia/engine/pm_sampling_engine.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include "gpufl/backends/nvidia/cupti_utils.hpp" @@ -63,9 +62,7 @@ void PmSamplingEngine::start() { // reset. A deep-window open therefore performs only the bounded arm. if (PreparePmSampling_()) { if (opts_.deep_arm_mode == DeepArmMode::WindowOnly) { - std::fprintf( - stderr, - "[gpufl] deep engine prepared: nvidia.pm_sampling\n"); + GFL_LOG_INFO("deep engine prepared: nvidia.pm_sampling"); } GFL_LOG_DEBUG("[PmSamplingEngine] prepared; sampling remains " "idle until a deep window opens"); @@ -523,9 +520,7 @@ void PmSamplingEngine::DecodeAndEmit_() { Monitor::PushPmSamples(rows); if (!produced_data_.exchange(true, std::memory_order_acq_rel) && opts_.deep_arm_mode == DeepArmMode::WindowOnly) { - std::fprintf(stderr, - "[gpufl] deep engine produced data: " - "nvidia.pm_sampling\n"); + GFL_LOG_INFO("deep engine produced data: nvidia.pm_sampling"); } // Release the scopes no later sample can reach - but only when CUPTI @@ -614,8 +609,7 @@ void PmSamplingEngine::StartPmSampling_() { Monitor::BeginPmScopeAttribution(attribution_start_ns); operational_.store(true, std::memory_order_relaxed); if (opts_.deep_arm_mode == DeepArmMode::WindowOnly) { - std::fprintf(stderr, - "[gpufl] deep window armed: nvidia.pm_sampling\n"); + GFL_LOG_INFO("deep window armed: nvidia.pm_sampling"); } GFL_LOG_DEBUG("[PmSamplingEngine] >>> STARTED (Scope Begin) <<<"); } diff --git a/include/gpufl/core/debug_logger.hpp b/include/gpufl/core/debug_logger.hpp index bfcce74..fc49c71 100644 --- a/include/gpufl/core/debug_logger.hpp +++ b/include/gpufl/core/debug_logger.hpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace gpufl { @@ -21,6 +22,22 @@ class DebugLogger { } } + template + static void info(const char* prefix, Args&&... args) { + std::stringstream ss; + ss << prefix; + (ss << ... << std::forward(args)); + std::cerr << ss.str() << std::endl; + } + + template + static void warn(const char* prefix, Args&&... args) { + std::stringstream ss; + ss << prefix; + (ss << ... << std::forward(args)); + std::cerr << ss.str() << std::endl; + } + template static void error(const char* prefix, Args&&... args) { // Errors ALWAYS print, regardless of the debug-output flag. @@ -39,6 +56,8 @@ class DebugLogger { }; #define GFL_LOG_DEBUG(...) ::gpufl::DebugLogger::log("[GPUFL] ", __VA_ARGS__) +#define GFL_LOG_INFO(...) ::gpufl::DebugLogger::info("[GPUFL] ", __VA_ARGS__) +#define GFL_LOG_WARN(...) ::gpufl::DebugLogger::warn("[GPUFL-WARN] ", __VA_ARGS__) #define GFL_LOG_ERROR(...) \ ::gpufl::DebugLogger::error("[GPUFL-ERROR] " __FILE__ ":", __LINE__, ": ", __VA_ARGS__) diff --git a/include/gpufl/core/deep_window_rule.cpp b/include/gpufl/core/deep_window_rule.cpp index 91ca2da..b33147d 100644 --- a/include/gpufl/core/deep_window_rule.cpp +++ b/include/gpufl/core/deep_window_rule.cpp @@ -536,6 +536,10 @@ RuleSummary RuleEvaluator::snapshot(const int64_t now_ns) const { s.last_observed_ns = last_observed_ns_; s.last_metric_state = last_metric_state_; s.truncated_samples = truncated_samples_; + if (source_ != nullptr) { + s.metric_quality_resets = source_->qualityResets(); + s.last_quality_reason = source_->lastQualityReason(); + } s.reason = reason_; s.state_sequence = state_sequence_; s.emitted_ns = now_ns; diff --git a/include/gpufl/core/deep_window_rule.hpp b/include/gpufl/core/deep_window_rule.hpp index 0a18634..3332930 100644 --- a/include/gpufl/core/deep_window_rule.hpp +++ b/include/gpufl/core/deep_window_rule.hpp @@ -178,6 +178,14 @@ struct RuleSummary { * the workloads this feature targets actually reach. */ uint64_t truncated_samples = 0; + /** + * Rate windows THIS rule's metric discarded for failed reads, and why the + * last one was. Per rule, never session-wide: copying a session's whole + * tally onto every rule would make an unrelated counter's errors look + * like they broke this rule. + */ + uint64_t metric_quality_resets = 0; + std::string last_quality_reason; std::string reason; uint64_t state_sequence = 0; int64_t emitted_ns = 0; diff --git a/include/gpufl/core/deep_window_rules.cpp b/include/gpufl/core/deep_window_rules.cpp index a90b2a6..0bc8627 100644 --- a/include/gpufl/core/deep_window_rules.cpp +++ b/include/gpufl/core/deep_window_rules.cpp @@ -16,6 +16,7 @@ #include "gpufl/core/events.hpp" #include "gpufl/core/logger/logger.hpp" #include "gpufl/core/metric_registry.hpp" +#include "gpufl/core/nvtx_counters.hpp" #include "gpufl/core/model/deep_window_model.hpp" #include "gpufl.hpp" #include "gpufl/core/monitor.hpp" @@ -221,6 +222,8 @@ void EmitSummary(const RuleSummary& summary, const std::string& expression) { ev.samples_seen = summary.samples_seen; ev.windows_opened = summary.windows_opened; ev.truncated_samples = summary.truncated_samples; + ev.metric_quality_resets = summary.metric_quality_resets; + ev.last_quality_reason = summary.last_quality_reason; ev.has_last_value = summary.last_value.has_value(); if (ev.has_last_value) { ev.last_value = *summary.last_value; @@ -411,6 +414,53 @@ void DeepWindowRules::Service() { EmitSummary(terminal, expression); } +void DeepWindowRules::EmitCounterQuality() { + // Separate from Finish() because this event exists WITHOUT a rule: NVTX + // counters flow through the bridge whether or not anything watches them, + // and a refused registration with no rule configured still needs a place + // to be reported. + NvtxCounterBridge::QualitySnapshot snap = + NvtxCounterBridge::instance().takeSessionSnapshot(); + + uint64_t discarded = 0; + { + std::lock_guard lk(g_mu); + if (g_source) discarded = g_source->qualityResets(); + } + + // Silent when this SESSION had nothing to say. Gating on trackedCount() + // was wrong: the table is process-lifetime, so one registration in an + // earlier embedded session would emit an all-zero row for every session + // after it - and an all-zero row with no denominator cannot tell "clean" + // from "nothing was watched". samples_observed is session-scoped, so this + // gate is too. + if (!snap.any() && discarded == 0 && snap.samples_observed == 0) { + return; + } + + const Runtime* rt = runtime(); + if (!rt || !rt->logger) return; + + CounterDataQualitySummaryEvent ev; + ev.pid = detail::GetPid(); + ev.app = rt->app_name; + ev.session_id = rt->session_id; + ev.tracked_counters = NvtxCounterBridge::instance().trackedCount(); + ev.samples_observed = snap.samples_observed; + ev.registration_rejected = snap.registration_rejected; + ev.unknown_id_samples = snap.unknown_id_samples; + ev.unavailable_samples = snap.unavailable_samples; + ev.negative_delta_samples = snap.negative_delta_samples; + ev.rate_windows_discarded = discarded; + ev.emitted_ns = detail::GetTimestampNs(); + rt->logger->write(model::CounterDataQualitySummaryModel(ev)); + GFL_LOG_DEBUG("[NvtxCounters] quality summary rejected=", + ev.registration_rejected, " unknown=", ev.unknown_id_samples, + " unavailable=", ev.unavailable_samples, + " negative=", ev.negative_delta_samples, + " discarded=", ev.rate_windows_discarded); +} + void DeepWindowRules::Finish() { RuleSummary summary; std::string expression; diff --git a/include/gpufl/core/deep_window_rules.hpp b/include/gpufl/core/deep_window_rules.hpp index 243535e..ed3b092 100644 --- a/include/gpufl/core/deep_window_rules.hpp +++ b/include/gpufl/core/deep_window_rules.hpp @@ -53,6 +53,17 @@ class DeepWindowRules { */ static void Finish(); + /** + * @brief Write the session's counter data-quality summary, if any. + * + * Called at shutdown beside Finish(), but independent of it: the event + * reports what the APPLICATION sent (refused registrations, failed reads, + * negative deltas) and exists whether or not a rule was configured. + * Advances the bridge's session baseline, so an embedded re-init reports + * only its own session's problems. + */ + static void EmitCounterQuality(); + /** @brief True when a rule is installed - valid or refused. */ static bool Installed(); diff --git a/include/gpufl/core/events.hpp b/include/gpufl/core/events.hpp index 7dd5254..df5667c 100644 --- a/include/gpufl/core/events.hpp +++ b/include/gpufl/core/events.hpp @@ -730,11 +730,57 @@ struct DeepWindowRuleSummaryEvent { * a value alone can never show. */ uint64_t truncated_samples = 0; + /// Rate windows THIS rule's metric discarded for failed reads (NVTX + /// SAMPLE_UNAVAILABLE), and why the last one was. Per rule, never the + /// session total - an unrelated counter's errors must not look like they + /// broke this rule. + uint64_t metric_quality_resets = 0; + std::string last_quality_reason; // Monotonic, so a redelivered or late record cannot overwrite a newer one. uint64_t state_sequence = 0; int64_t emitted_ns = 0; }; +/** + * Session-scoped data quality of application-fed counters. + * + * NOT capture capability: none of these say what the GPU or driver supports. + * They say what the APPLICATION sent - refused registrations, samples for ids + * nobody issued, reads the application itself failed, deltas that went + * backwards - and how often the metric layer had to discard a rate window + * because of it. Each field has ONE meaning; a combined tally is a number + * nobody can act on. + * + * Values are this SESSION's, not the process totals: the tallies live for the + * process (like counter slots) and an embedded host re-initialises, so raw + * totals would re-report session one's problems as session two's. + */ +struct CounterDataQualitySummaryEvent { + int pid = 0; + std::string app; + std::string session_id; + /** + * Which counter path these tallies observed. Only "nvtx" today: the + * gpufl::counter() API's own rejections are not routed through the + * bridge, and a generic-looking row would claim coverage it does not + * have - a gpufl::counter registration failure beside + * registration_rejected: 0 would read as "nothing went wrong". + */ + std::string source = "nvtx"; + int schema_version = 1; + /// Registration-table size at emit (process-lifetime context). + uint64_t tracked_counters = 0; + /// Valid samples THIS session - the denominator that distinguishes + /// "0 failures out of many" from "0 out of 0". + uint64_t samples_observed = 0; + uint64_t registration_rejected = 0; + uint64_t unknown_id_samples = 0; + uint64_t unavailable_samples = 0; + uint64_t negative_delta_samples = 0; + uint64_t rate_windows_discarded = 0; + int64_t emitted_ns = 0; +}; + /** * One CUDA graph launch event captured by CUPTI's * CUPTI_ACTIVITY_KIND_GRAPH_TRACE stream. diff --git a/include/gpufl/core/metric_registry.cpp b/include/gpufl/core/metric_registry.cpp index f5eaa5f..b0f3384 100644 --- a/include/gpufl/core/metric_registry.cpp +++ b/include/gpufl/core/metric_registry.cpp @@ -4,6 +4,7 @@ #include #include "gpufl/core/events.hpp" +#include "gpufl/core/nvtx_counters.hpp" namespace gpufl::detail { namespace { @@ -210,6 +211,7 @@ void MetricSource::closeBucket(const int64_t boundary_ns) { switch (id_.shape()) { case MetricShape::Rate: { uint64_t total = 0; + bool source_reported_failure = false; if (id_.kind == MetricKind::KernelLaunchRate) { total = feeds_->launchFeed().count; } else if (resolveCustomHandle()) { @@ -220,6 +222,19 @@ void MetricSource::closeBucket(const int64_t boundary_ns) { // this run never saw. total = counters_->load_since_baseline(handle_); if (total > 0) first_tick_seen_ = true; + // An NVTX-fed counter can report that the APPLICATION failed to + // read it (SAMPLE_UNAVAILABLE). The delta over that stretch is + // unknown, not zero; treating it as zero makes the rate sag and + // fires a stall rule on a workload that never slowed down. + // Returns 0 for counters the bridge does not track, so + // gpufl::counter() metrics never pay this branch. + const uint64_t unavailable = + NvtxCounterBridge::instance().unavailableCountFor( + id_.custom_name); + if (unavailable != last_unavailable_seen_) { + last_unavailable_seen_ = unavailable; + source_reported_failure = true; + } } // Unsigned delta: correct across a wrap, which is why the counter is // allowed to wrap rather than saturate. @@ -228,6 +243,18 @@ void MetricSource::closeBucket(const int64_t boundary_ns) { baselined_ = true; buckets_[head_] = delta; if (delta > 0) last_tick_ns_ = boundary_ns; + if (source_reported_failure) { + ++quality_resets_; + last_quality_reason_ = "counter_unavailable"; + // Discard the whole window, not just this bucket: the failure + // is only observed at bucket close, so any bucket since the + // last close may straddle the gap. The window refills from + // post-failure data, the sample reads WarmingUp until it does, + // and the evaluator already treats a non-Fresh sample as + // broken evidence - Pending drops back to Armed. + std::fill(buckets_.begin(), buckets_.end(), 0); + buckets_closed_ = 0; + } break; } case MetricShape::Percentile: { diff --git a/include/gpufl/core/metric_registry.hpp b/include/gpufl/core/metric_registry.hpp index cdf3db0..df7e056 100644 --- a/include/gpufl/core/metric_registry.hpp +++ b/include/gpufl/core/metric_registry.hpp @@ -227,6 +227,11 @@ class MetricSource { /** @brief True once a custom counter has been resolved to a live handle. */ bool customResolved() const { return handle_ != nullptr; } + /** @brief Rate windows discarded because the source reported failed reads. */ + uint64_t qualityResets() const { return quality_resets_; } + /** @brief Why the last window was discarded; empty when none ever was. */ + const char* lastQualityReason() const { return last_quality_reason_; } + private: void closeBucket(int64_t boundary_ns); bool resolveCustomHandle(); @@ -278,6 +283,20 @@ class MetricSource { * a dead counter would read as a steady rate of 0 forever. */ int64_t last_tick_ns_ = 0; + /** + * Failed-read count last seen from the NVTX counter bridge. + * + * An UNAVAILABLE sample means the application could not read its own + * counter, so the true delta over that stretch is unknown - NOT zero. A + * rate computed across the gap sags below any threshold, which is a false + * stall. When this advances, the current window is discarded and refills + * from post-failure buckets only. + */ + uint64_t last_unavailable_seen_ = 0; + /// Rate windows this source discarded because its counter reported a + /// failed read. Per source, so a rule only ever wears its own problems. + uint64_t quality_resets_ = 0; + const char* last_quality_reason_ = ""; uint64_t last_published_close_ = 0; /// Durations the feed had to refuse. Surfaced so a truncated percentile is /// not presented as a complete one. diff --git a/include/gpufl/core/model/deep_window_model.cpp b/include/gpufl/core/model/deep_window_model.cpp index d387b10..e5b9d17 100644 --- a/include/gpufl/core/model/deep_window_model.cpp +++ b/include/gpufl/core/model/deep_window_model.cpp @@ -67,6 +67,8 @@ std::string DeepWindowRuleSummaryModel::buildJson() const { << ",\"samples_seen\":" << e_.samples_seen << ",\"windows_opened\":" << e_.windows_opened << ",\"truncated_samples\":" << e_.truncated_samples + << ",\"metric_quality_resets\":" << e_.metric_quality_resets + << ",\"last_quality_reason\":\"" << jsonEscape(e_.last_quality_reason) << "\"" << ",\"state_sequence\":" << e_.state_sequence << ",\"emitted_ns\":" << e_.emitted_ns; // Written only when there is one. A null would have to be distinguished @@ -79,4 +81,24 @@ std::string DeepWindowRuleSummaryModel::buildJson() const { return oss.str(); } +std::string CounterDataQualitySummaryModel::buildJson() const { + std::ostringstream oss; + oss << "{\"type\":\"counter_data_quality_summary\"" + << ",\"pid\":" << e_.pid + << ",\"app\":\"" << jsonEscape(e_.app) << "\"" + << ",\"session_id\":\"" << jsonEscape(e_.session_id) << "\"" + << ",\"source\":\"" << jsonEscape(e_.source) << "\"" + << ",\"schema_version\":" << e_.schema_version + << ",\"tracked_counters\":" << e_.tracked_counters + << ",\"samples_observed\":" << e_.samples_observed + << ",\"registration_rejected\":" << e_.registration_rejected + << ",\"unknown_id_samples\":" << e_.unknown_id_samples + << ",\"unavailable_samples\":" << e_.unavailable_samples + << ",\"negative_delta_samples\":" << e_.negative_delta_samples + << ",\"rate_windows_discarded\":" << e_.rate_windows_discarded + << ",\"emitted_ns\":" << e_.emitted_ns + << "}"; + return oss.str(); +} + } // namespace gpufl::model diff --git a/include/gpufl/core/model/deep_window_model.hpp b/include/gpufl/core/model/deep_window_model.hpp index a7634b6..9a7c51e 100644 --- a/include/gpufl/core/model/deep_window_model.hpp +++ b/include/gpufl/core/model/deep_window_model.hpp @@ -25,6 +25,19 @@ struct DeepWindowModel final : IJsonSerializable { * itself: the two are read together, and a summary that arrived on a different * channel could be ingested after the windows it explains. */ +// Lives beside the rule-summary model because its consumer is the same +// conditional-window feature; the event itself is counter data quality. +struct CounterDataQualitySummaryModel final : IJsonSerializable { + explicit CounterDataQualitySummaryModel(const CounterDataQualitySummaryEvent& e) + : e_(e) {} + std::string buildJson() const override; + // Scope channel, like the rule summary it is read next to. + Channel channel() const override { return Channel::Scope; } + + private: + const CounterDataQualitySummaryEvent& e_; +}; + struct DeepWindowRuleSummaryModel final : IJsonSerializable { explicit DeepWindowRuleSummaryModel(const DeepWindowRuleSummaryEvent& e) : e_(e) {} diff --git a/include/gpufl/core/monitor.cpp b/include/gpufl/core/monitor.cpp index 2a59597..a9e770f 100644 --- a/include/gpufl/core/monitor.cpp +++ b/include/gpufl/core/monitor.cpp @@ -538,6 +538,10 @@ void Monitor::Shutdown() { // AFTER the collector has stopped, and before the logger goes away. Writing // the summary while the collector still runs would let the evaluator open a // window the recorded summary never mentions. + // Quality BEFORE Finish: Finish releases the rule session, which destroys + // the metric source whose discard count the quality event reports. The + // collector is already stopped, so nothing advances between the two. + detail::DeepWindowRules::EmitCounterQuality(); detail::DeepWindowRules::Finish(); while (RecordProcessor::processNext()) {} @@ -571,6 +575,10 @@ void Monitor::DrainAndFinalizeForExit() { // Same ordering as Shutdown: the collector is stopped first, so nothing can // advance the rule past what this summary reports. + // Quality BEFORE Finish: Finish releases the rule session, which destroys + // the metric source whose discard count the quality event reports. The + // collector is already stopped, so nothing advances between the two. + detail::DeepWindowRules::EmitCounterQuality(); detail::DeepWindowRules::Finish(); while (RecordProcessor::processNext()) {} diff --git a/include/gpufl/core/nvtx_counters.cpp b/include/gpufl/core/nvtx_counters.cpp new file mode 100644 index 0000000..931945e --- /dev/null +++ b/include/gpufl/core/nvtx_counters.cpp @@ -0,0 +1,381 @@ +#include "gpufl/core/nvtx_counters.hpp" + +#include +#include + +#include "gpufl/core/counter_provider.hpp" +#include "gpufl/core/counter_registry.hpp" +#include "gpufl/core/debug_logger.hpp" + +namespace gpufl::detail { +namespace { + +// NVTX_COUNTER_SAMPLE_* from nvToolsExtCounters.h. Duplicated as plain +// constants so this file stays free of NVTX headers and can be unit-tested +// without them; the values are part of a released ABI and pinned by a test. +constexpr uint8_t kSampleZero = 0; +constexpr uint8_t kSampleUnchanged = 1; +constexpr uint8_t kSampleUnavailable = 2; + +// Serialises registration only. The sample path never takes it. +std::mutex g_mu; + +std::atomic g_registration_rejected{0}; +std::atomic g_unknown_id_samples{0}; +std::atomic g_unavailable_samples{0}; +std::atomic g_negative_samples{0}; +std::atomic g_samples_observed{0}; + +// Where the previous session's report ended. Guarded by g_mu; the snapshot is +// taken once per session at shutdown, never on a hot path. +uint64_t g_base_registration_rejected = 0; +uint64_t g_base_unknown_id = 0; +uint64_t g_base_unavailable = 0; +uint64_t g_base_negative = 0; +uint64_t g_base_observed = 0; + +bool CharsetOk(const char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '.' || c == '_' || c == '-'; +} + +// Logged once each: a rejected registration usually repeats, and a sample the +// bridge does not know about can arrive on every iteration of a decode loop. +bool g_logged_static_id = false; +bool g_logged_value_type = false; +bool g_logged_bad_name = false; +bool g_logged_collision = false; +bool g_logged_limit = false; +std::atomic g_logged_unknown_id{false}; +std::atomic g_logged_negative{false}; + +} // namespace + +NvtxCounterBridge& NvtxCounterBridge::instance() { + static auto* bridge = new NvtxCounterBridge(); + return *bridge; +} + +std::string NvtxCounterBridge::canonicalName(const std::string& domain_name, + const std::string& counter_name) { + std::string joined; + if (!domain_name.empty()) { + joined = domain_name; + joined += '.'; + } + joined += counter_name; + + std::string out; + out.reserve(joined.size()); + for (const char c : joined) out.push_back(CharsetOk(c) ? c : '_'); + + // A name of nothing but separators would produce a metric like + // `custom.___rate`, which names no counter anybody could have meant. + bool has_alnum = false; + for (const char c : out) { + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9')) { + has_alnum = true; + break; + } + } + if (!has_alnum) return {}; + if (out.size() > CounterRegistry::kMaxNameLength) return {}; + return out; +} + +NvtxCounterBridge::RegisterResult NvtxCounterBridge::registerCounter( + const std::string& domain_name, const std::string& counter_name, + const uint64_t requested_id, const ValueType type) { + RegisterResult result = registerCounterInner(domain_name, counter_name, + requested_id, type); + if (result.status != RegisterStatus::Accepted) { + g_registration_rejected.fetch_add(1, std::memory_order_relaxed); + } + return result; +} + +NvtxCounterBridge::RegisterResult NvtxCounterBridge::registerCounterInner( + const std::string& domain_name, const std::string& counter_name, + const uint64_t requested_id, const ValueType type) { + RegisterResult result; + + // An application-chosen ID is unique only inside its domain, so binding it + // to a slot needs a (domain, id) table. Without one, two domains that both + // use id 123 would share a slot and add their rates together - a wrong + // number that looks like a real one. Refused until that table exists. + if (requested_id != 0) { + result.status = RegisterStatus::StaticIdUnsupported; + std::lock_guard lk(g_mu); + if (!g_logged_static_id) { + g_logged_static_id = true; + GFL_LOG_WARN("[NvtxCounters] counter '", counter_name, + "' uses an application-assigned id; gpufl only " + "tracks tool-assigned ids (pass " + "NVTX_COUNTER_ID_NONE). This counter is not " + "available to rules."); + } + return result; + } + + // The only shape the registry can represent. It accumulates unsigned, so + // an absolute series read as deltas would sum a monotonic curve, and a + // counter with no semantics at all is not enough to tell the two apart. + if (type != ValueType::Delta) { + result.status = RegisterStatus::UnsupportedValueType; + std::lock_guard lk(g_mu); + if (!g_logged_value_type) { + g_logged_value_type = true; + GFL_LOG_WARN("[NvtxCounters] counter '", counter_name, + "' is not a DELTA counter (attach " + "nvtxSemanticsCounter_t with " + "NVTX_COUNTER_FLAG_VALUETYPE_DELTA). Absolute and " + "unspecified counters are not converted, because " + "reading one as deltas produces a plausible wrong " + "rate rather than an obvious failure."); + } + return result; + } + + const std::string name = canonicalName(domain_name, counter_name); + if (name.empty()) { + result.status = RegisterStatus::BadName; + std::lock_guard lk(g_mu); + if (!g_logged_bad_name) { + g_logged_bad_name = true; + GFL_LOG_WARN("[NvtxCounters] counter name has nothing usable in " + "it after canonicalisation; rules address counters " + "as custom._rate over [A-Za-z0-9._-]"); + } + return result; + } + + std::lock_guard lk(g_mu); + const size_t count = count_.load(std::memory_order_relaxed); + + // Canonicalisation maps every out-of-charset byte to '_' and joins the + // domain with '.', so it is not injective: "a b" and "a/b" meet at "a_b", + // and ("a", "b.c") meets ("a.b", "c"). The ORIGINAL pair decides which + // case this is: the same pair again is an idempotent re-registration; a + // different pair is two counters, and merging them would silently add two + // unrelated workloads into a single rate. + for (size_t i = 0; i < count; ++i) { + if (entries_[i].name != name) continue; + if (entries_[i].domain_original == domain_name && + entries_[i].counter_original == counter_name) { + result.id = kDynamicIdBase + i; + result.status = RegisterStatus::Accepted; + result.metric = "custom." + name + "_rate"; + return result; + } + result.status = RegisterStatus::BadName; + if (!g_logged_collision) { + g_logged_collision = true; + GFL_LOG_WARN("[NvtxCounters] counter '", counter_name, + "' in domain '", domain_name, "' canonicalises to '", + name, "', which already belongs to counter '", + entries_[i].counter_original, "' in domain '", + entries_[i].domain_original, + "'. Refused rather than merged; rename one of them."); + } + return result; + } + + if (count >= kMaxTracked) { + result.status = RegisterStatus::LimitReached; + if (!g_logged_limit) { + g_logged_limit = true; + GFL_LOG_WARN("[NvtxCounters] tracking limit (", kMaxTracked, + ") reached; '", name, "' is not available to rules"); + } + return result; + } + + // Through the provider: the evaluator reads whatever + // ActiveCounterProvider() resolves, and registering into this module's own + // registry instead leaves the counter invisible to it wherever a shared + // runtime is bound - proven by mutation on the 3090, where exactly this + // bypass turned a firing rule into custom_metric_never_registered. + const gpufl_counter_provider_v1* provider = ActiveCounterProvider(); + gpufl_counter_handle handle = + provider->register_counter(name.c_str(), name.size()); + if (handle == nullptr) { + result.status = RegisterStatus::LimitReached; + if (!g_logged_limit) { + g_logged_limit = true; + GFL_LOG_WARN("[NvtxCounters] the counter registry refused '", name, + "'; it is not available to rules"); + } + return result; + } + + // Fields first, count last with release: the sample path reads count_ + // with acquire and no lock, so the store below is what publishes the + // entry. The provider travels with the handle it issued - a handle is a + // pointer into that provider's registry, and the pair must never split. + entries_[count].provider = provider; + entries_[count].handle = handle; + entries_[count].name = name; + entries_[count].domain_original = domain_name; + entries_[count].counter_original = counter_name; + entries_[count].unavailable.store(0, std::memory_order_relaxed); + count_.store(count + 1, std::memory_order_release); + + result.id = kDynamicIdBase + count; + result.status = RegisterStatus::Accepted; + result.metric = "custom." + name + "_rate"; + // Printed because nothing else connects the NVTX name the application + // wrote to the metric name a rule has to be written against. + GFL_LOG_INFO("[NvtxCounters] counter '", counter_name, "' -> rule metric ", + result.metric); + return result; +} + +void NvtxCounterBridge::sampleDelta(const uint64_t id, const int64_t value) { + if (id < kDynamicIdBase) { + g_unknown_id_samples.fetch_add(1, std::memory_order_relaxed); + if (!g_logged_unknown_id.exchange(true, std::memory_order_relaxed)) { + GFL_LOG_WARN("[NvtxCounters] sample for a counter this build did " + "not assign an id to; it is not reaching any rule"); + } + return; + } + const uint64_t index = id - kDynamicIdBase; + if (index >= count_.load(std::memory_order_acquire)) { + g_unknown_id_samples.fetch_add(1, std::memory_order_relaxed); + return; + } + if (value < 0) { + // The registry accumulates unsigned. Wrapping it backwards turns the + // very next rate into an enormous number instead of a small one, so a + // negative delta is dropped and counted rather than applied. + g_negative_samples.fetch_add(1, std::memory_order_relaxed); + if (!g_logged_negative.exchange(true, std::memory_order_relaxed)) { + GFL_LOG_WARN("[NvtxCounters] negative delta dropped; a DELTA " + "counter feeding a rate must not go backwards"); + } + return; + } + g_samples_observed.fetch_add(1, std::memory_order_relaxed); + if (value == 0) return; // a real observation of "no traffic"; nothing to add + + const Entry& entry = entries_[index]; + entry.provider->add(entry.handle, static_cast(value)); +} + +void NvtxCounterBridge::sampleNoValue(const uint64_t id, const uint8_t reason) { + if (id < kDynamicIdBase || + id - kDynamicIdBase >= count_.load(std::memory_order_acquire)) { + g_unknown_id_samples.fetch_add(1, std::memory_order_relaxed); + return; + } + switch (reason) { + case kSampleZero: + case kSampleUnchanged: + // Both say the delta is zero. For a rate that is precisely "do + // nothing" - adding 1 here would invent traffic out of a sample + // that exists to report the absence of it. Still a valid + // OBSERVATION, so it counts toward the denominator. + g_samples_observed.fetch_add(1, std::memory_order_relaxed); + return; + case kSampleUnavailable: + default: + // The application could not read its own counter, so the true + // delta over this stretch is unknown - NOT zero. Recorded per + // counter; MetricSource discards the rate window it lands in, + // because a rate computed over the gap sags below any threshold + // and fires the rule on a workload that never slowed down. + entries_[id - kDynamicIdBase].unavailable.fetch_add( + 1, std::memory_order_relaxed); + g_unavailable_samples.fetch_add(1, std::memory_order_relaxed); + return; + } +} + +uint64_t NvtxCounterBridge::unavailableCountFor( + const std::string& canonical_name) const { + // Lock-free on purpose: names are immutable once their entry is published, + // and count_ is acquire-loaded, so this is safe against a concurrent + // registration of a LATER entry. + const size_t count = count_.load(std::memory_order_acquire); + for (size_t i = 0; i < count; ++i) { + if (entries_[i].name == canonical_name) { + return entries_[i].unavailable.load(std::memory_order_relaxed); + } + } + return 0; +} + +NvtxCounterBridge::QualitySnapshot NvtxCounterBridge::takeSessionSnapshot() { + std::lock_guard lk(g_mu); + QualitySnapshot snap; + const uint64_t rr = g_registration_rejected.load(std::memory_order_relaxed); + const uint64_t ui = g_unknown_id_samples.load(std::memory_order_relaxed); + const uint64_t ua = g_unavailable_samples.load(std::memory_order_relaxed); + const uint64_t ng = g_negative_samples.load(std::memory_order_relaxed); + const uint64_t ob = g_samples_observed.load(std::memory_order_relaxed); + snap.registration_rejected = rr - g_base_registration_rejected; + snap.unknown_id_samples = ui - g_base_unknown_id; + snap.unavailable_samples = ua - g_base_unavailable; + snap.negative_delta_samples = ng - g_base_negative; + snap.samples_observed = ob - g_base_observed; + g_base_registration_rejected = rr; + g_base_unknown_id = ui; + g_base_unavailable = ua; + g_base_negative = ng; + g_base_observed = ob; + return snap; +} + +uint64_t NvtxCounterBridge::registrationRejected() const { + return g_registration_rejected.load(std::memory_order_relaxed); +} + +uint64_t NvtxCounterBridge::unknownIdSamples() const { + return g_unknown_id_samples.load(std::memory_order_relaxed); +} + +uint64_t NvtxCounterBridge::unavailableSamples() const { + return g_unavailable_samples.load(std::memory_order_relaxed); +} + +uint64_t NvtxCounterBridge::negativeSamples() const { + return g_negative_samples.load(std::memory_order_relaxed); +} + +size_t NvtxCounterBridge::trackedCount() const { + return count_.load(std::memory_order_acquire); +} + +void NvtxCounterBridge::resetForTesting() { + std::lock_guard lk(g_mu); + const size_t count = count_.load(std::memory_order_relaxed); + for (size_t i = 0; i < count; ++i) { + entries_[i].provider = nullptr; + entries_[i].handle = nullptr; + entries_[i].name.clear(); + entries_[i].domain_original.clear(); + entries_[i].counter_original.clear(); + entries_[i].unavailable.store(0, std::memory_order_relaxed); + } + count_.store(0, std::memory_order_release); + g_registration_rejected.store(0, std::memory_order_relaxed); + g_unknown_id_samples.store(0, std::memory_order_relaxed); + g_unavailable_samples.store(0, std::memory_order_relaxed); + g_negative_samples.store(0, std::memory_order_relaxed); + g_base_registration_rejected = 0; + g_base_unknown_id = 0; + g_base_unavailable = 0; + g_base_negative = 0; + g_samples_observed.store(0, std::memory_order_relaxed); + g_base_observed = 0; + g_logged_static_id = false; + g_logged_value_type = false; + g_logged_bad_name = false; + g_logged_collision = false; + g_logged_limit = false; + g_logged_unknown_id.store(false, std::memory_order_relaxed); + g_logged_negative.store(false, std::memory_order_relaxed); +} + +} // namespace gpufl::detail diff --git a/include/gpufl/core/nvtx_counters.hpp b/include/gpufl/core/nvtx_counters.hpp new file mode 100644 index 0000000..d763410 --- /dev/null +++ b/include/gpufl/core/nvtx_counters.hpp @@ -0,0 +1,226 @@ +#pragma once + +#include +#include +#include + +struct gpufl_counter_provider_v1; + +namespace gpufl::detail { + +/** + * @brief Routes NVTX Counters extension samples into the counter registry. + * + * Lets an application drive a conditional deep window through the STANDARD + * NVIDIA API instead of gpufl's own: `nvtxCounterRegister` + `nvtxCounterSample*` + * reach us as direct calls into the injection library (the launcher already + * owns NVTX_INJECTION64_PATH), so nothing goes through a CUPTI activity + * buffer and the per-sample cost stays one relaxed atomic. + * + * Deliberately free of NVTX types so it is testable without an injected + * process; inject_entry.cpp does the NVTX-shaped translation. + */ +class NvtxCounterBridge { +public: + /** How the application says its samples should be read. */ + enum class ValueType { + /// No counter semantics were attached. NOT assumed to be anything. + Unspecified, + Absolute, + Delta, + DeltaSinceStart, + }; + + /** Why a registration was refused, for the capability report. */ + enum class RegisterStatus { + Accepted, + /// Value type absent or not Delta. Reading an absolute series as + /// deltas would sum a monotonic curve into nonsense, so it is refused + /// rather than guessed. + UnsupportedValueType, + /// An application-chosen (static) ID. Unique only WITHIN its domain, + /// so honouring it needs a (domain, id) table this build does not + /// have; two domains reusing one id would share a slot and silently + /// add their rates together. + StaticIdUnsupported, + /// The name could not be turned into a metric name, or two DIFFERENT + /// NVTX names canonicalise to the same one - merging those would + /// silently add two unrelated workloads into a single rate. + BadName, + /// The registry is full, or this bridge's table is. + LimitReached, + }; + + struct RegisterResult { + /// Non-zero only when Accepted. Encodes the table index, so a sample + /// resolves in constant time with no lookup and no lock. + uint64_t id = 0; + RegisterStatus status = RegisterStatus::BadName; + /// The `custom._rate` a rule would be written against. Reported + /// so the link between the NVTX name and the CLI name is visible + /// rather than something the user has to infer. + std::string metric; + }; + + /** Base of the IDs this bridge hands out. Matches NVTX_COUNTER_ID_DYNAMIC_START. */ + static constexpr uint64_t kDynamicIdBase = static_cast(1) << 32; + /** Bound on tracked counters. The registry has its own, lower, limit. */ + static constexpr size_t kMaxTracked = 64; + + static NvtxCounterBridge& instance(); + + /** + * @brief Bind an NVTX counter to a registry slot. + * + * @param domain_name Domain name, or empty. Prefixed onto the counter name + * so two domains using the same counter name stay distinct. + * @param requested_id The application's counterId; 0 means "assign one". + */ + RegisterResult registerCounter(const std::string& domain_name, + const std::string& counter_name, + uint64_t requested_id, ValueType type); + + /** + * @brief A delta sample. Ignores anything this bridge did not hand out. + * + * Lock-free: the entry holds the provider that issued its handle, taken + * once at registration, so the sample path never resolves the provider - + * resolving it takes a mutex, and a mutex per sample in a decode loop + * changes the throughput the rule is measuring. + * + * Negative deltas are counted and dropped: the registry is an unsigned + * monotonic accumulator, and wrapping one backwards would turn the next + * rate into an astronomically large number rather than a small one. + */ + void sampleDelta(uint64_t id, int64_t value); + + /** + * @brief A sample that carries no value. + * + * `zero` and `unchanged` both mean the delta is 0, which for a rate is + * exactly "do nothing" - NOT an event. `unavailable` means the application + * failed to read its own counter: the true delta is UNKNOWN, not zero, and + * a rate computed over the gap would sag below any threshold and fire the + * rule on a workload that never slowed down. It is recorded per counter so + * the metric layer can discard the affected window. + */ + void sampleNoValue(uint64_t id, uint8_t reason); + + /** + * @brief Failed reads recorded for the counter behind @p canonical_name. + * + * Polled by MetricSource at bucket close: when this advances, the current + * rate window contains a gap of unknown size and is discarded rather than + * evaluated. 0 for names this bridge does not track, so non-NVTX custom + * counters are unaffected. + */ + uint64_t unavailableCountFor(const std::string& canonical_name) const; + + /** + * @brief One session's data-quality tallies, each with ONE meaning. + * + * Kept apart on purpose: "registration was refused" is a configuration + * problem, "an id we never issued" is an application bug, "unavailable" + * is a read the application itself failed, and a negative delta is a + * value that cannot be applied. Folding any two together produces a + * number nobody can act on. + */ + struct QualitySnapshot { + uint64_t registration_rejected = 0; + uint64_t unknown_id_samples = 0; + uint64_t unavailable_samples = 0; + uint64_t negative_delta_samples = 0; + /** + * Valid samples this session: accepted deltas (a zero delta is a real + * observation) plus ZERO/UNCHANGED no-value samples. The denominator + * that makes an all-zero failure row mean something - "0 failures out + * of 12,000 samples" is a clean bill; "0 out of 0" is a session where + * nothing was watched, and without this field the two are the same + * row. + */ + uint64_t samples_observed = 0; + + /// Failures only, on purpose: samples_observed is the denominator, + /// not a problem. + bool any() const { + return registration_rejected != 0 || unknown_id_samples != 0 || + unavailable_samples != 0 || negative_delta_samples != 0; + } + }; + + /** + * @brief This session's tallies, advancing the session baseline. + * + * The tallies are process-lifetime (like the counter slots), and an + * embedded host re-initialises in one process - exporting the raw values + * would re-report session one's problems as session two's. Each call + * returns what accrued since the previous call; the FIRST call returns + * everything since process start, which is where pre-init NVTX + * registrations belong: they happened during this run's startup and no + * other session can report them. + * + * Single consumer: the session-summary emit at shutdown. + */ + QualitySnapshot takeSessionSnapshot(); + + /** @brief Process-lifetime registrations refused, any reason. */ + uint64_t registrationRejected() const; + /** @brief Samples whose id this bridge never issued. */ + uint64_t unknownIdSamples() const; + /** @brief Samples the application itself could not read, all counters. */ + uint64_t unavailableSamples() const; + /** @brief Samples dropped for carrying a negative delta. */ + uint64_t negativeSamples() const; + /** @brief Counters bound so far. */ + size_t trackedCount() const; + + /** @brief Test seam. A real process never drops these bindings. */ + void resetForTesting(); + + /** + * @brief NVTX name to registry name: `.`, canonicalised. + * + * Counter names reach a rule as `custom._rate`, whose charset is + * [A-Za-z0-9._-]; NVTX names are free-form. Everything outside the charset + * becomes '_'. That is not injective - and the domain joins with '.', so + * ("a", "b.c") and ("a.b", "c") also meet - which is why the entry keeps + * the ORIGINAL pair and a collision is refused at registration instead of + * silently merging two counters into one slot. + */ + static std::string canonicalName(const std::string& domain_name, + const std::string& counter_name); + +private: + NvtxCounterBridge() = default; + + /// The actual registration; the public wrapper counts refusals in one + /// place so no refusal path can forget to. + RegisterResult registerCounterInner(const std::string& domain_name, + const std::string& counter_name, + uint64_t requested_id, ValueType type); + + struct Entry { + /// The provider that issued `handle`. A handle is a pointer into the + /// issuing provider's registry, so the pair must never be split. + const gpufl_counter_provider_v1* provider = nullptr; + void* handle = nullptr; // gpufl_counter_handle + std::string name; // canonical registry name + /// The NVTX names as the application wrote them. What tells an + /// idempotent re-registration apart from a canonicalisation collision. + std::string domain_original; + std::string counter_original; + /// Failed reads (SAMPLE_UNAVAILABLE) for this counter. + std::atomic unavailable{0}; + }; + + Entry entries_[kMaxTracked]; + /** + * Published with release AFTER an entry's fields are written, and read + * with acquire on the sample path. The mutex only serialises writers; a + * plain size_t here was a data race with the lock-free readers, and the + * race window is exactly a counter's first samples. + */ + std::atomic count_{0}; +}; + +} // namespace gpufl::detail diff --git a/include/gpufl/inject/cuda_interpose_linux.cpp b/include/gpufl/inject/cuda_interpose_linux.cpp new file mode 100644 index 0000000..ed81e29 --- /dev/null +++ b/include/gpufl/inject/cuda_interpose_linux.cpp @@ -0,0 +1,290 @@ +// Linux/LD_PRELOAD CUDA activity-boundary interposition. +// +// Keep this in a separate translation unit from inject_entry.cpp so every +// exported CUDA symbol is compiled against NVIDIA's official CUDA 13 headers. +// That makes the compiler enforce the ABI for opaque handles and 2D/3D +// parameter structures instead of relying on hand-written void* equivalents. + +#if !defined(_WIN32) + +#include +#include + +#include +#include + +extern "C" { +void GpuFlightWaitAtCudaLaunchBoundary(); +void GpuFlightWaitAtCudaSyncBoundary(); +void GpuFlightWaitAtCudaMemoryBoundary(); +} + +namespace { + +template +Fn ResolveNext(const char* symbol) noexcept { + // RTLD_NEXT skips libgpufl_inject itself. Resolution is performed once per + // wrapper by a function-local static below, after the readiness wait; no + // CUDA API is called while the loader is resolving the real symbol. + return reinterpret_cast(dlsym(RTLD_NEXT, symbol)); +} + +} // namespace + +#define GPUFL_CUDA_INTERPOSE(WAIT, RETURN_TYPE, NAME, PARAMS, ARGS, FAILURE) \ + extern "C" __attribute__((visibility("default"))) \ + RETURN_TYPE NAME PARAMS { \ + WAIT(); \ + using Function = RETURN_TYPE (*) PARAMS; \ + static Function resolved_function = ResolveNext(#NAME); \ + return resolved_function ? resolved_function ARGS : FAILURE; \ + } + +// Launch and synchronization boundaries previously lived in inject_entry.cpp +// with ABI-compatible opaque types. Keeping them here means the same official +// headers now validate every interposed CUDA function. +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaLaunchBoundary, cudaError_t, __cudaLaunchKernel, + (const void* function_address, dim3 grid_dim, dim3 block_dim, void** args, + std::size_t shared_mem, cudaStream_t stream), + (function_address, grid_dim, block_dim, args, shared_mem, stream), + cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaLaunchBoundary, cudaError_t, cudaLaunchKernel, + (const void* function_address, dim3 grid_dim, dim3 block_dim, void** args, + std::size_t shared_mem, cudaStream_t stream), + (function_address, grid_dim, block_dim, args, shared_mem, stream), + cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaLaunchBoundary, cudaError_t, cudaLaunchKernelExC, + (const cudaLaunchConfig_t* config, const void* function_address, void** args), + (config, function_address, args), + cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaLaunchBoundary, CUresult, cuLaunchKernel, + (CUfunction function, unsigned int grid_x, unsigned int grid_y, + unsigned int grid_z, unsigned int block_x, unsigned int block_y, + unsigned int block_z, unsigned int shared_mem, CUstream stream, + void** kernel_params, void** extra), + (function, grid_x, grid_y, grid_z, block_x, block_y, block_z, shared_mem, + stream, kernel_params, extra), + CUDA_ERROR_UNKNOWN) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaSyncBoundary, cudaError_t, cudaDeviceSynchronize, + (), (), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaSyncBoundary, cudaError_t, cudaStreamSynchronize, + (cudaStream_t stream), (stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaSyncBoundary, CUresult, cuCtxSynchronize, + (), (), CUDA_ERROR_UNKNOWN) + +// CUDA Runtime API: synchronous, asynchronous, pitched, 3D, and peer copies. +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy, + (void* dst, const void* src, std::size_t bytes, cudaMemcpyKind kind), + (dst, src, bytes, kind), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpyAsync, + (void* dst, const void* src, std::size_t bytes, cudaMemcpyKind kind, + cudaStream_t stream), + (dst, src, bytes, kind, stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpyAsync_ptsz, + (void* dst, const void* src, std::size_t bytes, cudaMemcpyKind kind, + cudaStream_t stream), + (dst, src, bytes, kind, stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy2D, + (void* dst, std::size_t dst_pitch, const void* src, std::size_t src_pitch, + std::size_t width, std::size_t height, cudaMemcpyKind kind), + (dst, dst_pitch, src, src_pitch, width, height, kind), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy2DAsync, + (void* dst, std::size_t dst_pitch, const void* src, std::size_t src_pitch, + std::size_t width, std::size_t height, cudaMemcpyKind kind, + cudaStream_t stream), + (dst, dst_pitch, src, src_pitch, width, height, kind, stream), + cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy2DAsync_ptsz, + (void* dst, std::size_t dst_pitch, const void* src, std::size_t src_pitch, + std::size_t width, std::size_t height, cudaMemcpyKind kind, + cudaStream_t stream), + (dst, dst_pitch, src, src_pitch, width, height, kind, stream), + cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy3D, + (const cudaMemcpy3DParms* params), (params), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy3DAsync, + (const cudaMemcpy3DParms* params, cudaStream_t stream), + (params, stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy3DAsync_ptsz, + (const cudaMemcpy3DParms* params, cudaStream_t stream), + (params, stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpyPeer, + (void* dst, int dst_device, const void* src, int src_device, + std::size_t bytes), + (dst, dst_device, src, src_device, bytes), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpyPeerAsync, + (void* dst, int dst_device, const void* src, int src_device, + std::size_t bytes, cudaStream_t stream), + (dst, dst_device, src, src_device, bytes, stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpyPeerAsync_ptsz, + (void* dst, int dst_device, const void* src, int src_device, + std::size_t bytes, cudaStream_t stream), + (dst, dst_device, src, src_device, bytes, stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy3DPeer, + (const cudaMemcpy3DPeerParms* params), (params), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy3DPeerAsync, + (const cudaMemcpy3DPeerParms* params, cudaStream_t stream), + (params, stream), cudaErrorUnknown) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, cudaError_t, cudaMemcpy3DPeerAsync_ptsz, + (const cudaMemcpy3DPeerParms* params, cudaStream_t stream), + (params, stream), cudaErrorUnknown) + +// CUDA Driver API. The non-stream variants have PTDS aliases; asynchronous +// variants have PTSZ aliases when an application opts into per-thread default +// streams. Export both spellings so the readiness barrier cannot be bypassed +// by that compile-time mode. +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, CUresult, cuMemcpy, + (CUdeviceptr dst, CUdeviceptr src, std::size_t bytes), + (dst, src, bytes), CUDA_ERROR_UNKNOWN) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, CUresult, cuMemcpy_ptds, + (CUdeviceptr dst, CUdeviceptr src, std::size_t bytes), + (dst, src, bytes), CUDA_ERROR_UNKNOWN) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, CUresult, cuMemcpyAsync, + (CUdeviceptr dst, CUdeviceptr src, std::size_t bytes, CUstream stream), + (dst, src, bytes, stream), CUDA_ERROR_UNKNOWN) + +GPUFL_CUDA_INTERPOSE( + GpuFlightWaitAtCudaMemoryBoundary, CUresult, cuMemcpyAsync_ptsz, + (CUdeviceptr dst, CUdeviceptr src, std::size_t bytes, CUstream stream), + (dst, src, bytes, stream), CUDA_ERROR_UNKNOWN) + +#define GPUFL_DRIVER_COPY_SYNC_PAIR(NAME, PARAMS, ARGS) \ + GPUFL_CUDA_INTERPOSE( \ + GpuFlightWaitAtCudaMemoryBoundary, CUresult, NAME, PARAMS, ARGS, \ + CUDA_ERROR_UNKNOWN) \ + GPUFL_CUDA_INTERPOSE( \ + GpuFlightWaitAtCudaMemoryBoundary, CUresult, NAME##_ptds, PARAMS, \ + ARGS, CUDA_ERROR_UNKNOWN) + +#define GPUFL_DRIVER_COPY_ASYNC_PAIR(NAME, PARAMS, ARGS) \ + GPUFL_CUDA_INTERPOSE( \ + GpuFlightWaitAtCudaMemoryBoundary, CUresult, NAME, PARAMS, ARGS, \ + CUDA_ERROR_UNKNOWN) \ + GPUFL_CUDA_INTERPOSE( \ + GpuFlightWaitAtCudaMemoryBoundary, CUresult, NAME##_ptsz, PARAMS, \ + ARGS, CUDA_ERROR_UNKNOWN) + +GPUFL_DRIVER_COPY_SYNC_PAIR( + cuMemcpyHtoD_v2, + (CUdeviceptr dst, const void* src, std::size_t bytes), + (dst, src, bytes)) + +GPUFL_DRIVER_COPY_SYNC_PAIR( + cuMemcpyDtoH_v2, + (void* dst, CUdeviceptr src, std::size_t bytes), + (dst, src, bytes)) + +GPUFL_DRIVER_COPY_SYNC_PAIR( + cuMemcpyDtoD_v2, + (CUdeviceptr dst, CUdeviceptr src, std::size_t bytes), + (dst, src, bytes)) + +GPUFL_DRIVER_COPY_ASYNC_PAIR( + cuMemcpyHtoDAsync_v2, + (CUdeviceptr dst, const void* src, std::size_t bytes, CUstream stream), + (dst, src, bytes, stream)) + +GPUFL_DRIVER_COPY_ASYNC_PAIR( + cuMemcpyDtoHAsync_v2, + (void* dst, CUdeviceptr src, std::size_t bytes, CUstream stream), + (dst, src, bytes, stream)) + +GPUFL_DRIVER_COPY_ASYNC_PAIR( + cuMemcpyDtoDAsync_v2, + (CUdeviceptr dst, CUdeviceptr src, std::size_t bytes, CUstream stream), + (dst, src, bytes, stream)) + +GPUFL_DRIVER_COPY_SYNC_PAIR( + cuMemcpy2D_v2, + (const CUDA_MEMCPY2D* params), + (params)) + +GPUFL_DRIVER_COPY_ASYNC_PAIR( + cuMemcpy2DAsync_v2, + (const CUDA_MEMCPY2D* params, CUstream stream), + (params, stream)) + +GPUFL_DRIVER_COPY_SYNC_PAIR( + cuMemcpy3D_v2, + (const CUDA_MEMCPY3D* params), + (params)) + +GPUFL_DRIVER_COPY_ASYNC_PAIR( + cuMemcpy3DAsync_v2, + (const CUDA_MEMCPY3D* params, CUstream stream), + (params, stream)) + +GPUFL_DRIVER_COPY_SYNC_PAIR( + cuMemcpyPeer, + (CUdeviceptr dst, CUcontext dst_context, CUdeviceptr src, + CUcontext src_context, std::size_t bytes), + (dst, dst_context, src, src_context, bytes)) + +GPUFL_DRIVER_COPY_ASYNC_PAIR( + cuMemcpyPeerAsync, + (CUdeviceptr dst, CUcontext dst_context, CUdeviceptr src, + CUcontext src_context, std::size_t bytes, CUstream stream), + (dst, dst_context, src, src_context, bytes, stream)) + +GPUFL_DRIVER_COPY_SYNC_PAIR( + cuMemcpy3DPeer, + (const CUDA_MEMCPY3D_PEER* params), + (params)) + +GPUFL_DRIVER_COPY_ASYNC_PAIR( + cuMemcpy3DPeerAsync, + (const CUDA_MEMCPY3D_PEER* params, CUstream stream), + (params, stream)) + +#undef GPUFL_DRIVER_COPY_ASYNC_PAIR +#undef GPUFL_DRIVER_COPY_SYNC_PAIR +#undef GPUFL_CUDA_INTERPOSE + +#endif // !defined(_WIN32) diff --git a/include/gpufl/inject/inject_entry.cpp b/include/gpufl/inject/inject_entry.cpp index 9a44f51..dc0c645 100644 --- a/include/gpufl/inject/inject_entry.cpp +++ b/include/gpufl/inject/inject_entry.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -53,12 +54,34 @@ #define NVTX_NO_IMPL #endif #include +// Counters extension types (nvtxCounterAttr_t, the CBID list, module id) and +// the injection-side nvtxExtModuleInfo_t it pulls in. NVTX_NO_IMPL above keeps +// this types-only, same as the core header. The semantics header is separate +// and NOT pulled in by the counters one, but it is what says whether a sample +// is a delta or an absolute reading - the difference between a correct rate +// and a plausible wrong one. +// +// Guarded because the header-only NVTX3 distribution CMake falls back to when +// the toolkit ships no NVTX predates the counters extension. Without counters +// the injection simply does not export the extension entry, and NVTX counters +// stay tool-less no-ops in the target. +#if defined(__has_include) +# if __has_include() && \ + __has_include() +# define GPUFL_HAS_NVTX_COUNTERS 1 +# endif +#endif +#ifdef GPUFL_HAS_NVTX_COUNTERS +# include +# include +#endif #include "gpufl/gpufl.hpp" #include "gpufl/core/activity_record.hpp" #include "gpufl/core/common.hpp" #include "gpufl/core/debug_logger.hpp" // GFL_LOG_DEBUG (teardown tracing) #include "gpufl/core/monitor.hpp" +#include "gpufl/core/nvtx_counters.hpp" #include "gpufl/core/teardown_flag.hpp" // setProcessExitTeardown (Windows) #include "gpufl/upload/upload_logs.hpp" // gpufl::uploadLogs for --upload @@ -73,6 +96,11 @@ std::atomic g_deferred_init_finished{false}; std::atomic g_shutdown_started{false}; std::mutex g_deferred_init_mutex; std::condition_variable g_deferred_init_cv; +// A CUDA call made by gpufl's own deferred-init worker must never wait for +// that worker to finish. Most CUPTI setup does not call the runtime copy +// APIs, but the interpose layer must be safe if a driver/toolkit revision +// does: waiting here would be a self-deadlock. +thread_local bool g_is_deferred_init_worker = false; // Captured during init for the atexit upload path (--upload). g_log_path // mirrors InitOptions.log_path; both are read only in shutdownAndSignal, @@ -242,6 +270,7 @@ void waitForDeferredInit() { // these are compiled out to avoid unused-function diagnostics. void waitForDeferredInitForMs(const int wait_ms) { if (wait_ms <= 0) return; + if (g_is_deferred_init_worker) return; if (g_deferred_init_started.load(std::memory_order_acquire) && !g_deferred_init_finished.load(std::memory_order_acquire)) { std::unique_lock lock(g_deferred_init_mutex); @@ -259,6 +288,14 @@ void waitAtCudaSyncBoundary() { void waitAtCudaLaunchBoundary() { waitForDeferredInitForMs(envIntOrDefault(gpufl::env::kInjectLaunchWaitMs, 15000)); } + +void waitAtCudaMemoryBoundary() { + // A memory transfer is an activity-capture boundary for the same reason a + // launch is: if it crosses the target before CUPTI subscription completes, + // it is gone permanently. Reuse the launch timeout rather than introduce a + // second knob with identical semantics. + waitForDeferredInitForMs(envIntOrDefault(gpufl::env::kInjectLaunchWaitMs, 15000)); +} #endif // !_WIN32 @@ -394,6 +431,101 @@ int DomainRangePop(nvtxDomainHandle_t) { } } // namespace nvtx_injection_impl +// ── NVTX Counters extension (module 4) ──────────────────────────── +// +// The extension loader resolves InitializeInjectionNvtxExtension from the SAME +// NVTX_INJECTION64_PATH the launcher already sets for range injection +// (nvtxDetail/nvtxExtInit.h), so counters arrive as direct function calls into +// this library - no CUPTI activity records, and no second injection tool. That +// is what makes the standard NVIDIA API usable for a rule that fires thousands +// of times a second. +// +// These are the NVTX-shaped half only. Everything that decides what a sample +// MEANS lives in NvtxCounterBridge, which has no NVTX types in it and is unit +// tested without an injected process. +#ifdef GPUFL_HAS_NVTX_COUNTERS +namespace nvtx_counters_impl { + +using gpufl::detail::NvtxCounterBridge; + +// Every struct here was written by the CLIENT's copy of the NVTX headers, +// which can be a different version from ours. structSize gates every read: a +// smaller struct simply does not contain the field, and reading past it is +// reading someone else's memory that happens to parse. +bool AttrReadable(const nvtxCounterAttr_t* attr) { + return attr != nullptr && attr->structSize >= sizeof(nvtxCounterAttr_t); +} + +// Walk the semantics list for the counters extension and read the value type. +// Absent semantics stay Unspecified rather than defaulting to Delta: an +// absolute series accumulated as deltas produces a plausible wrong rate, which +// is worse than refusing the counter. +NvtxCounterBridge::ValueType valueTypeOf(const nvtxCounterAttr_t* attr) { + if (!AttrReadable(attr)) return NvtxCounterBridge::ValueType::Unspecified; + // Bounded walk: the list is application-built, and a cycle or a garbage + // `next` must not hang the register call. Nobody chains more than a + // handful of semantics; 16 is generous. + // + // The HEADER has to be readable before anything else is touched - + // including `next`, which the previous shape read in the loop increment + // even for a node it had just rejected as too short to contain it. A node + // that cannot hold its own header ends the walk rather than being skipped. + int walked = 0; + const nvtxSemanticsHeader_t* h = attr->semantics; + while (h != nullptr && walked++ < 16) { + if (h->structSize < sizeof(nvtxSemanticsHeader_t)) break; + const nvtxSemanticsHeader_t* next = h->next; + + // Exactly the version whose layout this build was compiled against. + // `>=` would read a future version's bytes through today's struct - + // the same plausible-wrong-value failure the DELTA check refuses. + if (h->semanticId == NVTX_SEMANTIC_ID_COUNTERS_V1 && + h->version == NVTX_COUNTER_SEMANTIC_VERSION && + h->structSize >= sizeof(nvtxSemanticsCounter_t)) { + const auto* sem = reinterpret_cast(h); + switch (sem->flags & NVTX_COUNTER_FLAG_VALUETYPE_DELTA_SINCE_START) { + case NVTX_COUNTER_FLAG_VALUETYPE_ABSOLUTE: + return NvtxCounterBridge::ValueType::Absolute; + case NVTX_COUNTER_FLAG_VALUETYPE_DELTA: + return NvtxCounterBridge::ValueType::Delta; + case NVTX_COUNTER_FLAG_VALUETYPE_DELTA_SINCE_START: + return NvtxCounterBridge::ValueType::DeltaSinceStart; + default: + break; // a counters semantic with no value type; keep walking + } + } + h = next; + } + return NvtxCounterBridge::ValueType::Unspecified; +} + +uint64_t NVTX_API CounterRegister(nvtxDomainHandle_t domain, + const nvtxCounterAttr_t* attr) { + const bool readable = AttrReadable(attr); + const char* name = (readable && attr->name != nullptr) ? attr->name : ""; + const uint64_t requested = + readable ? attr->counterId : NVTX_COUNTER_ID_NONE; + const auto result = NvtxCounterBridge::instance().registerCounter( + nvtxDomainName(domain), name, requested, valueTypeOf(attr)); + // 0 is a refusal the application can see: the header documents a register + // that could not be honoured, and every later sample carrying 0 is then + // dropped by the bridge instead of landing on someone else's slot. + return result.id; +} + +void NVTX_API CounterSampleInt64(nvtxDomainHandle_t, const uint64_t id, + const int64_t value) { + NvtxCounterBridge::instance().sampleDelta(id, value); +} + +void NVTX_API CounterSampleNoValue(nvtxDomainHandle_t, const uint64_t id, + const uint8_t reason) { + NvtxCounterBridge::instance().sampleNoValue(id, reason); +} + +} // namespace nvtx_counters_impl +#endif // GPUFL_HAS_NVTX_COUNTERS + void registerDeferredWaitAtexit() { std::call_once(g_deferred_wait_atexit_once, [] { // atexit handlers run in reverse registration order. Register this @@ -546,6 +678,7 @@ void startDeferredInjectInit() { } std::thread([] { + g_is_deferred_init_worker = true; // NVIDIA calls InitializeInjection from inside the CUDA driver's own // initialization. cuptiSubscribe probes driver state, so reaching it // while that initialization is still running faults inside libcuda. @@ -567,6 +700,7 @@ void startDeferredInjectInit() { // Injection must never throw through libcuda's callback path. } markDeferredInitFinished(); + g_is_deferred_init_worker = false; }).detach(); } @@ -574,6 +708,24 @@ void startDeferredInjectInit() { extern "C" { +#ifndef _WIN32 +// Cross-TU boundary hooks used by the typed CUDA interpose translation unit. +// They remain hidden inside libgpufl_inject; only CUDA ABI symbols are public. +__attribute__((visibility("hidden"))) +void GpuFlightWaitAtCudaLaunchBoundary() { + waitAtCudaLaunchBoundary(); +} + +__attribute__((visibility("hidden"))) +void GpuFlightWaitAtCudaSyncBoundary() { + waitAtCudaSyncBoundary(); +} + +__attribute__((visibility("hidden"))) +void GpuFlightWaitAtCudaMemoryBoundary() { + waitAtCudaMemoryBoundary(); +} +#endif GPUFL_INJECT_EXPORT int InitializeInjectionNvtx2( const NvtxGetExportTableFunc_t getExportTable) { @@ -607,6 +759,59 @@ GPUFL_INJECT_EXPORT int InitializeInjectionNvtx2( return 1; } +// NVTX extension-module entry: called once per extension module, on that +// module's first API call in the target, with a slot table to fill. Slots left +// untouched are turned into no-ops by the loader. +// +// Always returns success. Returning 0 makes the loader dlclose an injection +// library it loaded dynamically, and in a no-CUDA target this library can be +// held by exactly that reference while the deferred-init thread is still +// running. A module this build does not recognize gets no slots filled, which +// ends in the same no-ops as a failure return - without the unload. +#ifdef GPUFL_HAS_NVTX_COUNTERS +GPUFL_INJECT_EXPORT +int NVTX_API InitializeInjectionNvtxExtension(nvtxExtModuleInfo_t* module_info) { + // Every field below is written by the CLIENT's copy of the NVTX headers, + // which can be a different version from ours. Nothing is trusted before it + // is checked: structSize first, because it is what says the rest of the + // struct is even there to read. + if (module_info == nullptr || + module_info->structSize < sizeof(nvtxExtModuleInfo_t)) { + return 1; + } + if (module_info->moduleId != NVTX_EXT_COUNTERS_MODULEID || + module_info->compatId != NVTX_EXT_COUNTERS_COMPATID || + module_info->segments == nullptr) { + return 1; + } + + for (size_t s = 0; s < module_info->segmentsCount; ++s) { + nvtxExtModuleSegment_t* segment = module_info->segments + s; + // Counters declares exactly one segment (id 0). Filling slots in a + // segment we cannot name would be writing function pointers into a + // table whose layout we are guessing at. + if (segment->segmentId != 0 || segment->functionSlots == nullptr) { + continue; + } + const auto install = [segment](const size_t cbid, const intptr_t fn) { + if (cbid < segment->slotCount) segment->functionSlots[cbid] = fn; + }; + install(NVTX3EXT_CBID_nvtxCounterRegister, + reinterpret_cast(&nvtx_counters_impl::CounterRegister)); + install(NVTX3EXT_CBID_nvtxCounterSampleInt64, + reinterpret_cast(&nvtx_counters_impl::CounterSampleInt64)); + install(NVTX3EXT_CBID_nvtxCounterSampleNoValue, + reinterpret_cast(&nvtx_counters_impl::CounterSampleNoValue)); + // Float64, by-reference and batch stay unfilled: the registry is an + // unsigned integer accumulator, and the loader turns an unfilled slot + // into a no-op, which is the honest answer for a shape we cannot + // represent. Filling them to "not lose data" is how a float counter + // would start reporting a truncated rate nobody asked for. + } + return 1; +} +#endif // GPUFL_HAS_NVTX_COUNTERS + // First-chance entry: ld.so runs us before main(). // // **Disabled by default.** Phase 0.1 spike (2026-05-11) confirmed @@ -688,7 +893,7 @@ int GpuFlightInitializeInjectionAfterCuda() { return g_init_ok.load(std::memory_order_acquire) ? 1 : 0; } -#ifndef _WIN32 +#if !defined(_WIN32) && !defined(GPUFL_TYPED_CUDA_INTERPOSE) // Launch/sync symbol interposition (wait-for-init + forward) is Linux/glibc // only: it relies on LD_PRELOAD shadowing libcudart's symbols. Windows has // no preload interposition, so these wrappers don't exist there - Windows @@ -772,6 +977,6 @@ int cudaStreamSynchronize(void* stream) { static auto* fn = reinterpret_cast(dlsym(RTLD_NEXT, "cudaStreamSynchronize")); return fn ? fn(stream) : 0; } -#endif // !_WIN32 +#endif // !defined(_WIN32) && !defined(GPUFL_TYPED_CUDA_INTERPOSE) } // extern "C" diff --git a/include/gpufl/upload/upload_logs.cpp b/include/gpufl/upload/upload_logs.cpp index f7b7843..eac0963 100644 --- a/include/gpufl/upload/upload_logs.cpp +++ b/include/gpufl/upload/upload_logs.cpp @@ -1280,14 +1280,12 @@ UploadResult uploadLogs(const UploadOptions& opts) { // "%zu/%zu session(s)" which read like "F of S sessions // complete" but was actually mixing files (numerator) and // sessions (denominator) - fixed by labeling both axes. - std::fprintf(stderr, - "[gpufl::upload] %zu events uploaded (%zu MB), " - "%zu file(s), %zu session(s), %llds elapsed\n", - result.events_uploaded, + GFL_LOG_INFO("[upload] ", result.events_uploaded, + " events uploaded (", result.bytes_uploaded / (1024 * 1024), - result.files_processed, - targets.size(), - static_cast(total_elapsed)); + " MB), ", result.files_processed, " file(s), ", + targets.size(), " session(s), ", total_elapsed, + "s elapsed"); last_progress_time = now; bytes_since_last_progress = 0; }; diff --git a/scripts/counter_workload_bench.cu b/scripts/counter_workload_bench.cu new file mode 100644 index 0000000..fe49937 --- /dev/null +++ b/scripts/counter_workload_bench.cu @@ -0,0 +1,70 @@ +// Fixed-work decode-shaped target for the workload benchmark. +// +// Every configuration runs THIS binary with THESE arguments; only the rule +// configuration around it changes. Which config fires is decided by the +// threshold, not by reshaping the workload - a benchmark that slows the +// workload down to make the rule fire is measuring two changes at once. +// +// Fixed WORK (iterations), not fixed time: throughput and CPU time are only +// comparable when every run did the same thing. The kernel is deliberately +// small - a high iteration rate is the WORST case for per-iteration overhead, +// which is the thing being measured. +#include +#include +#include +#include + +#include +#include +#include + +__global__ void decode_step(float* x, int inner) { + float v = x[threadIdx.x]; + for (int i = 0; i < inner; ++i) v = v * 1.0001f + 0.5f; + x[threadIdx.x] = v; +} + +int main(int argc, char** argv) { + const long iters = argc > 1 ? std::atol(argv[1]) : 200000L; + const int tokens_per_step = argc > 2 ? std::atoi(argv[2]) : 32; + const int inner = argc > 3 ? std::atoi(argv[3]) : 200; + + nvtxDomainHandle_t domain = nvtxDomainCreateA("bench"); + nvtxSemanticsCounter_t sem = {}; + sem.header.structSize = sizeof(sem); + sem.header.semanticId = NVTX_SEMANTIC_ID_COUNTERS_V1; + sem.header.version = NVTX_COUNTER_SEMANTIC_VERSION; + sem.flags = NVTX_COUNTER_FLAG_VALUETYPE_DELTA; + sem.unit = "tokens"; + sem.unitScaleNumerator = 1; + sem.unitScaleDenominator = 1; + nvtxCounterAttr_t attr = {}; + attr.structSize = sizeof(attr); + attr.name = "tokens"; + attr.counterId = NVTX_COUNTER_ID_NONE; + attr.semantics = &sem.header; + const uint64_t counter = nvtxCounterRegister(domain, &attr); + + float* buf = nullptr; + cudaMalloc(&buf, 64 * sizeof(float)); + // Warm the context and JIT outside the timed region. + for (int i = 0; i < 200; ++i) decode_step<<<1, 64>>>(buf, inner); + cudaDeviceSynchronize(); + + const auto t0 = std::chrono::steady_clock::now(); + for (long i = 0; i < iters; ++i) { + decode_step<<<1, 64>>>(buf, inner); + cudaDeviceSynchronize(); + nvtxCounterSampleInt64(domain, counter, tokens_per_step); + } + const auto t1 = std::chrono::steady_clock::now(); + cudaFree(buf); + + const double secs = + std::chrono::duration_cast>(t1 - t0) + .count(); + std::printf("WL iters=%ld secs=%.3f iters_per_sec=%.1f tokens_per_sec=%.0f\n", + iters, secs, iters / secs, + iters / secs * tokens_per_step); + return 0; +} diff --git a/scripts/counter_workload_bench.py b/scripts/counter_workload_bench.py new file mode 100644 index 0000000..2bad094 --- /dev/null +++ b/scripts/counter_workload_bench.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""Workload-mode benchmark: evaluator cost on a fixed-work decode loop. + +Five configurations, identical target and arguments; only the rule differs. +Which one fires is decided by the threshold, never by reshaping the workload. + + bare no gpufl at all - the floor + no_evaluator gpufl trace, PM prepared+dormant (--deep-after far future), + NO rule: the profiling baseline the evaluator adds onto + rule_missing rule on a counter that never registers: evaluator polls a + Missing source all run + rule_armed rule on the live counter, threshold below any real rate: + full metric pipeline, condition never true + rule_fires threshold above the steady rate: condition true from warmup, + exactly one PM deep window fires mid-run + +Paired randomized blocks: each block runs all five in a shuffled order, and +ratios are computed within the block before aggregating. +""" +import csv +import gzip +import json +import os +import random +import re +import shutil +import statistics +import subprocess +import sys + +HOME = os.path.expanduser("~") +GPUFL = HOME + "/sources/gpufl-client/build/daemon/launcher/gpufl" +TARGET = "/tmp/wl/target" +OUTROOT = "/tmp/wl/runs" +CSV_PATH = "/tmp/wl/results.csv" +BLOCKS = int(sys.argv[1]) if len(sys.argv) > 1 else 10 +ITERS = sys.argv[2] if len(sys.argv) > 2 else "12000" +TOKENS = "32" +RULE_LIVE = "custom.bench.tokens_rate" +DEEP = ["--deep-for", "2s", "--deep-cooldown", "600s"] + + +def config_cmd(name, outdir): + base = ["/usr/bin/time", "-v"] + tgt = [TARGET, ITERS, TOKENS, "400000"] + if name == "bare": + return base + tgt + launch = base + [GPUFL, "trace", "-o", outdir] + if name == "no_evaluator": + return launch + ["--deep-after", "100000s"] + DEEP + ["--"] + tgt + if name == "rule_missing": + return launch + ["--deep-when", "custom.bench.missing_rate<1 for 1s"] + DEEP + ["--"] + tgt + if name == "rule_armed": + return launch + ["--deep-when", RULE_LIVE + "<1 for 1s"] + DEEP + ["--"] + tgt + if name == "rule_fires": + return launch + ["--deep-when", RULE_LIVE + "<999999999 for 1s"] + DEEP + ["--"] + tgt + raise ValueError(name) + + +CONFIGS = ["bare", "no_evaluator", "rule_missing", "rule_armed", "rule_fires"] + + +def parse_time_v(err): + user = sys_t = rss_kb = None + for line in err.splitlines(): + if "User time (seconds):" in line: + user = float(line.split(":")[-1]) + elif "System time (seconds):" in line: + sys_t = float(line.split(":")[-1]) + elif "Maximum resident set size" in line: + rss_kb = int(line.split(":")[-1]) + return user, sys_t, rss_kb + + +def scan_trace(outdir): + windows = 0 + outcome = "" + if not os.path.isdir(outdir): + return windows, outcome + for root, _dirs, files in os.walk(outdir): + for f in files: + if not f.endswith(".log.gz"): + continue + try: + with gzip.open(os.path.join(root, f), "rt", errors="replace") as fh: + for line in fh: + if '"type":"deep_window_event"' in line: + windows += 1 + elif '"type":"deep_window_rule_summary"' in line: + m = re.search(r'"outcome":"([a-z_]+)"', line) + if m: + outcome = m.group(1) + except OSError: + pass + return windows, outcome + + +def one_run(name, block): + outdir = "%s/%s_b%d" % (OUTROOT, name, block) + shutil.rmtree(outdir, ignore_errors=True) + cmd = config_cmd(name, outdir) + proc = subprocess.run(cmd, capture_output=True, text=True) + m = re.search(r"iters_per_sec=([0-9.]+)", proc.stdout) + if proc.returncode != 0 or not m: + print("RUN FAILED", name, "rc=", proc.returncode, file=sys.stderr) + print(proc.stdout[-2000:], file=sys.stderr) + print(proc.stderr[-2000:], file=sys.stderr) + sys.exit(1) + ips = float(m.group(1)) + wall = float(re.search(r"secs=([0-9.]+)", proc.stdout).group(1)) + user, sys_t, rss_kb = parse_time_v(proc.stderr) + windows, outcome = scan_trace(outdir) + shutil.rmtree(outdir, ignore_errors=True) + return { + "block": block, "config": name, "iters_per_sec": ips, "wall_s": wall, + "cpu_s": round((user or 0) + (sys_t or 0), 3), + "maxrss_mb": round((rss_kb or 0) / 1024.0, 1), + "windows": windows, "outcome": outcome, + } + + +def governor(): + try: + with open("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor") as f: + return f.read().strip() + except OSError: + return "unknown" + + +def boot_ci(values, iters=10000): + n = len(values) + meds = sorted(statistics.median(random.choices(values, k=n)) + for _ in range(iters)) + return meds[int(0.025 * iters)], meds[int(0.975 * iters)] + + +def main(): + random.seed(20260728) + os.makedirs(OUTROOT, exist_ok=True) + print("governor:", governor(), flush=True) + rows = [] + for block in range(BLOCKS): + order = CONFIGS[:] + random.shuffle(order) + print("block %d order: %s" % (block, " ".join(order)), flush=True) + for name in order: + row = one_run(name, block) + rows.append(row) + print(" %-13s %9.1f it/s cpu %6.2fs rss %7.1fMB win %d %s" + % (row["config"], row["iters_per_sec"], row["cpu_s"], + row["maxrss_mb"], row["windows"], row["outcome"]), + flush=True) + + # ── gates ─────────────────────────────────────────────────────────── + failures = [] + expected_rows = BLOCKS * len(CONFIGS) + if len(rows) != expected_rows: + failures.append("row count %d != %d" % (len(rows), expected_rows)) + + # windows / outcome each configuration MUST show. no_evaluator and bare + # have no rule, so no rule summary may exist at all. + expect = { + "bare": (0, ""), + "no_evaluator": (0, ""), + "rule_missing": (0, "never_true"), + "rule_armed": (0, "never_true"), + "rule_fires": (1, "fired"), + } + for r in rows: + want_win, want_outcome = expect[r["config"]] + if r["windows"] != want_win or r["outcome"] != want_outcome: + failures.append( + "block %d %s: windows=%d outcome=%r (want %d %r)" % + (r["block"], r["config"], r["windows"], r["outcome"], + want_win, want_outcome)) + + if failures: + for f in failures: + print("GATE FAILED:", f, file=sys.stderr) + sys.exit(1) + + with open(CSV_PATH, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) + + by = {} + for r in rows: + by.setdefault(r["config"], []).append(r) + + print("\n== medians over %d blocks (iters=%s) ==" % (BLOCKS, ITERS)) + print("%-13s %10s %8s %9s %7s %s" % + ("config", "it/s", "cpu_s", "rss_MB", "windows", "outcome")) + for name in CONFIGS: + rs = by.get(name, []) + if not rs: + continue + print("%-13s %10.1f %8.2f %9.1f %7d %s" % ( + name, + statistics.median(x["iters_per_sec"] for x in rs), + statistics.median(x["cpu_s"] for x in rs), + statistics.median(x["maxrss_mb"] for x in rs), + max(x["windows"] for x in rs), + rs[0]["outcome"])) + + print("\n== paired throughput ratios (per block, vs no_evaluator) ==") + base = {r["block"]: r["iters_per_sec"] for r in by.get("no_evaluator", [])} + for name in CONFIGS: + if name == "no_evaluator": + continue + ratios = [r["iters_per_sec"] / base[r["block"]] + for r in by.get(name, []) if r["block"] in base] + if not ratios: + continue + med = statistics.median(ratios) + lo, hi = boot_ci(ratios) + print(" %-13s x%.4f [%.4f, %.4f]" % (name, med, lo, hi)) + + print("\ngovernor after:", governor()) + print("WL_DRIVER_DONE all_gates_passed rows=%d" % len(rows)) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_early_memcpy_capture.py b/scripts/verify_early_memcpy_capture.py new file mode 100644 index 0000000..6688ea4 --- /dev/null +++ b/scripts/verify_early_memcpy_capture.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Verify the injected early-memcpy regression capture from raw NDJSON logs.""" + +from __future__ import annotations + +import argparse +import gzip +import json +from pathlib import Path +from typing import Iterable, TextIO + + +EXPECTED_BYTES = 4_194_304 + + +def open_text(path: Path) -> TextIO: + if path.suffix == ".gz": + return gzip.open(path, "rt", encoding="utf-8") + return path.open("r", encoding="utf-8") + + +def events(root: Path) -> Iterable[dict]: + for path in sorted(root.rglob("*.log")) + sorted(root.rglob("*.log.gz")): + with open_text(path) as stream: + for line_number, line in enumerate(stream, 1): + line = line.strip() + if not line: + continue + try: + yield json.loads(line) + except json.JSONDecodeError as exc: + raise RuntimeError(f"{path}:{line_number}: invalid JSON: {exc}") from exc + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("log_root", type=Path) + args = parser.parse_args() + + matching = {1: [], 2: []} + batch_count = 0 + kernel_rows = 0 + synchronization_rows = 0 + for event in events(args.log_root): + event_type = event.get("type") + if event_type == "kernel_event_batch": + kernel_rows += len(event.get("rows", [])) + continue + if event_type == "synchronization_event_batch": + synchronization_rows += len(event.get("rows", [])) + continue + if event_type != "memcpy_event_batch": + continue + batch_count += 1 + columns = event.get("columns", []) + try: + bytes_index = columns.index("bytes") + kind_index = columns.index("copy_kind") + except ValueError as exc: + raise RuntimeError("memcpy_event_batch is missing bytes/copy_kind") from exc + for row in event.get("rows", []): + kind = int(row[kind_index]) + size = int(row[bytes_index]) + if kind in matching and size == EXPECTED_BYTES: + matching[kind].append(size) + + print( + "memcpy capture:" + f" batches={batch_count}" + f" H2D_count={len(matching[1])} H2D_bytes={sum(matching[1])}" + f" D2H_count={len(matching[2])} D2H_bytes={sum(matching[2])}" + f" kernel_rows={kernel_rows}" + f" synchronization_rows={synchronization_rows}" + ) + if matching[1] != [EXPECTED_BYTES] or matching[2] != [EXPECTED_BYTES]: + print("VERIFY FAIL: expected exactly one 4 MiB H2D and one 4 MiB D2H") + return 1 + if kernel_rows < 1 or synchronization_rows < 1: + print("VERIFY FAIL: expected kernel and synchronization capture to remain active") + return 1 + print("VERIFY PASS") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3ec7bc7..7ca2d86 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -19,6 +19,8 @@ set(GPUFL_TEST_SOURCES core/test_disabled.cpp core/test_wire_contract.cpp core/test_counter_registry.cpp + core/test_nvtx_counters.cpp + core/test_debug_logger.cpp core/test_metric_registry.cpp core/test_deep_window_rule.cpp core/test_deep_window_rules_install.cpp @@ -75,6 +77,18 @@ endif() add_executable(gpufl_tests ${GPUFL_TEST_SOURCES}) +# Standalone target for the Linux injection readiness regression. It +# intentionally performs its H2D copy immediately after two allocations, before +# the first launch boundary, so GPUFL_INJECT_INIT_DELAY_MS exposes any missing +# memcpy wait. The verifier is run explicitly on a CUDA host because this is an +# injected-process E2E, not a hermetic unit test. +if(UNIX AND NOT APPLE AND GPUFL_HAS_CUDA AND TARGET gpufl_inject) + add_executable(gpufl_early_memcpy_target + common/early_memcpy_target.cu + ) + target_link_libraries(gpufl_early_memcpy_target PRIVATE CUDA::cudart) +endif() + # 3. Enable CUDA-specific test target behavior only when CUDA is available if(GPUFL_HAS_CUDA) set_target_properties(gpufl_tests PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON) diff --git a/tests/common/early_memcpy_target.cu b/tests/common/early_memcpy_target.cu new file mode 100644 index 0000000..baf0bce --- /dev/null +++ b/tests/common/early_memcpy_target.cu @@ -0,0 +1,69 @@ +#include + +#include +#include +#include + +namespace { + +constexpr std::size_t kElements = 1u << 20; +constexpr std::size_t kBytes = kElements * sizeof(float); + +__global__ void AddOne(const float* input, float* output, std::size_t count) { + const std::size_t index = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (index < count) output[index] = input[index] + 1.0f; +} + +bool Check(cudaError_t result, const char* operation) { + if (result == cudaSuccess) return true; + std::fprintf(stderr, "%s failed: %s\n", operation, cudaGetErrorString(result)); + return false; +} + +} // namespace + +int main() { + std::vector input(kElements, 2.0f); + std::vector output(kElements, 0.0f); + float* device_input = nullptr; + float* device_output = nullptr; + + // The ordering is load-bearing for the regression: + // first allocation -> cuInit/injection callback -> deferred init; + // second allocation -> immediate H2D before any launch/sync wrapper. + if (!Check(cudaMalloc(&device_input, kBytes), "cudaMalloc(input)") || + !Check(cudaMalloc(&device_output, kBytes), "cudaMalloc(output)") || + !Check(cudaMemcpy(device_input, input.data(), kBytes, + cudaMemcpyHostToDevice), + "cudaMemcpy(H2D)")) { + cudaFree(device_input); + cudaFree(device_output); + return 1; + } + + AddOne<<<(kElements + 255) / 256, 256>>>( + device_input, device_output, kElements); + if (!Check(cudaGetLastError(), "AddOne launch") || + !Check(cudaDeviceSynchronize(), "cudaDeviceSynchronize") || + !Check(cudaMemcpy(output.data(), device_output, kBytes, + cudaMemcpyDeviceToHost), + "cudaMemcpy(D2H)")) { + cudaFree(device_input); + cudaFree(device_output); + return 1; + } + + const bool correct = + std::fabs(output.front() - 3.0f) < 1e-6f && + std::fabs(output.back() - 3.0f) < 1e-6f; + cudaFree(device_input); + cudaFree(device_output); + if (!correct) { + std::fprintf(stderr, "VERIFY FAIL\n"); + return 2; + } + + std::printf("VERIFY PASS bytes=%zu\n", kBytes); + return 0; +} diff --git a/tests/core/test_debug_logger.cpp b/tests/core/test_debug_logger.cpp new file mode 100644 index 0000000..5d7c5d1 --- /dev/null +++ b/tests/core/test_debug_logger.cpp @@ -0,0 +1,42 @@ +#include + +#include "gpufl/core/debug_logger.hpp" + +namespace { + +class DebugLoggerTest : public ::testing::Test { + protected: + void SetUp() override { gpufl::DebugLogger::setEnabled(false); } + void TearDown() override { gpufl::DebugLogger::setEnabled(false); } +}; + +TEST_F(DebugLoggerTest, InfoIsAlwaysVisibleWithoutSourceLocation) { + testing::internal::CaptureStderr(); + GFL_LOG_INFO("prepared ", 3, " engines"); + const std::string output = testing::internal::GetCapturedStderr(); + + EXPECT_EQ(output, "[GPUFL] prepared 3 engines\n"); + EXPECT_EQ(output.find(__FILE__), std::string::npos); +} + +TEST_F(DebugLoggerTest, WarningIsAlwaysVisibleWithoutSourceLocation) { + testing::internal::CaptureStderr(); + GFL_LOG_WARN("sampling returned no data"); + const std::string output = testing::internal::GetCapturedStderr(); + + EXPECT_EQ(output, "[GPUFL-WARN] sampling returned no data\n"); + EXPECT_EQ(output.find(__FILE__), std::string::npos); +} + +TEST_F(DebugLoggerTest, DebugRemainsControlledByTheVerboseFlag) { + testing::internal::CaptureStdout(); + GFL_LOG_DEBUG("hidden"); + EXPECT_TRUE(testing::internal::GetCapturedStdout().empty()); + + gpufl::DebugLogger::setEnabled(true); + testing::internal::CaptureStdout(); + GFL_LOG_DEBUG("visible"); + EXPECT_EQ(testing::internal::GetCapturedStdout(), "[GPUFL] visible\n"); +} + +} // namespace diff --git a/tests/core/test_nvtx_counters.cpp b/tests/core/test_nvtx_counters.cpp new file mode 100644 index 0000000..7bfd1dd --- /dev/null +++ b/tests/core/test_nvtx_counters.cpp @@ -0,0 +1,453 @@ +#include + +#include + +#include "gpufl/core/counter_provider.hpp" +#include "gpufl/core/counter_registry.hpp" +#include "gpufl/core/metric_id.hpp" +#include "gpufl/core/metric_registry.hpp" +#include "gpufl/core/model/deep_window_model.hpp" +#include "gpufl/core/nvtx_counters.hpp" + +using gpufl::detail::ActiveCounterProvider; +using gpufl::detail::CounterRegistry; +using gpufl::detail::NvtxCounterBridge; +using ValueType = NvtxCounterBridge::ValueType; +using RegisterStatus = NvtxCounterBridge::RegisterStatus; + +namespace { + +// NVTX_COUNTER_SAMPLE_* from nvToolsExtCounters.h, restated so a change to the +// released ABI shows up here rather than as a counter that quietly stops +// matching. +constexpr uint8_t kZero = 0; +constexpr uint8_t kUnchanged = 1; +constexpr uint8_t kUnavailable = 2; + +class NvtxCounterBridgeTest : public ::testing::Test { + protected: + void SetUp() override { + CounterRegistry::instance().resetForTesting(); + NvtxCounterBridge::instance().resetForTesting(); + } + void TearDown() override { + NvtxCounterBridge::instance().resetForTesting(); + CounterRegistry::instance().resetForTesting(); + } + + NvtxCounterBridge& bridge() { return NvtxCounterBridge::instance(); } + + /// What a rule would read: the value accrued since this session started. + uint64_t observed(const std::string& name) { + const auto* provider = ActiveCounterProvider(); + gpufl_counter_handle h = provider->lookup(name.c_str(), name.size()); + return h == nullptr ? 0 : provider->load_since_baseline(h); + } + + NvtxCounterBridge::RegisterResult acceptDelta(const char* domain, + const char* name) { + auto r = bridge().registerCounter(domain, name, 0, ValueType::Delta); + EXPECT_EQ(r.status, RegisterStatus::Accepted); + return r; + } +}; + +// ── what a sample MEANS ───────────────────────────────────────────────────── + +TEST_F(NvtxCounterBridgeTest, DeltaSamplesAccumulateIntoTheRegistry) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + + bridge().sampleDelta(r.id, 8); + bridge().sampleDelta(r.id, 16); + + // A rate is derived from this by the metric registry; what the bridge owes + // is the running total the application actually reported. + EXPECT_EQ(observed("inference.tokens"), 24u); +} + +TEST_F(NvtxCounterBridgeTest, ANoValueSampleIsNotAnEvent) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + bridge().sampleDelta(r.id, 10); + + bridge().sampleNoValue(r.id, kZero); + bridge().sampleNoValue(r.id, kUnchanged); + + // ZERO and UNCHANGED both say the delta was zero. Counting either as +1 + // would invent traffic out of a sample whose entire purpose is to report + // that there was none - and would make an idle workload look busy to the + // rule watching it. + EXPECT_EQ(observed("inference.tokens"), 10u); + EXPECT_EQ(bridge().unavailableSamples(), 0u); +} + +TEST_F(NvtxCounterBridgeTest, AnUnavailableSampleIsRecordedNotCounted) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + bridge().sampleDelta(r.id, 10); + + bridge().sampleNoValue(r.id, kUnavailable); + + // The application could not read its own counter. That must not move the + // total, but it must not vanish either: the resulting rate rests on less + // data than its sample count suggests. + EXPECT_EQ(observed("inference.tokens"), 10u); + EXPECT_EQ(bridge().unavailableSamples(), 1u); +} + +TEST_F(NvtxCounterBridgeTest, ANegativeDeltaIsDroppedRatherThanWrapped) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + bridge().sampleDelta(r.id, 10); + + bridge().sampleDelta(r.id, -4); + + // The registry accumulates unsigned and rates are unsigned deltas, so + // subtracting here would not read as -4; the very next bucket would report + // a rate near 2^64. + EXPECT_EQ(observed("inference.tokens"), 10u); + EXPECT_EQ(bridge().negativeSamples(), 1u); +} + +// ── what is refused, and why ──────────────────────────────────────────────── + +TEST_F(NvtxCounterBridgeTest, ACounterWithNoSemanticsIsRefused) { + // nvtxCounterSampleInt64 says the value is an int64; it does not say + // whether it is a delta or an absolute reading. Accumulating an absolute + // series would produce a rate that looks reasonable and is wrong. + const auto r = bridge().registerCounter("d", "c", 0, ValueType::Unspecified); + EXPECT_EQ(r.status, RegisterStatus::UnsupportedValueType); + EXPECT_EQ(r.id, 0u); +} + +TEST_F(NvtxCounterBridgeTest, AnAbsoluteCounterIsRefused) { + EXPECT_EQ(bridge().registerCounter("d", "c", 0, ValueType::Absolute).status, + RegisterStatus::UnsupportedValueType); + EXPECT_EQ(bridge() + .registerCounter("d", "c", 0, ValueType::DeltaSinceStart) + .status, + RegisterStatus::UnsupportedValueType); +} + +TEST_F(NvtxCounterBridgeTest, AnApplicationAssignedIdIsRefused) { + // An NVTX counter id is unique only WITHIN its domain. Binding one to a + // slot without a (domain, id) table would let two domains that both chose + // id 123 share a slot and add their rates together. + const auto r = bridge().registerCounter("d", "c", 1u << 24, ValueType::Delta); + EXPECT_EQ(r.status, RegisterStatus::StaticIdUnsupported); + EXPECT_EQ(r.id, 0u); +} + +TEST_F(NvtxCounterBridgeTest, SamplesForAnIdWeNeverIssuedGoNowhere) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + + // A refused registration returns 0, and the application keeps sampling. + bridge().sampleDelta(0, 99); + bridge().sampleDelta(1u << 24, 99); // a static id + bridge().sampleDelta(r.id + 1000, 99); // past the table + + EXPECT_EQ(observed("inference.tokens"), 0u) + << "an unknown id landed on someone else's slot"; + EXPECT_EQ(bridge().unknownIdSamples(), 3u); +} + +// ── naming ────────────────────────────────────────────────────────────────── + +TEST_F(NvtxCounterBridgeTest, TheDomainIsPartOfTheName) { + // Two teams each calling their counter "tokens" in their own domain are + // two counters, and a rule has to be able to name one of them. + const auto a = acceptDelta("inference", "tokens"); + const auto b = acceptDelta("training", "tokens"); + EXPECT_NE(a.id, b.id); + EXPECT_EQ(a.metric, "custom.inference.tokens_rate"); + EXPECT_EQ(b.metric, "custom.training.tokens_rate"); +} + +TEST_F(NvtxCounterBridgeTest, AnUndomainedCounterKeepsItsBareName) { + EXPECT_EQ(NvtxCounterBridge::canonicalName("", "tokens"), "tokens"); +} + +TEST_F(NvtxCounterBridgeTest, OutOfCharsetBytesAreMappedNotDropped) { + // Rules address counters as custom._rate over [A-Za-z0-9._-], and + // NVTX names are free-form. + EXPECT_EQ(NvtxCounterBridge::canonicalName("My Server", "tokens/sec"), + "My_Server.tokens_sec"); + // Nothing usable left is a refusal, not a metric called custom.___rate. + EXPECT_TRUE(NvtxCounterBridge::canonicalName("", "///").empty()); + EXPECT_TRUE(NvtxCounterBridge::canonicalName("", "").empty()); +} + +TEST_F(NvtxCounterBridgeTest, TwoNamesThatCanonicaliseAlikeAreRefused) { + ActiveCounterProvider()->begin_session(); + // "a b" and "a/b" both canonicalise to "a_b", but they are DIFFERENT NVTX + // counters. Sharing a binding would silently add two unrelated workloads + // into one rate - a wrong number that looks like a real one - so the + // second registration is refused and told to rename. + const auto a = acceptDelta("", "a b"); + const auto b = bridge().registerCounter("", "a/b", 0, ValueType::Delta); + EXPECT_EQ(b.status, RegisterStatus::BadName); + EXPECT_EQ(b.id, 0u); + EXPECT_EQ(bridge().trackedCount(), 1u); + + bridge().sampleDelta(a.id, 3); + EXPECT_EQ(observed("a_b"), 3u) << "the refused counter leaked into the slot"; +} + +TEST_F(NvtxCounterBridgeTest, TheDomainJoinIsACollisionAxisToo) { + // The domain joins with '.', so ("a", "b.c") and ("a.b", "c") meet at + // "a.b.c" without any out-of-charset byte involved. Only the original + // (domain, counter) pair can tell them apart. + const auto first = acceptDelta("a", "b.c"); + const auto second = bridge().registerCounter("a.b", "c", 0, ValueType::Delta); + EXPECT_EQ(first.metric, "custom.a.b.c_rate"); + EXPECT_EQ(second.status, RegisterStatus::BadName); + EXPECT_EQ(bridge().trackedCount(), 1u); +} + +TEST_F(NvtxCounterBridgeTest, RegisteringTheSameCounterTwiceIsIdempotent) { + // Same ORIGINAL pair - not merely the same canonical name - is the one + // case that returns the existing binding. + const auto first = acceptDelta("inference", "tokens"); + const auto again = acceptDelta("inference", "tokens"); + EXPECT_EQ(first.id, again.id); + EXPECT_EQ(bridge().trackedCount(), 1u); +} + +// ── the failed-read contract, all the way to the metric layer ─────────────── + +TEST_F(NvtxCounterBridgeTest, AnUnavailableSampleDiscardsTheRateWindow) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + + const auto parsed = gpufl::detail::parseMetric("custom.inference.tokens_rate"); + ASSERT_EQ(parsed.error, gpufl::detail::MetricParseError::None); + gpufl::detail::MetricWindowConfig cfg; + cfg.rate_window_ms = 100; // bucket = 10ms, 10 buckets + cfg.stale_after_ms = 20000; + gpufl::detail::MetricFeeds feeds; + gpufl::detail::MetricSource src(parsed.id, cfg, &feeds, + ActiveCounterProvider()); + + constexpr int64_t kStep = 10 * 1000000; // one bucket interval + int64_t t = 0; + gpufl::detail::MetricSample s; + + // Steady traffic until the window is full and the metric is usable. + int polls = 0; + do { + bridge().sampleDelta(r.id, 5); + s = src.poll(t); + t += kStep; + } while (s.state != gpufl::detail::MetricState::Fresh && ++polls < 40); + ASSERT_EQ(s.state, gpufl::detail::MetricState::Fresh) + << "never reached a usable reading; the fixture is wrong"; + + // The application fails to read its counter. Real throughput has NOT + // dropped - only the observation did. Without the discard, the missing + // deltas read as a rate collapse and a stall rule fires on a workload + // that never slowed down. + bridge().sampleNoValue(r.id, kUnavailable); + s = src.poll(t); + t += kStep; + EXPECT_EQ(s.state, gpufl::detail::MetricState::WarmingUp) + << "a window containing a failed read was presented as evidence"; + + // Recovery: the window must refill from post-failure data before the + // metric is usable again - a new baseline, not a resumed one. + int refill_polls = 0; + bool fresh_before_refill = false; + do { + bridge().sampleDelta(r.id, 5); + s = src.poll(t); + t += kStep; + ++refill_polls; + if (s.state == gpufl::detail::MetricState::Fresh && refill_polls < 10) { + fresh_before_refill = true; + } + } while (s.state != gpufl::detail::MetricState::Fresh && refill_polls < 40); + EXPECT_EQ(s.state, gpufl::detail::MetricState::Fresh) + << "the metric never recovered after the failed read"; + EXPECT_FALSE(fresh_before_refill) + << "usable again before a full window of post-failure data"; +} + +// ── the routing that keeps the evaluator and the target together ──────────── + +TEST_F(NvtxCounterBridgeTest, TheCounterIsVisibleThroughTheActiveProvider) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + bridge().sampleDelta(r.id, 5); + + // The evaluator reads whatever ActiveCounterProvider() resolves. Writing + // straight to this module's CounterRegistry instead would split the two + // apart wherever a shared runtime is present - being in the same injection + // module as the evaluator is not enough to assume otherwise. + const std::string name = "inference.tokens"; + gpufl_counter_handle via_provider = + ActiveCounterProvider()->lookup(name.c_str(), name.size()); + ASSERT_NE(via_provider, nullptr); + EXPECT_EQ(ActiveCounterProvider()->load_since_baseline(via_provider), 5u); +} + +TEST_F(NvtxCounterBridgeTest, TicksBeforeTheSessionStartsAreNotCounted) { + // Registration and the first samples can legitimately arrive before + // gpufl::init(): NVTX fires on the application's first NVTX call, which + // may precede any CUDA call. + const auto r = acceptDelta("inference", "tokens"); + bridge().sampleDelta(r.id, 100); + + ActiveCounterProvider()->begin_session(); + bridge().sampleDelta(r.id, 7); + + EXPECT_EQ(observed("inference.tokens"), 7u) + << "pre-session traffic leaked into this session's rate"; +} + +// ── the session data-quality contract ─────────────────────────────────────── + +TEST_F(NvtxCounterBridgeTest, EveryRefusedRegistrationIsCountedOnce) { + bridge().registerCounter("d", "c1", 1u << 24, ValueType::Delta); // static id + bridge().registerCounter("d", "c2", 0, ValueType::Absolute); // value type + bridge().registerCounter("", "///", 0, ValueType::Delta); // bad name + acceptDelta("", "a b"); + bridge().registerCounter("", "a/b", 0, ValueType::Delta); // collision + EXPECT_EQ(bridge().registrationRejected(), 4u) + << "a refusal path forgot to count itself"; + // The accepted one, and its idempotent repeat, are not refusals. + acceptDelta("", "a b"); + EXPECT_EQ(bridge().registrationRejected(), 4u); +} + +TEST_F(NvtxCounterBridgeTest, ASessionSnapshotReportsOnlyItsOwnSession) { + // The tallies live for the process, like the counter slots; an embedded + // host re-initialises in one process, and exporting raw totals would + // re-report session one's problems as session two's. + bridge().registerCounter("d", "c", 0, ValueType::Absolute); + const auto r = acceptDelta("inference", "tokens"); + bridge().sampleDelta(r.id, -1); + bridge().sampleNoValue(r.id, kUnavailable); + + auto first = bridge().takeSessionSnapshot(); + EXPECT_EQ(first.registration_rejected, 1u); + EXPECT_EQ(first.negative_delta_samples, 1u); + EXPECT_EQ(first.unavailable_samples, 1u); + EXPECT_TRUE(first.any()); + + // Session two: only what happened after the previous report. + bridge().sampleDelta(0, 5); // unknown id + auto second = bridge().takeSessionSnapshot(); + EXPECT_EQ(second.registration_rejected, 0u) + << "session one's refusal was reported twice"; + EXPECT_EQ(second.unknown_id_samples, 1u); + EXPECT_EQ(second.unavailable_samples, 0u); + + // Session three: nothing happened, nothing to say. + EXPECT_FALSE(bridge().takeSessionSnapshot().any()); +} + +TEST_F(NvtxCounterBridgeTest, PreInitEventsBelongToTheFirstSession) { + // NVTX registration legitimately runs before gpufl::init() - proven on + // hardware - so a refusal during startup happens before any session + // exists. It belongs to the first session that reports: no other session + // can, and dropping it hides exactly the config error the event is for. + bridge().registerCounter("d", "c", 0, ValueType::Unspecified); + + // ... gpufl::init() happens here ... + const auto snap = bridge().takeSessionSnapshot(); + EXPECT_EQ(snap.registration_rejected, 1u) + << "a startup config error was attributed to no session at all"; +} + +// ── the wire shape ────────────────────────────────────────────────────────── +// +// The backend parses these by field name from hand-built JSON; nothing in the +// type system connects the two sides. Pinned here so a renamed field fails a +// test instead of silently landing as an empty column. + +TEST(CounterDataQualityWireTest, TheSummaryCarriesEveryTallyByName) { + gpufl::CounterDataQualitySummaryEvent ev; + ev.pid = 7; + ev.app = "serve"; + ev.session_id = "s1"; + ev.tracked_counters = 1; + ev.samples_observed = 12000; + ev.registration_rejected = 2; + ev.unknown_id_samples = 3; + ev.unavailable_samples = 4; + ev.negative_delta_samples = 1; + ev.rate_windows_discarded = 2; + ev.emitted_ns = 42; + + const std::string json = + gpufl::model::CounterDataQualitySummaryModel(ev).buildJson(); + EXPECT_NE(json.find("\"type\":\"counter_data_quality_summary\""), + std::string::npos) << json; + // Only "nvtx" is observed today; a generic-looking row would claim + // coverage of gpufl::counter() failures it does not have. + EXPECT_NE(json.find("\"source\":\"nvtx\""), std::string::npos); + EXPECT_NE(json.find("\"schema_version\":1"), std::string::npos); + // The denominators that make an all-zero row readable at all. + EXPECT_NE(json.find("\"tracked_counters\":1"), std::string::npos); + EXPECT_NE(json.find("\"samples_observed\":12000"), std::string::npos); + EXPECT_NE(json.find("\"registration_rejected\":2"), std::string::npos); + EXPECT_NE(json.find("\"unknown_id_samples\":3"), std::string::npos); + EXPECT_NE(json.find("\"unavailable_samples\":4"), std::string::npos); + EXPECT_NE(json.find("\"negative_delta_samples\":1"), std::string::npos); + EXPECT_NE(json.find("\"rate_windows_discarded\":2"), std::string::npos); +} + +TEST(CounterDataQualityWireTest, TheRuleSummaryCarriesItsOwnQualityFields) { + gpufl::DeepWindowRuleSummaryEvent ev; + ev.session_id = "s1"; + ev.outcome = "never_true"; + ev.metric_quality_resets = 2; + ev.last_quality_reason = "counter_unavailable"; + + const std::string json = + gpufl::model::DeepWindowRuleSummaryModel(ev).buildJson(); + EXPECT_NE(json.find("\"metric_quality_resets\":2"), std::string::npos) + << json; + EXPECT_NE(json.find("\"last_quality_reason\":\"counter_unavailable\""), + std::string::npos); +} + +TEST_F(NvtxCounterBridgeTest, SamplesObservedCountsValidObservationsOnly) { + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + + bridge().sampleDelta(r.id, 8); + bridge().sampleDelta(r.id, 0); // real observation of "no traffic" + bridge().sampleNoValue(r.id, kZero); // ditto + bridge().sampleNoValue(r.id, kUnchanged); // ditto + bridge().sampleNoValue(r.id, kUnavailable); // a FAILURE, not an observation + bridge().sampleDelta(r.id, -1); // ditto + bridge().sampleDelta(0, 5); // unknown id: ditto + + const auto snap = bridge().takeSessionSnapshot(); + // 8 + 0 + zero + unchanged = 4 observations; the three failures are + // tallied on their own axes, or the failure RATE would be diluted by its + // own failures. + EXPECT_EQ(snap.samples_observed, 4u); + EXPECT_EQ(snap.unavailable_samples, 1u); + EXPECT_EQ(snap.negative_delta_samples, 1u); + EXPECT_EQ(snap.unknown_id_samples, 1u); +} + +TEST_F(NvtxCounterBridgeTest, SamplesObservedIsSessionScopedToo) { + // The stale-clean-row problem: a counter registered in session one must + // not make every later session claim it observed something. + ActiveCounterProvider()->begin_session(); + const auto r = acceptDelta("inference", "tokens"); + bridge().sampleDelta(r.id, 8); + EXPECT_EQ(bridge().takeSessionSnapshot().samples_observed, 1u); + + // Session two: the counter is still tracked, but nothing sampled it. + const auto second = bridge().takeSessionSnapshot(); + EXPECT_EQ(second.samples_observed, 0u) + << "an idle session inherited the previous session's denominator"; + EXPECT_FALSE(second.any()); +} + +} // namespace diff --git a/tests/package/consumer/CMakeLists.txt b/tests/package/consumer/CMakeLists.txt new file mode 100644 index 0000000..549aa2a --- /dev/null +++ b/tests/package/consumer/CMakeLists.txt @@ -0,0 +1,13 @@ +# The documented consumer, verbatim. This project is configured against a +# scratch INSTALL tree by tests/package/counters_consumer_check.cmake - it is +# the fixture that keeps `find_package(gpufl_client)` + `gpufl::counters` +# honest. If EXPORT_NAME, the Config file install, or the counters export +# set's isolation from the SDK's dependencies regresses, this is the test +# that fails; the unit suite links the build tree and cannot see any of it. +cmake_minimum_required(VERSION 3.20) +project(gpufl_counters_consumer CXX) + +find_package(gpufl_client 1.2 REQUIRED) + +add_executable(app main.cpp) +target_link_libraries(app PRIVATE gpufl::counters) diff --git a/tests/package/consumer/main.cpp b/tests/package/consumer/main.cpp new file mode 100644 index 0000000..0ec4b53 --- /dev/null +++ b/tests/package/consumer/main.cpp @@ -0,0 +1,13 @@ +// Ticks a counter through the INSTALLED package. The check script greps the +// line below, so a silent link against a stub or a handle that failed to +// register fails the test rather than passing quietly. +#include + +#include "gpufl.hpp" + +int main() { + auto tokens = gpufl::counter("tokens"); + for (int i = 0; i < 1000; ++i) tokens.add(8); + std::printf("consumer ok valid=%d\n", tokens.valid() ? 1 : 0); + return tokens.valid() ? 0 : 1; +} diff --git a/tests/package/counters_consumer_check.cmake b/tests/package/counters_consumer_check.cmake new file mode 100644 index 0000000..6cab723 --- /dev/null +++ b/tests/package/counters_consumer_check.cmake @@ -0,0 +1,102 @@ +# Steps the counters package through the exact flow a user follows: +# +# cmake --build --target gpufl_counters +# cmake --install --prefix --component counters +# cmake -S consumer -DCMAKE_PREFIX_PATH= +# cmake --build consumer +# ./app -> "consumer ok valid=1" +# +# Run by CTest (see the add_test in the root CMakeLists), not by hand: the one +# manual run that validated this flow proved nothing about the NEXT change. +# Every step's failure names the step, so a regression reads as "install-tree +# consumer broke at configure" rather than a bare non-zero exit. +# +# The install is the COUNTERS COMPONENT, and the build is the counters target, +# so this test responds to exactly what the package promises - the archive, +# the public headers, the Config/Version files, the gpufl::counters export and +# its Threads/dl propagation - and to nothing else. A full-tree install tied +# it to zlib's install rules, httplib's export set and whether +# gpufl_counter_runtime happened to be built in this configuration; a Debug +# run failed on that last one without a single counters file being wrong. + +foreach(var GPUFL_BINARY_DIR GPUFL_SOURCE_DIR GPUFL_CONFIG GPUFL_GENERATOR) + if(NOT DEFINED ${var}) + message(FATAL_ERROR "counters_consumer_check: ${var} not set") + endif() +endforeach() + +set(scratch "${GPUFL_BINARY_DIR}/package_check") +file(REMOVE_RECURSE "${scratch}") + +# Single-config generators expand $ to an empty string, and +# `--config ""` is an error rather than a no-op. The flag exists only when +# there is a configuration to name. +set(config_args) +if(GPUFL_CONFIG) + set(config_args --config "${GPUFL_CONFIG}") +endif() + +function(run_step name) + execute_process(COMMAND ${ARGN} + RESULT_VARIABLE rc + OUTPUT_VARIABLE out + ERROR_VARIABLE err) + if(NOT rc EQUAL 0) + message(FATAL_ERROR + "install-tree consumer broke at ${name} (exit ${rc}):\n${out}\n${err}") + endif() + set(step_output "${out}" PARENT_SCOPE) +endfunction() + +# Built here rather than assumed: CTest promises nothing about what ran +# before this test, and an install of a never-built target is a confusing +# missing-file error instead of a compile error. +run_step("counters build" + ${CMAKE_COMMAND} --build "${GPUFL_BINARY_DIR}" + --target gpufl_counters ${config_args}) + +run_step("install" + ${CMAKE_COMMAND} --install "${GPUFL_BINARY_DIR}" + --prefix "${scratch}/prefix" --component counters ${config_args}) + +# The parent's platform/toolset travel along when they exist: a VS parent +# configured for a specific -A/-T must not have its consumer silently probe a +# different one. +set(gen_args) +if(DEFINED GPUFL_PLATFORM AND GPUFL_PLATFORM) + list(APPEND gen_args -A "${GPUFL_PLATFORM}") +endif() +if(DEFINED GPUFL_TOOLSET AND GPUFL_TOOLSET) + list(APPEND gen_args -T "${GPUFL_TOOLSET}") +endif() + +run_step("consumer configure" + ${CMAKE_COMMAND} -G "${GPUFL_GENERATOR}" ${gen_args} + -S "${GPUFL_SOURCE_DIR}/tests/package/consumer" + -B "${scratch}/consumer" + "-DCMAKE_PREFIX_PATH=${scratch}/prefix" + "-DCMAKE_BUILD_TYPE=${GPUFL_CONFIG}") + +run_step("consumer build" + ${CMAKE_COMMAND} --build "${scratch}/consumer" ${config_args}) + +# Single-config generators put the binary at the top; multi-config nests it. +set(app "${scratch}/consumer/app") +foreach(candidate "${scratch}/consumer/app" + "${scratch}/consumer/app.exe" + "${scratch}/consumer/${GPUFL_CONFIG}/app.exe" + "${scratch}/consumer/${GPUFL_CONFIG}/app") + if(EXISTS "${candidate}") + set(app "${candidate}") + break() + endif() +endforeach() + +run_step("consumer run" "${app}") +if(NOT step_output MATCHES "consumer ok valid=1") + message(FATAL_ERROR + "install-tree consumer ran but did not report a valid counter:\n" + "${step_output}") +endif() + +message(STATUS "install-tree counters consumer: ok") From 841de4f31e2182d4b84af7168bbda7536ee19c2c Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Tue, 28 Jul 2026 16:08:41 -0700 Subject: [PATCH 26/28] fix(monitor): omit kernel rows when real activity records never arrived --- .../gpufl/backends/nvidia/cupti_backend.hpp | 25 ++++++++++++++++ include/gpufl/core/monitor.cpp | 30 +++++++++++++++++-- include/gpufl/core/monitor_backend.hpp | 13 ++++++++ tests/backends/nvidia/test_nvidia_backend.cpp | 29 ++++++++++++++++++ 4 files changed, 95 insertions(+), 2 deletions(-) diff --git a/include/gpufl/backends/nvidia/cupti_backend.hpp b/include/gpufl/backends/nvidia/cupti_backend.hpp index b85e988..36cb1d7 100644 --- a/include/gpufl/backends/nvidia/cupti_backend.hpp +++ b/include/gpufl/backends/nvidia/cupti_backend.hpp @@ -27,6 +27,21 @@ namespace gpufl { class ICuptiHandler; +// Decision for Monitor::Shutdown's orphan-synthesis suppression, kept as a +// pure function so the truth table is unit-testable without a CUPTI session. +// True = this session expected real kernel activity records, launches +// happened, and none arrived - so orphan launch metas must NOT be resurrected +// as synthetic kernel rows (their "durations" would be host dispatch gaps). +// Modes that synthesize by design (PC sampling / SASS safe mode) pass +// will_emit_synthetic=true and are never suppressed here. +inline bool KernelActivityExpectedButMissing(bool collects_kernel_events, + bool will_emit_synthetic, + uint64_t launch_callbacks, + uint64_t activity_records_seen) { + return collects_kernel_events && !will_emit_synthetic && + launch_callbacks > 0 && activity_records_seen == 0; +} + /** * @brief CUPTI-based monitoring backend for NVIDIA GPUs. * @@ -90,6 +105,16 @@ class CuptiBackend : public IMonitorBackend { return opts_.profiling_engine == ProfilingEngine::PcSampling || !AllowSassKernelActivity(); } + // See IMonitorBackend. Observed live (Linux 3090, Trace+PM adaptive, + // NVTX-counter targets, ~1/4 runs): every cuptiActivityEnable succeeds, + // SYNCHRONIZATION records flow, but zero KERNEL/CONCURRENT_KERNEL + // records ever arrive. Valid only after stop()'s disable + flush. + bool kernelActivityExpectedButMissing() const override { + return KernelActivityExpectedButMissing( + collectsKernelEvents(), WillEmitSyntheticKernels(), + kernel_launch_callback_count_.load(std::memory_order_acquire), + kernel_activity_seen_.load(std::memory_order_relaxed)); + } bool AllowSassMarkerActivity() const { return resolved_plan_.allow_sass_marker_activity; } diff --git a/include/gpufl/core/monitor.cpp b/include/gpufl/core/monitor.cpp index a9e770f..0d6445b 100644 --- a/include/gpufl/core/monitor.cpp +++ b/include/gpufl/core/monitor.cpp @@ -158,8 +158,11 @@ struct MonitorState { detail::MonitorBatchManager batches; MetadataManager metadata; - bool suppressOrphanSyntheticKernels = false; - bool drainSyntheticMidRun = false; + // Atomic: set by the backend's start() thread and (since the + // missing-records suppression) by Monitor::Shutdown while the collector + // thread may still be reading it at a drain site. + std::atomic suppressOrphanSyntheticKernels{false}; + std::atomic drainSyntheticMidRun{false}; }; MonitorState g_state; @@ -529,6 +532,24 @@ void Monitor::Shutdown() { if (g_state.adapter) { g_state.adapter->stop(); + // stop() has disabled and flushed activity, so "zero kernel records" + // is final. When a real-record session lost every record (observed: + // Trace+PM adaptive on Linux, ~1/4 of NVTX-counter-target runs - + // enables succeed, sync records flow, kernel records never arrive), + // the teardown drains below would resurrect every launch meta as a + // synthetic row whose "duration" is the host gap to the next launch, + // fabricating a full kernel timeline. Suppress instead: no kernel + // rows, and the capability event already says + // enabled_but_no_records, so the session stays self-describing. + if (IMonitorBackend* b = g_state.adapter->backend(); + b && b->kernelActivityExpectedButMissing()) { + GFL_LOG_ERROR( + "[Monitor] Kernel activity was enabled and launches happened, " + "but no kernel activity record arrived this session. Kernel " + "rows are omitted (a synthetic fallback would carry host-gap " + "durations, not kernel time)."); + SetSuppressOrphanSyntheticKernels(true); + } g_state.adapter->shutdown(); } @@ -563,6 +584,11 @@ void Monitor::Shutdown() { void Monitor::DrainAndFinalizeForExit() { if (!g_state.initialized.exchange(false)) return; + // No missing-records suppression here (unlike Shutdown): this path runs + // BEFORE the backend's stop(), so activity was never flushed and + // "zero records seen" may just mean a short run whose only buffer is + // still pending - suppressing would drop the synthetic fallback that is + // this path's whole reason to exist. // The backend's PC sampling cycle thread stops issuing CUPTI reads as soon // as process-exit teardown is flagged (PcSamplingEngine::drainData), so it // sits idle here and can't fault mid-flush. We deliberately do NOT join it diff --git a/include/gpufl/core/monitor_backend.hpp b/include/gpufl/core/monitor_backend.hpp index 049753a..f002b54 100644 --- a/include/gpufl/core/monitor_backend.hpp +++ b/include/gpufl/core/monitor_backend.hpp @@ -87,6 +87,19 @@ class IMonitorBackend { */ virtual bool IsProfilingOperational() const { return true; } + /** + * @brief True when this session expected REAL kernel activity records + * (kernel activity enabled, not a synthesize-by-design mode), + * launches happened, and yet zero records arrived. Only + * meaningful after stop() has disabled + flushed activity, when + * "zero" is final. Monitor::Shutdown consults this to suppress + * the orphan synthetic-kernel drain: with every record missing, + * that drain would fabricate a full kernel timeline out of + * host launch-to-launch gaps (sleeps included) and present it + * as measured kernel time. Default: false (nothing to suppress). + */ + virtual bool kernelActivityExpectedButMissing() const { return false; } + virtual void OnScopeStart(const char* name) {} virtual void OnScopeStop(const char* name) {} diff --git a/tests/backends/nvidia/test_nvidia_backend.cpp b/tests/backends/nvidia/test_nvidia_backend.cpp index 8338043..18be97b 100644 --- a/tests/backends/nvidia/test_nvidia_backend.cpp +++ b/tests/backends/nvidia/test_nvidia_backend.cpp @@ -11,6 +11,35 @@ class CuptiBackendTest : public ::testing::Test { void SetUp() override { SKIP_IF_NO_CUDA(); } }; +// Pure decision for Monitor::Shutdown's orphan-synthesis suppression. No CUDA +// needed. The case this exists for: a real-record session (kernel activity +// enabled, not PC/SASS synthesize-by-design) where launches happened but zero +// activity records arrived - the orphan drain would otherwise fabricate every +// kernel row from host launch-to-launch gaps. +TEST(KernelActivityExpectedButMissing, TruthTable) { + using gpufl::KernelActivityExpectedButMissing; + + // The broken session: real records expected, launches happened, none came. + EXPECT_TRUE(KernelActivityExpectedButMissing(true, false, 100000, 0)); + // One single missing-record launch is still a fully-lost session. + EXPECT_TRUE(KernelActivityExpectedButMissing(true, false, 1, 0)); + + // Any real record arrived: keep the drain - tail orphans stay best-effort. + EXPECT_FALSE(KernelActivityExpectedButMissing(true, false, 100000, 1)); + EXPECT_FALSE(KernelActivityExpectedButMissing(true, false, 100000, 99999)); + + // No launches: nothing was lost (and nothing to suppress). + EXPECT_FALSE(KernelActivityExpectedButMissing(true, false, 0, 0)); + + // Synthesize-by-design modes (PC sampling / SASS safe): synthetic rows + // are the product, never suppressed here. + EXPECT_FALSE(KernelActivityExpectedButMissing(true, true, 100000, 0)); + EXPECT_FALSE(KernelActivityExpectedButMissing(false, true, 100000, 0)); + + // Kernel activity was never enabled: absence of records is expected. + EXPECT_FALSE(KernelActivityExpectedButMissing(false, false, 100000, 0)); +} + TEST_F(CuptiBackendTest, Lifecycle) { gpufl::MonitorOptions opts; opts.enable_debug_output = true; From a5be2099bfd8a160addeb0e08fef51d22eca8c72 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Tue, 28 Jul 2026 16:41:16 -0700 Subject: [PATCH 27/28] fix(monitor): never infer real kernel timing from host gaps --- .../gpufl/backends/nvidia/cupti_backend.cpp | 14 ++++----- .../gpufl/backends/nvidia/cupti_backend.hpp | 31 ++++++++++++------- include/gpufl/core/monitor.cpp | 10 +++++- include/gpufl/core/monitor.hpp | 13 ++++---- include/gpufl/core/trace_type.hpp | 7 +++-- tests/backends/nvidia/test_nvidia_backend.cpp | 19 +++++++++++- tests/core/test_monitor.cpp | 24 ++++++++++++++ 7 files changed, 88 insertions(+), 30 deletions(-) diff --git a/include/gpufl/backends/nvidia/cupti_backend.cpp b/include/gpufl/backends/nvidia/cupti_backend.cpp index 79b9c26..86ca9a2 100644 --- a/include/gpufl/backends/nvidia/cupti_backend.cpp +++ b/include/gpufl/backends/nvidia/cupti_backend.cpp @@ -221,14 +221,14 @@ CUptiResult (*CuptiBackend::get_value())(CUpti_ActivityKind) { void CuptiBackend::start() { if (!initialized_) return; - // SASS safe mode keeps kernel-activity OFF (it deadlocks), so orphan launch - // metas would become synthetic kernels with host-dispatch timing only. Keep - // suppressing that path for SASS. PC sampling intentionally leaves the - // synthetic drain enabled so callback-derived kernel rows can be emitted - // when kernel activity is unavailable; drainSyntheticKernels filters - // non-kernel memcpy/memset API metas before emitting rows. + // Establish the timing-provenance contract before any launch callback can + // arrive. Real-record modes never convert an unmatched launch into a kernel + // row: the launch-to-launch host gap is not GPU execution time. Synthetic + // rows remain available only in modes that explicitly use callback-derived + // kernels (for example PC sampling). SetSuppressOrphanSyntheticKernels( - opts_.profiling_engine == ProfilingEngine::SassMetrics); + ShouldSuppressOrphanKernelSynthesis(collectsKernelEvents(), + WillEmitSyntheticKernels())); // Windows-injection PC sampling AND Deep both lose their final flush to the // process-exit teardown race (gpufl::shutdown runs during DLL detach, after // the OS starts tearing the process down, so the shutdown drain can be cut diff --git a/include/gpufl/backends/nvidia/cupti_backend.hpp b/include/gpufl/backends/nvidia/cupti_backend.hpp index 36cb1d7..adf5b17 100644 --- a/include/gpufl/backends/nvidia/cupti_backend.hpp +++ b/include/gpufl/backends/nvidia/cupti_backend.hpp @@ -37,9 +37,18 @@ class ICuptiHandler; inline bool KernelActivityExpectedButMissing(bool collects_kernel_events, bool will_emit_synthetic, uint64_t launch_callbacks, - uint64_t activity_records_seen) { + uint64_t valid_activity_rows) { return collects_kernel_events && !will_emit_synthetic && - launch_callbacks > 0 && activity_records_seen == 0; + launch_callbacks > 0 && valid_activity_rows == 0; +} + +// Host launch-to-launch gaps are not kernel durations. A mode that requested +// real CUPTI kernel activity must therefore drop every unmatched launch meta, +// even if some other launches produced valid activity rows. Only modes whose +// explicit product is callback-derived kernel rows may synthesize orphans. +inline bool ShouldSuppressOrphanKernelSynthesis(bool collects_kernel_events, + bool will_emit_synthetic) { + return collects_kernel_events || !will_emit_synthetic; } /** @@ -94,16 +103,14 @@ class CuptiBackend : public IMonitorBackend { } return opts_.profiling_engine == ProfilingEngine::RangeProfilerKernelReplay; } - // True when CUPTI kernel ACTIVITY records won't be collected, so every - // launch must be reported from its callback as a synthetic kernel (PC - // Sampling, or SASS profiler safe mode without GPUFL_SASS_ALLOW_KERNEL_ - // ACTIVITY). Mirrors KernelLaunchHandler::requiredActivityKinds() returning - // {}. In these modes the launch callback precomputes the simplified kernel - // occupancy (the activity record that would otherwise carry it never - // arrives); see KernelLaunchHandler::handle + drainSyntheticKernels. + // True only when callback-derived kernel rows are the selected mode's + // deliberate product. If real CUPTI kernel activity is enabled, unmatched + // launches are data loss and must never be turned into host-gap durations. + // SASSMetrics safe mode also suppresses synthetic rows because its + // execution signature is carried separately. bool WillEmitSyntheticKernels() const { - return opts_.profiling_engine == ProfilingEngine::PcSampling || - !AllowSassKernelActivity(); + if (collectsKernelEvents()) return false; + return opts_.profiling_engine != ProfilingEngine::SassMetrics; } // See IMonitorBackend. Observed live (Linux 3090, Trace+PM adaptive, // NVTX-counter targets, ~1/4 runs): every cuptiActivityEnable succeeds, @@ -113,7 +120,7 @@ class CuptiBackend : public IMonitorBackend { return KernelActivityExpectedButMissing( collectsKernelEvents(), WillEmitSyntheticKernels(), kernel_launch_callback_count_.load(std::memory_order_acquire), - kernel_activity_seen_.load(std::memory_order_relaxed)); + kernel_activity_emitted_.load(std::memory_order_relaxed)); } bool AllowSassMarkerActivity() const { return resolved_plan_.allow_sass_marker_activity; diff --git a/include/gpufl/core/monitor.cpp b/include/gpufl/core/monitor.cpp index 0d6445b..92317ed 100644 --- a/include/gpufl/core/monitor.cpp +++ b/include/gpufl/core/monitor.cpp @@ -500,6 +500,12 @@ void CollectorLoop() { void Monitor::Initialize(const MonitorOptions& opts) { if (g_state.initialized.exchange(true)) return; + // Process-lifetime state must not leak a prior session's capture policy. + // The active backend's start() replaces these defaults before callbacks + // begin. Monitor/no-backend sessions keep both disabled. + SetSuppressOrphanSyntheticKernels(false); + SetDrainSyntheticKernelsMidRun(false); + // The engine AFTER env overrides. InitOptions::profiling_engine is the // pre-override request, and `gpufl trace --passes X` overrides only the // MonitorOptions copy - so anything reporting which engine ran must read @@ -548,7 +554,6 @@ void Monitor::Shutdown() { "but no kernel activity record arrived this session. Kernel " "rows are omitted (a synthetic fallback would carry host-gap " "durations, not kernel time)."); - SetSuppressOrphanSyntheticKernels(true); } g_state.adapter->shutdown(); } @@ -812,5 +817,8 @@ void Monitor::EmitPmSamplingConfig(uint32_t device_id, uint32_t interval_us, uin void SetSuppressOrphanSyntheticKernels(const bool suppress) { g_state.suppressOrphanSyntheticKernels = suppress; } void SetDrainSyntheticKernelsMidRun(const bool enable) { g_state.drainSyntheticMidRun = enable; } +bool SuppressOrphanSyntheticKernelsForTesting() { + return g_state.suppressOrphanSyntheticKernels.load(std::memory_order_acquire); +} } // namespace gpufl diff --git a/include/gpufl/core/monitor.hpp b/include/gpufl/core/monitor.hpp index c091515..4986965 100644 --- a/include/gpufl/core/monitor.hpp +++ b/include/gpufl/core/monitor.hpp @@ -231,12 +231,9 @@ struct PmSampleInput { }; // Session-level switch (set by the active backend at start): when true the -// collector DROPS orphaned launch metas at shutdown instead of emitting them as -// synthetic kernel rows. Used for engines where kernel-activity is intentionally -// off and the synthetic host-dispatch durations would mislead - SASS safe mode, -// where real kernel activity deadlocks (NVIDIA CUPTI/driver bug). The Execution -// Signature is accumulated separately, so a multi-pass merge is unaffected. -// Default false: normal / PC modes keep best-effort synthesis. +// collector DROPS unmatched launch metas instead of emitting host launch gaps +// as kernel durations. All real-record modes enable it; callback-derived rows +// remain available only for synthesize-by-design modes such as PC sampling. void SetSuppressOrphanSyntheticKernels(bool suppress); // Session-level switch (set by the active backend at start): when true the @@ -246,6 +243,10 @@ void SetSuppressOrphanSyntheticKernels(bool suppress); // those kernel rows survive. Default false (other modes emit at shutdown). void SetDrainSyntheticKernelsMidRun(bool enable); +// Test seam for the process-lifetime session policy. Production code should +// only set the policy through the two functions above. +bool SuppressOrphanSyntheticKernelsForTesting(); + /** * @brief The central monitoring engine. */ diff --git a/include/gpufl/core/trace_type.hpp b/include/gpufl/core/trace_type.hpp index af79da4..5b41495 100644 --- a/include/gpufl/core/trace_type.hpp +++ b/include/gpufl/core/trace_type.hpp @@ -99,8 +99,9 @@ enum class TraceType : uint8_t { // API_ENTER. The collector stores it in g_launchMetaByCorr keyed by // corr_id; the matching KERNEL / MEMCPY / MEMSET activity record later // joins scope path, stack id, and API timestamps from it. Entries with no - // activity record by shutdown become synthetic kernels (drainSynthetic - // Kernels). Fields used on ActivityRecord: corr_id, name (raw), device_id, + // activity record by shutdown are dropped in real-record modes. Only + // synthesize-by-design modes turn them into callback-derived kernel rows. + // Fields used on ActivityRecord: corr_id, name (raw), device_id, // api_start_ns (= API_ENTER ns), user_scope, scope_depth, stack_id, plus // has_details + grid/block/dyn_shared and the precomputed simplified // occupancy (synthetic-kernel modes only). @@ -127,4 +128,4 @@ enum class TraceType : uint8_t { // synthetic rows. Fields used on ActivityRecord: corr_id. KERNEL_META_DISCARD, }; -} \ No newline at end of file +} diff --git a/tests/backends/nvidia/test_nvidia_backend.cpp b/tests/backends/nvidia/test_nvidia_backend.cpp index 18be97b..af4bef3 100644 --- a/tests/backends/nvidia/test_nvidia_backend.cpp +++ b/tests/backends/nvidia/test_nvidia_backend.cpp @@ -24,7 +24,11 @@ TEST(KernelActivityExpectedButMissing, TruthTable) { // One single missing-record launch is still a fully-lost session. EXPECT_TRUE(KernelActivityExpectedButMissing(true, false, 1, 0)); - // Any real record arrived: keep the drain - tail orphans stay best-effort. + // The final argument is accepted/valid rows, not records merely observed + // before timestamp validation. A rejected record therefore leaves it zero. + EXPECT_TRUE(KernelActivityExpectedButMissing(true, false, 100000, 0)); + // Any valid real row means the all-records-missing diagnostic is false. + // Orphan synthesis is still independently disabled for the whole session. EXPECT_FALSE(KernelActivityExpectedButMissing(true, false, 100000, 1)); EXPECT_FALSE(KernelActivityExpectedButMissing(true, false, 100000, 99999)); @@ -40,6 +44,19 @@ TEST(KernelActivityExpectedButMissing, TruthTable) { EXPECT_FALSE(KernelActivityExpectedButMissing(false, false, 100000, 0)); } +TEST(OrphanKernelSynthesisPolicy, RealRecordModesAlwaysSuppressOrphans) { + using gpufl::ShouldSuppressOrphanKernelSynthesis; + + EXPECT_TRUE(ShouldSuppressOrphanKernelSynthesis(true, false)); + // Even a contradictory caller cannot opt a real-record mode into + // host-gap timing. + EXPECT_TRUE(ShouldSuppressOrphanKernelSynthesis(true, true)); + // SASS metrics-only: neither real rows nor synthetic timing is wanted. + EXPECT_TRUE(ShouldSuppressOrphanKernelSynthesis(false, false)); + // PC sampling / other synthesize-by-design modes. + EXPECT_FALSE(ShouldSuppressOrphanKernelSynthesis(false, true)); +} + TEST_F(CuptiBackendTest, Lifecycle) { gpufl::MonitorOptions opts; opts.enable_debug_output = true; diff --git a/tests/core/test_monitor.cpp b/tests/core/test_monitor.cpp index 2e494ca..9770cf8 100644 --- a/tests/core/test_monitor.cpp +++ b/tests/core/test_monitor.cpp @@ -77,6 +77,30 @@ TEST_F(MonitorTest, MultipleInitialize) { gpufl::Monitor::Shutdown(); // Should be safe } +TEST_F(MonitorTest, InitializeClearsThePreviousSessionsSynthesisPolicy) { + gpufl::SetSuppressOrphanSyntheticKernels(true); + + gpufl::MonitorOptions opts; + opts.profiling_engine = gpufl::ProfilingEngine::Monitor; + opts.backend_kind = gpufl::MonitorBackendKind::None; + gpufl::Monitor::Initialize(opts); + + EXPECT_FALSE(gpufl::SuppressOrphanSyntheticKernelsForTesting()) + << "a prior session's suppression state must not leak into a new one"; +} + +TEST_F(MonitorTest, TraceEnablesTrustworthyOrphanPolicyBeforeCollection) { + SKIP_IF_NO_CUDA(); + + gpufl::MonitorOptions opts; + opts.profiling_engine = gpufl::ProfilingEngine::Trace; + gpufl::Monitor::Initialize(opts); + gpufl::Monitor::Start(); + + EXPECT_TRUE(gpufl::SuppressOrphanSyntheticKernelsForTesting()) + << "Trace must drop unmatched launches instead of inventing GPU time"; +} + // ── scope name stack ──────────────────────────────────────────────────────── // // The active scope name is what stamps every profile and PM sample. It has to From 6d90f0463939f715c6683789eb3fd58474290a65 Mon Sep 17 00:00:00 2001 From: Myoungho Shin Date: Tue, 28 Jul 2026 18:04:58 -0700 Subject: [PATCH 28/28] build: stop ignoring the cmake/ source directory --- .gitignore | 1 - cmake/gpufl_clientConfig.cmake.in | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 cmake/gpufl_clientConfig.cmake.in diff --git a/.gitignore b/.gitignore index 111d35e..d23e90c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ build/ build-*/ build_tests/ cmake-build-*/ -cmake/ CMakeFiles/ CMakeCache.txt wget-log* diff --git a/cmake/gpufl_clientConfig.cmake.in b/cmake/gpufl_clientConfig.cmake.in new file mode 100644 index 0000000..7e57836 --- /dev/null +++ b/cmake/gpufl_clientConfig.cmake.in @@ -0,0 +1,24 @@ +@PACKAGE_INIT@ + +# find_package(gpufl_client) entry point. What it guarantees is the counters +# layer: +# +# find_package(gpufl_client REQUIRED) +# target_link_libraries(app PRIVATE gpufl::counters) +# +# Threads is the one real dependency in its link interface, resolved here so +# the consumer does not have to know. +# +# The full SDK target is NOT loaded by this file. Its targets file records +# private dependencies (httplib, zlib, OpenSSL, CUDA) and refuses to load +# unless the consumer already provides those imported targets - publishing it +# as a component would be documenting a flow that only works by accident. +# Consuming the SDK from an install tree needs find_dependency() wiring that +# does not exist yet. + +include(CMakeFindDependencyMacro) +find_dependency(Threads) + +include("${CMAKE_CURRENT_LIST_DIR}/gpufl_countersTargets.cmake") + +check_required_components(gpufl_client)