Bounded deep-profiling window + PC sampling collection fixes

CHANGELOG.md

@@ -9,6 +9,53 @@ versioning follows PEP 440 for the Python wheel and semver-style
 
 ### Added
 
+- **Bounded deep-profiling windows.** `gpufl::deepWindow(ms, max_launches)`
+  (Python: `gpufl.deep_window(seconds, max_launches)`) arms the deep engines
+  for a short region and disarms them automatically, so a long-running job
+  can profile the moment it went wrong without carrying replay cost for its
+  whole lifetime:
+
+  ```cpp
+  if (tokens_per_sec < 1000) gpufl::deepWindow(3000);
+  ```
+
+  Calling it again while a window is open is ignored rather than treated as
+  an extension, so a check inside a training loop can run every step. The
+  duration and launch bounds combine with OR; prefer a launch budget for the
+  replay engines, where wall time and work done diverge sharply. Each window
+  emits a `deep_window` event carrying the launches it actually covered and
+  which bound closed it.
+
+  Set `InitOptions::deep_window_only` (Python `deep_window_only=True`, env
+  `GPUFL_DEEP_ARM=window`) to keep PC sampling, SASS metrics, PM sampling and
+  the Range profiler idle outside windows. `GPUFL_DEEP_WINDOW_MS` and
+  `GPUFL_DEEP_WINDOW_MAX_LAUNCHES` supply defaults for bounds left at 0.
+
+  `DeepWindowSpec::cooldown_ms` (`GPUFL_DEEP_WINDOW_COOLDOWN_MS`) sets the
+  minimum quiet time between windows. Without it a condition that stays true
+  reopens a window the instant the last one expired; only the library knows
+  when that was, so the bound lives there rather than in your trigger.
+- **`gpufl trace --deep-after` / `--deep-for` / `--deep-launches` /
+  `--deep-cooldown`.** Arms a deep window inside a target whose source you
+  can't edit, which otherwise has no way to call `deepWindow()`. Unlike
+  `--window`, which bounds the target's LIFETIME, these leave it running and
+  bound only how long the deep engines stay armed:
+
+  ```bash
+  gpufl trace --deep-after 30s --deep-for 3s --passes PmSampling -- python train.py
+  ```
+
+  Any `--deep-*` flag implies `GPUFL_DEEP_ARM=window`. A window with neither
+  a duration nor a launch bound is rejected.
+
+### Fixed
+
+- **PM/PC sample timestamps under `gpufl trace`.** The deferred engine start
+  (the path Windows injection always takes, since gpufl initializes before
+  the target creates a CUDA context) built its `EngineContext` without the
+  CUPTI-to-wall-clock anchor. Engines that stamp their own samples emitted
+  raw CUPTI timestamps, putting every PM sample days away from the kernel
+  timeline it should line up with.
 - **Shared-memory bank-conflict profiling.**
   `RangeProfilerKernelReplay` now emits per-kernel shared load/store/total
   conflict counts, shared wavefronts, conflict overhead, and average N-way

CMakeLists.txt

@@ -101,6 +101,8 @@ target_sources(gpufl PRIVATE
     include/gpufl/core/model/memory_alloc_event_model.cpp
     include/gpufl/core/model/graph_launch_event_model.cpp
     include/gpufl/core/model/system_event_model.cpp
+    include/gpufl/core/model/deep_window_model.cpp
+    include/gpufl/core/deep_window.cpp
     include/gpufl/core/sampler.cpp
     include/gpufl/core/runtime.cpp
     include/gpufl/core/backend_factory.cpp

README.md

@@ -77,6 +77,9 @@ Toolkit, Python virtual environment, or wheel ABI.
 # Build a wheel into ./dist
 ./build.sh --wheel
 
+# Build the native gpufl trace launcher and injection library
+./build.sh --trace
+
 # Use an explicit Python and CUDA Toolkit
 ./build-ubuntu.sh --wheel \
   --python .venv/bin/python \
@@ -89,6 +92,7 @@ Useful options:
 |---|---|
 | `--install` | Install the package into the selected Python environment. This is the default. |
 | `--wheel` | Build a wheel into `./dist` or `--wheel-dir`. |
+| `--trace` | Build the native `gpufl` launcher and `libgpufl_inject.so` into `./build-ubuntu`. |
 | `--python PATH` | Python executable to use. Use your target virtual environment's Python when building wheels. |
 | `--cuda-root PATH` | CUDA Toolkit root, for example `/usr/local/cuda-13.2`. |
 | `--wheel-dir PATH` | Output directory for built wheels. |

build-ubuntu.sh

@@ -9,7 +9,7 @@ WHEEL_DIR="$ROOT_DIR/dist"
 
 usage() {
   cat <<'EOF'
-Usage: ./build-ubuntu.sh [--install|--wheel] [--python PATH] [--cuda-root PATH] [--wheel-dir PATH]
+Usage: ./build-ubuntu.sh [--install|--wheel|--trace] [--python PATH] [--cuda-root PATH] [--wheel-dir PATH]
 
 Defaults:
   --install
@@ -20,6 +20,7 @@ Defaults:
 Examples:
   ./build-ubuntu.sh
   ./build-ubuntu.sh --wheel
+  ./build-ubuntu.sh --trace
   ./build-ubuntu.sh --python .venv/bin/python --cuda-root /usr/local/cuda-13.2
 EOF
 }
@@ -34,6 +35,10 @@ while [[ $# -gt 0 ]]; do
       MODE="wheel"
       shift
       ;;
+    --trace)
+      MODE="trace"
+      shift
+      ;;
     --python)
       PYTHON_BIN="$2"
       shift 2
@@ -99,6 +104,33 @@ echo "  cuda root: $CUDA_ROOT"
 if [[ "$MODE" == "wheel" ]]; then
   mkdir -p "$WHEEL_DIR"
   "$PYTHON_BIN" -m pip wheel "$ROOT_DIR" -w "$WHEEL_DIR" --no-deps -v "${COMMON_CONFIG[@]}"
+elif [[ "$MODE" == "trace" ]]; then
+  BUILD_DIR="$ROOT_DIR/build-ubuntu"
+  TRACE_CONFIG=(
+    -DCMAKE_BUILD_TYPE=Release
+    -DGPUFL_ENABLE_NVIDIA=ON
+    -DGPUFL_ENABLE_AMD=OFF
+    -DBUILD_PYTHON=OFF
+    -DBUILD_TESTING=OFF
+    -DBUILD_GPUFL_EXAMPLE=OFF
+    -DBUILD_GPUFL_LAUNCHER=ON
+    -DBUILD_GPUFL_INJECT=ON
+    "-DCUDAToolkit_ROOT=$CUDA_ROOT"
+    "-DCMAKE_CUDA_COMPILER=$CUDA_ROOT/bin/nvcc"
+  )
+
+  cmake -S "$ROOT_DIR" -B "$BUILD_DIR" "${TRACE_CONFIG[@]}"
+  cmake --build "$BUILD_DIR" --target gpufl_launcher gpufl_inject -j
+
+  LAUNCHER="$BUILD_DIR/daemon/launcher/gpufl"
+  INJECT_LIBRARY="$BUILD_DIR/libgpufl_inject.so"
+  echo
+  echo "Built native trace tooling:"
+  echo "  launcher: $LAUNCHER"
+  echo "  inject:   $INJECT_LIBRARY"
+  echo
+  echo "Run:  \"$LAUNCHER\" trace --passes=Trace -- \"$PYTHON_BIN\" <script.py>"
+  echo "      \"$LAUNCHER\" trace --passes=PcSampling -- \"$PYTHON_BIN\" <script.py>"
 else
   "$PYTHON_BIN" -m pip install "$ROOT_DIR" -v "${COMMON_CONFIG[@]}"
 fi

daemon/launcher/cli_parse.cpp

@@ -168,6 +168,24 @@ const char* traceHelp() {
         "                            Hard cap on total target runtime (safety).\n"
         "        --after-window=<WHAT>\n"
         "                            What to do at window end. Only 'stop' today.\n"
+        "        --deep-after=<DUR>  Arm the DEEP engines this long into the run,\n"
+        "                            then disarm. Unlike --window the target keeps\n"
+        "                            running. Needs a bound below. Default: 0 (arm\n"
+        "                            at the first kernel launch).\n"
+        "        --deep-for=<DUR>    How long the deep window stays armed. Note this\n"
+        "                            bounds TIME, which does not bound how much the\n"
+        "                            engines actually collect - see --deep-launches.\n"
+        "        --deep-launches=<N> Kernel-launch bound on the deep window; ends it\n"
+        "                            at whichever bound is hit first. PREFER THIS.\n"
+        "                            For SASS / Range replay re-runs every kernel, so a\n"
+        "                            second of wall time covers ~25x less work there\n"
+        "                            than under PM sampling. For PcSampling it is what\n"
+        "                            decides whether you get data at all: samples only\n"
+        "                            become readable after a few thousand launches, so\n"
+        "                            a short --deep-for window (or any window over\n"
+        "                            slow kernels) can collect nothing.\n"
+        "        --deep-cooldown=<DUR>\n"
+        "                            Quiet time before another window may open.\n"
         "        --pc-sample-period=<N>\n"
         "                            PC sampling period: log2 of GPU cycles per sample\n"
         "                            (5..31; default 10). Lower = more frequent — for\n"
@@ -369,6 +387,53 @@ TraceParseResult parseTraceArgs(const std::vector<std::string>& argv) {
                         " (expected a duration like 30s, 5m, 1h, "
                         "or a bare number of seconds)"};
             }
+        } else if (key == "--deep-after" || key == "--deep-for" ||
+                   key == "--deep-cooldown") {
+            std::string v;
+            auto err = take_value(v);
+            if (!err.empty()) return {std::nullopt, err};
+            int64_t ms = 0;
+            if (!parseDurationMs(v, ms)) {
+                return {std::nullopt,
+                        "invalid " + key + " value: " + v +
+                        " (expected a duration like 30s, 500ms, 5m, 1h, "
+                        "or a bare number of seconds)"};
+            }
+            // A bare number is seconds, which collides with --deep-launches:
+            // `--deep-for=2000` meaning "2000 launches" silently becomes a
+            // 33-minute window, and the no-bound check below can't catch it
+            // because a bound *was* given. Reject the unit-less form once it
+            // is too large to plausibly be a duration someone typed on
+            // purpose - naming the alternative, since that is the mistake.
+            const bool unitless =
+                !v.empty() &&
+                v.find_first_not_of("0123456789") == std::string::npos;
+            if (key == "--deep-for" && unitless && ms >= 600'000) {
+                return {std::nullopt,
+                        "--deep-for=" + v + " means " + std::to_string(ms / 1000) +
+                        " SECONDS (a bare number is seconds), which is almost "
+                        "certainly not what you meant. Add a unit (e.g. " + v +
+                        "s, 5m) if you really want that long a window, or use "
+                        "--deep-launches " + v + " to bound it by kernel "
+                        "launches instead"};
+            }
+            if (key == "--deep-after")         out.deep_after_ms = ms;
+            else if (key == "--deep-for")      out.deep_for_ms = ms;
+            else                               out.deep_cooldown_ms = ms;
+            out.deep_requested = true;
+        } else if (key == "--deep-launches") {
+            std::string v;
+            auto err = take_value(v);
+            if (!err.empty()) return {std::nullopt, err};
+            char* end = nullptr;
+            const unsigned long long n = std::strtoull(v.c_str(), &end, 10);
+            if (end == v.c_str() || (end && *end != '\0') || n == 0) {
+                return {std::nullopt,
+                        "invalid --deep-launches value: " + v +
+                        " (expected a positive number of kernel launches)"};
+            }
+            out.deep_launches = static_cast<uint64_t>(n);
+            out.deep_requested = true;
         } else if (key == "--after-window") {
             auto err = take_value(out.after_window);
             if (!err.empty()) return {std::nullopt, err};
@@ -398,6 +463,16 @@ TraceParseResult parseTraceArgs(const std::vector<std::string>& argv) {
     if (out.command.empty()) {
         return {std::nullopt, "no command specified after `--`"};
     }
+    // A deep window with neither bound would arm and never disarm, which is
+    // just "profile deeply for the whole run" with extra steps.
+    if (out.deep_requested && out.deep_for_ms == 0 && out.deep_launches == 0) {
+        return {std::nullopt,
+                "a deep window needs a bound: pass --deep-launches <n> or "
+                "--deep-for <duration> (prefer --deep-launches: it is what "
+                "the engines actually scale with. The replay engines cover "
+                "far less work per second of wall time, and PC sampling "
+                "returns nothing at all below a few thousand launches)"};
+    }
     return {out, ""};
 }
 

daemon/launcher/cli_parse.hpp

@@ -37,6 +37,17 @@ struct TraceArgs {
     int64_t window_ms = 0;              // --window; 0 = run to the target's natural exit
     int64_t window_timeout_ms = 0;      // --window-timeout; hard cap on total runtime (0 = warmup+window)
     std::string after_window = "stop";  // --after-window; "stop" is the only value today
+    // Bounded DEEP window - unrelated to --window above, which bounds the
+    // target's LIFETIME. These keep the target running and instead bound how
+    // long the deep engines (PC sampling / SASS / PM / Range) stay armed
+    // inside it. A target whose source can't be edited has no way to call
+    // gpufl::deepWindow(), so time is the trigger the launcher can offer.
+    // Any of them turns on window-only arming (GPUFL_DEEP_ARM=window).
+    int64_t  deep_after_ms = 0;      // --deep-after; 0 = arm at the first launch
+    int64_t  deep_for_ms = 0;        // --deep-for; duration bound, 0 = none
+    uint64_t deep_launches = 0;      // --deep-launches; launch bound, 0 = none
+    int64_t  deep_cooldown_ms = 0;   // --deep-cooldown; quiet time between windows
+    bool     deep_requested = false; // any --deep-* flag was given
     // PC sampling period as a log2 exponent (2^N GPU cycles/sample, valid 5..31;
     // lower = more frequent → catches shorter kernels). 0 = leave the engine
     // default. Plumbed to the injected target via GPUFL_PC_SAMPLING_PERIOD.

daemon/launcher/trace_command_common.cpp

@@ -616,6 +616,34 @@ int runTraceCommon(const TraceArgs& args, const TracePlatform& platform) {
         return 2;
     }
 
+    // --deep-*: bound how long the DEEP engines stay armed inside a target
+    // that keeps running. Distinct from --window above, which bounds the
+    // target's lifetime. Asking for a deep window implies window-only
+    // arming, or the engines would be armed from the first kernel and the
+    // window would bound nothing.
+    if (args.deep_requested) {
+        if (!setEnvOrPrint(platform, env::kDeepArm, "window") ||
+            !setEnvOrPrint(platform, env::kDeepAfterMs,
+                           std::to_string(args.deep_after_ms))) {
+            return 2;
+        }
+        if (args.deep_for_ms > 0 &&
+            !setEnvOrPrint(platform, env::kDeepWindowMs,
+                           std::to_string(args.deep_for_ms))) {
+            return 2;
+        }
+        if (args.deep_launches > 0 &&
+            !setEnvOrPrint(platform, env::kDeepWindowMaxLaunches,
+                           std::to_string(args.deep_launches))) {
+            return 2;
+        }
+        if (args.deep_cooldown_ms > 0 &&
+            !setEnvOrPrint(platform, env::kDeepWindowCooldownMs,
+                           std::to_string(args.deep_cooldown_ms))) {
+            return 2;
+        }
+    }
+
     // A bounded window stops the target after warmup+window wall-clock;
     // run_ms == 0 keeps the historical "run until the target exits" behavior.
     RunOptions run_opts;

example/cuda/CMakeLists.txt

@@ -42,6 +42,7 @@ add_executable(memory_coalescing_demo memory_coalescing_demo.cu)
 add_executable(shared_bank_conflicts_demo shared_bank_conflicts_demo.cu)
 add_executable(deep_deadlock_repro deep_deadlock_repro.cu)
 add_executable(multi_engine_demo multi_engine_demo.cu)
+add_executable(deep_window_demo deep_window_demo.cu)
 
 target_compile_options(gfl_block_example PRIVATE
         $<$<COMPILE_LANGUAGE:CUDA>:
@@ -183,6 +184,17 @@ target_link_libraries(multi_engine_demo PRIVATE
         CUDA::cudart
 )
 
+target_link_libraries(deep_window_demo PRIVATE
+        gpufl::gpufl
+        CUDA::cupti
+        CUDA::cudart
+)
+# PC / SASS want the cubin embedded for source correlation.
+target_compile_options(deep_window_demo PRIVATE
+        $<$<COMPILE_LANGUAGE:CUDA>:
+        -lineinfo
+        >)
+
 
 # Properties
 set_target_properties(gfl_block_example PROPERTIES