From f8a83ad67faf549365f056b2545401bb2af4b99e Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Wed, 5 Aug 2026 13:24:09 -0700 Subject: [PATCH 1/2] Add check and agent guidance about uncapped mempools --- cuda_core/tests/AGENTS.md | 67 ++++++++++++++ cuda_core/tests/test_memory.py | 3 + cuda_core/tests/test_mempool_hygiene.py | 90 +++++++++++++++++++ .../tests/test_multiprocessing_warning.py | 10 ++- 4 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 cuda_core/tests/AGENTS.md create mode 100644 cuda_core/tests/test_mempool_hygiene.py diff --git a/cuda_core/tests/AGENTS.md b/cuda_core/tests/AGENTS.md new file mode 100644 index 00000000000..16e54264040 --- /dev/null +++ b/cuda_core/tests/AGENTS.md @@ -0,0 +1,67 @@ +# cuda.core test suite + +Package-wide conventions live in `../AGENTS.md`; repository-wide ones in +`../../AGENTS.md`. This file covers conventions specific to the tests. + +## Never create an uncapped memory pool + +A memory pool created without `max_size` reserves a virtual address window +sized from device memory (roughly 1x device memory) regardless of what the +test actually allocates. The reservation is charged to the process address +space even though it is not backed by physical memory, and it is not returned +until the pool is destroyed *and* the stream-ordered frees of its outstanding +allocations retire. The whole suite shares one process and one device, so +these reservations accumulate across tests. + +When a test needs its own pool, use the suite-wide cap: + +```python +POOL_SIZE = 2097152 # 2 MiB + +mr = DeviceMemoryResource(dev, DeviceMemoryResourceOptions(max_size=POOL_SIZE)) +``` + +Use a larger value only if a test genuinely requires it. + +### Passing no options is different from passing empty options + +`DeviceMemoryResource(dev)` with no options does **not** create a pool. It +wraps the device's existing default mempool (`_mempool_owned` is false) and +costs no additional address space. Passing *any* options object creates a new +owned pool, and a new pool without `max_size` is uncapped: + +```python +DeviceMemoryResource(dev) # wraps default pool, free +DeviceMemoryResource(dev, DeviceMemoryResourceOptions()) # NEW uncapped pool, expensive +DeviceMemoryResource(dev, {"ipc_enabled": True}) # NEW uncapped pool, expensive +``` + +Do not add `max_size` to a call that currently passes no options: that +converts a free default-pool wrapper into a new pool and makes things worse. + +### Managed pools are exempt + +`cuMemPoolCreate` requires `CUmemPoolProps.maxSize` to be zero for managed +pools, so `ManagedMemoryResourceOptions` has no `max_size` option. Managed +pools cannot be right-sized and are not checked. + +### Enforcement + +`test_mempool_hygiene.py` statically scans this directory and fails on +`DeviceMemoryResourceOptions` / `PinnedMemoryResourceOptions` constructions +that omit `max_size`. When a call is deliberately exempt -- most often because +it sits inside `pytest.raises` and no pool is ever created -- annotate it: + +```python +with pytest.raises(RuntimeError, match="IPC is not available"): + # uncapped-pool-ok: raises before the pool is created + DeviceMemoryResource(mempool_device, DeviceMemoryResourceOptions(ipc_enabled=True)) +``` + +## Release resources at test boundaries + +The `_init_cuda_context` fixture in `conftest.py` runs `gc.collect()` followed +by `cuCtxSynchronize()` before popping the context. Tests should not rely on +that as a substitute for cleaning up explicitly: prefer context managers for +resources whose lifetime fits a single scope, and keep pool lifetimes inside +the test that creates them. diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index 98baa521ef2..267946d29e4 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -1519,9 +1519,11 @@ def test_pinned_mr_numa_id_negative_error(init_cuda): skip_if_pinned_memory_unsupported(device) with pytest.raises(ValueError, match="numa_id must be >= 0"): + # uncapped-pool-ok: numa_id is validated before the pool is created PinnedMemoryResource(PinnedMemoryResourceOptions(numa_id=-1)) with pytest.raises(ValueError, match="numa_id must be >= 0"): + # uncapped-pool-ok: numa_id is validated before the pool is created PinnedMemoryResource(PinnedMemoryResourceOptions(numa_id=-42)) @@ -1971,4 +1973,5 @@ def test_dmr_ipc_enabled_unsupported_raises(mempool_device): if not IS_WINDOWS: pytest.skip("memory IPC is supported on this platform; unsupported-raise path is Windows-only") with pytest.raises(RuntimeError, match="IPC is not available"): + # uncapped-pool-ok: IPC support is checked before the pool is created DeviceMemoryResource(mempool_device, DeviceMemoryResourceOptions(ipc_enabled=True)) diff --git a/cuda_core/tests/test_mempool_hygiene.py b/cuda_core/tests/test_mempool_hygiene.py new file mode 100644 index 00000000000..bb89f858857 --- /dev/null +++ b/cuda_core/tests/test_mempool_hygiene.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Static guard against uncapped memory pools in the test suite. + +A pool created without ``max_size`` reserves an address-space window sized from +device memory rather than from what the test allocates, and the whole suite +shares one process. Enough of those reservations exhaust the address space and +the rest of the session fails with ``CUDA_ERROR_OUT_OF_MEMORY`` on a device with +free physical memory (issue #2381). See AGENTS.md in this directory. + +This check is static rather than runtime so that it also covers pools created +by tests that are skipped on the current platform. +""" + +import ast +import pathlib + +import pytest + +TESTS_ROOT = pathlib.Path(__file__).parent + +# Managed pools cannot be right-sized: cuMemPoolCreate requires maxSize == 0 for +# managed pools, so ManagedMemoryResourceOptions has no max_size to set. +CAPPABLE_OPTIONS = frozenset({"DeviceMemoryResourceOptions", "PinnedMemoryResourceOptions"}) +CAPPABLE_RESOURCES = frozenset({"DeviceMemoryResource", "PinnedMemoryResource"}) + +OPT_OUT_MARKER = "uncapped-pool-ok" + + +def _callee_name(node: ast.Call) -> str: + func = node.func + if isinstance(func, ast.Attribute): + return func.attr + if isinstance(func, ast.Name): + return func.id + return "" + + +def _is_capped(node: ast.Call) -> bool: + # ``**kwargs`` (arg is None) may carry max_size; do not guess. + return any(kw.arg is None or kw.arg == "max_size" for kw in node.keywords) + + +def _dict_is_capped(node: ast.Dict) -> bool: + for key in node.keys: + if key is None: # ``**other`` inside the literal + return True + if isinstance(key, ast.Constant) and key.value == "max_size": + return True + return False + + +def _opted_out(lines: list[str], node: ast.AST) -> bool: + """True if the call, or the line above it, carries the opt-out marker.""" + start = max(node.lineno - 2, 0) # -1 for 0-based, -1 more for a preceding comment + end = getattr(node, "end_lineno", node.lineno) + return any(OPT_OUT_MARKER in line for line in lines[start:end]) + + +def _violations_in(path: pathlib.Path) -> list[str]: + source = path.read_text(encoding="utf-8") + lines = source.splitlines() + found = [] + for node in ast.walk(ast.parse(source, filename=str(path))): + if not isinstance(node, ast.Call): + continue + name = _callee_name(node) + if name in CAPPABLE_OPTIONS: + uncapped = not _is_capped(node) + elif name in CAPPABLE_RESOURCES: + # The options may also be given as a dict literal. + dicts = [arg for arg in [*node.args, *(kw.value for kw in node.keywords)] if isinstance(arg, ast.Dict)] + uncapped = any(not _dict_is_capped(d) for d in dicts) + else: + continue + if uncapped and not _opted_out(lines, node): + found.append(f"{path.relative_to(TESTS_ROOT).as_posix()}:{node.lineno}: {name} without max_size") + return found + + +@pytest.mark.agent_authored(model="claude-opus-5") +def test_no_uncapped_memory_pools(): + violations = sorted(v for path in TESTS_ROOT.rglob("*.py") for v in _violations_in(path)) + assert not violations, ( + "Memory pools created by tests must set max_size (use POOL_SIZE = 2097152).\n" + f"Annotate a deliberate exception with a '# {OPT_OUT_MARKER}: ' comment.\n" + "See cuda_core/tests/AGENTS.md.\n" + "\n".join(violations) + ) diff --git a/cuda_core/tests/test_multiprocessing_warning.py b/cuda_core/tests/test_multiprocessing_warning.py index 0f96e0abfbc..2bd17bcd5b8 100644 --- a/cuda_core/tests/test_multiprocessing_warning.py +++ b/cuda_core/tests/test_multiprocessing_warning.py @@ -18,12 +18,14 @@ from cuda.core._memory._ipc import _reduce_allocation_handle from cuda.core._utils.cuda_utils import check_multiprocessing_start_method, reset_fork_warning +POOL_SIZE = 2097152 # 2MB size + def test_warn_on_fork_method_device_memory_resource(ipc_device): """Test that warning is emitted when DeviceMemoryResource is pickled with fork method.""" device = ipc_device device.set_current() - options = DeviceMemoryResourceOptions(max_size=2097152, ipc_enabled=True) + options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True) mr = DeviceMemoryResource(device, options=options) with patch("multiprocessing.get_start_method", return_value="fork"), warnings.catch_warnings(record=True) as w: @@ -50,7 +52,7 @@ def test_warn_on_fork_method_allocation_handle(ipc_device): """Test that warning is emitted when IPCAllocationHandle is pickled with fork method.""" device = ipc_device device.set_current() - options = DeviceMemoryResourceOptions(max_size=2097152, ipc_enabled=True) + options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True) mr = DeviceMemoryResource(device, options=options) alloc_handle = mr.allocation_handle @@ -102,7 +104,7 @@ def test_no_warning_with_spawn_method(ipc_device): """Test that no warning is emitted when start method is 'spawn'.""" device = ipc_device device.set_current() - options = DeviceMemoryResourceOptions(max_size=2097152, ipc_enabled=True) + options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True) mr = DeviceMemoryResource(device, options=options) with patch("multiprocessing.get_start_method", return_value="spawn"), warnings.catch_warnings(record=True) as w: @@ -125,7 +127,7 @@ def test_warning_emitted_only_once(ipc_device): """Test that warning is only emitted once even when multiple objects are pickled.""" device = ipc_device device.set_current() - options = DeviceMemoryResourceOptions(max_size=2097152, ipc_enabled=True) + options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True) mr1 = DeviceMemoryResource(device, options=options) mr2 = DeviceMemoryResource(device, options=options) From 81e803bc435bb56f0891a4f566e33c663a62ef18 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Wed, 5 Aug 2026 16:34:57 -0700 Subject: [PATCH 2/2] Address review: move mempool check to pre-commit, share POOL_SIZE Review feedback on #2514: - Move the uncapped-pool check out of the live test suite into a check-mempool-hygiene pre-commit hook. The rule is about source text and needs no GPU, so a hook catches it earlier and for free. Its tests move to ci/tools/tests, alongside the other check scripts'. - Add helpers/constants.py and route the eleven ad-hoc POOL_SIZE definitions through it. - Qualify "device memory" as installed/physical where the doc explains what an uncapped pool reserves. --- .pre-commit-config.yaml | 6 ++ .../tools/check_mempool_hygiene.py | 65 +++++++++---- ci/tools/tests/test_check_mempool_hygiene.py | 96 +++++++++++++++++++ cuda_core/tests/AGENTS.md | 36 +++---- cuda_core/tests/conftest.py | 2 +- cuda_core/tests/helpers/constants.py | 14 +++ cuda_core/tests/memory_ipc/test_errors.py | 2 +- .../memory_ipc/test_ipc_duplicate_import.py | 1 - .../tests/memory_ipc/test_peer_access.py | 2 +- .../tests/memory_ipc/test_send_buffers.py | 2 +- cuda_core/tests/memory_ipc/test_serialize.py | 1 - cuda_core/tests/memory_ipc/test_workerpool.py | 2 +- cuda_core/tests/test_memory.py | 3 +- cuda_core/tests/test_memory_peer_access.py | 11 +-- .../tests/test_multiprocessing_warning.py | 4 +- cuda_core/tests/test_object_protocols.py | 3 +- 16 files changed, 190 insertions(+), 60 deletions(-) rename cuda_core/tests/test_mempool_hygiene.py => ci/tools/check_mempool_hygiene.py (56%) create mode 100644 ci/tools/tests/test_check_mempool_hygiene.py create mode 100644 cuda_core/tests/helpers/constants.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d677b7ea7fe..30faa009fc2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,6 +58,12 @@ repos: files: '^(ci/versions\.yml|cuda_bindings/pixi\.toml|cuda_core/pixi\.toml)$' pass_filenames: false + - id: check-mempool-hygiene + name: Check tests do not create uncapped memory pools + entry: python ./ci/tools/check_mempool_hygiene.py + language: python + files: '^cuda_core/tests/.*\.py$' + - id: no-markdown-in-docs-source name: Prevent markdown files in docs/source directories entry: bash -c diff --git a/cuda_core/tests/test_mempool_hygiene.py b/ci/tools/check_mempool_hygiene.py similarity index 56% rename from cuda_core/tests/test_mempool_hygiene.py rename to ci/tools/check_mempool_hygiene.py index bb89f858857..b200aba1ebb 100644 --- a/cuda_core/tests/test_mempool_hygiene.py +++ b/ci/tools/check_mempool_hygiene.py @@ -1,25 +1,26 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# # SPDX-License-Identifier: Apache-2.0 -"""Static guard against uncapped memory pools in the test suite. +"""Check that tests do not create uncapped CUDA memory pools. A pool created without ``max_size`` reserves an address-space window sized from -device memory rather than from what the test allocates, and the whole suite -shares one process. Enough of those reservations exhaust the address space and -the rest of the session fails with ``CUDA_ERROR_OUT_OF_MEMORY`` on a device with -free physical memory (issue #2381). See AGENTS.md in this directory. +installed device memory rather than from what the test allocates, and the whole +cuda_core suite shares one process. Enough of those reservations exhaust the +address space, after which the rest of the session fails with +CUDA_ERROR_OUT_OF_MEMORY on a device with free physical memory. -This check is static rather than runtime so that it also covers pools created -by tests that are skipped on the current platform. +See cuda_core/tests/AGENTS.md for the rule this enforces. """ -import ast -import pathlib +from __future__ import annotations -import pytest +import argparse +import ast +import sys +from pathlib import Path -TESTS_ROOT = pathlib.Path(__file__).parent +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_TREE = ROOT / "cuda_core" / "tests" # Managed pools cannot be right-sized: cuMemPoolCreate requires maxSize == 0 for # managed pools, so ManagedMemoryResourceOptions has no max_size to set. @@ -59,7 +60,8 @@ def _opted_out(lines: list[str], node: ast.AST) -> bool: return any(OPT_OUT_MARKER in line for line in lines[start:end]) -def _violations_in(path: pathlib.Path) -> list[str]: +def violations_in(path: Path) -> list[str]: + """Return one message per uncapped pool construction in ``path``.""" source = path.read_text(encoding="utf-8") lines = source.splitlines() found = [] @@ -76,15 +78,36 @@ def _violations_in(path: pathlib.Path) -> list[str]: else: continue if uncapped and not _opted_out(lines, node): - found.append(f"{path.relative_to(TESTS_ROOT).as_posix()}:{node.lineno}: {name} without max_size") + found.append(f"{path.as_posix()}:{node.lineno}: {name} without max_size") return found -@pytest.mark.agent_authored(model="claude-opus-5") -def test_no_uncapped_memory_pools(): - violations = sorted(v for path in TESTS_ROOT.rglob("*.py") for v in _violations_in(path)) - assert not violations, ( - "Memory pools created by tests must set max_size (use POOL_SIZE = 2097152).\n" - f"Annotate a deliberate exception with a '# {OPT_OUT_MARKER}: ' comment.\n" - "See cuda_core/tests/AGENTS.md.\n" + "\n".join(violations) +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "paths", + nargs="*", + type=Path, + help=f"Files to check. Defaults to every .py under {DEFAULT_TREE.relative_to(ROOT).as_posix()}.", + ) + args = parser.parse_args(argv) + + paths = args.paths or sorted(DEFAULT_TREE.rglob("*.py")) + violations = sorted(v for path in paths if path.suffix == ".py" for v in violations_in(path)) + if not violations: + return 0 + + print("error: memory pools created by tests must set max_size:", file=sys.stderr) + for violation in violations: + print(f" - {violation}", file=sys.stderr) + print( + f"Use the suite-wide POOL_SIZE from cuda_core/tests/helpers/constants.py, or annotate a\n" + f"deliberate exception with a '# {OPT_OUT_MARKER}: ' comment.\n" + f"See cuda_core/tests/AGENTS.md.", + file=sys.stderr, ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/tools/tests/test_check_mempool_hygiene.py b/ci/tools/tests/test_check_mempool_hygiene.py new file mode 100644 index 00000000000..5ff3562059b --- /dev/null +++ b/ci/tools/tests/test_check_mempool_hygiene.py @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from check_mempool_hygiene import DEFAULT_TREE, main, violations_in + + +def write(tmp_path, source): + path = tmp_path / "test_sample.py" + path.write_text(source, encoding="utf-8") + return path + + +UNCAPPED = [ + pytest.param("DeviceMemoryResource(dev, DeviceMemoryResourceOptions(ipc_enabled=True))", id="options-kwarg"), + pytest.param("PinnedMemoryResource(PinnedMemoryResourceOptions())", id="options-empty"), + pytest.param('DeviceMemoryResource(dev, {"ipc_enabled": True})', id="options-dict"), +] + +CAPPED = [ + pytest.param("DeviceMemoryResource(dev, DeviceMemoryResourceOptions(max_size=POOL_SIZE))", id="capped-kwarg"), + pytest.param('DeviceMemoryResource(dev, {"max_size": POOL_SIZE})', id="capped-dict"), + pytest.param("DeviceMemoryResource(dev, DeviceMemoryResourceOptions(**opts))", id="opaque-kwargs"), + # No options at all wraps the device's default pool and reserves nothing, so + # capping it would convert a free wrapper into a new pool. + pytest.param("DeviceMemoryResource(dev)", id="default-pool-wrapper"), + # cuMemPoolCreate requires maxSize == 0 for managed pools, so these have no + # max_size to set. + pytest.param("ManagedMemoryResource(ManagedMemoryResourceOptions(preferred_location=0))", id="managed-exempt"), +] + + +@pytest.mark.agent_authored(model="claude-opus-5") +@pytest.mark.parametrize("source", UNCAPPED) +def test_uncapped_pool_is_reported(tmp_path, source): + assert violations_in(write(tmp_path, source)) + + +@pytest.mark.agent_authored(model="claude-opus-5") +@pytest.mark.parametrize("source", CAPPED) +def test_acceptable_construction_is_not_reported(tmp_path, source): + assert violations_in(write(tmp_path, source)) == [] + + +@pytest.mark.agent_authored(model="claude-opus-5") +@pytest.mark.parametrize("comment_line", [0, 1], ids=["marker-above", "marker-inline"]) +def test_marker_opts_a_call_out(tmp_path, comment_line): + # The escape hatch exists mainly for pytest.raises cases, where validation + # rejects the arguments before any pool is created. + call = "PinnedMemoryResource(PinnedMemoryResourceOptions())" + marker = "# uncapped-pool-ok: raises before the pool is created" + source = f"{marker}\n{call}" if comment_line == 0 else f"{call} {marker}" + + assert violations_in(write(tmp_path, source)) == [] + + +@pytest.mark.agent_authored(model="claude-opus-5") +def test_reported_message_names_file_line_and_symbol(tmp_path): + path = write(tmp_path, "x = 1\nDeviceMemoryResource(dev, DeviceMemoryResourceOptions())\n") + + (violation,) = violations_in(path) + + assert violation.startswith(path.as_posix()) + assert ":2:" in violation + assert "DeviceMemoryResourceOptions without max_size" in violation + + +@pytest.mark.agent_authored(model="claude-opus-5") +def test_main_reports_failure_for_the_files_it_is_given(tmp_path, capsys): + path = write(tmp_path, "DeviceMemoryResource(dev, DeviceMemoryResourceOptions())") + + assert main([str(path)]) == 1 + assert "must set max_size" in capsys.readouterr().err + + +@pytest.mark.agent_authored(model="claude-opus-5") +def test_main_ignores_non_python_files(tmp_path): + unrelated = tmp_path / "notes.txt" + unrelated.write_text("DeviceMemoryResourceOptions()", encoding="utf-8") + + assert main([str(unrelated)]) == 0 + + +@pytest.mark.agent_authored(model="claude-opus-5") +def test_the_live_test_suite_is_clean(): + # Without a default the hook would only ever see changed files, so a + # violation could ride in on a rename or a merge. + assert DEFAULT_TREE.is_dir() + assert main([]) == 0 diff --git a/cuda_core/tests/AGENTS.md b/cuda_core/tests/AGENTS.md index 16e54264040..39472d745b5 100644 --- a/cuda_core/tests/AGENTS.md +++ b/cuda_core/tests/AGENTS.md @@ -5,23 +5,25 @@ Package-wide conventions live in `../AGENTS.md`; repository-wide ones in ## Never create an uncapped memory pool -A memory pool created without `max_size` reserves a virtual address window -sized from device memory (roughly 1x device memory) regardless of what the -test actually allocates. The reservation is charged to the process address -space even though it is not backed by physical memory, and it is not returned -until the pool is destroyed *and* the stream-ordered frees of its outstanding -allocations retire. The whole suite shares one process and one device, so -these reservations accumulate across tests. +A memory pool created without `max_size` reserves virtual address space similar +in size to the installed physical device memory regardless of what the test +actually allocates. The reservation is charged to the process address space +even though it is not backed by physical memory, and it is not returned until +the pool is destroyed *and* the stream-ordered frees of its outstanding +allocations retire. The whole suite shares one process and one device, so these +reservations accumulate across tests. -When a test needs its own pool, use the suite-wide cap: +When a test needs its own pool, use the suite-wide cap from +`helpers/constants.py`: ```python -POOL_SIZE = 2097152 # 2 MiB +from helpers.constants import POOL_SIZE mr = DeviceMemoryResource(dev, DeviceMemoryResourceOptions(max_size=POOL_SIZE)) ``` -Use a larger value only if a test genuinely requires it. +Use a larger value only if a test genuinely requires it, and prefer adding a +shared constant to `helpers/constants.py` over redefining one per module. ### Passing no options is different from passing empty options @@ -31,9 +33,9 @@ costs no additional address space. Passing *any* options object creates a new owned pool, and a new pool without `max_size` is uncapped: ```python -DeviceMemoryResource(dev) # wraps default pool, free -DeviceMemoryResource(dev, DeviceMemoryResourceOptions()) # NEW uncapped pool, expensive -DeviceMemoryResource(dev, {"ipc_enabled": True}) # NEW uncapped pool, expensive +DeviceMemoryResource(dev) # wraps default pool, free +DeviceMemoryResource(dev, DeviceMemoryResourceOptions()) # NEW uncapped pool, expensive +DeviceMemoryResource(dev, {"ipc_enabled": True}) # NEW uncapped pool, expensive ``` Do not add `max_size` to a call that currently passes no options: that @@ -45,12 +47,10 @@ converts a free default-pool wrapper into a new pool and makes things worse. pools, so `ManagedMemoryResourceOptions` has no `max_size` option. Managed pools cannot be right-sized and are not checked. -### Enforcement +### Document exemptions -`test_mempool_hygiene.py` statically scans this directory and fails on -`DeviceMemoryResourceOptions` / `PinnedMemoryResourceOptions` constructions -that omit `max_size`. When a call is deliberately exempt -- most often because -it sits inside `pytest.raises` and no pool is ever created -- annotate it: +When a call is deliberately exempt -- most often because it sits inside +`pytest.raises` and no pool is ever created -- annotate it: ```python with pytest.raises(RuntimeError, match="IPC is not available"): diff --git a/cuda_core/tests/conftest.py b/cuda_core/tests/conftest.py index e012e349d27..dfe97b265eb 100644 --- a/cuda_core/tests/conftest.py +++ b/cuda_core/tests/conftest.py @@ -31,6 +31,7 @@ from cuda_python_test_helpers.marks import skipif_need_cuda_headers # noqa: F401 (re-exported for tests) from cuda_python_test_helpers.mempool import xfail_if_mempool_oom +from helpers.constants import POOL_SIZE import cuda.core from cuda.bindings import driver @@ -316,7 +317,6 @@ def ipc_device(init_cuda): ) def ipc_memory_resource(request, ipc_device): """Provides IPC-enabled memory resource (either Device or Pinned).""" - POOL_SIZE = 2097152 mr_type = request.param if mr_type == "device": diff --git a/cuda_core/tests/helpers/constants.py b/cuda_core/tests/helpers/constants.py new file mode 100644 index 00000000000..f4ea61b1938 --- /dev/null +++ b/cuda_core/tests/helpers/constants.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Constants shared across the cuda_core test suite.""" + +# Cap for memory pools created by tests. A pool created without an explicit +# max_size instead reserves a system-dependent window that scales with +# installed device memory -- hundreds of GiB on large-memory GPUs. The +# per-process virtual address budget is bounded (~1 TB on Windows MCDM), and a +# reservation is not returned until the pool is torn down and its +# stream-ordered frees retire, so oversized windows accumulate across a session +# and eventually starve later pool creations with CUDA_ERROR_OUT_OF_MEMORY +# (issue #2381). See AGENTS.md in the tests directory. +POOL_SIZE = 2097152 # 2 MiB diff --git a/cuda_core/tests/memory_ipc/test_errors.py b/cuda_core/tests/memory_ipc/test_errors.py index 40162fab01d..0aac9f9a297 100644 --- a/cuda_core/tests/memory_ipc/test_errors.py +++ b/cuda_core/tests/memory_ipc/test_errors.py @@ -7,6 +7,7 @@ import pytest from helpers.child_processes import child_timeout_sec, kill_subprocesses +from helpers.constants import POOL_SIZE from cuda.core import Buffer, Device, DeviceMemoryResource, DeviceMemoryResourceOptions from cuda.core._memory._ipc import IPCBufferDescriptor @@ -14,7 +15,6 @@ CHILD_TIMEOUT_SEC = child_timeout_sec() NBYTES = 64 -POOL_SIZE = 2097152 # these tests spawn new processes and files which fails for very many threads diff --git a/cuda_core/tests/memory_ipc/test_ipc_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_duplicate_import.py index eaa6ddec92f..dc3f5e57c33 100644 --- a/cuda_core/tests/memory_ipc/test_ipc_duplicate_import.py +++ b/cuda_core/tests/memory_ipc/test_ipc_duplicate_import.py @@ -20,7 +20,6 @@ CHILD_TIMEOUT_SEC = child_timeout_sec() NBYTES = 64 -POOL_SIZE = 2097152 ENABLE_LOGGING = False # Set True for test debugging and development diff --git a/cuda_core/tests/memory_ipc/test_peer_access.py b/cuda_core/tests/memory_ipc/test_peer_access.py index ac7f71a88e9..4dc04a8bd0b 100644 --- a/cuda_core/tests/memory_ipc/test_peer_access.py +++ b/cuda_core/tests/memory_ipc/test_peer_access.py @@ -6,13 +6,13 @@ import pytest from helpers.buffers import PatternGen from helpers.child_processes import child_timeout_sec, kill_subprocesses +from helpers.constants import POOL_SIZE from cuda.core import Device, DeviceMemoryResource, DeviceMemoryResourceOptions from cuda.core._utils.cuda_utils import CUDAError CHILD_TIMEOUT_SEC = child_timeout_sec() NBYTES = 64 -POOL_SIZE = 2097152 # these tests spawn new processes and files which fails for very many threads pytestmark = pytest.mark.parallel_threads_limit(4) diff --git a/cuda_core/tests/memory_ipc/test_send_buffers.py b/cuda_core/tests/memory_ipc/test_send_buffers.py index 59216cd9cce..efa4d8b2abc 100644 --- a/cuda_core/tests/memory_ipc/test_send_buffers.py +++ b/cuda_core/tests/memory_ipc/test_send_buffers.py @@ -7,6 +7,7 @@ import pytest from helpers.buffers import PatternGen from helpers.child_processes import child_timeout_sec, kill_subprocesses +from helpers.constants import POOL_SIZE from cuda.core import Device, DeviceMemoryResource, DeviceMemoryResourceOptions @@ -14,7 +15,6 @@ NBYTES = 64 NMRS = 3 NTASKS = 7 -POOL_SIZE = 2097152 # these tests spawn new processes and files which fails for very many threads pytestmark = pytest.mark.parallel_threads_limit(4) diff --git a/cuda_core/tests/memory_ipc/test_serialize.py b/cuda_core/tests/memory_ipc/test_serialize.py index 4289de4b5a9..22596582c49 100644 --- a/cuda_core/tests/memory_ipc/test_serialize.py +++ b/cuda_core/tests/memory_ipc/test_serialize.py @@ -13,7 +13,6 @@ CHILD_TIMEOUT_SEC = child_timeout_sec() NBYTES = 64 -POOL_SIZE = 2097152 # these tests spawn new processes and files which fails for very many threads pytestmark = pytest.mark.parallel_threads_limit(4) diff --git a/cuda_core/tests/memory_ipc/test_workerpool.py b/cuda_core/tests/memory_ipc/test_workerpool.py index 358c16fd7bf..e358c043b00 100644 --- a/cuda_core/tests/memory_ipc/test_workerpool.py +++ b/cuda_core/tests/memory_ipc/test_workerpool.py @@ -7,6 +7,7 @@ import pytest from helpers.buffers import PatternGen +from helpers.constants import POOL_SIZE from cuda.core import Buffer, Device, DeviceMemoryResource, DeviceMemoryResourceOptions @@ -14,7 +15,6 @@ NWORKERS = 2 NMRS = 3 NTASKS = 20 -POOL_SIZE = 2097152 # these tests spawn new processes and files which fails for very many threads pytestmark = pytest.mark.parallel_threads_limit(4) diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index 267946d29e4..6af4d025b03 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -22,6 +22,7 @@ ) from helpers import supports_ipc_mempool from helpers.buffers import DummyDeviceMemoryResource, DummyUnifiedMemoryResource, TrackingMR +from helpers.constants import POOL_SIZE from cuda.core import ( Buffer, @@ -55,8 +56,6 @@ from cuda.core.utils import StridedMemoryView from cuda_python_test_helpers import IS_WINDOWS -POOL_SIZE = 2097152 # 2MB size - def _allocate_pinned_buffer_or_xfail(mr, size, *, device): try: diff --git a/cuda_core/tests/test_memory_peer_access.py b/cuda_core/tests/test_memory_peer_access.py index 2cbfbbd302f..4763b761ad4 100644 --- a/cuda_core/tests/test_memory_peer_access.py +++ b/cuda_core/tests/test_memory_peer_access.py @@ -4,6 +4,7 @@ import pytest from helpers.buffers import PatternGen, compare_buffer_to_constant, make_scratch_buffer from helpers.collection_interface_testers import assert_single_member_mutable_set_interface +from helpers.constants import POOL_SIZE from cuda.core import Device, DeviceMemoryResource, DeviceMemoryResourceOptions, system from cuda.core._memory import _peer_access_utils @@ -11,14 +12,8 @@ from cuda.core._utils.cuda_utils import CUDAError NBYTES = 1024 -# Every owned pool below holds at most NBYTES, but a pool created without an -# explicit max_size reserves a system-dependent window that scales with device -# memory -- hundreds of GiB on large-memory GPUs. The per-process virtual -# address budget is bounded (~1 TB on Windows MCDM), and reservations are not -# returned until a pool is torn down and its stream-ordered frees retire, so -# oversized windows accumulate across a session and eventually starve later -# pool creations with CUDA_ERROR_OUT_OF_MEMORY (issue #2381). Cap them. -POOL_SIZE = 2097152 # 2MB size +# Every owned pool below holds at most NBYTES, so they are all capped at the +# suite-wide POOL_SIZE; see helpers/constants.py for why that matters. pytestmark = pytest.mark.thread_unsafe(reason="peer access tests mutate process-global CUDA memory-pool access state") diff --git a/cuda_core/tests/test_multiprocessing_warning.py b/cuda_core/tests/test_multiprocessing_warning.py index 2bd17bcd5b8..1ddb53edb0f 100644 --- a/cuda_core/tests/test_multiprocessing_warning.py +++ b/cuda_core/tests/test_multiprocessing_warning.py @@ -12,14 +12,14 @@ import warnings from unittest.mock import patch +from helpers.constants import POOL_SIZE + from cuda.core import DeviceMemoryResource, DeviceMemoryResourceOptions, EventOptions from cuda.core._event import _reduce_event from cuda.core._memory._device_memory_resource import _deep_reduce_device_memory_resource from cuda.core._memory._ipc import _reduce_allocation_handle from cuda.core._utils.cuda_utils import check_multiprocessing_start_method, reset_fork_warning -POOL_SIZE = 2097152 # 2MB size - def test_warn_on_fork_method_device_memory_resource(ipc_device): """Test that warning is emitted when DeviceMemoryResource is pickled with fork method.""" diff --git a/cuda_core/tests/test_object_protocols.py b/cuda_core/tests/test_object_protocols.py index f843f451683..f7f4c854313 100644 --- a/cuda_core/tests/test_object_protocols.py +++ b/cuda_core/tests/test_object_protocols.py @@ -14,6 +14,7 @@ import pytest from conftest import xfail_on_graph_mempool_oom +from helpers.constants import POOL_SIZE from helpers.graph_kernels import compile_common_kernels from helpers.misc import try_create_condition @@ -224,8 +225,6 @@ def sample_kernel_alt(sample_object_code_alt): # Fixtures - IPC samples (for pickle tests) # ============================================================================= -POOL_SIZE = 2097152 - @pytest.fixture def sample_ipc_buffer_descriptor(ipc_device):