From 3ec71593ded87c3c54f368eddd700e277304e4a0 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 6 Feb 2026 09:34:12 -0500 Subject: [PATCH 01/10] Skip NVML tests if the hardware doesn't support NVML --- .../cuda/bindings/_test_helpers/arch_check.py | 19 +++++++++++++++++++ cuda_bindings/tests/nvml/__init__.py | 9 ++++++++- cuda_core/tests/system/conftest.py | 6 ++++-- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py b/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py index e063fd4ff8..61a9f5adbc 100644 --- a/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py +++ b/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py @@ -3,11 +3,30 @@ from contextlib import contextmanager +from functools import cache import pytest from cuda.bindings import nvml +@cache +def hardware_supports_nvml(): + """ + Tries to call the simplest NVML API possible to see if just the basics + works. If not we are probably on one of the platforms where NVML is not + supported at all (e.g. Jetson Orin). + """ + nvml.init_v2() + try: + nvml.system_get_driver_branch() + except nvml.NotSupportedError: + return False + else: + return True + finally: + nvml.shutdown() + + @contextmanager def unsupported_before(device: int, expected_device_arch: nvml.DeviceArch | str | None): device_arch = nvml.device_get_architecture(device) diff --git a/cuda_bindings/tests/nvml/__init__.py b/cuda_bindings/tests/nvml/__init__.py index 830c1bf0de..996effce9c 100644 --- a/cuda_bindings/tests/nvml/__init__.py +++ b/cuda_bindings/tests/nvml/__init__.py @@ -1,2 +1,9 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + + +import pytest +from cuda.bindings._test_helpers.arch_check import hardware_supports_nvml + +if not hardware_supports_nvml(): + pytest.skip("NVML not supported on this platform", allow_module_level=True) diff --git a/cuda_core/tests/system/conftest.py b/cuda_core/tests/system/conftest.py index cb6974806d..70b4ee2faa 100644 --- a/cuda_core/tests/system/conftest.py +++ b/cuda_core/tests/system/conftest.py @@ -1,13 +1,15 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 import pytest +from cuda.bindings._test_helpers.arch_check import hardware_supports_nvml from cuda.core import system skip_if_nvml_unsupported = pytest.mark.skipif( - not system.CUDA_BINDINGS_NVML_IS_COMPATIBLE, reason="NVML support requires cuda.bindings version 12.9.6+ or 13.1.2+" + not system.CUDA_BINDINGS_NVML_IS_COMPATIBLE or not hardware_supports_nvml(), + reason="NVML support requires cuda.bindings version 12.9.6+ or 13.1.2+, and hardware that supports NVML", ) From 6bcd42be7dff1aa0127a047f26a61435f43c9320 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 6 Feb 2026 09:37:26 -0500 Subject: [PATCH 02/10] Also skip converting between Device types --- cuda_core/tests/test_device.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cuda_core/tests/test_device.py b/cuda_core/tests/test_device.py index 09f33b92d2..10ddd4dbbc 100644 --- a/cuda_core/tests/test_device.py +++ b/cuda_core/tests/test_device.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 try: @@ -8,6 +8,7 @@ from cuda import cudart as runtime import cuda.core import pytest +from cuda.bindings._test_helpers.arch_check import hardware_supports_nvml from cuda.core import Device from cuda.core._utils.cuda_utils import ComputeCapability, get_binding_version, handle_return @@ -30,7 +31,7 @@ def test_to_system_device(deinit_cuda): device = Device() - if not _system.CUDA_BINDINGS_NVML_IS_COMPATIBLE: + if not _system.CUDA_BINDINGS_NVML_IS_COMPATIBLE or not hardware_supports_nvml(): with pytest.raises(RuntimeError): device.to_system_device() pytest.skip("NVML support requires cuda.bindings version 12.9.6+ or 13.1.2+") From 0ac140f2bdd0e0b1d0a4a24972e7113917954612 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 6 Feb 2026 10:47:48 -0500 Subject: [PATCH 03/10] Fix test_to_system_device test --- cuda_core/tests/test_device.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cuda_core/tests/test_device.py b/cuda_core/tests/test_device.py index 10ddd4dbbc..903516f5c4 100644 --- a/cuda_core/tests/test_device.py +++ b/cuda_core/tests/test_device.py @@ -8,7 +8,6 @@ from cuda import cudart as runtime import cuda.core import pytest -from cuda.bindings._test_helpers.arch_check import hardware_supports_nvml from cuda.core import Device from cuda.core._utils.cuda_utils import ComputeCapability, get_binding_version, handle_return @@ -31,11 +30,18 @@ def test_to_system_device(deinit_cuda): device = Device() - if not _system.CUDA_BINDINGS_NVML_IS_COMPATIBLE or not hardware_supports_nvml(): + if not _system.CUDA_BINDINGS_NVML_IS_COMPATIBLE: with pytest.raises(RuntimeError): device.to_system_device() pytest.skip("NVML support requires cuda.bindings version 12.9.6+ or 13.1.2+") + from cuda.bindings._test_helpers.arch_check import hardware_supports_nvml + + if not hardware_supports_nvml(): + with pytest.raises(RuntimeError): + device.to_system_device() + pytest.skip("NVML not supported on this platform") + from cuda.core.system import Device as SystemDevice system_device = device.to_system_device() From 86cd074ae84e6a189c2c73cad57585e063e1c582 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 6 Feb 2026 10:49:02 -0500 Subject: [PATCH 04/10] Update cuda_bindings/cuda/bindings/_test_helpers/arch_check.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../cuda/bindings/_test_helpers/arch_check.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py b/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py index 61a9f5adbc..af2dc5994f 100644 --- a/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py +++ b/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py @@ -16,15 +16,29 @@ def hardware_supports_nvml(): works. If not we are probably on one of the platforms where NVML is not supported at all (e.g. Jetson Orin). """ - nvml.init_v2() + initialized = False try: - nvml.system_get_driver_branch() - except nvml.NotSupportedError: - return False - else: - return True + try: + nvml.init_v2() + except Exception: + # NVML cannot be initialized in this environment; treat as unsupported. + return False + else: + initialized = True + + try: + nvml.system_get_driver_branch() + except nvml.NotSupportedError: + return False + else: + return True finally: - nvml.shutdown() + if initialized: + # Ignore shutdown errors in this lightweight capability check. + try: + nvml.shutdown() + except Exception: + pass @contextmanager From 9a020b21f43fbc91f989159943f32d124564b908 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 6 Feb 2026 10:50:08 -0500 Subject: [PATCH 05/10] Fix test --- cuda_core/tests/test_device.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cuda_core/tests/test_device.py b/cuda_core/tests/test_device.py index 903516f5c4..d561f92b9e 100644 --- a/cuda_core/tests/test_device.py +++ b/cuda_core/tests/test_device.py @@ -38,8 +38,6 @@ def test_to_system_device(deinit_cuda): from cuda.bindings._test_helpers.arch_check import hardware_supports_nvml if not hardware_supports_nvml(): - with pytest.raises(RuntimeError): - device.to_system_device() pytest.skip("NVML not supported on this platform") from cuda.core.system import Device as SystemDevice From aae5a3ea7b76b1410d7ef0536c8d8bb58d0f30a5 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 6 Feb 2026 10:52:09 -0500 Subject: [PATCH 06/10] Simplify check --- .../cuda/bindings/_test_helpers/arch_check.py | 28 +++++-------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py b/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py index af2dc5994f..61a9f5adbc 100644 --- a/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py +++ b/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py @@ -16,29 +16,15 @@ def hardware_supports_nvml(): works. If not we are probably on one of the platforms where NVML is not supported at all (e.g. Jetson Orin). """ - initialized = False + nvml.init_v2() try: - try: - nvml.init_v2() - except Exception: - # NVML cannot be initialized in this environment; treat as unsupported. - return False - else: - initialized = True - - try: - nvml.system_get_driver_branch() - except nvml.NotSupportedError: - return False - else: - return True + nvml.system_get_driver_branch() + except nvml.NotSupportedError: + return False + else: + return True finally: - if initialized: - # Ignore shutdown errors in this lightweight capability check. - try: - nvml.shutdown() - except Exception: - pass + nvml.shutdown() @contextmanager From 3efb2d5ab59ffd7070f7cb72be0daebd0220b770 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 6 Feb 2026 11:33:08 -0500 Subject: [PATCH 07/10] Fix import when cuda_bindings is incompatible --- cuda_core/tests/system/conftest.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cuda_core/tests/system/conftest.py b/cuda_core/tests/system/conftest.py index 70b4ee2faa..800677c74f 100644 --- a/cuda_core/tests/system/conftest.py +++ b/cuda_core/tests/system/conftest.py @@ -4,11 +4,19 @@ import pytest -from cuda.bindings._test_helpers.arch_check import hardware_supports_nvml from cuda.core import system +SHOULD_SKIP_NVML_TESTS = not system.CUDA_BINDINGS_NVML_IS_COMPATIBLE + + +if system.CUDA_BINDINGS_NVML_IS_COMPATIBLE: + from cuda.bindings._test_helpers.arch_check import hardware_supports_nvml + + SHOULD_SKIP_NVML_TESTS &= not hardware_supports_nvml() + + skip_if_nvml_unsupported = pytest.mark.skipif( - not system.CUDA_BINDINGS_NVML_IS_COMPATIBLE or not hardware_supports_nvml(), + SHOULD_SKIP_NVML_TESTS, reason="NVML support requires cuda.bindings version 12.9.6+ or 13.1.2+, and hardware that supports NVML", ) From 0570c1cebfad272feead65c97cbf44ef66405c63 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 6 Feb 2026 11:40:18 -0500 Subject: [PATCH 08/10] Fix logic --- cuda_core/tests/system/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuda_core/tests/system/conftest.py b/cuda_core/tests/system/conftest.py index 800677c74f..594f2ba925 100644 --- a/cuda_core/tests/system/conftest.py +++ b/cuda_core/tests/system/conftest.py @@ -12,7 +12,7 @@ if system.CUDA_BINDINGS_NVML_IS_COMPATIBLE: from cuda.bindings._test_helpers.arch_check import hardware_supports_nvml - SHOULD_SKIP_NVML_TESTS &= not hardware_supports_nvml() + SHOULD_SKIP_NVML_TESTS |= not hardware_supports_nvml() skip_if_nvml_unsupported = pytest.mark.skipif( From aa0578cd5bd524796b83635f6b46fdf9b58ee50b Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 6 Feb 2026 11:48:40 -0500 Subject: [PATCH 09/10] Move over a single test from #1583 --- cuda_core/tests/test_graph_mem.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cuda_core/tests/test_graph_mem.py b/cuda_core/tests/test_graph_mem.py index bcb8a800a1..c80dd8c982 100644 --- a/cuda_core/tests/test_graph_mem.py +++ b/cuda_core/tests/test_graph_mem.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE @@ -13,6 +13,7 @@ ProgramOptions, launch, ) +from cuda.core._utils.cuda_utils import CUDAError from helpers import IS_WINDOWS, IS_WSL from helpers.buffers import compare_buffer_to_constant, make_scratch_buffer, set_buffer @@ -166,7 +167,12 @@ def test_graph_alloc_with_output(mempool_device, mode): out.copy_from(in_, stream=gb) launch(gb, LaunchConfig(grid=1, block=1), add_one, out, NBYTES) options = GraphCompleteOptions(auto_free_on_launch=True) - graph = gb.end_building().complete(options) + try: + graph = gb.end_building().complete(options) + except CUDAError as exc: + if "CUDA_ERROR_INVALID_VALUE" in str(exc): + pytest.skip("auto_free_on_launch not supported on this platform") + raise # Launch the graph. The output buffer is allocated and set to one. graph.upload(stream) From eaa2e6282b68796a478ca8e843f41401cbc225f2 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 6 Feb 2026 12:35:24 -0500 Subject: [PATCH 10/10] Fix check --- cuda_bindings/cuda/bindings/_test_helpers/arch_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py b/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py index 61a9f5adbc..bf66250d1e 100644 --- a/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py +++ b/cuda_bindings/cuda/bindings/_test_helpers/arch_check.py @@ -19,7 +19,7 @@ def hardware_supports_nvml(): nvml.init_v2() try: nvml.system_get_driver_branch() - except nvml.NotSupportedError: + except (nvml.NotSupportedError, nvml.UnknownError): return False else: return True