From f8730f43394bb06bb68630d5ed1337b5782fa895 Mon Sep 17 00:00:00 2001 From: oobabooga Date: Tue, 4 Aug 2026 10:55:29 -0300 Subject: [PATCH] Installer: select CUDA wheels that cover the host's GPUs (#7814) * Installer: select CUDA wheels that cover the host's GPUs * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix Windows venv wipe and warning dedupe for PR #7814 - Windows pins torch<2.11, whose cu128 still ships sm_70, so capping a Volta to cu126 there rewrote a working family. The stale-venv check then read that as drift and deleted the venv on a direct "unsloth studio update", which cannot recreate it. Make the pre-Turing floor per-family (70 for cu128). - Repair an unpinned cu* -> cu* move in place instead of rebuilding the venv. - Decide the cu126 advice before deduping the uncovered-host warning: the host facts are release invariant but the artifact list is not, so the release walk-back let an unhelpful release swallow the remedy. - Gate the new coverage repair and the cu126 advice on x86_64, matching the cap. - Add tests/studio/test_pre_turing_cap.ps1: the parity test only greps for the call spelling, so neither PowerShell copy had behavioural coverage. * Tighten comments for PR #7814 --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: danielhanchen --- install.ps1 | 65 ++++- install.sh | 63 ++++- studio/install_llama_prebuilt.py | 74 ++++++ studio/install_python_stack.py | 156 ++++++++++- studio/setup.ps1 | 79 +++++- tests/python/test_cross_platform_parity.py | 51 +++- tests/sh/test_get_torch_index_url.sh | 102 +++++++- tests/studio/install/test_cuda_repair.py | 258 ++++++++++++++++++- tests/studio/install/test_selection_logic.py | 126 +++++++++ tests/studio/test_pre_turing_cap.ps1 | 117 +++++++++ 10 files changed, 1049 insertions(+), 42 deletions(-) create mode 100644 tests/studio/test_pre_turing_cap.ps1 diff --git a/install.ps1 b/install.ps1 index 8e1d7063d4..077325b07f 100644 --- a/install.ps1 +++ b/install.ps1 @@ -2100,7 +2100,10 @@ exit 0 param( [Parameter(Mandatory = $true, Position = 0)][string]$Exe, [Parameter(Position = 1)][string[]]$SmiArgs = @(), - [int]$TimeoutSec = 10 + [int]$TimeoutSec = 10, + # Driver warnings on stderr would corrupt machine-readable --query-gpu + # output; the human-readable probes keep the default merge. + [switch]$StdoutOnly ) try { $psi = New-Object System.Diagnostics.ProcessStartInfo @@ -2119,6 +2122,7 @@ exit 0 return "" } $global:LASTEXITCODE = $proc.ExitCode + if ($StdoutOnly) { return $outTask.Result } return ($outTask.Result + "`n" + $errTask.Result) } catch { $global:LASTEXITCODE = 1 @@ -2688,6 +2692,52 @@ exit 0 return ((($Url -split '[?#]', 2)[0].TrimEnd('/') -split '/')[-1]).ToLowerInvariant() } + # Classify the physical NVIDIA inventory for a cu126 fallback: "cu126" when it + # covers every GPU, "uncovered" for an incompatible mix, empty when no fallback is + # needed or the inventory is unreadable. CUDA_VISIBLE_DEVICES is ignored because + # the wheel must support the host. Mirrors _nvidia_cu126_verdict in install.sh. + function Get-NvidiaCu126Verdict { + # Floor is per-release, not fixed: only 2.11 dropped sm_70 from cu128. + param([string]$SmiExe, [int]$LegacyFloorSm = 75) + if (-not $SmiExe) { return '' } + $raw = Invoke-NvidiaSmiBounded $SmiExe @('--query-gpu=compute_cap', '--format=csv,noheader,nounits') -StdoutOnly + if ($LASTEXITCODE -ne 0 -or -not $raw) { return '' } + $legacy = $false + $outsideCu126 = $false + $seen = $false + foreach ($line in ($raw -split "`n")) { + $value = $line.Trim() + if (-not $value) { continue } + if ($value -notmatch '^(\d+)\.(\d+)$') { return '' } + $sm = ([int]$Matches[1] * 10) + [int]$Matches[2] + if ($sm -lt $LegacyFloorSm) { $legacy = $true } + if ($sm -lt 50 -or $sm -gt 90) { $outsideCu126 = $true } + $seen = $true + } + if (-not $seen -or -not $legacy) { return '' } + if ($outsideCu126) { return 'uncovered' } + return 'cu126' + } + + function Get-CudaFamilyCappedForPreTuring { + param([string]$Family, [string]$SmiExe) + if ($Family -notin @('cu128', 'cu130')) { return $Family } + # Windows pins torch<2.11, whose cu128 still ships sm_70, so only cu130 + # strands a Volta here. Raise to 75 when that pin reaches 2.11. + $legacyFloorSm = if ($Family -eq 'cu128') { 70 } else { 75 } + switch (Get-NvidiaCu126Verdict $SmiExe $legacyFloorSm) { + 'cu126' { + substep "pre-Turing NVIDIA GPUs (sm_<75) are present -- selecting cu126, because PyTorch 2.11's $Family wheels start at sm_75" "Yellow" + return 'cu126' + } + 'uncovered' { + substep "this host mixes pre-Turing NVIDIA GPUs with GPUs that cu126 cannot serve; no PyTorch 2.11 CUDA family covers both" "Yellow" + substep "keeping $Family, so the pre-Turing GPUs will be unusable; set UNSLOTH_TORCH_INDEX_FAMILY=cu126 to choose the other way" "Yellow" + } + } + return $Family + } + # ── Choose the correct PyTorch index URL based on driver CUDA version ── # Mirrors Get-PytorchCudaTag in setup.ps1. function Get-TorchIndexUrl { @@ -2709,12 +2759,13 @@ exit 0 # Accept both spellings so we don't fall through to the cu126 default. if ($output -match 'CUDA(?: UMD)? Version:\s+(\d+)\.(\d+)') { $major = [int]$Matches[1]; $minor = [int]$Matches[2] - if ($major -ge 13) { return "$baseUrl/cu130" } - if ($major -eq 12 -and $minor -ge 8) { return "$baseUrl/cu128" } - if ($major -eq 12 -and $minor -ge 6) { return "$baseUrl/cu126" } - if ($major -ge 12) { return "$baseUrl/cu124" } - if ($major -ge 11) { return "$baseUrl/cu118" } - return "$baseUrl/cpu" + if ($major -ge 13) { $family = "cu130" } + elseif ($major -eq 12 -and $minor -ge 8) { $family = "cu128" } + elseif ($major -eq 12 -and $minor -ge 6) { $family = "cu126" } + elseif ($major -ge 12) { $family = "cu124" } + elseif ($major -ge 11) { $family = "cu118" } + else { return "$baseUrl/cpu" } + return "$baseUrl/$(Get-CudaFamilyCappedForPreTuring $family $NvidiaSmiExe)" } } catch {} substep "could not determine CUDA version from nvidia-smi, defaulting to cu126" "Yellow" diff --git a/install.sh b/install.sh index 25d71224e2..ce99f34738 100755 --- a/install.sh +++ b/install.sh @@ -2686,6 +2686,56 @@ _probe_amd_gfx_arch() { printf '%s\n' "$_pg" } +# Classify the physical NVIDIA inventory for a cu126 fallback: "cu126" when it covers +# every GPU, "uncovered" for an incompatible mix, empty when no fallback is needed or the +# inventory is unreadable. CUDA_VISIBLE_DEVICES is ignored because the wheel must support +# the host. Shared decision with install.ps1 / setup.ps1 / install_python_stack.py. +_nvidia_cu126_verdict() { + [ -n "$1" ] || return 0 + _ncv_caps=$(_run_bounded "$1" --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null) || return 0 + printf '%s\n' "$_ncv_caps" | awk ' + { gsub(/^[[:space:]]+|[[:space:]]+$/, "") } # match the .Trim()/.strip() siblings + /^[0-9]+\.[0-9]+$/ { + split($0, _sm, ".") + _n = (_sm[1] * 10) + _sm[2] + seen = 1 + if (_n < 75) legacy = 1 + if (_n < 50 || _n > 90) outside_cu126 = 1 + next + } + /./ { unreadable = 1 } + END { + if (!seen || unreadable || !legacy) exit + print outside_cu126 ? "uncovered" : "cu126" + } + ' +} + +# Cap cu128/cu130 at cu126 when it covers every physical GPU: PyTorch 2.11's cu128/cu130 +# start at sm_75, cu126 spans sm_50-90. Non-x86_64 keeps driver-only selection. +_cap_cuda_family_for_pre_turing() { + case "$_ARCH" in + x86_64|amd64) ;; + *) printf '%s\n' "$1"; return ;; + esac + case "$1" in + cu128|cu130) ;; + *) printf '%s\n' "$1"; return ;; + esac + case "$(_nvidia_cu126_verdict "$2")" in + cu126) + echo "[WARN] Pre-Turing NVIDIA GPUs (sm_<75) are present -- selecting cu126, because PyTorch 2.11's $1 wheels start at sm_75." >&2 + printf '%s\n' "cu126" + return + ;; + uncovered) + echo "[WARN] This host mixes pre-Turing NVIDIA GPUs with GPUs that cu126 cannot serve; no PyTorch 2.11 CUDA family covers both." >&2 + echo "[WARN] Keeping $1, so the pre-Turing GPUs will be unusable. Set UNSLOTH_TORCH_INDEX_FAMILY=cu126 to choose the other way." >&2 + ;; + esac + printf '%s\n' "$1" +} + # ── Detect GPU and choose PyTorch index URL ── # Mirrors Get-TorchIndexUrl in install.ps1. # On CPU-only machines this returns the cpu index, avoiding the solver @@ -2861,12 +2911,13 @@ get_torch_index_url() { fi _major=${_cuda_ver%%.*} _minor=${_cuda_ver#*.} - if [ "$_major" -ge 13 ]; then echo "$_base/cu130" - elif [ "$_major" -eq 12 ] && [ "$_minor" -ge 8 ]; then echo "$_base/cu128" - elif [ "$_major" -eq 12 ] && [ "$_minor" -ge 6 ]; then echo "$_base/cu126" - elif [ "$_major" -ge 12 ]; then echo "$_base/cu124" - elif [ "$_major" -ge 11 ]; then echo "$_base/cu118" - else echo "$_base/cpu"; fi + if [ "$_major" -ge 13 ]; then _cuda_tag=cu130 + elif [ "$_major" -eq 12 ] && [ "$_minor" -ge 8 ]; then _cuda_tag=cu128 + elif [ "$_major" -eq 12 ] && [ "$_minor" -ge 6 ]; then _cuda_tag=cu126 + elif [ "$_major" -ge 12 ]; then _cuda_tag=cu124 + elif [ "$_major" -ge 11 ]; then _cuda_tag=cu118 + else echo "$_base/cpu"; return; fi + echo "$_base/$(_cap_cuda_family_for_pre_turing "$_cuda_tag" "$_smi")" } # ── Torch flavor helpers (to repair a stale CPU / wrong-CUDA wheel) ── diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index bcbc3d99ce..dda8c28099 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -295,6 +295,13 @@ FORCE_COMPILE_DEFAULT_REF = os.environ.get("UNSLOTH_LLAMA_FORCE_COMPILE_REF", "m _MIN_CUDA_MAJOR = 12 _MAX_PROBE_CUDA_MAJOR = 19 +# CUDA 13 dropped every target below sm_75, so no cuda13 bundle can serve a +# Maxwell, Pascal or Volta card (issue #7765). +_TURING_MIN_SM = 75 +# Span of PyTorch's cu126 wheels, the build the remedy below points at. +# Mirrors _CU126_SM_RANGE in install_python_stack.py. +_CU126_SM_RANGE = (50, 90) + # Blackwell floor is sm_100: data-center parts (B100/B200 sm_100, B300/GB300 # sm_103) sit below consumer Blackwell (RTX 50 sm_120); the family needs toolkit # >= 12.8, except sm_103/sm_121 which need 12.9. (120 here wrongly excluded the @@ -1515,6 +1522,59 @@ _sm_range = _core.sm_range _blackwell_capable_linux_runtime_lines = _core.blackwell_capable_linux_runtime_lines +_UNCOVERED_CUDA_HOST_WARNINGS: set[tuple[tuple[str, ...], ...]] = set() + + +def _warn_uncovered_cuda_host( + host_sms: list[str], + detected_runtime_lines: list[str], + driver_runtime_lines: list[str], + artifacts: list[PublishedLlamaArtifact], + is_arm64: bool = False, +) -> None: + """Log an uncovered CUDA host once, with cu126 advice only when it can help. + + Selection logs cover only viable attempts, so this handles the no-attempt + path. The explicit pin remains valid when llama.cpp visibility differs from + the installer's physical inventory. + """ + # Decided before the dedupe: `artifacts` varies per release, so the release walk-back + # could let an unhelpful release swallow the remedy. The cap is x86_64 only. + advise_cu126 = ( + not is_arm64 + and any(int(sm) < _TURING_MIN_SM for sm in host_sms) + and all(_CU126_SM_RANGE[0] <= int(sm) <= _CU126_SM_RANGE[1] for sm in host_sms) + and "cuda12" in driver_runtime_lines + and "cuda12" not in detected_runtime_lines + and any( + artifact.runtime_line == "cuda12" and _artifact_covers_sms(artifact, host_sms) + for artifact in artifacts + ) + ) + reason = ( + tuple(host_sms), + tuple(detected_runtime_lines), + tuple(driver_runtime_lines), + advise_cu126, + ) + if reason in _UNCOVERED_CUDA_HOST_WARNINGS: + return + _UNCOVERED_CUDA_HOST_WARNINGS.add(reason) + message = ( + "no published CUDA bundle covers this host " + f"(GPUs={','.join(f'sm_{sm}' for sm in host_sms) if host_sms else 'unknown'}" + f", CUDA runtimes on disk={','.join(detected_runtime_lines) or 'none'}" + f", runnable by this driver={','.join(driver_runtime_lines) or 'none'})" + " -- GGUF inference will fall back to a source build or the CPU" + ) + if advise_cu126: + message += ( + ". CUDA 13 dropped pre-Turing GPUs, so the venv needs a CUDA 12 runtime:" + " re-run the Unsloth installer with UNSLOTH_TORCH_INDEX_FAMILY=cu126" + ) + log(message) + + def linux_cuda_choice_from_release( host: HostInfo, release: PublishedReleaseBundle, @@ -1575,6 +1635,13 @@ def linux_cuda_choice_from_release( selection_log.append( "linux_cuda_selection: no Linux CUDA runtime line satisfied both runtime libraries and driver compatibility" ) + _warn_uncovered_cuda_host( + host_sms, + detected_runtime_lines, + driver_runtime_lines, + published_artifacts, + host.is_arm64, + ) return None blackwell_lines = ( @@ -1723,6 +1790,13 @@ def linux_cuda_choice_from_release( add_attempt(artifact, url, "portable fallback for runtime line") if not attempts: + _warn_uncovered_cuda_host( + host_sms, + detected_runtime_lines, + driver_runtime_lines, + published_artifacts, + host.is_arm64, + ) return None selection_log.append( diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py index 4378f11118..7f0fcf5ccc 100644 --- a/studio/install_python_stack.py +++ b/studio/install_python_stack.py @@ -1328,6 +1328,106 @@ def _install_bnb_windows_rocm() -> bool: return True +def _nvidia_smi_path() -> "str | None": + """nvidia-smi from PATH, falling back to the canonical Linux install path a + stripped-down PATH (systemd units, cron) can miss.""" + exe = shutil.which("nvidia-smi") + if not exe and os.path.isfile("/usr/bin/nvidia-smi"): + exe = "/usr/bin/nvidia-smi" + return exe + + +def _nvidia_compute_sms(exe: str) -> "list[int] | None": + """Every GPU's sm_NN as nvidia-smi reports it, or None when the inventory is + unreadable. One unparseable row (an "N/A" capability on a vGPU, a driver too + old for --query-gpu=compute_cap) poisons the whole answer, so a partial + reading can never drive a wheel decision.""" + try: + result = subprocess.run( + [exe, "--query-gpu=compute_cap", "--format=csv,noheader,nounits"], + stdout = subprocess.PIPE, + stderr = subprocess.DEVNULL, + text = True, + timeout = 10, + ) + except Exception: + return None + if result.returncode != 0: + return None + sms: list[int] = [] + for line in result.stdout.splitlines(): + value = line.strip() + if not value: + continue + match = re.fullmatch(r"(\d+)\.(\d+)", value) + if match is None: + return None + sms.append((int(match.group(1)) * 10) + int(match.group(2))) + return sms or None + + +# PyTorch 2.11's cu126 spans sm_50-90 (Maxwell to Hopper) with no PTX above that. It is +# the fallback family, so a Kepler or Blackwell card in the mix leaves the host uncovered. +_CU126_SM_RANGE = (50, 90) + + +def _cuda_family_sm_range(family: str, torch_release: str = "") -> "tuple[int, int] | None": + """Return the supported SM span for a CUDA wheel family. + + cu128 and cu129 include sm_70 only for torch 2.8 through 2.10. + An empty release models a fresh torch 2.11 installation. + """ + if not _is_cuda_family_leaf(family): + return None + number = int(family[len("cu") :]) + if number < 124: + return (37, 90) + if number < 128: + return _CU126_SM_RANGE + if number < 130: + release = re.match(r"(\d+)\.(\d+)", torch_release) + if release and (2, 8) <= (int(release.group(1)), int(release.group(2))) < (2, 11): + return (70, 120) + return (75, 120) + + +def _span_covers(span: "tuple[int, int]", sms: "list[int]") -> bool: + """Whether a wheel family's sm span holds every GPU on the host.""" + return all(span[0] <= sm <= span[1] for sm in sms) + + +def _cap_cuda_family_for_pre_turing(family: str, exe: "str | None") -> str: + """Use cu126 when it covers every physical GPU missed by the selected family. + + CUDA_VISIBLE_DEVICES is intentionally ignored. Non-x86_64 hosts retain the + driver-derived family because their wheel matrices differ. + """ + if platform.machine().lower() not in ("x86_64", "amd64"): + return family + span = _cuda_family_sm_range(family) + if span is None or exe is None: + return family + if span[0] <= _CU126_SM_RANGE[0]: + return family # nothing lower to fall back to + floor = span[0] + sms = _nvidia_compute_sms(exe) + if not sms or all(sm >= floor for sm in sms): + return family # no GPU here sits under the family's floor + if not _span_covers(_CU126_SM_RANGE, sms): + print( + f" NVIDIA GPUs below sm_{floor} are present, but no PyTorch 2.11 CUDA " + f"family covers this mix -- keeping {family}, which cannot use " + + ",".join(f"sm_{sm}" for sm in sorted(set(sms)) if sm < floor) + + ". Set UNSLOTH_TORCH_INDEX_FAMILY=cu126 to choose the other way" + ) + return family + print( + f" NVIDIA GPUs below sm_{floor} are present -- selecting cu126, because " + f"PyTorch 2.11's {family} wheels ship no kernels for them" + ) + return "cu126" + + def _detect_cuda_torch_index_url() -> str: """Return the pytorch.org CUDA wheel index URL for the host's NVIDIA driver. @@ -1335,7 +1435,9 @@ def _detect_cuda_torch_index_url() -> str: to the same wheel family a fresh install would pick. Honours the explicit overrides first (UNSLOTH_TORCH_INDEX_URL / _FAMILY) so a headless / CI install never lets the host GPU decide. Otherwise probes nvidia-smi (parsing both "CUDA - Version:" and "CUDA UMD Version:"), defaulting to cu126 when unreadable. + Version:" and "CUDA UMD Version:"), defaulting to cu126 when unreadable. The + driver version is only an upper bound, so the GPU architectures can cap the + result at cu126 (see _cap_cuda_family_for_pre_turing). """ _override_url = os.environ.get("UNSLOTH_TORCH_INDEX_URL", "").strip() if _override_url: @@ -1343,9 +1445,7 @@ def _detect_cuda_torch_index_url() -> str: _override_family = os.environ.get("UNSLOTH_TORCH_INDEX_FAMILY", "").strip() if _override_family: return f"{_PYTORCH_WHL_BASE}/{_override_family.strip('/')}" - exe = shutil.which("nvidia-smi") - if not exe and os.path.isfile("/usr/bin/nvidia-smi"): - exe = "/usr/bin/nvidia-smi" + exe = _nvidia_smi_path() tag = "cu126" # default when the driver CUDA version cannot be read if exe: try: @@ -1374,6 +1474,7 @@ def _detect_cuda_torch_index_url() -> str: tag = "cpu" # ancient driver: no usable CUDA wheels except Exception: pass + tag = _cap_cuda_family_for_pre_turing(tag, exe) return f"{_PYTORCH_WHL_BASE}/{tag}" @@ -1558,8 +1659,9 @@ def _ensure_cuda_torch() -> None: satisfies the version constraint and nothing force-reinstalls it. This detects that exact case and reinstalls CUDA torch. - Only repairs when torch actually links against HIP/ROCm. Healthy CUDA - torch and deliberate CPU-only torch are left untouched. + Also repairs a CUDA torch whose wheel family ships no kernels for the host's + GPUs (a pre-Turing box that the driver-only ladder sent to cu128/cu130). + Healthy CUDA torch and deliberate CPU-only torch are left untouched. """ # Respect install.sh's backend: only "" (standalone update) or "cuda" force CUDA # wheels; "rocm"/"cpu"/unrecognised are deliberate. @@ -1601,7 +1703,8 @@ def _ensure_cuda_torch() -> None: "ver = getattr(torch, '__version__', '').lower(); " "m = re.search(r'\\+(cu\\d+)', ver); " "marker = 'hip' if (hip or 'rocm' in ver) else ('cuda' if cuda else 'cpu'); " - "print(marker + '|' + (m.group(1) if m else ''))" + "print('|'.join((marker, m.group(1) if m else '', ver.split('+', 1)[0], " + "('cu' + cuda.replace('.', '')) if cuda else '')))" ), ], stdout = subprocess.PIPE, @@ -1640,13 +1743,19 @@ def _ensure_cuda_torch() -> None: ] if not _marker_lines: return - _marker, _, _installed_cu = _marker_lines[-1].partition("|") - # Reinstall CUDA torch on a ROCm build on an NVIDIA host (poisoning signature), or when a - # CUDA index is pinned but the venv has the wrong family (CPU or a different cuXXX). A - # healthy match, or a CPU wheel with no CUDA pin, is left alone. + # marker | +cuXXX local tag | release | family from torch.version.cuda. The last is the + # only CUDA clue an untagged wheel gives: PyPI forbids the local +cuXXX version. + _marker, _installed_cu, _installed_release, _runtime_cu = ( + _marker_lines[-1].split("|") + ["", "", ""] + )[:4] + # Reinstall on a ROCm build on an NVIDIA host (poisoning signature), when a CUDA index + # is pinned but the venv has the wrong family (CPU or a different cuXXX), or when the + # installed family ships no kernels for this host's GPUs. A healthy match, or a CPU + # wheel with no CUDA pin, is left alone. _pin = _explicit_torch_index_url() _pin_leaf = _torch_index_leaf(_pin) if _pin else "" _pinned_cuda = _is_cuda_family_leaf(_pin_leaf) + index_url: "str | None" = None if _marker == "hip": _why = "torch is a ROCm build on an NVIDIA host" elif _marker == "cpu" and _pinned_cuda: @@ -1656,10 +1765,33 @@ def _ensure_cuda_torch() -> None: # the family can't be confirmed, so reinstall to enforce it (idempotent). _installed_desc = _installed_cu if _installed_cu else "an untagged CUDA build" _why = f"torch is {_installed_desc} but the pinned CUDA index is {_pin_leaf}" + elif _marker == "cuda" and not _pinned_cuda: + # x86_64 only, like the cap: the spans below are the x86_64 build matrix. + if platform.machine().lower() not in ("x86_64", "amd64"): + return + _family = _installed_cu or _runtime_cu + _span = _cuda_family_sm_range(_family, _installed_release) + if _span is None: + return # untagged or unrecognised build: not this check's business + _smi = _nvidia_smi_path() + _sms = _nvidia_compute_sms(_smi) if _smi else None + if not _sms or _span_covers(_span, _sms): + return # healthy CUDA torch this host can use + # Never trade one partial family for another, or reinstall the same one forever. + index_url = _detect_cuda_torch_index_url() + _target = _torch_index_leaf(index_url) + _target_span = _cuda_family_sm_range(_target) + if _target_span is None or not _span_covers(_target_span, _sms): + return + _why = ( + f"torch is {_family} but this host has GPUs outside its " + f"sm_{_span[0]}-{_span[1]} range" + ) else: return # healthy CUDA torch matching the pin, or a deliberate CPU wheel - index_url = _detect_cuda_torch_index_url() + if index_url is None: + index_url = _detect_cuda_torch_index_url() _torch_pkg, _vision_pkg, _audio_pkg = _CUDA_TORCH_PKG_SPEC print( f" {_why} -- reinstalling CUDA torch from {_strip_index_url_credentials(index_url)}\n" diff --git a/studio/setup.ps1 b/studio/setup.ps1 index 433db7c46d..e043ed4f3b 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -563,6 +563,57 @@ function Get-NvccMaxArch { return $null } +# Classify the physical NVIDIA inventory for a cu126 fallback: "cu126" when it covers +# every GPU, "uncovered" for an incompatible mix, empty when no fallback is needed or the +# inventory is unreadable. CUDA_VISIBLE_DEVICES is ignored because the wheel must support +# the host. Mirrors _nvidia_cu126_verdict in install.sh. +function Get-NvidiaCu126Verdict { + # Floor is per-release, not fixed: only 2.11 dropped sm_70 from cu128. + param([string]$SmiExe, [int]$LegacyFloorSm = 75) + if (-not $SmiExe) { return '' } + $raw = Invoke-NvidiaSmiBounded $SmiExe @('--query-gpu=compute_cap', '--format=csv,noheader,nounits') -StdoutOnly + if ($LASTEXITCODE -ne 0 -or -not $raw) { return '' } + $legacy = $false + $outsideCu126 = $false + $seen = $false + foreach ($line in ($raw -split "`n")) { + $value = $line.Trim() + if (-not $value) { continue } + if ($value -notmatch '^(\d+)\.(\d+)$') { return '' } + $sm = ([int]$Matches[1] * 10) + [int]$Matches[2] + if ($sm -lt $LegacyFloorSm) { $legacy = $true } + if ($sm -lt 50 -or $sm -gt 90) { $outsideCu126 = $true } + $seen = $true + } + if (-not $seen -or -not $legacy) { return '' } + if ($outsideCu126) { return 'uncovered' } + return 'cu126' +} + +function Get-CudaFamilyCappedForPreTuring { + param([string]$Family, [string]$SmiExe) + if ($Family -notin @('cu128', 'cu130')) { return $Family } + # Windows pins torch<2.11, whose cu128 still ships sm_70, so only cu130 + # strands a Volta here. Raise to 75 when that pin reaches 2.11. + $legacyFloorSm = if ($Family -eq 'cu128') { 70 } else { 75 } + $verdict = Get-NvidiaCu126Verdict $SmiExe $legacyFloorSm + if (-not $verdict) { return $Family } + # This runs twice per setup; announce once without polluting pipeline output. + $announce = -not $script:PreTuringCapAnnounced + $script:PreTuringCapAnnounced = $true + if ($verdict -eq 'cu126') { + if ($announce) { + substep "pre-Turing NVIDIA GPUs (sm_<75) are present -- selecting cu126, because PyTorch 2.11's $Family wheels start at sm_75" "Yellow" + } + return 'cu126' + } + if ($announce) { + substep "this host mixes pre-Turing NVIDIA GPUs with GPUs that cu126 cannot serve; no PyTorch 2.11 CUDA family covers both" "Yellow" + substep "keeping $Family, so the pre-Turing GPUs will be unusable; set UNSLOTH_TORCH_INDEX_FAMILY=cu126 to choose the other way" "Yellow" + } + return $Family +} + # Detect driver's max CUDA version from nvidia-smi and return the highest # compatible PyTorch CUDA index tag (e.g. "cu128"). # PyTorch on Windows ships CPU-only by default from PyPI; CUDA wheels live at @@ -587,12 +638,13 @@ function Get-PytorchCudaTag { $major = [int]$Matches[1] $minor = [int]$Matches[2] # PyTorch 2.10 offers: cu124, cu126, cu128, cu130 - if ($major -ge 13) { return "cu130" } - if ($major -eq 12 -and $minor -ge 8) { return "cu128" } - if ($major -eq 12 -and $minor -ge 6) { return "cu126" } - if ($major -ge 12) { return "cu124" } - if ($major -ge 11) { return "cu118" } - return "cpu" + if ($major -ge 13) { $family = "cu130" } + elseif ($major -eq 12 -and $minor -ge 8) { $family = "cu128" } + elseif ($major -eq 12 -and $minor -ge 6) { $family = "cu126" } + elseif ($major -ge 12) { $family = "cu124" } + elseif ($major -ge 11) { $family = "cu118" } + else { return "cpu" } + return (Get-CudaFamilyCappedForPreTuring $family $smiExe) } } catch { } @@ -1568,7 +1620,10 @@ function Invoke-NvidiaSmiBounded { param( [Parameter(Mandatory = $true, Position = 0)][string]$Exe, [Parameter(Position = 1)][string[]]$SmiArgs = @(), - [int]$TimeoutSec = 10 + [int]$TimeoutSec = 10, + # Driver warnings on stderr would corrupt machine-readable --query-gpu + # output; the human-readable probes keep the default merge. + [switch]$StdoutOnly ) try { $psi = New-Object System.Diagnostics.ProcessStartInfo @@ -1587,6 +1642,7 @@ function Invoke-NvidiaSmiBounded { return "" } $global:LASTEXITCODE = $proc.ExitCode + if ($StdoutOnly) { return $outTask.Result } return ($outTask.Result + "`n" + $errTask.Result) } catch { $global:LASTEXITCODE = 1 @@ -3449,6 +3505,15 @@ if ((Test-Path -LiteralPath $VenvDir -PathType Container) -and -not $NoTorchMode $script:PinChangedForceReinstall = $true $shouldRebuild = $false } + # Same for an unpinned cu* -> cu* move: the cap can change the expected family on a + # healthy venv, and only install.ps1 creates venvs, so wiping here strands a direct + # `studio update`. CPU/ROCm/XPU drift still rebuilds. + if ($shouldRebuild -and -not $_pinnedIdx -and $installedTorchTag -and + (Test-CudaFamilyLeaf $installedTorchTag) -and (Test-CudaFamilyLeaf $expectedTorchTag)) { + substep "CUDA family $installedTorchTag does not cover this host -- reinstalling $expectedTorchTag in place." "Cyan" + $script:PinChangedForceReinstall = $true + $shouldRebuild = $false + } # A +xpu venv is never wiped by a DIRECT update: on a hybrid NVIDIA+Arc box the promotion # above is gated on -not $HasNvidiaSmi, so a later pinless update expects a cu* tag, calls the # working Arc venv stale and deletes it -- then exits, because only install.ps1 creates venvs. diff --git a/tests/python/test_cross_platform_parity.py b/tests/python/test_cross_platform_parity.py index 06e444314b..43d384b611 100644 --- a/tests/python/test_cross_platform_parity.py +++ b/tests/python/test_cross_platform_parity.py @@ -68,6 +68,53 @@ class TestInstallShHasGpuDetection: ), "install.sh should assign TORCH_INDEX_URL from get_torch_index_url()" +class TestPreTuringCapParity: + """Every wheel-selection site caps cu128/cu130 on a pre-Turing host (issue #7765). + + PyTorch 2.11 builds those families for sm_75 and newer, so a Maxwell/Pascal/Volta + box needs cu126 -- both for torch itself and for the CUDA 12 runtime that gets it + a llama.cpp GGUF bundle. Four scripts pick the family; none may be left behind. + """ + + # (file, call spelling, selection function that must invoke it, its end marker). The + # spelling carries the first argument, so a prose mention cannot satisfy the assertion. + _SITES = ( + (INSTALL_SH, '_cap_cuda_family_for_pre_turing "', "get_torch_index_url() {", "\n}"), + ( + INSTALL_PS1, + "Get-CudaFamilyCappedForPreTuring $", + "function Get-TorchIndexUrl", + "\n }", + ), + (SETUP_PS1, "Get-CudaFamilyCappedForPreTuring $", "function Get-PytorchCudaTag", "\n}"), + ( + STACK_PY, + "_cap_cuda_family_for_pre_turing(", + "def _detect_cuda_torch_index_url", + "\ndef ", + ), + ) + + def test_cu126_span_agrees_across_the_python_modules(self): + # Neither module imports the other (the installer runs before dependencies + # exist), so assert the shared span here rather than let it drift silently. + span = "_CU126_SM_RANGE = (50, 90)" + for path in (STACK_PY, REPO_ROOT / "studio" / "install_llama_prebuilt.py"): + assert span in path.read_text(encoding = "utf-8"), f"{path.name} lost {span}" + + @pytest.mark.parametrize("path,call,start,end", _SITES) + def test_selection_function_applies_the_cap(self, path, call, start, end): + text = path.read_text(encoding = "utf-8") + assert start in text, f"{path.name} no longer defines {start!r}" + body = text.split(start, 1)[1].split(end, 1)[0] + assert call in body, f"{path.name}'s selection function never applies {call!r}" + + +# A ladder rung names its family either as an index-URL suffix ("$base/cu128") or as a +# variable a later step can still cap ("_cuda_tag=cu128"), so accept both spellings. +_CUDA_LEAF_RE = r"""[/=]\s*["']?(cu\d+|cpu)""" + + class TestCudaMappingParity: """CUDA version thresholds must match between install.sh and install.ps1.""" @@ -84,7 +131,7 @@ class TestCudaMappingParity: if in_func and line.startswith("}"): break if in_func and ("_major" in line or "_minor" in line): - m = re.search(r"/(cu\d+|cpu)", line) + m = re.search(_CUDA_LEAF_RE, line) if m: results.append(m.group(1)) return results @@ -106,7 +153,7 @@ class TestCudaMappingParity: break # Only match the if-chain lines that compare $major/$minor if "$major" in line or "$minor" in line: - m = re.search(r"/(cu\d+|cpu)", line) + m = re.search(_CUDA_LEAF_RE, line) if m: results.append(m.group(1)) return results diff --git a/tests/sh/test_get_torch_index_url.sh b/tests/sh/test_get_torch_index_url.sh index 633d8ca17b..0d51e6c21f 100755 --- a/tests/sh/test_get_torch_index_url.sh +++ b/tests/sh/test_get_torch_index_url.sh @@ -49,6 +49,10 @@ _FAKE_ROCM_DIR=$(mktemp -d) echo "" sed -n '/^_trim_index_path_slashes()/,/^}/p' "$INSTALL_SH" echo "" + sed -n '/^_nvidia_cu126_verdict()/,/^}/p' "$INSTALL_SH" + echo "" + sed -n '/^_cap_cuda_family_for_pre_turing()/,/^}/p' "$INSTALL_SH" + echo "" sed -n '/^get_torch_index_url()/,/^}/p' "$INSTALL_SH" } | sed -e "s|/usr/bin/nvidia-smi|$_FAKE_SMI_DIR/nvidia-smi-absent|g" \ -e "s|/opt/rocm|$_FAKE_ROCM_DIR|g" \ @@ -68,17 +72,24 @@ assert_eq() { fi } -# Helper: create a mock nvidia-smi that prints a given CUDA version string. -# Handles both default output (version header) and -L (GPU listing) so that -# _has_usable_nvidia_gpu sees a valid GPU. +# Helper: create a mock nvidia-smi answering the version header, -L (so +# _has_usable_nvidia_gpu sees a GPU) and --query-gpu=compute_cap. $1 is the CUDA version, +# $2 a space-separated capability list (default one Ampere card), $3 the compute_cap +# query's exit code. make_mock_smi() { _dir=$(mktemp -d) + _caps="${2-8.6}" + _caps_rc="${3:-0}" cat > "$_dir/nvidia-smi" </dev/null @@ -372,6 +386,88 @@ _result=$(run_func "$_dir") assert_eq "CUDA Version 13.7 -> cu130" "https://download.pytorch.org/whl/cu130" "$_result" rm -rf "$_dir" +# ── Pre-Turing hosts cap at cu126 (issue #7765) ── +# PyTorch 2.11's cu128/cu130 start at sm_75, and their CUDA 13 runtime also costs +# these GPUs their llama.cpp GGUF bundle. +_dir=$(make_mock_smi "13.0" "7.0") +_result=$(run_func "$_dir") +assert_eq "CUDA 13.0 + Volta -> cu126" "https://download.pytorch.org/whl/cu126" "$_result" +# The mask is irrelevant: an all-pre-Turing host stays pre-Turing under any mask. +_result=$(run_func "$_dir" "0") +assert_eq "CUDA 13.0 + Volta + CVD=0 -> cu126" "https://download.pytorch.org/whl/cu126" "$_result" +rm -rf "$_dir" + +_dir=$(make_mock_smi "12.8" "6.1") +_result=$(run_func "$_dir") +assert_eq "CUDA 12.8 + Pascal -> cu126" "https://download.pytorch.org/whl/cu126" "$_result" +rm -rf "$_dir" + +_dir=$(make_mock_smi "13.0" "7.0 7.0") +_result=$(run_func "$_dir") +assert_eq "CUDA 13.0 + two Voltas -> cu126" "https://download.pytorch.org/whl/cu126" "$_result" +rm -rf "$_dir" + +# Turing is the floor of cu128/cu130, so it keeps the driver family. +_dir=$(make_mock_smi "13.0" "7.5") +_result=$(run_func "$_dir") +assert_eq "CUDA 13.0 + Turing -> cu130" "https://download.pytorch.org/whl/cu130" "$_result" +rm -rf "$_dir" + +# cu126 spans sm_50-90, so a mixed host is served whole while its newest card is Hopper. +_dir=$(make_mock_smi "13.0" "7.0 8.6") +_result=$(run_func "$_dir") +assert_eq "CUDA 13.0 + Volta and Ampere -> cu126" "https://download.pytorch.org/whl/cu126" "$_result" +rm -rf "$_dir" + +_dir=$(make_mock_smi "13.0" "6.1 9.0") +_result=$(run_func "$_dir") +assert_eq "CUDA 13.0 + Pascal and Hopper -> cu126" "https://download.pytorch.org/whl/cu126" "$_result" +rm -rf "$_dir" + +# Blackwell is past cu126's ceiling and Kepler is under its floor, so no family covers +# either mix whole. The newer card keeps its wheels. +_dir=$(make_mock_smi "13.0" "7.0 12.0") +_result=$(run_func "$_dir") +assert_eq "CUDA 13.0 + Volta and Blackwell -> cu130" "https://download.pytorch.org/whl/cu130" "$_result" +rm -rf "$_dir" + +_dir=$(make_mock_smi "13.0" "3.7 8.6") +_result=$(run_func "$_dir") +assert_eq "CUDA 13.0 + Kepler and Ampere -> cu130" "https://download.pytorch.org/whl/cu130" "$_result" +rm -rf "$_dir" + +# cu126 and older already ship the legacy kernels; nothing to cap. +_dir=$(make_mock_smi "12.6" "7.0") +_result=$(run_func "$_dir") +assert_eq "CUDA 12.6 + Volta -> cu126" "https://download.pytorch.org/whl/cu126" "$_result" +rm -rf "$_dir" + +# An unreadable or partial inventory keeps the driver-only choice. +_dir=$(make_mock_smi "13.0" "7.0 N/A") +_result=$(run_func "$_dir") +assert_eq "unreadable capability row -> cu130" "https://download.pytorch.org/whl/cu130" "$_result" +rm -rf "$_dir" + +_dir=$(make_mock_smi "13.0" "7.0" 1) +_result=$(run_func "$_dir") +assert_eq "failed capability query -> cu130" "https://download.pytorch.org/whl/cu130" "$_result" +rm -rf "$_dir" + +_dir=$(make_mock_smi "13.0" "") +_result=$(run_func "$_dir") +assert_eq "empty capability inventory -> cu130" "https://download.pytorch.org/whl/cu130" "$_result" +rm -rf "$_dir" + +# No aarch64 CUDA family ships sm_<80 kernels, so cu126 cannot help there. +_dir=$(make_mock_smi "13.0" "7.0") +_result=$(PATH="$_dir:$_TOOLS_DIR" bash -c \ + "_ARCH=aarch64; . '$_FUNC_FILE'; _cap_cuda_family_for_pre_turing cu130 nvidia-smi" 2>/dev/null) +assert_eq "aarch64 keeps the driver family" "cu130" "$_result" +_result=$(PATH="$_dir:$_TOOLS_DIR" bash -c \ + "_ARCH=x86_64; . '$_FUNC_FILE'; _cap_cuda_family_for_pre_turing cu130 nvidia-smi" 2>/dev/null) +assert_eq "x86_64 caps the driver family" "cu126" "$_result" +rm -rf "$_dir" + # 34) CUDA_VISIBLE_DEVICES="" hides the NVIDIA GPU -> cpu (no AMD present) _dir=$(make_mock_smi "12.8") _result=$(run_func "$_dir" "") diff --git a/tests/studio/install/test_cuda_repair.py b/tests/studio/install/test_cuda_repair.py index c6d2b95316..a5ee4c16e6 100644 --- a/tests/studio/install/test_cuda_repair.py +++ b/tests/studio/install/test_cuda_repair.py @@ -30,9 +30,11 @@ def _make_run( cuda_version = "12.8", torch_rc = 0, smi_rc = 0, + compute_caps = ("8.6",), ): """subprocess.run side_effect: torch-classify probe (sys.executable, bytes - stdout) vs nvidia-smi version probe (smi path, text=True), keyed on the executable.""" + stdout) vs the nvidia-smi version / compute-capability probes (smi path, + text=True), keyed on the executable.""" def _run(cmd, *args, **kwargs): result = MagicMock() @@ -41,9 +43,11 @@ def _make_run( result.returncode = torch_rc result.stdout = (torch_state + "\n").encode() return result - # nvidia-smi version probe (text = True) result.returncode = smi_rc - out = f"CUDA Version: {cuda_version}\n" if cuda_version else "No devices found\n" + if len(cmd) > 1 and str(cmd[1]) == "--query-gpu=compute_cap": + out = "".join(f"{cap}\n" for cap in compute_caps) + else: + out = f"CUDA Version: {cuda_version}\n" if cuda_version else "No devices found\n" result.stdout = out if kwargs.get("text") else out.encode() return result @@ -58,6 +62,8 @@ def _run_cuda_repair( cuda_version = "12.8", torch_rc = 0, smi_rc = 0, + compute_caps = ("8.6",), + machine = "x86_64", is_macos = False, is_windows = False, no_torch = False, @@ -71,7 +77,10 @@ def _run_cuda_repair( cvd controls CUDA_VISIBLE_DEVICES: None removes it from the env, any string sets it. index_family sets UNSLOTH_TORCH_INDEX_FAMILY (the explicit wheel-index pin). - index_url sets UNSLOTH_TORCH_INDEX_URL (the full-URL pin form).""" + index_url sets UNSLOTH_TORCH_INDEX_URL (the full-URL pin form). + compute_caps is what nvidia-smi reports for --query-gpu=compute_cap; machine + pins platform.machine() so the architecture policy behaves the same on any test + host.""" env = {} if rocm_marker: env["UNSLOTH_ROCM_TORCH_INSTALLED"] = "1" @@ -92,6 +101,7 @@ def _run_cuda_repair( patch.object(stack_mod, "IS_MACOS", is_macos), patch.object(stack_mod, "IS_WINDOWS", is_windows), patch.object(stack_mod, "NO_TORCH", no_torch), + patch.object(stack_mod.platform, "machine", return_value = machine), patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = nvidia), patch.object(stack_mod.shutil, "which", side_effect = _which), patch.object(stack_mod.os.path, "isfile", return_value = bool(smi_path)), @@ -99,7 +109,7 @@ def _run_cuda_repair( patch.object( stack_mod.subprocess, "run", - side_effect = _make_run(torch_state, cuda_version, torch_rc, smi_rc), + side_effect = _make_run(torch_state, cuda_version, torch_rc, smi_rc, compute_caps), ), patch.dict(stack_mod.os.environ, env, clear = False), ): @@ -409,5 +419,243 @@ class TestCudaIndexResolution: assert url == f"{stack_mod._PYTORCH_WHL_BASE}/cu126" +# PyTorch 2.11's cu128/cu130 start at sm_75, and their CUDA 13 runtime also costs a +# pre-Turing GPU its llama.cpp GGUF bundle, so such hosts get cu126 (#7765). + + +class TestPreTuringWheelFamily: + def test_volta_host_selects_cu126_over_the_driver_family(self): + assert "cu126" in _index_url(_run_cuda_repair(cuda_version = "13.0", compute_caps = ("7.0",))) + + def test_pascal_host_selects_cu126_over_the_driver_family(self): + assert "cu126" in _index_url(_run_cuda_repair(cuda_version = "12.8", compute_caps = ("6.1",))) + + def test_turing_host_keeps_the_driver_family(self): + assert "cu130" in _index_url(_run_cuda_repair(cuda_version = "13.0", compute_caps = ("7.5",))) + + def test_mixed_host_within_cu126_range_is_capped(self): + # cu126 spans sm_50-90, so serving the older card costs the newer one nothing. + for caps in (("7.0", "8.6"), ("6.1", "9.0"), ("5.0", "7.5")): + assert "cu126" in _index_url(_run_cuda_repair(cuda_version = "13.0", compute_caps = caps)) + + def test_mixed_host_outside_cu126_range_keeps_the_driver_family(self): + # Blackwell is past cu126's ceiling and Kepler is under its floor, so no family + # covers either mix whole. Capping would strand the newer card entirely. + for caps in (("7.0", "12.0"), ("3.7", "8.6")): + assert "cu130" in _index_url(_run_cuda_repair(cuda_version = "13.0", compute_caps = caps)) + + def test_cu126_venv_is_repaired_after_a_blackwell_upgrade(self): + # The span cuts both ways: a cu126 venv predating a GPU swap has nothing for + # sm_120, and a fresh install on that host would pick cu130. + mock_pip = _run_cuda_repair( + torch_state = "cuda|cu126|2.11.0", + cuda_version = "13.0", + compute_caps = ("12.0",), + ) + assert mock_pip.call_count == 1 + assert "cu130" in _index_url(mock_pip) + + def test_cu126_venv_is_kept_when_the_driver_allows_nothing_newer(self): + # Same host, CUDA 12.6 driver: cu130 is not installable, so leave it rather + # than reinstall cu126 over itself on every update. + mock_pip = _run_cuda_repair( + torch_state = "cuda|cu126|2.11.0", + cuda_version = "12.6", + compute_caps = ("12.0",), + ) + mock_pip.assert_not_called() + + def test_partial_family_is_not_traded_for_another_partial_family(self): + # A working V100 + cu126 box gains a Blackwell card. Neither family covers both, + # so swapping to cu130 would kill the Volta to revive the Blackwell. + mock_pip = _run_cuda_repair( + torch_state = "cuda|cu126|2.11.0", + cuda_version = "13.0", + compute_caps = ("7.0", "12.0"), + ) + mock_pip.assert_not_called() + + def test_cu118_kepler_build_is_kept(self): + # torch 2.7's cu118 still built sm_37 and nothing newer does, so the replacement + # would strand the GPU that works today. + mock_pip = _run_cuda_repair( + torch_state = "cuda|cu118|2.7.1", + cuda_version = "13.0", + compute_caps = ("3.7",), + ) + mock_pip.assert_not_called() + + def test_uncovered_mix_is_not_repaired_in_a_loop(self): + # The cap declines, so the replacement equals the installed family. + mock_pip = _run_cuda_repair( + torch_state = "cuda|cu130|2.11.0", + cuda_version = "13.0", + compute_caps = ("7.0", "12.0"), + ) + mock_pip.assert_not_called() + + def test_mixed_host_within_cu126_range_is_repaired(self): + mock_pip = _run_cuda_repair( + torch_state = "cuda|cu130|2.11.0", + cuda_version = "13.0", + compute_caps = ("7.0", "8.6"), + ) + assert mock_pip.call_count == 1 + assert "cu126" in _index_url(mock_pip) + + def test_partial_inventory_keeps_the_driver_family(self): + assert "cu130" in _index_url( + _run_cuda_repair(cuda_version = "13.0", compute_caps = ("7.0", "N/A")) + ) + + def test_incompatible_family_is_repaired(self): + mock_pip = _run_cuda_repair( + torch_state = "cuda|cu130|2.11.0", + cuda_version = "13.0", + compute_caps = ("7.0",), + ) + assert mock_pip.call_count == 1 + assert "cu126" in _index_url(mock_pip) + + def test_pre_211_cu128_volta_build_is_kept(self): + # torch 2.10's cu128 wheels still shipped sm_70; no reinstall is warranted. + mock_pip = _run_cuda_repair( + torch_state = "cuda|cu128|2.10.0", + cuda_version = "13.0", + compute_caps = ("7.0",), + ) + mock_pip.assert_not_called() + + def test_pre_211_cu128_pascal_build_is_repaired(self): + # ... but they never shipped sm_61, which only cu126 carries. + mock_pip = _run_cuda_repair( + torch_state = "cuda|cu128|2.10.0", + cuda_version = "13.0", + compute_caps = ("6.1",), + ) + assert mock_pip.call_count == 1 + assert "cu126" in _index_url(mock_pip) + + def test_compatible_family_is_kept(self): + for state, caps in ( + ("cuda|cu126|2.11.0", ("7.0",)), + ("cuda|cu130|2.11.0", ("7.5",)), + ("cuda|cu130|2.11.0", ("7.0", "12.0")), + ("cuda|cu130|2.11.0", ("7.0", "N/A")), + ): + mock_pip = _run_cuda_repair(torch_state = state, cuda_version = "13.0", compute_caps = caps) + mock_pip.assert_not_called() + + def test_explicit_pin_wins_over_the_architecture_policy(self): + for pin in ("index_family", "index_url"): + value = "cu130" if pin == "index_family" else "https://example.test/whl/cu130" + mock_pip = _run_cuda_repair( + torch_state = "cuda|cu130|2.11.0", + cuda_version = "13.0", + compute_caps = ("7.0",), + **{pin: value}, + ) + mock_pip.assert_not_called() + + def test_untagged_cuda_build_is_left_alone(self): + mock_pip = _run_cuda_repair( + torch_state = "cuda||2.11.0", + cuda_version = "13.0", + compute_caps = ("7.0",), + ) + mock_pip.assert_not_called() + + def test_non_x86_host_keeps_the_driver_family(self): + # No aarch64 CUDA family ships sm_<80 kernels, so cu126 cannot help there. + volta = MagicMock(returncode = 0, stdout = "7.0\n") + with patch.object(stack_mod.subprocess, "run", return_value = volta): + with patch.object(stack_mod.platform, "machine", return_value = "aarch64"): + assert stack_mod._cap_cuda_family_for_pre_turing("cu130", "smi") == "cu130" + with patch.object(stack_mod.platform, "machine", return_value = "x86_64"): + assert stack_mod._cap_cuda_family_for_pre_turing("cu130", "smi") == "cu126" + + def test_permissive_family_is_never_probed(self): + # cu126 has nothing older to fall back to, so it must not spawn nvidia-smi. + with ( + patch.object(stack_mod.platform, "machine", return_value = "x86_64"), + patch.object(stack_mod.subprocess, "run") as mock_run, + ): + assert stack_mod._cap_cuda_family_for_pre_turing("cu126", "smi") == "cu126" + assert stack_mod._cap_cuda_family_for_pre_turing("cpu", "smi") == "cpu" + mock_run.assert_not_called() + + def test_family_spans_track_the_pytorch_wheel_matrix(self): + # Read off pytorch's .ci/manywheel/build_cuda.sh at each release tag. cu118 kept + # Kepler: torch 2.7 still built sm_37 for it. + assert stack_mod._cuda_family_sm_range("cu118") == (37, 90) + assert stack_mod._cuda_family_sm_range("cu124") == (50, 90) + assert stack_mod._cuda_family_sm_range("cu126") == (50, 90) + assert stack_mod._cuda_family_sm_range("cu126", "2.10.0") == (50, 90) + assert stack_mod._cuda_family_sm_range("cu128") == (75, 120) + assert stack_mod._cuda_family_sm_range("cu128", "2.11.0") == (75, 120) + assert stack_mod._cuda_family_sm_range("cu129", "2.9.0") == (70, 120) + assert stack_mod._cuda_family_sm_range("cu130", "2.10.0") == (75, 120) + assert stack_mod._cuda_family_sm_range("cpu") is None + assert stack_mod._cuda_family_sm_range("") is None + + def test_cu128_volta_window_opens_at_torch_28(self): + # 2.7's cu128 dropped sm_50-70 when CUDA 12.8 deprecated them; 2.8 put sm_70 + # back and 2.11 took it away again. + assert stack_mod._cuda_family_sm_range("cu128", "2.7.1")[0] == 75 + assert stack_mod._cuda_family_sm_range("cu128", "2.8.0")[0] == 70 + assert stack_mod._cuda_family_sm_range("cu128", "2.10.0")[0] == 70 + mock_pip = _run_cuda_repair( + torch_state = "cuda|cu128|2.7.1", + cuda_version = "13.0", + compute_caps = ("7.0",), + ) + assert mock_pip.call_count == 1 + assert "cu126" in _index_url(mock_pip) + + def test_untagged_pypi_wheel_is_classified_by_its_cuda_runtime(self): + # PyPI forbids local versions, so a torch from PyPI has no +cuXXX tag; + # torch.version.cuda is the only clue that it is a CUDA 13 build. + mock_pip = _run_cuda_repair( + torch_state = "cuda||2.11.0|cu130", + cuda_version = "13.0", + compute_caps = ("7.0",), + ) + assert mock_pip.call_count == 1 + assert "cu126" in _index_url(mock_pip) + + healthy = _run_cuda_repair( + torch_state = "cuda||2.11.0|cu126", + cuda_version = "13.0", + compute_caps = ("7.0",), + ) + healthy.assert_not_called() + + def test_repair_is_skipped_when_it_would_reinstall_the_same_family(self): + # aarch64 has no CUDA family below sm_80, so the cap declines and the replacement + # would be the condemned wheel itself, once per update forever. + mock_pip = _run_cuda_repair( + torch_state = "cuda|cu130|2.11.0", + cuda_version = "13.0", + compute_caps = ("7.0",), + machine = "aarch64", + ) + mock_pip.assert_not_called() + + def test_compute_sms_rejects_an_unreadable_inventory(self): + def _sms(stdout, returncode = 0): + with patch.object( + stack_mod.subprocess, + "run", + return_value = MagicMock(returncode = returncode, stdout = stdout), + ): + return stack_mod._nvidia_compute_sms("nvidia-smi") + + assert _sms("7.0\n12.0\n") == [70, 120] + assert _sms(" 8.6 \n\n") == [86] + assert _sms("7.0\nN/A\n") is None + assert _sms("") is None + assert _sms("7.0\n", returncode = 1) is None + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-q"])) diff --git a/tests/studio/install/test_selection_logic.py b/tests/studio/install/test_selection_logic.py index 15537bb2bf..2c18ad3e55 100644 --- a/tests/studio/install/test_selection_logic.py +++ b/tests/studio/install/test_selection_logic.py @@ -60,6 +60,13 @@ pinned_macos_release_tag = INSTALL_LLAMA_PREBUILT.pinned_macos_release_tag resolve_simple_install_release_plans = INSTALL_LLAMA_PREBUILT.resolve_simple_install_release_plans +@pytest.fixture(autouse = True) +def _reset_uncovered_cuda_warnings(monkeypatch): + # The warning dedupe is module-level state: without this reset, a test's verdict + # depends on which earlier test logged the same reason first. + monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "_UNCOVERED_CUDA_HOST_WARNINGS", set()) + + @pytest.fixture(autouse = True) def _disable_download_host_fast_path(monkeypatch): # This module exercises the GitHub API enumeration and asset selection against @@ -1177,6 +1184,125 @@ class TestLinuxCudaChoiceFromRelease: assert result is not None assert result.primary.runtime_line == "cuda13" + def test_volta_host_needs_the_cuda12_line(self, monkeypatch, capsys): + # Issue #7765: CUDA 13 dropped sm_70, so a V100 whose only runtime line is cuda13 + # (torch from the cu130 index) gets no bundle and GGUF inference silently moves to + # the CPU. The selection log only prints when an attempt survives, so the reason + # has to reach the user another way. + mock_linux_runtime(monkeypatch, ["cuda13"]) + host = make_host(driver_cuda_version = (13, 0), compute_caps = ["70"]) + art12 = make_artifact( + "bundle-cuda12-older.tar.gz", + runtime_line = "cuda12", + supported_sms = ["70", "75", "80", "86", "89"], + min_sm = 70, + max_sm = 89, + ) + art13 = make_artifact("bundle-cuda13-older.tar.gz", runtime_line = "cuda13") + release = make_release([art12, art13]) + + assert linux_cuda_choice_from_release(host, release) is None + warning = capsys.readouterr().err + assert "sm_70" in warning + # The pin, not a promise about what a re-run picks: these GPUs are the masked + # view, while the installer weighs every physical GPU. + assert "UNSLOTH_TORCH_INDEX_FAMILY=cu126" in warning + + # The same host with a CUDA 12 runtime in the venv reaches its bundle. + mock_linux_runtime(monkeypatch, ["cuda13", "cuda12"]) + result = linux_cuda_choice_from_release(host, release) + assert result is not None + assert result.primary.name == "bundle-cuda12-older.tar.gz" + + def test_cu126_advice_reaches_a_mixed_host_cu126_can_serve(self, monkeypatch, capsys): + # A Volta beside an Ampere is what cu126 exists for, and the cuda12 portable + # bundle covers both. Withholding the remedy would contradict the installer. + mock_linux_runtime(monkeypatch, ["cuda13"]) + host = make_host(driver_cuda_version = (13, 0), compute_caps = ["70", "86"]) + release = make_release( + [ + make_artifact( + "bundle-cuda12-portable.tar.gz", + runtime_line = "cuda12", + supported_sms = ["70", "75", "80", "86", "89", "90"], + min_sm = 70, + max_sm = 90, + ), + make_artifact("bundle-cuda13-older.tar.gz", runtime_line = "cuda13"), + ] + ) + + assert linux_cuda_choice_from_release(host, release) is None + assert "UNSLOTH_TORCH_INDEX_FAMILY=cu126" in capsys.readouterr().err + + @pytest.mark.parametrize( + "case,driver,caps,artifacts", + [ + # Kepler: no published bundle covers sm_37 at any runtime line. + ("kepler", (13, 0), ["37"], [("cuda12", ["50", "61"], 50, 61)]), + # Blackwell beside a Volta is past cu126's sm_90 ceiling. + ( + "blackwell", + (13, 0), + ["70", "120"], + [("cuda12", ["70", "86", "120"], 70, 120)], + ), + # Driver too old to run a CUDA 12 runtime at all. + ("old_driver", (11, 4), ["70"], [("cuda12", ["70", "75"], 70, 89)]), + # arm64 publishes no cuda12 bundle, and no aarch64 CUDA wheel goes under sm_80. + ("arm64", (13, 0), ["72"], [("cuda13", ["90", "121"], 90, 121)]), + ], + ) + def test_cu126_advice_is_withheld_when_it_cannot_help( + self, monkeypatch, capsys, case, driver, caps, artifacts + ): + mock_linux_runtime(monkeypatch, ["cuda13"]) + host = make_host( + driver_cuda_version = driver, + compute_caps = caps, + machine = "aarch64" if case == "arm64" else "x86_64", + ) + kind = "linux-arm64-cuda" if case == "arm64" else "linux-cuda" + release = make_release( + [ + make_artifact( + f"bundle-{line}.tar.gz", + install_kind = kind, + runtime_line = line, + supported_sms = sms, + min_sm = lo, + max_sm = hi, + ) + for line, sms, lo, hi in artifacts + ] + ) + + assert linux_cuda_choice_from_release(host, release) is None + warning = capsys.readouterr().err + assert "no published CUDA bundle covers this host" in warning + assert "cu126" not in warning + + def test_uncovered_host_warning_is_not_repeated(self, monkeypatch, capsys): + # The release walk-back re-runs the selection per release. + mock_linux_runtime(monkeypatch, ["cuda13"]) + host = make_host(driver_cuda_version = (13, 0), compute_caps = ["70"]) + release = make_release([make_artifact("bundle-cuda13.tar.gz", runtime_line = "cuda13")]) + + for _ in range(3): + assert linux_cuda_choice_from_release(host, release) is None + assert capsys.readouterr().err.count("no published CUDA bundle covers this host") == 1 + + def test_uncovered_modern_host_gets_no_cu126_hint(self, monkeypatch, capsys): + # cu126 only helps a pre-Turing host; never suggest it to anyone else. + mock_linux_runtime(monkeypatch, ["cuda13"]) + host = make_host(driver_cuda_version = (13, 0), compute_caps = ["120"]) + release = make_release([make_artifact("bundle-cuda13.tar.gz", runtime_line = "cuda13")]) + + assert linux_cuda_choice_from_release(host, release) is None + warning = capsys.readouterr().err + assert "sm_120" in warning + assert "cu126" not in warning + def test_blackwell_prefers_cuda14_over_lower_majors(self, monkeypatch): # The highest sm_120-capable CUDA major wins. mock_linux_runtime(monkeypatch, ["cuda14", "cuda13", "cuda12"]) diff --git a/tests/studio/test_pre_turing_cap.ps1 b/tests/studio/test_pre_turing_cap.ps1 new file mode 100644 index 0000000000..dcef77e459 --- /dev/null +++ b/tests/studio/test_pre_turing_cap.ps1 @@ -0,0 +1,117 @@ +#!/usr/bin/env pwsh +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 +# Behavioural test for the pre-Turing cu126 cap (Get-NvidiaCu126Verdict, +# Get-CudaFamilyCappedForPreTuring) in install.ps1 and studio/setup.ps1. +# test_cross_platform_parity.py only greps for the call spelling, so a selector that +# computes the verdict and discards it still passes there. This runs both copies. +# Run: pwsh -NoProfile -File tests/studio/test_pre_turing_cap.ps1 + +$ErrorActionPreference = "Stop" +$root = (Resolve-Path ([System.IO.Path]::Combine($PSScriptRoot, "..", ".."))).Path + +$failures = 0 +function Check($name, $cond) { + if ($cond) { Write-Host " PASS $name" } + else { Write-Host " FAIL $name" -ForegroundColor Red; $script:failures++ } +} + +# Returns the source text of each named function. The caller Invoke-Expression's it at +# script scope; doing that inside a function would lose the helpers on return. +function Get-HelperSources($path, $names) { + $tokens = $null; $errors = $null + $ast = [System.Management.Automation.Language.Parser]::ParseFile($path, [ref]$tokens, [ref]$errors) + if ($errors) { $errors | ForEach-Object { $_.ToString() }; throw "$path has parse errors" } + $out = @() + foreach ($name in $names) { + $fn = $ast.FindAll({ param($n) + $n -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $n.Name -eq $name + }, $true) + if ($fn.Count -lt 1) { throw "expected $name in $path, found none" } + $out += $fn[0].Extent.Text + } + return $out +} + +# Stubs for the installers' printers, so this file does not depend on the ANSI helpers. +# Both use Write-Host, so neither can pollute a function's return value. +function substep { param([string]$Message, [string]$Color = "DarkGray") } +function Write-StudioStdoutMirror { param([string]$Line) } + +# Drives Get-NvidiaCu126Verdict without spawning a process: $script:FakeSmiStdout is what +# a -StdoutOnly probe returns, $script:FakeSmiRc its exit code. +function Invoke-NvidiaSmiBounded { + param([string]$Exe, [string[]]$SmiArgs = @(), [int]$TimeoutSec = 10, [switch]$StdoutOnly) + $global:LASTEXITCODE = $script:FakeSmiRc + # The real helper appends stderr to stdout without -StdoutOnly. Reproducing that is + # the point: the caller MUST pass the switch. + if ($StdoutOnly) { return $script:FakeSmiStdout } + return ($script:FakeSmiStdout + "`n" + $script:FakeSmiStderr) +} + +foreach ($file in @("install.ps1", "studio/setup.ps1")) { + $path = Join-Path $root $file + Write-Host "" + Write-Host "=== $file ===" + foreach ($srcText in (Get-HelperSources $path @("Get-NvidiaCu126Verdict", + "Get-CudaFamilyCappedForPreTuring"))) { + Invoke-Expression $srcText + } + + # --- the verdict table ----------------------------------------------------- + $script:FakeSmiRc = 0 + $script:FakeSmiStderr = "" + function Verdict($rows, $floor = 75) { + $script:FakeSmiStdout = ($rows -join "`n") + return (Get-NvidiaCu126Verdict "nvidia-smi" $floor) + } + Check "V100 sm_70 under floor 75 -> cu126" ((Verdict @("7.0")) -eq 'cu126') + Check "V100 sm_70 under floor 70 -> no cap" ((Verdict @("7.0") 70) -eq '') + Check "GTX980 sm_52 -> cu126" ((Verdict @("5.2")) -eq 'cu126') + Check "GTX1080 sm_61 -> cu126" ((Verdict @("6.1")) -eq 'cu126') + Check "T4 sm_75 -> no cap" ((Verdict @("7.5")) -eq '') + Check "H100 sm_90 -> no cap" ((Verdict @("9.0")) -eq '') + Check "B200 sm_100 -> no cap" ((Verdict @("10.0")) -eq '') + Check "Volta + Ampere -> cu126" ((Verdict @("7.0", "8.6")) -eq 'cu126') + Check "Volta + Blackwell -> uncovered" ((Verdict @("7.0", "12.0")) -eq 'uncovered') + Check "Kepler sm_37 -> uncovered" ((Verdict @("3.7")) -eq 'uncovered') + Check "CRLF rows still parse" ((Verdict @("7.0`r", "8.6`r")) -eq 'cu126') + Check "padded rows still parse" ((Verdict @(" 7.0 ")) -eq 'cu126') + Check "blank rows are skipped" ((Verdict @("7.0", "", "8.6")) -eq 'cu126') + Check "'N/A' row poisons the inventory" ((Verdict @("7.0", "N/A")) -eq '') + Check "'[N/A]' row poisons the inventory" ((Verdict @("7.0", "[N/A]")) -eq '') + Check "'[Not Supported]' poisons the inventory" ((Verdict @("7.0", "[Not Supported]")) -eq '') + Check "decimal comma poisons the inventory" ((Verdict @("7,0")) -eq '') + Check "empty inventory -> no cap" ((Verdict @("")) -eq '') + Check "no exe -> no cap" ((Get-NvidiaCu126Verdict "" 75) -eq '') + $script:FakeSmiRc = 1 + Check "non-zero exit -> no cap" ((Verdict @("7.0")) -eq '') + $script:FakeSmiRc = 0 + + # --- -StdoutOnly is load-bearing ------------------------------------------ + # A driver warning on stderr is ordinary (corrupted infoROM, ECC pending). Without the + # switch it lands in the CSV, the inventory reads as unparseable, and a V100 silently + # keeps cu130 -- issue #7765 all over again. + $script:FakeSmiStdout = "7.0" + $script:FakeSmiStderr = "WARNING: infoROM is corrupted at gpu 0000:00:04.0" + Check "stderr noise does not reach the CSV parse" ((Get-NvidiaCu126Verdict "nvidia-smi" 75) -eq 'cu126') + $src = Get-Content -Raw $path + Check "the probe call passes -StdoutOnly" ($src -match 'compute_cap.*-StdoutOnly') + $script:FakeSmiStderr = "" + + # --- the cap only rewrites the families it can replace --------------------- + $script:FakeSmiStdout = "7.0" + Check "cap cu130 on a V100 -> cu126" ((Get-CudaFamilyCappedForPreTuring 'cu130' "nvidia-smi") -eq 'cu126') + Check "cap cu128 on a V100 -> cu128 (floor 70)" ((Get-CudaFamilyCappedForPreTuring 'cu128' "nvidia-smi") -eq 'cu128') + Check "cap cu126 is a no-op" ((Get-CudaFamilyCappedForPreTuring 'cu126' "nvidia-smi") -eq 'cu126') + Check "cap cu124 is a no-op" ((Get-CudaFamilyCappedForPreTuring 'cu124' "nvidia-smi") -eq 'cu124') + Check "cap cpu is a no-op" ((Get-CudaFamilyCappedForPreTuring 'cpu' "nvidia-smi") -eq 'cpu') + $r = Get-CudaFamilyCappedForPreTuring 'cu130' "nvidia-smi" + Check "returns a single string, not an array" (-not ($r -is [array])) + $script:FakeSmiStdout = "7.0`n12.0" + Check "uncovered mix keeps the driver family" ((Get-CudaFamilyCappedForPreTuring 'cu130' "nvidia-smi") -eq 'cu130') +} + +Write-Host "" +if ($failures -gt 0) { Write-Host "$failures check(s) FAILED" -ForegroundColor Red; exit 1 } +Write-Host "All checks passed" -ForegroundColor Green