From f19bf3a2eebc386a7cf13e0543e535e00a5ccb58 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 4 Aug 2026 03:56:51 -0700 Subject: [PATCH] Name the missing shared library when llama-server cannot start (#7782) * Name the missing shared library when llama-server cannot start * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Tell a missing library apart from an unloadable one, a bundled runtime, and a bare exit 127 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Keep permission-denied and custom-runtime libraries out of the missing-package advice * Apply the same provenance split to the bare exit-127 remedy * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Keep package-manager advice to libraries a package can actually provide * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Match glibc's slash rule when deciding a library name is a path --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- studio/backend/core/inference/llama_cpp.py | 188 ++++++++++++ ..._llama_cpp_start_failure_classification.py | 272 ++++++++++++++++++ 2 files changed, 460 insertions(+) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 688a3d5edb..8909a38161 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -7312,12 +7312,132 @@ class LlamaCppBackend: ) ) + # Distro package names for the shared libraries the Linux prebuilt links, + # so the loader failure below can say what to install. + _SHARED_LIB_PACKAGES = { + "libgomp.so.1": " (Debian/Ubuntu: libgomp1, Fedora/RHEL: libgomp)", + } + + # Shared objects Unsloth ships itself, next to llama-server in build/bin + # (see runtime_payload_health_groups in install_llama_prebuilt.py, and + # _llama_server_env_for_binary which puts that dir on LD_LIBRARY_PATH). No + # distro packages these, so a loader failure naming one means the managed + # runtime is incomplete or mismatched, not that something must be installed. + _BUNDLED_LIB_PREFIXES = ("libllama", "libggml", "libmtmd") + + @staticmethod + def _is_bundled_llama_library(lib: str) -> bool: + # The loader prints a bare soname when the file is absent but a full + # path when it found and rejected it, so compare on the basename. + return os.path.basename((lib or "").strip()).startswith( + LlamaCppBackend._BUNDLED_LIB_PREFIXES + ) + + @staticmethod + def _is_unsloth_managed_binary(binary: Optional[str]) -> bool: + """Would `unsloth studio update` actually replace ``binary``? + + A pinned LLAMA_SERVER_PATH, or a llama-server found on PATH, is + explicitly unmanaged (update_flow.managed_install_root returns None), + so telling the user to update Unsloth's runtime cannot repair it. + Unknown binary (callers that pass nothing) keeps the managed default. + """ + if not binary: + return True + try: + from utils.llama_cpp_update import _llama_install_root + return _llama_install_root(binary) is not None + except Exception: + return True + + @staticmethod + def _is_library_path(lib: str) -> bool: + # glibc's own discriminator is `strchr (name, '/') == NULL` + # (elf/dl-load.c): a slash anywhere means the name is a pathname and no + # search happens, so a relative DT_NEEDED is as exact as an absolute + # one. os.sep would be platform-bound and this string comes from + # whatever host ran llama-server, so match both separators. + lib = (lib or "").strip() + return "/" in lib or "\\" in lib + + @staticmethod + def _missing_library_message(lib: str, binary: Optional[str] = None) -> str: + """The loader could not FIND ``lib`` ("cannot open shared object file").""" + if LlamaCppBackend._is_bundled_llama_library(lib): + if LlamaCppBackend._is_unsloth_managed_binary(binary): + return ( + f"llama-server could not start: {lib} is part of Unsloth's own " + "llama.cpp runtime and is missing from the install. Run " + "`unsloth studio update` to reinstall it, then load the model " + "again." + ) + return ( + f"llama-server could not start: {lib} is part of the llama.cpp " + "runtime that the llama-server binary in use was built with. That " + "binary is a custom install Unsloth does not manage, so reinstall " + "or rebuild that llama.cpp, then load the model again." + ) + # A bare soname means the loader searched the standard directories, + # which is what a package populates. Any name carrying a separator is a + # pathname the loader opened directly (e.g. a vendor .so under /opt), so + # one exact file is absent and no package will put it there. + if LlamaCppBackend._is_library_path(lib): + _remedy = ( + "run `unsloth studio update`" + if LlamaCppBackend._is_unsloth_managed_binary(binary) + else "reinstall or rebuild that custom llama.cpp" + ) + return ( + f"llama-server could not start: {lib} is missing from that exact " + f"location. Restore it, or {_remedy}, then load the model again." + ) + return ( + f"llama-server could not start: the system library " + f"{lib or 'it needs'} is missing. Install it with your package " + f"manager{LlamaCppBackend._SHARED_LIB_PACKAGES.get(lib, '')}, " + "then load the model again." + ) + + @staticmethod + def _unloadable_library_message( + lib: str, + reason: str, + binary: Optional[str] = None, + ) -> str: + """The loader FOUND ``lib`` and refused it: the same "error while + loading shared libraries" line also carries "file too short", "invalid + ELF header", "wrong ELF class", "cannot open shared object file: + Permission denied", ... Installing a package is the wrong advice there + -- the file is already present.""" + detail = f" ({reason})" if reason else "" + if LlamaCppBackend._is_bundled_llama_library(lib): + if LlamaCppBackend._is_unsloth_managed_binary(binary): + return ( + f"llama-server could not start: {lib}, part of Unsloth's own " + f"llama.cpp runtime, could not be loaded{detail}. The install " + "is incomplete or mismatched. Run `unsloth studio update` to " + "reinstall it, then load the model again." + ) + return ( + f"llama-server could not start: {lib}, part of the llama.cpp " + "runtime that the llama-server binary in use was built with, " + f"could not be loaded{detail}. That binary is a custom install " + "Unsloth does not manage, so reinstall or rebuild that " + "llama.cpp, then load the model again." + ) + return ( + f"llama-server could not start: the library {lib or 'it needs'} " + f"could not be loaded{detail}. Reinstall or update whatever " + "provides it, then load the model again." + ) + @staticmethod def _classify_llama_start_failure( output: str, gguf_path: Optional[str], model_identifier: Optional[str], returncode: Optional[int] = None, + binary: Optional[str] = None, ) -> str: """Explain *why* llama-server failed to start, from its output. @@ -7329,6 +7449,51 @@ class LlamaCppBackend: """ lowered = (output or "").lower() + # The dynamic loader kills llama-server before main(), so nothing below + # matches and the fallback blames the file or memory instead. The Linux + # prebuilt links libgomp.so.1, which a stock container does not ship. + # glibc prints ": error while loading shared libraries: : + # [: ]" (elf/dl-catch.c fatal_error). is + # echoed verbatim, so an absolute dependency under a directory with + # spaces keeps them ("/opt/My Runtime/libfoo.so: file too short"): + # split on the ": " that introduces the diagnostic, not on whitespace. + missing_lib = re.search( + r"error while loading shared libraries:[ \t]*([^\r\n]+?)" + r"(?::[ \t]+([^\r\n]*))?[ \t]*\r?$", + output or "", + re.MULTILINE, + ) + if missing_lib: + _lib = missing_lib.group(1).strip() + _reason = (missing_lib.group(2) or "").strip() + # glibc omits the object name entirely on its own allocation + # failures ("...shared libraries: cannot create search path array"), + # so the first word is prose, not a library. Report it unnamed. + if not ( + "/" in _lib or "\\" in _lib or any(_e in _lib for _e in (".so", ".dylib", ".dll")) + ): + return LlamaCppBackend._unloadable_library_message( + "", f"{_lib} {_reason}".strip(), binary + ) + # Only "cannot open shared object file" means absent; the same line + # reports corrupt/incompatible libraries too, which are present. + # glibc appends strerror(errno) to that diagnostic, and only ENOENT + # ("No such file or directory") means the file is not there: EACCES + # ("Permission denied", seen with an absolute DT_NEEDED path, an + # unreadable parent directory, SELinux mislabels or container + # sandboxing) means it exists and cannot be opened, where installing + # a package is the wrong advice. A truncated tail keeps the + # missing-library reading. + _reason_l = _reason.lower() + _errno_text = "" + if "cannot open shared object file:" in _reason_l: + _errno_text = _reason_l.split("cannot open shared object file:", 1)[1].strip() + if not _reason_l or ( + "cannot open" in _reason_l and (not _errno_text or "no such file" in _errno_text) + ): + return LlamaCppBackend._missing_library_message(_lib, binary) + return LlamaCppBackend._unloadable_library_message(_lib, _reason, binary) + # Tensor parallelism (--split-mode tensor) is arch-gated in llama.cpp; # unsupported architectures abort the load with this marker. Point the # user at the toggle instead of a generic invalid-GGUF/OOM message. @@ -7389,6 +7554,27 @@ class LlamaCppBackend: "Ollama instead." ) + # 127 without any loader line above is not specific to a library: it is + # also what a shell wrapper entrypoint reports when its exec target is + # gone (_llama_lib_dir supports those, as does a custom + # LLAMA_SERVER_PATH), and what a "symbol lookup error" from a mismatched + # runtime exits with. Name both causes instead of blaming a distro + # package -- but still keep it off the GGUF and off memory. + if returncode == 127: + # Same provenance split as the library branches: the updater cannot + # touch a pinned binary, so do not send its owner there. + _remedy = ( + "run `unsloth studio update` to reinstall the llama.cpp runtime" + if LlamaCppBackend._is_unsloth_managed_binary(binary) + else "reinstall or rebuild that custom llama.cpp" + ) + return ( + "llama-server exited immediately (status 127): either the " + "llama-server executable could not be found or run, or one of " + f"the shared libraries it needs could not be loaded. Check the " + f"llama-server log, and {_remedy}." + ) + # SIGKILL with no diagnostic output is the OOM killer (e.g. a model too # large for the WSL VM's RAM cap); name it actionably. if returncode == -9: @@ -10724,6 +10910,7 @@ class LlamaCppBackend: gguf_path, self._model_identifier, _retry_rc, + binary, ) raise RuntimeError( self._mmproj_retry_failure_message( @@ -10738,6 +10925,7 @@ class LlamaCppBackend: gguf_path, self._model_identifier, _crash_rc, + binary, ) ) diff --git a/studio/backend/tests/test_llama_cpp_start_failure_classification.py b/studio/backend/tests/test_llama_cpp_start_failure_classification.py index 246d810602..342368d182 100644 --- a/studio/backend/tests/test_llama_cpp_start_failure_classification.py +++ b/studio/backend/tests/test_llama_cpp_start_failure_classification.py @@ -181,3 +181,275 @@ class TestOsKillReturncode: msg = _classify("", "/models/x.gguf", "local/x", -11) assert "GGUF file is valid" in msg assert "out of memory" not in msg.lower() + + +class TestMissingSharedLibrary: + """The dynamic loader stops llama-server before it prints anything of its + own, so a stock container missing libgomp.so.1 used to be reported as an + invalid GGUF or too little memory.""" + + _LOADER_OUT = ( + "/home/tester/.unsloth/llama.cpp/llama-server: error while loading " + "shared libraries: libgomp.so.1: cannot open shared object file: " + "No such file or directory" + ) + + def test_missing_libgomp_is_named_with_its_packages(self): + msg = _classify(self._LOADER_OUT, "/models/x.gguf", "local/x", 127) + assert "libgomp.so.1" in msg + assert "libgomp1" in msg + assert "Fedora/RHEL" in msg + assert "GGUF file is valid" not in msg + assert "enough memory" not in msg.lower() + + def test_unknown_library_is_still_named(self): + out = "llama-server: error while loading shared libraries: libfoo.so.7: cannot open" + msg = _classify(out, "/models/x.gguf", "local/x", 127) + assert "libfoo.so.7" in msg + assert "package manager" in msg + assert "libgomp1" not in msg + + def test_exit_127_with_no_output_names_both_causes(self): + # 127 is also a shell-wrapper entrypoint whose exec target is gone, so + # it must not claim a distro package is missing. The generic + # file/memory message is still wrong. + msg = _classify("", "/models/x.gguf", "local/x", 127) + assert "could not be found or run" in msg + assert "shared libraries" in msg + assert "package manager" not in msg + assert "GGUF file is valid" not in msg + assert "enough memory" not in msg.lower() + + def test_exit_127_on_a_pinned_binary_does_not_send_it_to_the_updater(self, monkeypatch): + # A wrapper whose exec target is gone exits 127 with no loader line, and + # the updater refuses to touch a LLAMA_SERVER_PATH pin, so the managed + # remedy is a dead end there too. + monkeypatch.setenv("LLAMA_SERVER_PATH", "/opt/custom/llama-server") + msg = _classify("", "/models/x.gguf", "local/x", 127, "/opt/custom/llama-server") + assert "unsloth studio update" not in msg + assert "custom llama.cpp" in msg + + def test_exit_127_on_a_managed_binary_still_points_at_the_updater(self): + msg = _classify("", "/models/x.gguf", "local/x", 127) + assert "unsloth studio update" in msg + + def test_wrapper_exec_failure_is_not_called_a_system_library(self): + # write_exec_wrapper's entrypoint: /bin/sh reports a missing exec + # target as "not found" and exits 127. + out = ( + "/home/t/.unsloth/llama.cpp/llama-server: 2: exec: " + "./build/bin/llama-server: not found" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127) + assert "package manager" not in msg + assert "could not be found or run" in msg + + def test_symbol_lookup_error_is_not_called_a_system_library(self): + # A mismatched bundled runtime exits 127 with this, not a loader line. + out = "llama-server: symbol lookup error: llama-server: undefined symbol: ggml_backend_init" + msg = _classify(out, "/models/x.gguf", "local/x", 127) + assert "package manager" not in msg + + def test_bundled_runtime_library_points_at_the_installer(self): + # libggml/libllama/libmtmd ship in build/bin (runtime_payload_health_groups) + # and no package manager can supply them. + out = ( + "/home/t/.unsloth/llama.cpp/build/bin/llama-server: error while loading " + "shared libraries: libggml.so.0: cannot open shared object file: " + "No such file or directory" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127) + assert "libggml.so.0" in msg + assert "unsloth studio update" in msg + assert "package manager" not in msg + + def test_corrupt_library_is_not_reported_as_missing(self): + # glibc reuses the same prefix for a present-but-unusable library. + out = ( + "/opt/llama/llama-server: error while loading shared libraries: " + "/usr/lib/x86_64-linux-gnu/libgomp.so.1: file too short" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127) + assert "file too short" in msg + assert "is missing" not in msg + assert "Install it with your package manager" not in msg + + def test_corrupt_bundled_library_points_at_the_installer(self): + out = ( + "llama-server: error while loading shared libraries: " + "/home/t/.unsloth/llama.cpp/build/bin/libggml-cuda.so: invalid ELF header" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127) + assert "invalid ELF header" in msg + assert "unsloth studio update" in msg + assert "is missing" not in msg + + # Verified on glibc 2.39: an absolute DT_NEEDED dependency that exists but + # cannot be opened exits 127 with the EACCES strerror appended. + def test_permission_denied_library_is_not_reported_as_missing(self): + out = ( + "/opt/llama/llama-server: error while loading shared libraries: " + "/opt/llama/lib/libgomp.so.1: cannot open shared object file: " + "Permission denied" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127) + assert "Permission denied" in msg + assert "is missing" not in msg + assert "package manager" not in msg + + def test_permission_denied_bundled_library_is_not_reported_as_missing(self): + out = ( + "/home/t/.unsloth/llama.cpp/build/bin/llama-server: error while loading " + "shared libraries: /home/t/.unsloth/llama.cpp/build/bin/libggml.so: " + "cannot open shared object file: Permission denied" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127) + assert "Permission denied" in msg + assert "is missing" not in msg + + # glibc echoes the object name verbatim, so a path with spaces must not be + # truncated at the first space (verified on glibc 2.39). + def test_library_path_with_spaces_is_named_in_full(self): + out = ( + "/opt/llama/llama-server: error while loading shared libraries: " + "/opt/My Runtime/libfoo.so: file too short" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127) + assert "/opt/My Runtime/libfoo.so" in msg + assert "file too short" in msg + + def test_missing_library_path_with_spaces_is_named_in_full(self): + out = ( + "/opt/llama/llama-server: error while loading shared libraries: " + "/opt/My Runtime/libbar.so: cannot open shared object file: " + "No such file or directory" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127) + assert "/opt/My Runtime/libbar.so" in msg + assert "is missing" in msg + + def test_an_absolute_path_is_not_offered_to_a_package_manager(self): + # An absolute DT_NEEDED names one exact file. No package puts a file at + # /opt/vendor, so apt/dnf is the wrong instruction whoever owns the + # binary. + out = ( + "llama-server: error while loading shared libraries: " + "/opt/vendor/libaccelerator.so: cannot open shared object file: " + "No such file or directory" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127) + assert "/opt/vendor/libaccelerator.so" in msg + assert "package manager" not in msg + assert "that exact location" in msg + + def test_an_absolute_path_on_a_pinned_binary_names_the_custom_runtime(self, monkeypatch): + monkeypatch.setenv("LLAMA_SERVER_PATH", "/opt/custom/llama-server") + out = ( + "llama-server: error while loading shared libraries: " + "/opt/vendor/libaccelerator.so: cannot open shared object file: " + "No such file or directory" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127, "/opt/custom/llama-server") + assert "custom llama.cpp" in msg + assert "unsloth studio update" not in msg + assert "package manager" not in msg + + def test_a_bare_soname_keeps_package_advice_even_on_a_pinned_binary(self, monkeypatch): + # The counter-case that stops the rule from being "unmanaged means never + # mention a package": a custom-built llama.cpp on a bare-bones host is + # still missing a distro library, and libgomp1 is exactly what fixes it. + monkeypatch.setenv("LLAMA_SERVER_PATH", "/opt/custom/llama-server") + out = ( + "llama-server: error while loading shared libraries: " + "libgomp.so.1: cannot open shared object file: No such file or directory" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127, "/opt/custom/llama-server") + assert "libgomp1" in msg + assert "package manager" in msg + + def test_a_relative_dt_needed_is_an_exact_path_too(self): + # glibc's rule is `strchr (name, '/') == NULL`: a slash anywhere means + # no search happened, so subdir/libfoo.so names one exact file just as + # an absolute path does. Reproduced on glibc 2.39 with a SONAME-less .so + # linked by relative path; it takes both to get here, so this is about + # matching the loader's rule rather than a case users hit. + out = ( + "llama-server: error while loading shared libraries: " + "subdir/libvendor.so: cannot open shared object file: " + "No such file or directory" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127) + assert "subdir/libvendor.so" in msg + assert "package manager" not in msg + assert "that exact location" in msg + + def test_a_windows_absolute_path_is_recognised_too(self): + out = ( + "llama-server: error while loading shared libraries: " + "C:\\vendor\\accel.dll: cannot open shared object file: " + "No such file or directory" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127) + assert "package manager" not in msg + assert "that exact location" in msg + + def test_bundled_library_under_a_spaced_path_still_points_at_the_installer(self): + out = ( + "llama-server: error while loading shared libraries: " + "/home/My User/.unsloth/llama.cpp/build/bin/libggml.so: invalid ELF header" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127) + assert "/home/My User/.unsloth/llama.cpp/build/bin/libggml.so" in msg + assert "unsloth studio update" in msg + + def test_pinned_custom_binary_is_not_called_unsloths_runtime(self, monkeypatch): + # LLAMA_SERVER_PATH pins an install update_flow.managed_install_root + # refuses to manage, so `unsloth studio update` cannot repair it. + monkeypatch.setenv("LLAMA_SERVER_PATH", "/opt/mybuild/bin/llama-server") + out = ( + "/opt/mybuild/bin/llama-server: error while loading shared libraries: " + "libggml.so: cannot open shared object file: No such file or directory" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127, "/opt/mybuild/bin/llama-server") + assert "libggml.so" in msg + assert "unsloth studio update" not in msg + assert "package manager" not in msg + assert "custom install" in msg + + def test_managed_binary_still_points_at_the_installer(self, monkeypatch, tmp_path): + monkeypatch.delenv("LLAMA_SERVER_PATH", raising = False) + binary = tmp_path / "llama.cpp" / "build" / "bin" / "llama-server" + binary.parent.mkdir(parents = True) + binary.write_text("") + monkeypatch.setenv("UNSLOTH_LLAMA_CPP_PATH", str(tmp_path / "llama.cpp")) + out = ( + f"{binary}: error while loading shared libraries: libggml.so: " + "cannot open shared object file: No such file or directory" + ) + msg = _classify(out, "/models/x.gguf", "local/x", 127, str(binary)) + assert "unsloth studio update" in msg + + def test_nameless_loader_error_does_not_invent_a_library(self): + # glibc's own allocation failures pass an empty object name, so the + # text right after the colon is prose, not a soname. + out = "llama-server: error while loading shared libraries: cannot create search path array" + msg = _classify(out, "/models/x.gguf", "local/x", 127) + assert "cannot create search path array" in msg + assert "the library cannot" not in msg + assert "is missing" not in msg + + def test_loader_error_wins_without_a_returncode(self): + msg = _classify(self._LOADER_OUT, "/models/x.gguf", "local/x") + assert "libgomp.so.1" in msg + + def test_a_normal_failure_is_untouched(self): + msg = _classify(_OOM_OUT, "/models/big.gguf", "local/big", 1) + assert "enough memory" in msg.lower() + assert "system library" not in msg + + def test_a_named_arch_wins_over_exit_127(self): + # The bare code is only a fallback, so it must not mask a diagnosis the + # output already gives. + msg = _classify(_QWEN_IMAGE_OUT, "/models/qwen-image.gguf", "local/qwen-image", 127) + assert "Images page" in msg + assert "system library" not in msg