From 9bcef39028a8d5579cfa9752c1584eb8d39cd4ca Mon Sep 17 00:00:00 2001 From: Alessandro <155005371+3clyp50@users.noreply.github.com> Date: Mon, 15 Jun 2026 05:37:28 +0200 Subject: [PATCH] Add llama.cpp and vLLM local providers Add llama.cpp / llama-server support: - Register llama.cpp chat and embedding providers through hosted_vllm defaults on host.docker.internal:8080. - Add onboarding metadata, a bundled icon, no-key provider metadata, model discovery coverage, setup docs, and tests. Add vLLM support: - Register vLLM chat and embedding providers through hosted_vllm defaults on host.docker.internal:8000. - Add onboarding metadata, a bundled icon, no-key provider metadata, model discovery coverage, setup docs, and tests. - Strip empty tools arrays from the Responses-to-chat fallback path so strict OpenAI-compatible servers accept local vLLM calls. --- conf/model_providers.yaml | 30 ++++++++ docs/setup/installation.md | 67 +++++++++++++++++ helpers/litellm_transport.py | 3 + helpers/litellm_transport.py.dox.md | 45 ++++++++++++ helpers/llm_result.py.dox.md | 39 ++++++++++ helpers/tunnel_origins.py.dox.md | 37 ++++++++++ plugins/_browser/helpers/connector_runtime.py | 2 +- plugins/_model_config/api/model_search.py | 10 ++- .../python/banners/_20_missing_api_key.py | 2 +- plugins/_model_config/helpers/model_config.py | 2 +- plugins/_model_config/provider_metadata.yaml | 8 +++ .../webui/assets/provider-logos/llama-cpp.svg | 17 +++++ .../webui/assets/provider-logos/vllm.svg | 18 +++++ .../_onboarding/webui/onboarding-providers.js | 20 +++++- tests/test_model_config_api_keys.py | 61 ++++++++++++++++ tests/test_model_search.py | 10 +++ tests/test_onboarding_static.py | 14 ++++ tests/test_stream_tool_early_stop.py | 71 +++++++++++++++++++ 18 files changed, 450 insertions(+), 6 deletions(-) create mode 100644 helpers/litellm_transport.py.dox.md create mode 100644 helpers/llm_result.py.dox.md create mode 100644 helpers/tunnel_origins.py.dox.md create mode 100644 plugins/_onboarding/webui/assets/provider-logos/llama-cpp.svg create mode 100644 plugins/_onboarding/webui/assets/provider-logos/vllm.svg diff --git a/conf/model_providers.yaml b/conf/model_providers.yaml index 0743a6512..541a4888b 100644 --- a/conf/model_providers.yaml +++ b/conf/model_providers.yaml @@ -86,6 +86,15 @@ chat: kwargs: api_base: "http://host.docker.internal:1234/v1" api_key: "lm-studio" + llama_cpp: + name: llama.cpp + litellm_provider: hosted_vllm + models_list: + endpoint_url: "/v1/models" + default_base: "http://host.docker.internal:8080" + kwargs: + api_base: "http://host.docker.internal:8080/v1" + api_key: "llama-cpp" mistral: name: Mistral AI litellm_provider: mistral @@ -167,6 +176,15 @@ chat: api_base: https://api.venice.ai/api/v1 venice_parameters: include_venice_system_prompt: false + vllm: + name: vLLM + litellm_provider: hosted_vllm + models_list: + endpoint_url: "/v1/models" + default_base: "http://host.docker.internal:8000" + kwargs: + api_base: "http://host.docker.internal:8000/v1" + api_key: "vllm" xai: name: xAI litellm_provider: xai @@ -203,6 +221,12 @@ embedding: kwargs: api_base: "http://host.docker.internal:1234/v1" api_key: "lm-studio" + llama_cpp: + name: llama.cpp + litellm_provider: hosted_vllm + kwargs: + api_base: "http://host.docker.internal:8080/v1" + api_key: "llama-cpp" mistral: name: Mistral AI litellm_provider: mistral @@ -250,6 +274,12 @@ embedding: endpoint_url: "https://api.venice.ai/api/v1/models" kwargs: api_base: https://api.venice.ai/api/v1 + vllm: + name: vLLM + litellm_provider: hosted_vllm + kwargs: + api_base: "http://host.docker.internal:8000/v1" + api_key: "vllm" other: name: Other OpenAI compatible litellm_provider: openai diff --git a/docs/setup/installation.md b/docs/setup/installation.md index 57c0b76b9..f4836a980 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -439,6 +439,8 @@ Use the naming format required by your selected provider: | OpenRouter | Provider prefix mostly required | `anthropic/claude-sonnet-4-5` | | Ollama | Model name only | `gpt-oss:20b` | | oMLX | API-visible model name from `/v1/models` | `Qwen3-0.6B-4bit` | +| llama.cpp | API-visible model name from `/v1/models` or `--alias` | `local-gguf` | +| vLLM | Hugging Face model ID or served model alias | `Qwen/Qwen2.5-1.5B-Instruct` | > [!TIP] > If you see "Invalid model ID," verify the provider and naming format on the provider website, or search the web for " model naming". @@ -504,6 +506,71 @@ omlx serve --model-dir ~/.omlx/models --paged-ssd-cache-dir ~/.omlx/cache --- +## Installing and Using llama.cpp (GGUF Local Models) + +llama.cpp provides `llama-server`, a lightweight OpenAI-compatible HTTP server for GGUF models. Agent Zero talks to it through the same `/v1` API used by OpenAI-compatible clients. + +### macOS llama.cpp Installation + +**Using Homebrew:** + +```bash +brew install llama.cpp +``` + +Start a server with a downloaded GGUF model: + +```bash +llama-server -m ~/models/model.gguf --port 8080 --alias local-gguf +``` + +By default, Agent Zero expects llama.cpp at `http://host.docker.internal:8080/v1`. The model name can be the model path returned by `/v1/models`, but using `--alias` gives you a short stable name such as `local-gguf`. + +### Configuring llama.cpp in Agent Zero + +1. Start `llama-server` and confirm `http://localhost:8080/v1/models` returns your model. +2. In Agent Zero Settings, choose **llama.cpp** as the Chat model, Utility model, or Embedding model provider. +3. Use the model ID shown by `/v1/models`, or the alias you passed with `--alias`. +4. Override the API base URL only if you started `llama-server` on another host or port. +5. Click `Save` to confirm your settings. + +> [!NOTE] +> If Agent Zero runs in Docker and cannot reach a host-side `llama-server`, start the server on an address Docker can reach, for example `--host 0.0.0.0`, and keep the port firewalled to trusted clients. + +--- + +## Installing and Using vLLM (Local OpenAI-Compatible Serving) + +vLLM is a high-throughput local inference server with an OpenAI-compatible API. It is most common on Linux GPU hosts, and can also run on Apple Silicon through the vLLM Apple Silicon path or vLLM-Metal. + +For Apple Silicon Macs, install and activate vLLM-Metal: + +```bash +curl -fsSL https://raw.githubusercontent.com/vllm-project/vllm-metal/main/install.sh | bash +source ~/.venv-vllm-metal/bin/activate +``` + +Start a basic OpenAI-compatible server: + +```bash +vllm serve Qwen/Qwen2.5-1.5B-Instruct --host 0.0.0.0 --port 8000 +``` + +By default, Agent Zero expects vLLM at `http://host.docker.internal:8000/v1`, matching vLLM's default HTTP port. If another local provider already uses port 8000, start vLLM on another port and update Agent Zero's API base, for example `http://host.docker.internal:8001/v1`. + +### Configuring vLLM in Agent Zero + +1. Start vLLM and confirm `http://localhost:8000/v1/models` returns the served model. +2. In Agent Zero Settings, choose **vLLM** as the Chat model, Utility model, or Embedding model provider. +3. Use the model ID returned by vLLM's model list endpoint. +4. If you started vLLM with `--api-key`, enter the same key in the advanced provider settings or environment. +5. Click `Save` to confirm your settings. + +> [!NOTE] +> vLLM serves one model at a time by default. Use a generation model for Chat and Utility slots, and a separate embedding-capable vLLM server if you want vLLM embeddings. + +--- + ## Installing and Using Ollama (Local Models) Ollama is a powerful tool that allows you to run various large language models locally. diff --git a/helpers/litellm_transport.py b/helpers/litellm_transport.py index de5a2b884..05b9c6b2a 100644 --- a/helpers/litellm_transport.py +++ b/helpers/litellm_transport.py @@ -459,6 +459,7 @@ class ChatCompletionsTransport: chat_kwargs = dict(kwargs) _drop_internal_transport_kwargs(chat_kwargs) if not _has_tools(chat_kwargs.get("tools")): + chat_kwargs.pop("tools", None) chat_kwargs.pop("tool_choice", None) chat_kwargs.pop("parallel_tool_calls", None) if _is_openai_prompt_cache_provider(model, chat_kwargs): @@ -1538,6 +1539,8 @@ def _is_responses_not_supported_error(exc: Exception) -> bool: "unsupportedparamserror", "does not support parameters", "no 'tools' defined while 'tool_choice' is specified", + "tools` must not be an empty array", + "tools must not be an empty array", ) ) diff --git a/helpers/litellm_transport.py.dox.md b/helpers/litellm_transport.py.dox.md new file mode 100644 index 000000000..4105d7ba0 --- /dev/null +++ b/helpers/litellm_transport.py.dox.md @@ -0,0 +1,45 @@ +# litellm_transport.py DOX + +## Purpose + +- Own Agent Zero's LiteLLM transport adapter for Chat Completions and Responses API calls. +- Normalize Agent Zero model-call kwargs into provider-safe LiteLLM requests. +- Preserve canonical response metadata for history, provider-state continuation, and fallback decisions. + +## Ownership + +- `litellm_transport.py` owns the runtime implementation. +- `litellm_transport.py.dox.md` owns durable notes about responsibilities, contracts, side effects, and verification for that implementation. +- Classes: +- `TransportMode` +- `TransportRecovery` +- `TransportPolicy` +- `LiteLLMTransport` +- `ChatCompletionsTransport` +- `ResponsesTransport` +- `ResponsesEventParser` +- Top-level functions include transport cache reset, request normalization, parsing, prompt-cache preparation, and response/error classifiers. + +## Runtime Contracts + +- Keep provider selection and provider-specific defaults outside this helper; callers pass a resolved LiteLLM model name and kwargs. +- Strip Agent Zero internal kwargs before sending requests to LiteLLM. +- Do not send orphan tool controls when no tools are present; strict OpenAI-compatible servers can reject empty `tools` arrays. +- Prefer Responses API when configured, but fallback to Chat Completions when the provider does not support Responses. +- Preserve provider-state metadata when Responses API calls succeed, and fall back to local replay when provider state is unsupported. +- Keep prompt-cache markers only for providers that accept them. + +## Work Guidance + +- Add provider-agnostic request cleanup here when multiple OpenAI-compatible providers can benefit. +- Treat fallback behavior as a shared transport contract, not a provider registry. +- Keep tool conversion symmetric between Chat Completions and Responses requests. + +## Verification + +- Run `pytest tests/test_stream_tool_early_stop.py tests/test_responses_architecture.py -q` after changing transport normalization or fallback behavior. +- Run local-provider smoke checks when changing OpenAI-compatible request cleanup. + +## Child DOX Index + +No child DOX files. diff --git a/helpers/llm_result.py.dox.md b/helpers/llm_result.py.dox.md new file mode 100644 index 000000000..b48cea191 --- /dev/null +++ b/helpers/llm_result.py.dox.md @@ -0,0 +1,39 @@ +# llm_result.py DOX + +## Purpose + +- Own canonical LLM result metadata shared by model transports, history, and tool-result processing. +- Preserve Responses API output items, provider response IDs, reasoning text, usage, and capability metadata in a serializable form. + +## Ownership + +- `llm_result.py` owns the runtime implementation. +- `llm_result.py.dox.md` owns durable notes about responsibilities, contracts, side effects, and verification for that implementation. +- Classes: +- `ResponseItem` +- `ResponseFunctionCall` +- `LLMResult` +- Top-level functions include metadata conversion, function-call output item construction, object normalization, output-text extraction, reasoning extraction, and function-call argument parsing. + +## Runtime Contracts + +- `LLMResult.metadata()` stores data under `RESPONSE_METADATA_KEY` so history can round-trip provider state. +- `from_response(...)` must preserve provider `response_id`, `previous_response_id`, raw output items, usage, and capability metadata. +- `from_chat(...)` must produce an equivalent chat-completions result with `mode="chat_completions"` and `state="off"`. +- Function-call output items must preserve `call_id` and optional acknowledged safety checks. +- Argument parsing must tolerate JSON strings, dictionaries, and malformed values without throwing. + +## Work Guidance + +- Keep metadata backward-compatible with existing serialized chat history. +- Treat unknown response item types as preserved built-in items unless they are local function calls, message text, or reasoning. +- Avoid provider-specific assumptions in result parsing. + +## Verification + +- Run `pytest tests/test_responses_architecture.py -q` after changing result metadata behavior. +- Run focused history/tool-processing tests when changing function-call serialization. + +## Child DOX Index + +No child DOX files. diff --git a/helpers/tunnel_origins.py.dox.md b/helpers/tunnel_origins.py.dox.md new file mode 100644 index 000000000..83c829d8b --- /dev/null +++ b/helpers/tunnel_origins.py.dox.md @@ -0,0 +1,37 @@ +# tunnel_origins.py DOX + +## Purpose + +- Own origin normalization for Remote Control tunnel URLs and CSRF/WebSocket same-origin checks. +- Provide a small helper boundary between tunnel discovery and security enforcement. + +## Ownership + +- `tunnel_origins.py` owns the runtime implementation. +- `tunnel_origins.py.dox.md` owns durable notes about responsibilities, contracts, side effects, and verification for that implementation. +- Top-level functions: +- `origin_from_url(value)` +- `origin_key(value)` +- `get_active_tunnel_origins()` + +## Runtime Contracts + +- Normalize URL and Origin header values to `scheme://host[:port]`, omitting default ports. +- Return comparable origin keys with default ports restored for same-origin checks. +- Treat invalid, missing, or malformed origins as `None`. +- Discover active tunnel origins from `TunnelManager` and the Docker tunnel API without raising if either source is unavailable. +- Keep tunnel service lookups short-timeout and local-only. + +## Work Guidance + +- Keep parsing based on `urllib.parse` rather than hand-rolled string checks. +- Preserve defensive exception handling because tunnel services are optional and may not be running. +- Coordinate security-sensitive changes with CSRF and WebSocket tests. + +## Verification + +- Run `pytest tests/test_csrf_tunnel_origins.py tests/test_ws_csrf.py -q` after changing tunnel origin behavior. + +## Child DOX Index + +No child DOX files. diff --git a/plugins/_browser/helpers/connector_runtime.py b/plugins/_browser/helpers/connector_runtime.py index 28c54b5fe..47b31512b 100644 --- a/plugins/_browser/helpers/connector_runtime.py +++ b/plugins/_browser/helpers/connector_runtime.py @@ -58,7 +58,7 @@ HOST_BROWSER_PROFILE_MODE_KEY = getattr( "host_browser_profile_mode", ) get_browser_config = browser_config.get_browser_config -_LOCAL_PROVIDERS = {"ollama", "lm_studio", "omlx"} +_LOCAL_PROVIDERS = {"ollama", "lm_studio", "llama_cpp", "omlx", "vllm"} _LOCAL_HOSTS = {"localhost", "127.0.0.1", "::1", "host.docker.internal"} _SENSITIVE_ACTIONS = {"content", "detail", "evaluate", "screenshot", "screenshot_file"} _KEY_ALIASES = { diff --git a/plugins/_model_config/api/model_search.py b/plugins/_model_config/api/model_search.py index ec5f50d24..a83ff68f5 100644 --- a/plugins/_model_config/api/model_search.py +++ b/plugins/_model_config/api/model_search.py @@ -26,6 +26,12 @@ _NON_CHAT_EXCLUDE = frozenset({ "omni-moderation", "vision-preview", }) +_LOCAL_PLACEHOLDER_KEYS = { + "lm_studio": {"lm-studio"}, + "llama_cpp": {"llama-cpp"}, + "omlx": {"omlx"}, + "vllm": {"vllm"}, +} class ModelSearch(ApiHandler): @@ -179,8 +185,8 @@ class ModelSearch(ApiHandler): elif provider == "azure": if has_key: headers["api-key"] = api_key - elif provider not in ("ollama", "lm_studio", "omlx"): - if has_key: + elif provider != "ollama": + if has_key and api_key not in _LOCAL_PLACEHOLDER_KEYS.get(provider, set()): headers["Authorization"] = f"Bearer {api_key}" extra = (cfg or {}).get("kwargs", {}).get("extra_headers", {}) diff --git a/plugins/_model_config/extensions/python/banners/_20_missing_api_key.py b/plugins/_model_config/extensions/python/banners/_20_missing_api_key.py index a3fc48301..c5722435c 100644 --- a/plugins/_model_config/extensions/python/banners/_20_missing_api_key.py +++ b/plugins/_model_config/extensions/python/banners/_20_missing_api_key.py @@ -6,7 +6,7 @@ from plugins._model_config.helpers import model_config class MissingApiKeyCheck(Extension): """Check if API keys are configured for selected model providers.""" - LOCAL_PROVIDERS = {"ollama", "lm_studio", "omlx"} + LOCAL_PROVIDERS = {"ollama", "lm_studio", "llama_cpp", "omlx", "vllm"} CONFIGURE_MODEL_SETTINGS_LINK = ( """
""" """