Add llama.cpp and vLLM local providers

Add llama.cpp / llama-server support:
- Register llama.cpp chat and embedding providers through hosted_vllm defaults on host.docker.internal:8080.
- Add onboarding metadata, a bundled icon, no-key provider metadata, model discovery coverage, setup docs, and tests.

Add vLLM support:
- Register vLLM chat and embedding providers through hosted_vllm defaults on host.docker.internal:8000.
- Add onboarding metadata, a bundled icon, no-key provider metadata, model discovery coverage, setup docs, and tests.
- Strip empty tools arrays from the Responses-to-chat fallback path so strict OpenAI-compatible servers accept local vLLM calls.
This commit is contained in:
Alessandro 2026-06-15 05:37:28 +02:00
parent 0450098117
commit 9bcef39028
18 changed files with 450 additions and 6 deletions

View file

@ -86,6 +86,15 @@ chat:
kwargs:
api_base: "http://host.docker.internal:1234/v1"
api_key: "lm-studio"
llama_cpp:
name: llama.cpp
litellm_provider: hosted_vllm
models_list:
endpoint_url: "/v1/models"
default_base: "http://host.docker.internal:8080"
kwargs:
api_base: "http://host.docker.internal:8080/v1"
api_key: "llama-cpp"
mistral:
name: Mistral AI
litellm_provider: mistral
@ -167,6 +176,15 @@ chat:
api_base: https://api.venice.ai/api/v1
venice_parameters:
include_venice_system_prompt: false
vllm:
name: vLLM
litellm_provider: hosted_vllm
models_list:
endpoint_url: "/v1/models"
default_base: "http://host.docker.internal:8000"
kwargs:
api_base: "http://host.docker.internal:8000/v1"
api_key: "vllm"
xai:
name: xAI
litellm_provider: xai
@ -203,6 +221,12 @@ embedding:
kwargs:
api_base: "http://host.docker.internal:1234/v1"
api_key: "lm-studio"
llama_cpp:
name: llama.cpp
litellm_provider: hosted_vllm
kwargs:
api_base: "http://host.docker.internal:8080/v1"
api_key: "llama-cpp"
mistral:
name: Mistral AI
litellm_provider: mistral
@ -250,6 +274,12 @@ embedding:
endpoint_url: "https://api.venice.ai/api/v1/models"
kwargs:
api_base: https://api.venice.ai/api/v1
vllm:
name: vLLM
litellm_provider: hosted_vllm
kwargs:
api_base: "http://host.docker.internal:8000/v1"
api_key: "vllm"
other:
name: Other OpenAI compatible
litellm_provider: openai