vrr/goose
2
0
Fork 0
mirror of https://github.com/block/goose.git synced 2026-07-09 16:09:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 22

local inference: stricter GGUF requirements, auto detection of tool calling support, fixed thinking output parsing (#9442)

Browse source 
Signed-off-by: jh-block <jhugo@block.xyz>
This commit is contained in:
jh-block 2026-05-27 20:00:30 +02:00 • committed by Jack Amadeo
parent 278948872e
commit 416942e430
23 changed files with 1844 additions and 492 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock generated
View file

@ -4497,6 +4497,7 @@ dependencies = [
"keyring",
"libc",
"llama-cpp-2",
"llama-cpp-sys-2",
"lru",
"minijinja",
"mockall",

35
crates/goose-cli/src/cli.rs
View file

@ -1816,7 +1816,7 @@ async fn handle_term_subcommand(command: TermCommand) -> Result<()> {
async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()> {
use goose::providers::local_inference::hf_models;
use goose::providers::local_inference::local_model_registry::{
get_registry, model_id_from_repo, LocalModelEntry,
get_registry, mmproj_local_path, model_id_from_repo, LocalModelEntry,
};
match command {
@ -1853,10 +1853,28 @@ async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()>
}
LocalModelsCommand::Download { spec } => {
println!("Resolving {}...", spec);
let (repo_id, file) = hf_models::resolve_model_spec(&spec).await?;
let (repo_id, resolved) = hf_models::resolve_model_spec_full(&spec).await?;
if resolved.files.len() > 1 {
anyhow::bail!(
"Model '{}' is sharded ({} files) — download it from the desktop UI",
spec,
resolved.files.len()
);
}
let mmproj = resolved.mmproj;
let file = resolved.files.into_iter().next().unwrap();
let model_id = model_id_from_repo(&repo_id, &file.quantization);
let local_path =
goose::config::paths::Paths::in_data_dir("models").join(&file.filename);
let mmproj_path = mmproj
.as_ref()
.map(|mmproj| mmproj_local_path(&repo_id, &mmproj.filename));
let mmproj_source_url = mmproj.as_ref().map(|mmproj| mmproj.download_url.clone());
let mmproj_size_bytes = mmproj.as_ref().map_or(0, |mmproj| mmproj.size_bytes);
let mut download_files = vec![(file.download_url.clone(), local_path.clone())];
if let Some(mmproj) = mmproj {
download_files.push((mmproj.download_url, mmproj_path.clone().unwrap()));
}
println!(
"Downloading {} ({})...",
@ -1881,9 +1899,10 @@ async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()>
source_url: file.download_url.clone(),
settings: Default::default(),
size_bytes: file.size_bytes,
mmproj_path: None,
mmproj_source_url: None,
mmproj_size_bytes: 0,
mmproj_path,
mmproj_source_url,
mmproj_size_bytes,
mmproj_checked: true,
shard_files: vec![],
};
@ -1897,10 +1916,10 @@ async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()>
// Download
let manager = goose::download_manager::get_download_manager();
manager
.download_model(
.download_model_sharded(
format!("{}-model", model_id),
file.download_url,
local_path,
download_files,
file.size_bytes + mmproj_size_bytes,
None,
)
.await?;

3
crates/goose-server/src/openapi.rs
View file

@ -683,6 +683,7 @@ pub struct ApiDoc;
super::routes::local_inference::list_local_models,
super::routes::local_inference::sync_featured_models,
super::routes::local_inference::search_hf_models,
super::routes::local_inference::list_builtin_chat_templates,
super::routes::local_inference::get_repo_files,
super::routes::local_inference::download_hf_model,
super::routes::local_inference::get_local_model_download_progress,
@ -701,7 +702,9 @@ pub struct ApiDoc;
goose::providers::local_inference::hf_models::HfQuantVariant,
super::routes::local_inference::RepoVariantsResponse,
goose::providers::local_inference::local_model_registry::ModelSettings,
goose::providers::local_inference::local_model_registry::ChatTemplate,
goose::providers::local_inference::local_model_registry::SamplingConfig,
goose::providers::local_inference::local_model_registry::ToolCallingMode,
))
)]
pub struct LocalInferenceApiDoc;

193
crates/goose-server/src/routes/local_inference.rs
View file

@ -13,10 +13,10 @@ use goose::config::paths::Paths;
use goose::download_manager::{get_download_manager, DownloadProgress};
use goose::providers::local_inference::hf_models::{self, HfModelInfo, HfQuantVariant};
use goose::providers::local_inference::{
available_inference_memory_bytes,
hf_models::{resolve_model_spec, resolve_model_spec_full, HfGgufFile},
available_inference_memory_bytes, builtin_chat_template_names,
hf_models::{resolve_model_spec_full, HfGgufFile},
local_model_registry::{
default_settings_for_model, featured_mmproj_spec, get_registry, is_featured_model,
default_settings_for_model, get_registry, is_featured_model, mmproj_local_path,
model_id_from_repo, LocalModelEntry, ModelDownloadStatus as RegistryDownloadStatus,
ModelSettings, ShardFile, FEATURED_MODELS,
},
@ -79,26 +79,18 @@ async fn ensure_featured_models_in_registry() -> Result<(), ErrorResponse> {
.lock()
.map_err(|_| ErrorResponse::internal("Failed to acquire registry lock"))?;
if let Some(existing) = registry.get_model(&model_id) {
let needs_backfill = existing.mmproj_path.is_none() && featured.mmproj.is_some();
let needs_download = existing.is_downloaded()
&& featured.mmproj.is_some()
&& !existing.mmproj_path.as_ref().is_some_and(|p| p.exists());
if needs_download {
if let Some(mmproj) = featured.mmproj.as_ref() {
let path = mmproj.local_path();
let url = format!(
"https://huggingface.co/{}/resolve/main/{}",
mmproj.repo, mmproj.filename
);
mmproj_downloads_needed.push((model_id.clone(), url, path));
if let Some(path) = &existing.mmproj_path {
if existing.is_downloaded() && !path.exists() {
if let Some(url) = &existing.mmproj_source_url {
mmproj_downloads_needed.push((
model_id.clone(),
url.clone(),
path.clone(),
));
}
}
}
if !needs_backfill {
continue;
}
// Fall through to resolve for backfill
}
}
@ -110,36 +102,45 @@ async fn ensure_featured_models_in_registry() -> Result<(), ErrorResponse> {
});
}
let resolved: Vec<(PendingResolve, HfGgufFile)> =
let resolved: Vec<(PendingResolve, HfGgufFile, Option<HfGgufFile>)> =
join_all(to_resolve.into_iter().map(|pending| async move {
let hf_file = match resolve_model_spec(pending.spec).await {
Ok((_repo, file)) => file,
let (hf_file, mmproj) = match resolve_model_spec_full(pending.spec).await {
Ok((_repo, resolved)) => (resolved.files[0].clone(), resolved.mmproj),
Err(_) => {
let filename = format!(
"{}-{}.gguf",
pending.repo_id.split('/').next_back().unwrap_or("model"),
pending.quantization
);
HfGgufFile {
filename: filename.clone(),
size_bytes: 0,
quantization: pending.quantization.to_string(),
download_url: format!(
"https://huggingface.co/{}/resolve/main/{}",
pending.repo_id, filename
),
}
(
HfGgufFile {
filename: filename.clone(),
size_bytes: 0,
quantization: pending.quantization.to_string(),
download_url: format!(
"https://huggingface.co/{}/resolve/main/{}",
pending.repo_id, filename
),
},
None,
)
}
};
(pending, hf_file)
(pending, hf_file, mmproj)
}))
.await;
let entries_to_add: Vec<LocalModelEntry> = resolved
.into_iter()
.map(|(pending, hf_file)| {
.map(|(pending, hf_file, mmproj)| {
let local_path = Paths::in_data_dir("models").join(&hf_file.filename);
let settings = default_settings_for_model(&pending.model_id);
let mmproj_path = mmproj
.as_ref()
.map(|mmproj| mmproj_local_path(&pending.repo_id, &mmproj.filename));
let mmproj_source_url = mmproj.as_ref().map(|mmproj| mmproj.download_url.clone());
let mmproj_size_bytes = mmproj.as_ref().map_or(0, |mmproj| mmproj.size_bytes);
let mmproj_checked = mmproj.is_some();
LocalModelEntry {
id: pending.model_id,
repo_id: pending.repo_id,
@ -149,9 +150,10 @@ async fn ensure_featured_models_in_registry() -> Result<(), ErrorResponse> {
source_url: hf_file.download_url,
settings,
size_bytes: hf_file.size_bytes,
mmproj_path: None,
mmproj_source_url: None,
mmproj_size_bytes: 0,
mmproj_path,
mmproj_source_url,
mmproj_size_bytes,
mmproj_checked,
shard_files: vec![],
}
})
@ -165,20 +167,80 @@ async fn ensure_featured_models_in_registry() -> Result<(), ErrorResponse> {
if !entries_to_add.is_empty() {
registry.sync_with_featured(entries_to_add);
}
}
let to_backfill: Vec<(String, String, String)> = {
let registry = get_registry()
.lock()
.map_err(|_| ErrorResponse::internal("Failed to acquire registry lock"))?;
registry
.list_models()
.iter()
.filter(|model| model.is_downloaded())
.filter(|model| model.mmproj_path.is_none())
.filter(|model| !model.mmproj_checked)
.map(|model| {
(
model.id.clone(),
model.repo_id.clone(),
model.quantization.clone(),
)
})
.collect()
};
let mmproj_backfills: Vec<(String, String, Option<Option<HfGgufFile>>)> = join_all(
to_backfill
.into_iter()
.map(|(id, repo_id, quantization)| async move {
let spec = format!("{repo_id}:{quantization}");
let mmproj = resolve_model_spec_full(&spec)
.await
.ok()
.map(|(_, resolved)| resolved.mmproj);
(id, repo_id, mmproj)
}),
)
.await;
{
let mut registry = get_registry()
.lock()
.map_err(|_| ErrorResponse::internal("Failed to acquire registry lock"))?;
for (model_id, repo_id, mmproj_result) in mmproj_backfills {
if let Some(model) = registry
.list_models_mut()
.iter_mut()
.find(|model| model.id == model_id)
{
let Some(mmproj) = mmproj_result else {
continue;
};
model.mmproj_checked = true;
if let Some(mmproj) = mmproj {
model.mmproj_path = Some(mmproj_local_path(&repo_id, &mmproj.filename));
model.mmproj_source_url = Some(mmproj.download_url);
model.mmproj_size_bytes = mmproj.size_bytes;
}
model.refresh_mmproj_metadata();
}
}
// Backfill mmproj data for all registry models and collect any
// needed mmproj downloads for models already on disk.
for model in registry.list_models_mut() {
model.enrich_with_featured_mmproj();
model.refresh_mmproj_metadata();
if model.is_downloaded() {
if let Some(mmproj) = featured_mmproj_spec(&model.id) {
let path = mmproj.local_path();
if let Some(path) = &model.mmproj_path {
if !path.exists() {
let url = format!(
"https://huggingface.co/{}/resolve/main/{}",
mmproj.repo, mmproj.filename
);
mmproj_downloads_needed.push((model.id.clone(), url, path));
if let Some(url) = &model.mmproj_source_url {
mmproj_downloads_needed.push((
model.id.clone(),
url.clone(),
path.clone(),
));
}
}
}
}
@ -431,6 +493,20 @@ pub async fn download_hf_model(
vec![]
};
let mmproj_path = resolved
.mmproj
.as_ref()
.map(|mmproj| mmproj_local_path(&repo_id, &mmproj.filename));
let mmproj_source_url = resolved
.mmproj
.as_ref()
.map(|mmproj| mmproj.download_url.clone());
let mmproj_size_bytes = resolved
.mmproj
.as_ref()
.map_or(0, |mmproj| mmproj.size_bytes);
let mmproj_checked = true;
let entry = LocalModelEntry {
id: model_id.clone(),
repo_id,
@ -440,13 +516,13 @@ pub async fn download_hf_model(
source_url: first_file.download_url.clone(),
settings: default_settings_for_model(&model_id),
size_bytes: resolved.total_size,
mmproj_path: None,
mmproj_source_url: None,
mmproj_size_bytes: 0,
mmproj_path,
mmproj_source_url,
mmproj_size_bytes,
mmproj_checked,
shard_files: shard_files.clone(),
};
// add_model enriches the entry with mmproj metadata from the featured table
let mmproj_path = {
let mut registry = get_registry()
.lock()
@ -649,6 +725,17 @@ pub async fn update_model_settings(
Ok(Json(settings))
}
#[utoipa::path(
get,
path = "/local-inference/chat-templates/builtin",
responses(
(status = 200, description = "llama.cpp built-in chat template names", body = Vec<String>)
)
)]
pub async fn list_builtin_chat_templates() -> Json<Vec<String>> {
Json(builtin_chat_template_names())
}
pub fn routes(state: Arc<AppState>) -> Router {
let registered_paths: std::collections::HashSet<std::path::PathBuf> = get_registry()
.lock()
@ -672,6 +759,10 @@ pub fn routes(state: Arc<AppState>) -> Router {
.route("/local-inference/models", get(list_local_models))
.route("/local-inference/sync-featured", post(sync_featured_models))
.route("/local-inference/search", get(search_hf_models))
.route(
"/local-inference/chat-templates/builtin",
get(list_builtin_chat_templates),
)
.route(
"/local-inference/repo/{author}/{repo}/files",
get(get_repo_files),

2
crates/goose/Cargo.toml
View file

@ -25,6 +25,7 @@ local-inference = [
"dep:candle-nn",
"dep:candle-transformers",
"dep:llama-cpp-2",
"dep:llama-cpp-sys-2",
"dep:tokenizers",
"dep:symphonia",
"dep:rubato",
@ -213,6 +214,7 @@ pctx_code_mode = { version = "0.3", default-features = false, optional = true }
# They are just here to pin the version, and can be removed if PCTX updates temporal_rs
icu_calendar = { version = "=2.1.1", default-features = false }
icu_locale = { version = "=2.1.1", default-features = false }
llama-cpp-sys-2 = { workspace = true, optional = true }
[target.'cfg(target_os = "windows")'.dependencies]
winapi = { workspace = true }

13
crates/goose/src/providers/base.rs
View file

@ -1419,6 +1419,19 @@ mod tests {
assert_eq!(out.thinking, "unfinished");
}
#[test]
fn test_think_filter_tracks_generation_prompt_open_block() {
let mut filter = ThinkFilter::new();
let _ = filter.push("<|assistant|><think>\n");
let mut out = filter.push("hidden reasoning</think>visible answer");
let final_out = filter.finish();
out.content.push_str(&final_out.content);
out.thinking.push_str(&final_out.thinking);
assert_eq!(out.content, "visible answer");
assert_eq!(out.thinking, "hidden reasoning");
}
#[test]
fn test_think_filter_preserves_tags_with_think_prefix() {
for input in [

49
crates/goose/src/providers/local_inference.rs
View file

@ -19,6 +19,7 @@ use async_trait::async_trait;
use backend::{BackendLoadedModel, LocalInferenceBackend};
use futures::future::BoxFuture;
use llamacpp::{LlamaCppBackend, LLAMACPP_BACKEND_ID};
use local_model_registry::ChatTemplate;
use rmcp::model::Tool;
use serde_json::{json, Value};
use std::collections::HashMap;
@ -33,13 +34,19 @@ type ModelSlot = Arc<Mutex<Option<Box<dyn BackendLoadedModel>>>>;
struct ModelCacheKey {
backend_id: &'static str,
model_id: String,
chat_template: ChatTemplate,
}
impl ModelCacheKey {
fn new(backend_id: &'static str, model_id: impl Into<String>) -> Self {
fn new(
backend_id: &'static str,
model_id: impl Into<String>,
chat_template: ChatTemplate,
) -> Self {
Self {
backend_id,
model_id: model_id.into(),
chat_template,
}
}
}
@ -49,6 +56,10 @@ pub struct InferenceRuntime {
backends: HashMap<&'static str, Arc<dyn LocalInferenceBackend>>,
}
pub fn builtin_chat_template_names() -> Vec<String> {
llamacpp::builtin_chat_template_names()
}
/// Global weak reference used to share a single `InferenceRuntime` across
/// all providers and server routes. Only a `Weak` is stored — strong `Arc`s
/// live in providers and `AppState`. When all strong refs drop (normal
@ -123,20 +134,12 @@ pub(super) struct ResolvedModelPaths {
/// Resolve model path, context limit, settings, and mmproj path for a model ID from the registry.
fn resolve_model_path(model_id: &str) -> Option<ResolvedModelPaths> {
use crate::providers::local_inference::local_model_registry::{
default_settings_for_model, get_registry,
};
use crate::providers::local_inference::local_model_registry::get_registry;
if let Ok(registry) = get_registry().lock() {
if let Some(entry) = registry.get_model(model_id) {
let ctx = entry.settings.context_size.unwrap_or(0) as usize;
let mut settings = entry.settings.clone();
// Capability flags are inherent to the model family, not user-configurable.
// Re-derive them so that registry entries persisted before a model was
// recognized (or with a different quantization) still get the right behavior.
let defaults = default_settings_for_model(model_id);
settings.native_tool_calling = defaults.native_tool_calling;
settings.vision_capable = defaults.vision_capable;
settings.mmproj_size_bytes = entry.mmproj_size_bytes;
let mmproj_path = entry.mmproj_path.as_ref().filter(|p| p.exists()).cloned();
return Some(ResolvedModelPaths {
@ -195,6 +198,22 @@ fn build_openai_messages_json(system: &str, messages: &[Message]) -> String {
serde_json::to_string(&arr).unwrap_or_else(|_| "[]".to_string())
}
fn build_openai_text_messages_json(system: &str, messages: &[Message]) -> String {
let mut arr: Vec<Value> = vec![json!({"role": "system", "content": system})];
arr.extend(messages.iter().filter_map(|m| {
let content = extract_text_content(m);
if content.trim().is_empty() {
return None;
}
let role = match m.role {
rmcp::model::Role::User => "user",
rmcp::model::Role::Assistant => "assistant",
};
Some(json!({"role": role, "content": content}))
}));
serde_json::to_string(&arr).unwrap_or_else(|_| "[]".to_string())
}
/// Remove `image_url` content parts from OpenAI-format messages JSON, replacing
/// each with a text note. This prevents an FFI crash in llama.cpp which does not
/// accept `image_url` content-part types.
@ -422,7 +441,11 @@ impl Provider for LocalInferenceProvider {
let backend = self.runtime.backend_for_model(&resolved)?;
let model_context_limit = resolved.context_limit;
let model_settings = resolved.settings.clone();
let cache_key = ModelCacheKey::new(backend.id(), model_config.model_name.clone());
let cache_key = ModelCacheKey::new(
backend.id(),
model_config.model_name.clone(),
model_settings.chat_template.clone(),
);
let model_slot = self.runtime.get_or_create_model_slot(cache_key.clone());
// Ensure model is loaded — unload any other models first to free memory.
@ -477,8 +500,8 @@ impl Provider for LocalInferenceProvider {
}).collect::<Vec<_>>(),
"tools": tools.iter().map(|t| &t.name).collect::<Vec<_>>(),
"settings": {
"use_jinja": settings.use_jinja,
"native_tool_calling": settings.native_tool_calling,
"tool_calling": settings.tool_calling,
"chat_template": settings.chat_template,
"context_size": settings.context_size,
"sampling": settings.sampling,
},

182
crates/goose/src/providers/local_inference/hf_models.rs
View file

@ -42,6 +42,7 @@ pub struct HfQuantVariant {
pub struct ResolvedModel {
pub files: Vec<HfGgufFile>,
pub total_size: u64,
pub mmproj: Option<HfGgufFile>,
}
#[derive(Debug, Deserialize)]
@ -183,6 +184,24 @@ fn parse_quantization(filename: &str) -> String {
"unknown".to_string()
}
fn quant_bits(quantization: &str) -> u8 {
let digits: String = quantization
.chars()
.skip_while(|c| !c.is_ascii_digit())
.take_while(|c| c.is_ascii_digit())
.collect();
digits.parse().unwrap_or(0)
}
fn mmproj_precision_preference(quantization: &str) -> u8 {
match quantization.to_uppercase().as_str() {
"BF16" => 3,
"F16" => 2,
"F32" => 1,
_ => 0,
}
}
fn looks_like_quant(s: &str) -> bool {
let upper = s.to_uppercase();
upper.starts_with("Q")
@ -226,6 +245,64 @@ fn build_download_url(repo_id: &str, filename: &str) -> String {
format!("{}/{}/resolve/main/{}", HF_DOWNLOAD_BASE, repo_id, filename)
}
fn parent_components(filename: &str) -> Vec<&str> {
filename.rsplit_once('/').map_or(Vec::new(), |(parent, _)| {
parent.split('/').filter(|part| !part.is_empty()).collect()
})
}
fn is_prefix(prefix: &[&str], parts: &[&str]) -> bool {
prefix.len() <= parts.len() && prefix.iter().zip(parts).all(|(a, b)| a == b)
}
fn select_best_mmproj(
repo_id: &str,
siblings: &[HfApiSibling],
model_filename: &str,
model_quantization: &str,
) -> Option<HfGgufFile> {
let model_dir = parent_components(model_filename);
let model_bits = quant_bits(model_quantization);
siblings
.iter()
.filter(|s| {
let lowercase = s.rfilename.to_lowercase();
lowercase.ends_with(".gguf") && lowercase.contains("mmproj")
})
.filter_map(|s| {
let mmproj_dir = parent_components(&s.rfilename);
if !is_prefix(&mmproj_dir, &model_dir) {
return None;
}
let quantization = parse_quantization(&s.rfilename);
let bits = quant_bits(&quantization);
let diff = bits.abs_diff(model_bits);
let proximity = u8::MAX - diff;
Some((
mmproj_dir.len(),
proximity,
mmproj_precision_preference(&quantization),
s,
quantization,
))
})
.max_by(|a, b| {
a.0.cmp(&b.0)
.then_with(|| a.1.cmp(&b.1))
.then_with(|| a.2.cmp(&b.2))
.then_with(|| b.3.rfilename.cmp(&a.3.rfilename))
})
.map(|(_, _, _, sibling, quantization)| HfGgufFile {
filename: sibling.rfilename.clone(),
size_bytes: sibling.size.unwrap_or(0),
quantization,
download_url: build_download_url(repo_id, &sibling.rfilename),
})
}
/// Derive the expected model filename stem from a repo_id.
/// e.g. "unsloth/gemma-4-26B-A4B-it-GGUF" → "gemma-4-26b-a4b-it" (lowercased)
fn model_stem_from_repo(repo_id: &str) -> String {
@ -500,7 +577,7 @@ pub async fn resolve_model_spec_full(spec: &str) -> Result<(String, ResolvedMode
// Collect all GGUF files matching the quantization
let matching: Vec<_> = siblings
.into_iter()
.iter()
.filter(|s| {
s.rfilename.ends_with(".gguf")
&& is_model_file(&s.rfilename, &stem)
@ -519,7 +596,7 @@ pub async fn resolve_model_spec_full(spec: &str) -> Result<(String, ResolvedMode
// Separate single files from shards
let mut single_files: Vec<&HfApiSibling> = Vec::new();
let mut shard_files: Vec<&HfApiSibling> = Vec::new();
for f in &matching {
for &f in &matching {
if is_shard_file(&f.rfilename) {
shard_files.push(f);
} else {
@ -529,6 +606,7 @@ pub async fn resolve_model_spec_full(spec: &str) -> Result<(String, ResolvedMode
// Prefer single file if available
if let Some(single) = single_files.first() {
let mmproj = select_best_mmproj(&repo_id, &siblings, &single.rfilename, &quant);
let file = HfGgufFile {
filename: single.rfilename.clone(),
size_bytes: single.size.unwrap_or(0),
@ -541,6 +619,7 @@ pub async fn resolve_model_spec_full(spec: &str) -> Result<(String, ResolvedMode
ResolvedModel {
files: vec![file],
total_size,
mmproj,
},
));
}
@ -600,7 +679,16 @@ pub async fn resolve_model_spec_full(spec: &str) -> Result<(String, ResolvedMode
.collect();
let total_size: u64 = files.iter().map(|f| f.size_bytes).sum();
Ok((repo_id, ResolvedModel { files, total_size }))
let mmproj = select_best_mmproj(&repo_id, &siblings, &files[0].filename, &quant);
Ok((
repo_id,
ResolvedModel {
files,
total_size,
mmproj,
},
))
}
/// Resolve a model spec to a specific GGUF file from the repo.
@ -816,4 +904,92 @@ mod tests {
assert_eq!(variants[1].quantization, "Q4_K_M");
assert_eq!(variants[2].quantization, "IQ1_S");
}
#[test]
fn test_select_best_mmproj_prefers_closest_precision() {
let files = vec![
HfApiSibling {
rfilename: "mmproj-F32.gguf".into(),
size: Some(3_000),
},
HfApiSibling {
rfilename: "mmproj-BF16.gguf".into(),
size: Some(2_000),
},
];
let mmproj =
select_best_mmproj("someone/model-GGUF", &files, "model-Q4_K_M.gguf", "Q4_K_M")
.unwrap();
assert_eq!(mmproj.filename, "mmproj-BF16.gguf");
assert_eq!(mmproj.quantization, "BF16");
}
#[test]
fn test_select_best_mmproj_prefers_bf16_over_f16_tie() {
let files = vec![
HfApiSibling {
rfilename: "mmproj-F16.gguf".into(),
size: Some(2_000),
},
HfApiSibling {
rfilename: "mmproj-BF16.gguf".into(),
size: Some(2_000),
},
];
let mmproj =
select_best_mmproj("someone/model-GGUF", &files, "model-Q8_0.gguf", "Q8_0").unwrap();
assert_eq!(mmproj.filename, "mmproj-BF16.gguf");
}
#[test]
fn test_select_best_mmproj_prefers_nearest_directory() {
let files = vec![
HfApiSibling {
rfilename: "mmproj-BF16.gguf".into(),
size: Some(2_000),
},
HfApiSibling {
rfilename: "Q4_K_M/mmproj-F32.gguf".into(),
size: Some(3_000),
},
];
let mmproj = select_best_mmproj(
"someone/model-GGUF",
&files,
"Q4_K_M/model-Q4_K_M.gguf",
"Q4_K_M",
)
.unwrap();
assert_eq!(mmproj.filename, "Q4_K_M/mmproj-F32.gguf");
}
#[test]
fn test_select_best_mmproj_ignores_sibling_directories() {
let files = vec![
HfApiSibling {
rfilename: "Q8_0/mmproj-BF16.gguf".into(),
size: Some(2_000),
},
HfApiSibling {
rfilename: "mmproj-F32.gguf".into(),
size: Some(3_000),
},
];
let mmproj = select_best_mmproj(
"someone/model-GGUF",
&files,
"Q4_K_M/model-Q4_K_M.gguf",
"Q4_K_M",
)
.unwrap();
assert_eq!(mmproj.filename, "mmproj-F32.gguf");
}
}

182
crates/goose/src/providers/local_inference/llamacpp/inference_emulated_tools.rs
View file

@ -22,7 +22,6 @@
use crate::conversation::message::{Message, MessageContent};
use crate::providers::errors::ProviderError;
use llama_cpp_2::model::AddBos;
use rmcp::model::{CallToolRequestParams, Tool};
use serde_json::json;
use std::borrow::Cow;
@ -30,8 +29,8 @@ use uuid::Uuid;
use super::super::{finalize_usage, StreamSender};
use super::inference_engine::{
create_and_prefill_context, create_and_prefill_multimodal, generation_loop,
validate_and_compute_context, GenerationContext, TokenAction,
generation_loop, prepare_generation, GenerationContext, StopSuffixTrimmer,
ThinkingOutputFilter, TokenAction,
};
const SHELL_TOOL: &str = "developer__shell";
@ -355,56 +354,26 @@ fn send_emulator_action(
pub(super) fn generate_with_emulated_tools(
ctx: &mut GenerationContext<'_>,
code_mode_enabled: bool,
oai_messages_json: &str,
) -> Result<(), ProviderError> {
// Use oaicompat variant — its C++ wrapper catches exceptions that would
// otherwise abort the process when other native libs disturb the C++ ABI.
let prompt = ctx
.loaded
.model
.apply_chat_template_with_tools_oaicompat(
&ctx.loaded.template,
ctx.chat_messages,
None, // no tools for emulated path
None, // no json_schema
true, // add_generation_prompt
)
.map(|r| r.prompt)
.map_err(|e| {
ProviderError::ExecutionError(format!("Failed to apply chat template: {}", e))
})?;
let (mut llama_ctx, prompt_token_count, effective_ctx) = if !ctx.images.is_empty() {
create_and_prefill_multimodal(
ctx.loaded,
ctx.backend,
&prompt,
ctx.images,
ctx.context_limit,
ctx.settings,
)?
} else {
let tokens = ctx
.loaded
.model
.str_to_token(&prompt, AddBos::Never)
.map_err(|e| ProviderError::ExecutionError(e.to_string()))?;
let (ptc, ectx) = validate_and_compute_context(
ctx.loaded,
ctx.backend,
tokens.len(),
ctx.context_limit,
ctx.settings,
)?;
let lctx =
create_and_prefill_context(ctx.loaded, ctx.backend, &tokens, ectx, ctx.settings)?;
(lctx, ptc, ectx)
};
let prepared = prepare_generation(ctx, oai_messages_json, None, None)?;
let template_result = prepared.template_result;
let mut llama_ctx = prepared.llama_ctx;
let prompt_token_count = prepared.prompt_token_count;
let effective_ctx = prepared.effective_ctx;
let message_id = ctx.message_id;
let tx = ctx.tx;
let mut emulator_parser = StreamingEmulatorParser::new(code_mode_enabled);
let mut output_filter = ThinkingOutputFilter::new(
ctx.settings.enable_thinking,
&template_result.generation_prompt,
);
let mut stop_trimmer = StopSuffixTrimmer::new(&template_result.additional_stops);
let mut generated_text = String::new();
let mut tool_call_emitted = false;
let mut send_failed = false;
let mut stop_string_emitted = false;
let output_token_count = generation_loop(
&ctx.loaded.model,
@ -413,7 +382,10 @@ pub(super) fn generate_with_emulated_tools(
prompt_token_count,
effective_ctx,
|piece| {
let actions = emulator_parser.process_chunk(piece);
generated_text.push_str(piece);
let filtered = output_filter.push_text(piece);
let (content, stop_seen) = stop_trimmer.push(&filtered.content);
let actions = emulator_parser.process_chunk(&content);
for action in actions {
match send_emulator_action(&action, message_id, tx) {
Ok(is_tool) => {
@ -429,12 +401,47 @@ pub(super) fn generate_with_emulated_tools(
}
if tool_call_emitted {
Ok(TokenAction::Stop)
} else if stop_seen
|| template_result
.additional_stops
.iter()
.any(|stop| generated_text.ends_with(stop))
{
stop_string_emitted = true;
Ok(TokenAction::Stop)
} else {
Ok(TokenAction::Continue)
}
},
)?;
if !send_failed {
let filtered = output_filter.finish();
if !filtered.thinking.is_empty() {
let mut message = Message::assistant().with_thinking(filtered.thinking, "");
message.id = Some(message_id.to_string());
send_failed = tx.blocking_send(Ok((Some(message), None))).is_err();
}
if !send_failed {
let content = if stop_string_emitted {
String::new()
} else {
let (content, stop_seen) = stop_trimmer.push(&filtered.content);
let mut content = content;
if !stop_seen {
content.push_str(&stop_trimmer.finish());
}
content
};
for action in emulator_parser.process_chunk(&content) {
if send_emulator_action(&action, message_id, tx).is_err() {
send_failed = true;
break;
}
}
}
}
if !send_failed {
for action in emulator_parser.flush() {
if send_emulator_action(&action, message_id, tx).is_err() {
@ -474,6 +481,50 @@ mod tests {
parse_chunks(&[input], code_mode)
}
fn trim_chunks(chunks: &[&str], stops: &[String]) -> (String, bool) {
let mut trimmer = StopSuffixTrimmer::new(stops);
let mut output = String::new();
let mut stopped = false;
for chunk in chunks {
let (content, stop_seen) = trimmer.push(chunk);
output.push_str(&content);
if stop_seen {
stopped = true;
break;
}
}
if !stopped {
output.push_str(&trimmer.finish());
}
(output, stopped)
}
fn parse_with_seeded_thinking(
chunks: &[&str],
code_mode: bool,
) -> (String, Vec<EmulatorAction>) {
let mut output_filter = ThinkingOutputFilter::new(true, "<|assistant|><think>\n");
let mut parser = StreamingEmulatorParser::new(code_mode);
let mut thinking = String::new();
let mut actions = Vec::new();
for chunk in chunks {
let filtered = output_filter.push_text(chunk);
thinking.push_str(&filtered.thinking);
actions.extend(parser.process_chunk(&filtered.content));
}
let filtered = output_filter.finish();
thinking.push_str(&filtered.thinking);
actions.extend(parser.process_chunk(&filtered.content));
actions.extend(parser.flush());
(thinking, actions)
}
fn assert_text(action: &EmulatorAction, expected: &str) {
match action {
EmulatorAction::Text(t) => assert_eq!(t.trim(), expected.trim(), "text mismatch"),
@ -507,6 +558,24 @@ mod tests {
}
}
#[test]
fn stop_suffix_trimmer_strips_split_stop() {
let stops = vec!["<|eom_id|>".to_string()];
let (content, stopped) = trim_chunks(&["The answer", "<|e", "om_id|>"], &stops);
assert!(stopped);
assert_eq!(content, "The answer");
}
#[test]
fn stop_suffix_trimmer_flushes_partial_non_stop() {
let stops = vec!["<|eom_id|>".to_string()];
let (content, stopped) = trim_chunks(&["Use the <", " symbol"], &stops);
assert!(!stopped);
assert_eq!(content, "Use the < symbol");
}
#[test]
fn plain_text_no_tools() {
let actions = parse_all("Hello, world!", false);
@ -667,6 +736,25 @@ mod tests {
assert_shell(shells[0], "echo hello");
}
#[test]
fn thinking_seeded_from_generation_prompt_is_not_emulated_text() {
let (thinking, actions) =
parse_with_seeded_thinking(&["reasoning\n$ echo hidden\n</think>The answer."], false);
assert_eq!(thinking.trim(), "reasoning\n$ echo hidden");
assert!(actions
.iter()
.all(|action| matches!(action, EmulatorAction::Text(_))));
let text: String = actions
.iter()
.filter_map(|action| match action {
EmulatorAction::Text(text) => Some(text.as_str()),
_ => None,
})
.collect();
assert_eq!(text.trim(), "The answer.");
}
#[test]
fn execute_block_with_multiline_code() {
let input = "```execute_typescript\nasync function run() {\n const r = await Developer.shell({ command: \"ls\" });\n return r;\n}\n```\n";

277
crates/goose/src/providers/local_inference/llamacpp/inference_engine.rs
View file

@ -1,3 +1,4 @@
use crate::providers::base::{FilterOut, ThinkFilter};
use crate::providers::errors::ProviderError;
use crate::providers::local_inference::backend::LocalInferenceBackend;
use crate::providers::local_inference::local_model_registry::ModelSettings;
@ -5,8 +6,9 @@ use crate::providers::local_inference::multimodal::ExtractedImage;
use crate::providers::utils::RequestLog;
use llama_cpp_2::context::params::LlamaContextParams;
use llama_cpp_2::llama_batch::LlamaBatch;
use llama_cpp_2::model::{LlamaChatMessage, LlamaChatTemplate, LlamaModel};
use llama_cpp_2::model::{AddBos, ChatTemplateResult, LlamaChatTemplate, LlamaModel};
use llama_cpp_2::mtmd::{MtmdBitmap, MtmdContext, MtmdInputText};
use llama_cpp_2::openai::OpenAIChatTemplateParams;
use llama_cpp_2::sampling::LlamaSampler;
use std::num::NonZeroU32;
@ -16,7 +18,7 @@ use super::LlamaCppBackend;
pub(super) struct GenerationContext<'a> {
pub loaded: &'a LoadedModel,
pub backend: &'a LlamaCppBackend,
pub chat_messages: &'a [LlamaChatMessage],
pub template: &'a LlamaChatTemplate,
pub settings: &'a ModelSettings,
pub context_limit: usize,
pub model_name: String,
@ -28,11 +30,165 @@ pub(super) struct GenerationContext<'a> {
pub(super) struct LoadedModel {
pub model: LlamaModel,
pub template: LlamaChatTemplate,
pub templates: LoadedChatTemplates,
/// Multimodal context for vision models. None for text-only models.
pub mtmd_ctx: Option<MtmdContext>,
}
pub(super) struct LoadedChatTemplates {
pub default: Option<LlamaChatTemplate>,
pub tool_use: Option<LlamaChatTemplate>,
pub force_default: bool,
}
pub(super) struct PreparedGeneration<'model> {
pub template_result: ChatTemplateResult,
pub llama_ctx: llama_cpp_2::context::LlamaContext<'model>,
pub prompt_token_count: usize,
pub effective_ctx: usize,
}
pub(super) struct ThinkingOutputFilter {
enabled: bool,
saw_structured_reasoning: bool,
think_filter: ThinkFilter,
pending_inline_thinking: String,
accumulated_thinking: String,
}
impl ThinkingOutputFilter {
pub(super) fn new(enable_thinking: bool, generation_prompt: &str) -> Self {
let mut think_filter = ThinkFilter::new();
if enable_thinking && !generation_prompt.is_empty() {
let _ = think_filter.push(generation_prompt);
}
Self {
enabled: enable_thinking,
saw_structured_reasoning: false,
think_filter,
pending_inline_thinking: String::new(),
accumulated_thinking: String::new(),
}
}
pub(super) fn push_structured_reasoning(&mut self, reasoning: &str) -> Option<String> {
if reasoning.is_empty() {
return None;
}
self.saw_structured_reasoning = true;
self.pending_inline_thinking.clear();
self.think_filter = ThinkFilter::new();
self.accumulated_thinking.push_str(reasoning);
Some(reasoning.to_string())
}
pub(super) fn push_text(&mut self, text: &str) -> FilterOut {
if !self.enabled {
return FilterOut {
content: text.to_string(),
thinking: String::new(),
};
}
let mut filtered = self.think_filter.push(text);
if self.saw_structured_reasoning {
filtered.thinking.clear();
} else if !filtered.thinking.is_empty() {
self.pending_inline_thinking.push_str(&filtered.thinking);
filtered.thinking.clear();
}
filtered
}
pub(super) fn finish(&mut self) -> FilterOut {
let mut filtered = if self.enabled && !self.saw_structured_reasoning {
std::mem::take(&mut self.think_filter).finish()
} else {
FilterOut::default()
};
if !self.saw_structured_reasoning {
let mut thinking = std::mem::take(&mut self.pending_inline_thinking);
thinking.push_str(&filtered.thinking);
if !thinking.is_empty() {
self.accumulated_thinking.push_str(&thinking);
}
filtered.thinking = thinking;
} else {
filtered.thinking.clear();
}
filtered
}
pub(super) fn accumulated_thinking(&self) -> &str {
&self.accumulated_thinking
}
}
pub(super) struct StopSuffixTrimmer {
pending: String,
stops: Vec<String>,
}
impl StopSuffixTrimmer {
pub(super) fn new(stops: &[String]) -> Self {
Self {
pending: String::new(),
stops: stops
.iter()
.filter(|stop| !stop.is_empty())
.cloned()
.collect(),
}
}
pub(super) fn push(&mut self, chunk: &str) -> (String, bool) {
if self.stops.is_empty() {
return (chunk.to_string(), false);
}
self.pending.push_str(chunk);
if let Some(stop) = self
.stops
.iter()
.filter(|stop| self.pending.ends_with(stop.as_str()))
.max_by_key(|stop| stop.len())
{
let emit_len = self.pending.len() - stop.len();
let _stop = self.pending.split_off(emit_len);
let emit = std::mem::take(&mut self.pending);
return (emit, true);
}
let hold_len = self
.pending
.char_indices()
.map(|(idx, _)| idx)
.chain(std::iter::once(self.pending.len()))
.filter(|idx| {
self.pending
.get(*idx..)
.is_some_and(|suffix| self.stops.iter().any(|stop| stop.starts_with(suffix)))
})
.map(|idx| self.pending.len() - idx)
.max()
.unwrap_or(0);
let emit_len = self.pending.len() - hold_len;
let keep = self.pending.split_off(emit_len);
let emit = std::mem::replace(&mut self.pending, keep);
(emit, false)
}
pub(super) fn finish(&mut self) -> String {
std::mem::take(&mut self.pending)
}
}
/// Estimate the maximum context length that can fit in available accelerator/CPU
/// memory based on the model's KV cache requirements.
///
@ -349,6 +505,121 @@ pub(super) fn create_and_prefill_multimodal<'model>(
Ok((llama_ctx, prompt_token_count, effective_ctx))
}
pub(super) fn prepare_generation<'model>(
ctx: &mut GenerationContext<'model>,
oai_messages_json: &str,
full_tools_json: Option<&str>,
compact_tools_json: Option<&str>,
) -> Result<PreparedGeneration<'model>, ProviderError> {
let apply_template = |tools: Option<&str>| {
let params = OpenAIChatTemplateParams {
messages_json: oai_messages_json,
tools_json: tools,
tool_choice: None,
json_schema: None,
grammar: None,
reasoning_format: if ctx.settings.enable_thinking {
Some("auto")
} else {
None
},
chat_template_kwargs: None,
add_generation_prompt: true,
use_jinja: true,
parallel_tool_calls: false,
enable_thinking: ctx.settings.enable_thinking,
add_bos: false,
add_eos: false,
parse_tool_calls: true,
};
ctx.loaded
.model
.apply_chat_template_oaicompat(ctx.template, &params)
};
let min_generation_headroom = 512;
let n_ctx_train = ctx.loaded.model.n_ctx_train() as usize;
let mmproj_overhead = if ctx.loaded.mtmd_ctx.is_some() {
ctx.settings.mmproj_size_bytes
} else {
0
};
let memory_max_ctx =
estimate_max_context_for_memory(&ctx.loaded.model, ctx.backend, mmproj_overhead);
let cap = context_cap(ctx.settings, ctx.context_limit, n_ctx_train, memory_max_ctx);
let token_budget = cap.saturating_sub(min_generation_headroom);
let estimated_image_tokens = ctx.images.len() * ctx.settings.image_token_estimate;
let template_result = match apply_template(full_tools_json) {
Ok(r) => {
let token_count = ctx
.loaded
.model
.str_to_token(&r.prompt, AddBos::Never)
.map(|t| t.len())
.unwrap_or(0);
if token_count + estimated_image_tokens > token_budget {
apply_template(compact_tools_json).unwrap_or(r)
} else {
r
}
}
Err(e) => {
tracing::warn!(
error = %e,
"Failed to apply llama.cpp OpenAI-compatible chat template"
);
match apply_template(compact_tools_json) {
Ok(r) => r,
Err(compact_err) => {
return Err(ProviderError::ExecutionError(format!(
"Failed to apply chat template with llama.cpp's Jinja renderer. This usually means the selected built-in template name does not exist, the embedded or custom template is invalid, or the template is incompatible with the current message shape. Select a valid llama.cpp built-in template name, configure a custom inline Jinja template, or use a GGUF with valid tokenizer.chat_template metadata. Full tools error: {e}; compact tools error: {compact_err}"
)));
}
}
}
};
let _ = ctx.log.write(
&serde_json::json!({"applied_prompt": &template_result.prompt}),
None,
);
let (llama_ctx, prompt_token_count, effective_ctx) = if !ctx.images.is_empty() {
create_and_prefill_multimodal(
ctx.loaded,
ctx.backend,
&template_result.prompt,
ctx.images,
ctx.context_limit,
ctx.settings,
)?
} else {
let tokens = ctx
.loaded
.model
.str_to_token(&template_result.prompt, AddBos::Never)
.map_err(|e| ProviderError::ExecutionError(e.to_string()))?;
let (ptc, ectx) = validate_and_compute_context(
ctx.loaded,
ctx.backend,
tokens.len(),
ctx.context_limit,
ctx.settings,
)?;
let lctx =
create_and_prefill_context(ctx.loaded, ctx.backend, &tokens, ectx, ctx.settings)?;
(lctx, ptc, ectx)
};
Ok(PreparedGeneration {
template_result,
llama_ctx,
prompt_token_count,
effective_ctx,
})
}
/// Action to take after processing a generated token piece.
pub(super) enum TokenAction {
Continue,

210
crates/goose/src/providers/local_inference/llamacpp/inference_native_tools.rs
View file

@ -1,7 +1,5 @@
use crate::conversation::message::{Message, MessageContent};
use crate::providers::errors::ProviderError;
use llama_cpp_2::model::AddBos;
use llama_cpp_2::openai::OpenAIChatTemplateParams;
use rmcp::model::CallToolRequestParams;
use serde_json::Value;
use std::borrow::Cow;
@ -9,121 +7,27 @@ use uuid::Uuid;
use super::super::finalize_usage;
use super::inference_engine::{
context_cap, create_and_prefill_context, create_and_prefill_multimodal,
estimate_max_context_for_memory, generation_loop, validate_and_compute_context,
GenerationContext, TokenAction,
generation_loop, prepare_generation, GenerationContext, StopSuffixTrimmer,
ThinkingOutputFilter, TokenAction,
};
pub(super) fn generate_with_native_tools(
ctx: &mut GenerationContext<'_>,
oai_messages_json: &Option<String>,
oai_messages_json: &str,
full_tools_json: Option<&str>,
compact_tools: Option<&str>,
) -> Result<(), ProviderError> {
let min_generation_headroom = 512;
let n_ctx_train = ctx.loaded.model.n_ctx_train() as usize;
let mmproj_overhead = if ctx.loaded.mtmd_ctx.is_some() {
ctx.settings.mmproj_size_bytes
} else {
0
};
let memory_max_ctx =
estimate_max_context_for_memory(&ctx.loaded.model, ctx.backend, mmproj_overhead);
let cap = context_cap(ctx.settings, ctx.context_limit, n_ctx_train, memory_max_ctx);
let token_budget = cap.saturating_sub(min_generation_headroom);
let apply_template = |tools: Option<&str>| {
if let Some(ref messages_json) = oai_messages_json {
let params = OpenAIChatTemplateParams {
messages_json: messages_json.as_str(),
tools_json: tools,
tool_choice: None,
json_schema: None,
grammar: None,
reasoning_format: if ctx.settings.enable_thinking {
Some("auto")
} else {
None
},
chat_template_kwargs: None,
add_generation_prompt: true,
use_jinja: true,
parallel_tool_calls: false,
enable_thinking: ctx.settings.enable_thinking,
add_bos: false,
add_eos: false,
parse_tool_calls: true,
};
ctx.loaded
.model
.apply_chat_template_oaicompat(&ctx.loaded.template, &params)
} else {
ctx.loaded.model.apply_chat_template_with_tools_oaicompat(
&ctx.loaded.template,
ctx.chat_messages,
tools,
None,
true,
)
}
};
let estimated_image_tokens = ctx.images.len() * ctx.settings.image_token_estimate;
let template_result = match apply_template(full_tools_json) {
Ok(r) => {
let token_count = ctx
.loaded
.model
.str_to_token(&r.prompt, AddBos::Never)
.map(|t| t.len())
.unwrap_or(0);
if token_count + estimated_image_tokens > token_budget {
apply_template(compact_tools).unwrap_or(r)
} else {
r
}
}
Err(_) => apply_template(compact_tools).map_err(|e| {
ProviderError::ExecutionError(format!("Failed to apply chat template: {}", e))
})?,
};
let _ = ctx.log.write(
&serde_json::json!({"applied_prompt": &template_result.prompt}),
None,
);
let (mut llama_ctx, prompt_token_count, effective_ctx) = if !ctx.images.is_empty() {
create_and_prefill_multimodal(
ctx.loaded,
ctx.backend,
&template_result.prompt,
ctx.images,
ctx.context_limit,
ctx.settings,
)?
} else {
let tokens = ctx
.loaded
.model
.str_to_token(&template_result.prompt, AddBos::Never)
.map_err(|e| ProviderError::ExecutionError(e.to_string()))?;
let (ptc, ectx) = validate_and_compute_context(
ctx.loaded,
ctx.backend,
tokens.len(),
ctx.context_limit,
ctx.settings,
)?;
let lctx =
create_and_prefill_context(ctx.loaded, ctx.backend, &tokens, ectx, ctx.settings)?;
(lctx, ptc, ectx)
};
let prepared = prepare_generation(ctx, oai_messages_json, full_tools_json, compact_tools)?;
let template_result = prepared.template_result;
let mut llama_ctx = prepared.llama_ctx;
let prompt_token_count = prepared.prompt_token_count;
let effective_ctx = prepared.effective_ctx;
let message_id = ctx.message_id;
let tx = ctx.tx;
let mut generated_text = String::new();
let mut stop_trimmer = StopSuffixTrimmer::new(&template_result.additional_stops);
let mut stop_string_emitted = false;
// Initialize streaming parser — handles thinking tokens, tool calls, etc.
let mut stream_parser = template_result.streaming_state_oaicompat().map_err(|e| {
@ -141,7 +45,10 @@ pub(super) fn generate_with_native_tools(
// Accumulate thinking/reasoning across the entire generation so we can
// attach it to the final tool-call message (mirroring what the OpenAI
// streaming path does). Streaming chunks are still sent for UI display.
let mut accumulated_thinking = String::new();
let mut output_filter = ThinkingOutputFilter::new(
ctx.settings.enable_thinking,
&template_result.generation_prompt,
);
let output_token_count = generation_loop(
&ctx.loaded.model,
@ -151,6 +58,7 @@ pub(super) fn generate_with_native_tools(
effective_ctx,
|piece| {
generated_text.push_str(piece);
let mut stop_seen = false;
// Feed the new piece to the streaming parser
match stream_parser.update(piece, true) {
@ -161,9 +69,10 @@ pub(super) fn generate_with_native_tools(
if let Some(reasoning) =
delta.get("reasoning_content").and_then(|v| v.as_str())
{
if !reasoning.is_empty() {
accumulated_thinking.push_str(reasoning);
let mut msg = Message::assistant().with_thinking(reasoning, "");
if let Some(thinking) =
output_filter.push_structured_reasoning(reasoning)
{
let mut msg = Message::assistant().with_thinking(thinking, "");
msg.id = Some(message_id.to_string());
if tx.blocking_send(Ok((Some(msg), None))).is_err() {
return Ok(TokenAction::Stop);
@ -173,10 +82,15 @@ pub(super) fn generate_with_native_tools(
// Stream content text to the UI
if let Some(content) = delta.get("content").and_then(|v| v.as_str()) {
if !content.is_empty() {
let mut msg = Message::assistant().with_text(content);
msg.id = Some(message_id.to_string());
if tx.blocking_send(Ok((Some(msg), None))).is_err() {
return Ok(TokenAction::Stop);
let filtered = output_filter.push_text(content);
let (content, seen) = stop_trimmer.push(&filtered.content);
stop_seen |= seen;
if !content.is_empty() {
let mut msg = Message::assistant().with_text(content);
msg.id = Some(message_id.to_string());
if tx.blocking_send(Ok((Some(msg), None))).is_err() {
return Ok(TokenAction::Stop);
}
}
}
}
@ -193,19 +107,26 @@ pub(super) fn generate_with_native_tools(
}
Err(e) => {
tracing::warn!("Streaming parser error: {}", e);
let mut msg = Message::assistant().with_text(piece);
msg.id = Some(message_id.to_string());
if tx.blocking_send(Ok((Some(msg), None))).is_err() {
return Ok(TokenAction::Stop);
let filtered = output_filter.push_text(piece);
let (content, seen) = stop_trimmer.push(&filtered.content);
stop_seen |= seen;
if !content.is_empty() {
let mut msg = Message::assistant().with_text(content);
msg.id = Some(message_id.to_string());
if tx.blocking_send(Ok((Some(msg), None))).is_err() {
return Ok(TokenAction::Stop);
}
}
}
}
let should_stop = template_result
.additional_stops
.iter()
.any(|stop| generated_text.ends_with(stop));
let should_stop = stop_seen
|| template_result
.additional_stops
.iter()
.any(|stop| generated_text.ends_with(stop));
if should_stop {
stop_string_emitted = true;
Ok(TokenAction::Stop)
} else {
Ok(TokenAction::Continue)
@ -218,18 +139,22 @@ pub(super) fn generate_with_native_tools(
for delta_json in final_deltas {
if let Ok(delta) = serde_json::from_str::<Value>(&delta_json) {
if let Some(reasoning) = delta.get("reasoning_content").and_then(|v| v.as_str()) {
if !reasoning.is_empty() {
accumulated_thinking.push_str(reasoning);
let mut msg = Message::assistant().with_thinking(reasoning, "");
if let Some(thinking) = output_filter.push_structured_reasoning(reasoning) {
let mut msg = Message::assistant().with_thinking(thinking, "");
msg.id = Some(message_id.to_string());
let _ = tx.blocking_send(Ok((Some(msg), None)));
}
}
if let Some(content) = delta.get("content").and_then(|v| v.as_str()) {
if !content.is_empty() {
let mut msg = Message::assistant().with_text(content);
msg.id = Some(message_id.to_string());
let _ = tx.blocking_send(Ok((Some(msg), None)));
let filtered = output_filter.push_text(content);
let (content, stop_seen) = stop_trimmer.push(&filtered.content);
stop_string_emitted |= stop_seen;
if !content.is_empty() {
let mut msg = Message::assistant().with_text(content);
msg.id = Some(message_id.to_string());
let _ = tx.blocking_send(Ok((Some(msg), None)));
}
}
}
if let Some(tool_calls) = delta.get("tool_calls").and_then(|v| v.as_array()) {
@ -241,6 +166,28 @@ pub(super) fn generate_with_native_tools(
}
}
let filtered = output_filter.finish();
if !filtered.thinking.is_empty() {
let mut msg = Message::assistant().with_thinking(&filtered.thinking, "");
msg.id = Some(message_id.to_string());
let _ = tx.blocking_send(Ok((Some(msg), None)));
}
let content = if stop_string_emitted {
String::new()
} else {
let (content, stop_seen) = stop_trimmer.push(&filtered.content);
let mut content = content;
if !stop_seen {
content.push_str(&stop_trimmer.finish());
}
content
};
if !content.is_empty() {
let mut msg = Message::assistant().with_text(content);
msg.id = Some(message_id.to_string());
let _ = tx.blocking_send(Ok((Some(msg), None)));
}
// Build a single message combining thinking + all tool calls, mirroring
// the structure produced by the OpenAI streaming path. The agent relies
// on this combined message to:
@ -250,8 +197,11 @@ pub(super) fn generate_with_native_tools(
let tool_call_contents = extract_oai_tool_call_contents(&accumulated_tool_calls);
if !tool_call_contents.is_empty() {
let mut contents: Vec<MessageContent> = Vec::new();
if !accumulated_thinking.is_empty() {
contents.push(MessageContent::thinking(&accumulated_thinking, ""));
if !output_filter.accumulated_thinking().is_empty() {
contents.push(MessageContent::thinking(
output_filter.accumulated_thinking(),
"",
));
}
contents.extend(tool_call_contents);
let mut msg = Message::new(

486
crates/goose/src/providers/local_inference/llamacpp/mod.rs
View file

@ -3,35 +3,312 @@ mod inference_engine;
mod inference_native_tools;
use std::any::Any;
use std::ffi::CStr;
use std::path::PathBuf;
use anyhow::Result;
use llama_cpp_2::llama_backend::LlamaBackend;
use llama_cpp_2::model::params::LlamaModelParams;
use llama_cpp_2::model::{LlamaChatMessage, LlamaChatTemplate, LlamaModel};
use llama_cpp_2::model::{ChatTemplateResult, LlamaChatTemplate, LlamaModel};
use llama_cpp_2::openai::OpenAIChatTemplateParams;
use llama_cpp_2::{list_llama_ggml_backend_devices, LlamaBackendDeviceType, LogOptions};
use rmcp::model::Role;
use self::inference_emulated_tools::{
build_emulator_tool_description, generate_with_emulated_tools, load_tiny_model_prompt,
};
use self::inference_engine::{GenerationContext, LoadedModel};
use self::inference_engine::{GenerationContext, LoadedChatTemplates, LoadedModel};
use self::inference_native_tools::generate_with_native_tools;
use crate::providers::errors::ProviderError;
use crate::providers::formats::openai::format_tools;
use crate::providers::local_inference::backend::{
BackendLoadedModel, LocalGenerationRequest, LocalInferenceBackend,
};
use crate::providers::local_inference::local_model_registry::{
ChatTemplate, ModelSettings, ToolCallingMode,
};
use crate::providers::local_inference::multimodal::ExtractedImage;
use crate::providers::local_inference::tool_parsing::compact_tools_json;
use crate::providers::local_inference::{
build_openai_messages_json, extract_text_content, ResolvedModelPaths,
build_openai_messages_json, build_openai_text_messages_json, ResolvedModelPaths,
};
pub(super) const LLAMACPP_BACKEND_ID: &str = "llamacpp";
const CODE_EXECUTION_TOOL: &str = "code_execution__execute_typescript";
pub(super) fn builtin_chat_template_names() -> Vec<String> {
let count = unsafe { llama_cpp_sys_2::llama_chat_builtin_templates(std::ptr::null_mut(), 0) };
if count <= 0 {
return Vec::new();
}
let mut templates = vec![std::ptr::null(); count as usize];
let written = unsafe {
llama_cpp_sys_2::llama_chat_builtin_templates(templates.as_mut_ptr(), templates.len())
};
templates.truncate(written.max(0) as usize);
templates
.into_iter()
.filter(|ptr| !ptr.is_null())
.filter_map(|ptr| {
unsafe { CStr::from_ptr(ptr) }
.to_str()
.ok()
.map(str::to_string)
})
.collect()
}
fn template_result_supports_native_tool_calling(result: &ChatTemplateResult) -> bool {
result.parse_tool_calls
&& result
.parser
.as_deref()
.is_some_and(|parser| !parser.trim().is_empty())
}
fn supports_native_tool_calling(
loaded: &LoadedModel,
settings: &ModelSettings,
template: &LlamaChatTemplate,
oai_messages_json: &str,
tools_json: Option<&str>,
) -> bool {
let Some(tools_json) = tools_json.filter(|tools| !tools.trim().is_empty()) else {
return false;
};
// llama.cpp exposes common_chat_templates_get_caps in C++, but llama-cpp-2
// 0.1.146 does not bind it yet. Replace this dry-run with that capability
// map once it is available through the Rust wrapper.
let params = OpenAIChatTemplateParams {
messages_json: oai_messages_json,
tools_json: Some(tools_json),
tool_choice: None,
json_schema: None,
grammar: None,
reasoning_format: if settings.enable_thinking {
Some("auto")
} else {
None
},
chat_template_kwargs: None,
add_generation_prompt: true,
use_jinja: true,
parallel_tool_calls: false,
enable_thinking: settings.enable_thinking,
add_bos: false,
add_eos: false,
parse_tool_calls: true,
};
match loaded
.model
.apply_chat_template_oaicompat(template, &params)
{
Ok(result) => template_result_supports_native_tool_calling(&result),
Err(e) => {
tracing::debug!(
error = %e,
"llama.cpp chat template dry-run did not support native tool calling"
);
false
}
}
}
fn should_use_native_tool_calling(
mode: ToolCallingMode,
has_tools: bool,
template_supports_native: bool,
) -> bool {
has_tools
&& match mode {
ToolCallingMode::Auto => template_supports_native,
ToolCallingMode::ForceNative => true,
ToolCallingMode::ForceEmulated => false,
}
}
fn is_legacy_builtin_template_name(template: &str) -> bool {
matches!(
template.trim(),
"bailing"
| "bailing-think"
| "bailing2"
| "chatglm3"
| "chatglm4"
| "command-r"
| "deepseek"
| "deepseek-ocr"
| "deepseek2"
| "deepseek3"
| "exaone-moe"
| "exaone3"
| "exaone4"
| "falcon3"
| "gemma"
| "gigachat"
| "glmedge"
| "gpt-oss"
| "granite"
| "granite-4.0"
| "grok-2"
| "hunyuan-dense"
| "hunyuan-moe"
| "hunyuan-ocr"
| "kimi-k2"
| "llama2"
| "llama2-sys"
| "llama2-sys-bos"
| "llama2-sys-strip"
| "llama3"
| "llama4"
| "megrez"
| "minicpm"
| "mistral-v1"
| "mistral-v3"
| "mistral-v3-tekken"
| "mistral-v7"
| "mistral-v7-tekken"
| "monarch"
| "openchat"
| "orion"
| "pangu-embedded"
| "phi3"
| "phi4"
| "rwkv-world"
| "seed_oss"
| "smolvlm"
| "solar-open"
| "vicuna"
| "vicuna-orca"
| "yandex"
| "zephyr"
)
}
fn missing_chat_template_error(
model_id: &str,
architecture: Option<&str>,
context: &str,
has_tool_use_template: bool,
) -> ProviderError {
let architecture = architecture
.map(str::trim)
.filter(|arch| !arch.is_empty())
.map(|arch| format!(" Detected GGUF general.architecture={arch}."))
.unwrap_or_default();
let tool_use_note = if has_tool_use_template {
" A named tool_use chat template is present, but that template is only used for native tool calls with tools present."
} else {
""
};
ProviderError::ExecutionError(format!(
"Model {model_id} does not contain GGUF tokenizer.chat_template metadata required for {context}.{architecture}{tool_use_note} \
Goose cannot safely infer the correct prompt format from architecture alone. Select a \
llama.cpp built-in chat template name, configure a custom inline chat template containing \
the full Jinja template source, or use a GGUF that includes tokenizer.chat_template metadata."
))
}
fn load_chat_templates(
model: &LlamaModel,
settings: &ModelSettings,
) -> Result<LoadedChatTemplates, ProviderError> {
match &settings.chat_template {
ChatTemplate::Embedded => Ok(LoadedChatTemplates {
default: model.chat_template(None).ok(),
tool_use: model.chat_template(Some("tool_use")).ok(),
force_default: false,
}),
ChatTemplate::Builtin { name } => {
let trimmed = name.trim();
if trimmed.is_empty() {
return Err(ProviderError::ExecutionError(
"Built-in chat template name is empty. Enter a llama.cpp built-in template name such as 'chatml', or use embedded chat template metadata.".to_string(),
));
}
LlamaChatTemplate::new(trimmed)
.map_err(|e| {
ProviderError::ExecutionError(format!(
"Built-in chat template name contains an invalid NUL byte: {e}"
))
})
.map(|template| LoadedChatTemplates {
default: Some(template),
tool_use: None,
force_default: true,
})
}
ChatTemplate::CustomInline { template } => {
let trimmed = template.trim();
if trimmed.is_empty() {
return Err(ProviderError::ExecutionError(
"Custom inline chat template is empty. Paste the full Jinja chat template source, use a llama.cpp built-in template name, or use embedded chat template metadata.".to_string(),
));
}
if trimmed == "chatml" || is_legacy_builtin_template_name(trimmed) {
return Err(ProviderError::ExecutionError(format!(
"Custom inline chat template is set to '{trimmed}', which is a llama.cpp template name rather than Jinja template source. Paste the full Jinja chat template source instead, or select Built-in and enter '{trimmed}' if that built-in template is intended."
)));
}
LlamaChatTemplate::new(template)
.map_err(|e| {
ProviderError::ExecutionError(format!(
"Custom inline chat template contains an invalid NUL byte: {e}"
))
})
.map(|template| LoadedChatTemplates {
default: Some(template),
tool_use: None,
force_default: true,
})
}
}
}
fn select_generation_template<'a>(
model_id: &str,
model: &LlamaModel,
templates: &'a LoadedChatTemplates,
native_tool_calling: bool,
has_tools: bool,
) -> Result<&'a LlamaChatTemplate, ProviderError> {
if templates.force_default {
return templates.default.as_ref().ok_or_else(|| {
ProviderError::ExecutionError(
"Configured chat template was not loaded correctly".to_string(),
)
});
}
if native_tool_calling && has_tools {
if let Some(template) = templates.tool_use.as_ref() {
return Ok(template);
}
}
templates.default.as_ref().ok_or_else(|| {
let architecture = model.meta_val_str("general.architecture").ok();
let context = if has_tools && native_tool_calling {
"native tool calling because no tool_use template is available"
} else if has_tools {
"emulated tool calling"
} else {
"chat without tools"
};
missing_chat_template_error(
model_id,
architecture.as_deref(),
context,
templates.tool_use.is_some(),
)
})
}
pub(super) struct LlamaCppBackend {
backend: LlamaBackend,
}
@ -134,18 +411,7 @@ impl LocalInferenceBackend for LlamaCppBackend {
let model = LlamaModel::load_from_file(&self.backend, model_path, &params)
.map_err(|e| ProviderError::ExecutionError(e.to_string()))?;
let template = match model.chat_template(None) {
Ok(t) => t,
Err(_) => {
tracing::warn!("Model has no embedded chat template, falling back to chatml");
LlamaChatTemplate::new("chatml").map_err(|e| {
ProviderError::ExecutionError(format!(
"Failed to create fallback chat template: {}",
e
))
})?
}
};
let templates = load_chat_templates(&model, settings)?;
let mtmd_ctx = Self::init_mtmd_context(&model, &resolved.mmproj_path, settings);
@ -157,7 +423,7 @@ impl LocalInferenceBackend for LlamaCppBackend {
Ok(Box::new(LoadedModel {
model,
template,
templates,
mtmd_ctx,
}))
}
@ -174,14 +440,6 @@ impl LocalInferenceBackend for LlamaCppBackend {
ProviderError::ExecutionError("Loaded model backend mismatch".to_string())
})?;
let native_tool_calling = request.settings.native_tool_calling;
let use_emulator = !native_tool_calling && !request.tools.is_empty();
let system_prompt = if use_emulator {
load_tiny_model_prompt()
} else {
request.system.to_string()
};
let has_vision = request.resolved_model.mmproj_path.is_some();
let marker = llama_cpp_2::mtmd::mtmd_default_marker();
let (images, vision_messages): (Vec<ExtractedImage>, Option<Vec<_>>) = if has_vision {
@ -193,45 +451,8 @@ impl LocalInferenceBackend for LlamaCppBackend {
};
let effective_messages = vision_messages.as_deref().unwrap_or(request.messages);
let mut chat_messages =
vec![
LlamaChatMessage::new("system".to_string(), system_prompt.clone()).map_err(
|e| {
ProviderError::ExecutionError(format!(
"Failed to create system message: {}",
e
))
},
)?,
];
let code_mode_enabled = request.tools.iter().any(|t| t.name == CODE_EXECUTION_TOOL);
if use_emulator && !request.tools.is_empty() {
let tool_desc = build_emulator_tool_description(request.tools, code_mode_enabled);
chat_messages = vec![LlamaChatMessage::new(
"system".to_string(),
format!("{}{}", system_prompt, tool_desc),
)
.map_err(|e| {
ProviderError::ExecutionError(format!("Failed to create system message: {}", e))
})?];
}
for msg in effective_messages {
let role = match msg.role {
Role::User => "user",
Role::Assistant => "assistant",
};
let content = extract_text_content(msg);
if !content.trim().is_empty() {
chat_messages.push(LlamaChatMessage::new(role.to_string(), content).map_err(
|e| ProviderError::ExecutionError(format!("Failed to create message: {}", e)),
)?);
}
}
let (full_tools_json, compact_tools) = if !use_emulator && !request.tools.is_empty() {
let (full_tools_json, compact_tools) = if !request.tools.is_empty() {
let full = format_tools(request.tools)
.ok()
.and_then(|spec| serde_json::to_string(&spec).ok());
@ -241,13 +462,53 @@ impl LocalInferenceBackend for LlamaCppBackend {
(None, None)
};
let oai_messages_json = if request.settings.use_jinja || native_tool_calling {
Some(build_openai_messages_json(
&system_prompt,
effective_messages,
))
let has_native_tool_payload = full_tools_json
.as_deref()
.is_some_and(|tools| !tools.trim().is_empty());
let template_supports_native =
if matches!(request.settings.tool_calling, ToolCallingMode::Auto)
&& has_native_tool_payload
{
let messages_json = build_openai_messages_json(request.system, effective_messages);
if let Some(template) = loaded.templates.tool_use.as_ref() {
supports_native_tool_calling(
loaded,
request.settings,
template,
&messages_json,
full_tools_json.as_deref(),
)
} else {
loaded.templates.default.as_ref().is_some_and(|template| {
supports_native_tool_calling(
loaded,
request.settings,
template,
&messages_json,
full_tools_json.as_deref(),
)
})
}
} else {
false
};
let native_tool_calling = should_use_native_tool_calling(
request.settings.tool_calling,
!request.tools.is_empty(),
template_supports_native,
);
let use_emulator = !native_tool_calling && !request.tools.is_empty();
let system_prompt = if use_emulator {
let tool_desc = build_emulator_tool_description(request.tools, code_mode_enabled);
format!("{}{}", load_tiny_model_prompt(), tool_desc)
} else {
None
request.system.to_string()
};
let oai_messages_json = if use_emulator {
build_openai_text_messages_json(&system_prompt, effective_messages)
} else {
build_openai_messages_json(&system_prompt, effective_messages)
};
if !images.is_empty() && loaded.mtmd_ctx.is_none() {
@ -258,10 +519,18 @@ impl LocalInferenceBackend for LlamaCppBackend {
);
}
let template = select_generation_template(
&request.model_name,
&loaded.model,
&loaded.templates,
native_tool_calling,
!request.tools.is_empty(),
)?;
let mut gen_ctx = GenerationContext {
loaded,
backend: self,
chat_messages: &chat_messages,
template,
settings: request.settings,
context_limit: request.context_limit,
model_name: request.model_name,
@ -272,7 +541,7 @@ impl LocalInferenceBackend for LlamaCppBackend {
};
if use_emulator {
generate_with_emulated_tools(&mut gen_ctx, code_mode_enabled)
generate_with_emulated_tools(&mut gen_ctx, code_mode_enabled, &oai_messages_json)
} else {
generate_with_native_tools(
&mut gen_ctx,
@ -353,3 +622,78 @@ fn log_inference_backend_devices() {
);
}
}
#[cfg(test)]
mod tests {
use super::*;
fn template_result(parser: Option<&str>, parse_tool_calls: bool) -> ChatTemplateResult {
ChatTemplateResult {
prompt: String::new(),
grammar: None,
grammar_lazy: false,
grammar_triggers: Vec::new(),
preserved_tokens: Vec::new(),
additional_stops: Vec::new(),
chat_format: 0,
parser: parser.map(str::to_string),
generation_prompt: String::new(),
parse_tool_calls,
}
}
#[test]
fn native_tool_calling_requires_generated_parser() {
assert!(template_result_supports_native_tool_calling(
&template_result(Some("parser"), true)
));
assert!(!template_result_supports_native_tool_calling(
&template_result(None, true)
));
assert!(!template_result_supports_native_tool_calling(
&template_result(Some("parser"), false)
));
assert!(!template_result_supports_native_tool_calling(
&template_result(Some(" "), true)
));
}
#[test]
fn tool_calling_mode_controls_path_selection() {
assert!(should_use_native_tool_calling(
ToolCallingMode::Auto,
true,
true
));
assert!(!should_use_native_tool_calling(
ToolCallingMode::Auto,
true,
false
));
assert!(should_use_native_tool_calling(
ToolCallingMode::ForceNative,
true,
false
));
assert!(!should_use_native_tool_calling(
ToolCallingMode::ForceEmulated,
true,
true
));
assert!(!should_use_native_tool_calling(
ToolCallingMode::ForceNative,
false,
true
));
}
#[test]
fn rejects_legacy_builtin_names_as_inline_templates() {
assert!(is_legacy_builtin_template_name("gemma"));
assert!(is_legacy_builtin_template_name("llama3"));
assert!(!is_legacy_builtin_template_name("chatml"));
assert!(!is_legacy_builtin_template_name(
"{% for message in messages %}{{ message.content }}{% endfor %}"
));
}
}

131
crates/goose/src/providers/local_inference/local_model_registry.rs
View file

@ -36,6 +36,29 @@ impl Default for SamplingConfig {
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum ToolCallingMode {
#[default]
Auto,
ForceNative,
ForceEmulated,
}
#[derive(Debug, Clone, Default, Hash, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ChatTemplate {
#[serde(alias = "auto")]
#[default]
Embedded,
Builtin {
name: String,
},
CustomInline {
template: String,
},
}
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
pub struct ModelSettings {
pub context_size: Option<u32>,
@ -57,13 +80,13 @@ pub struct ModelSettings {
pub flash_attention: Option<bool>,
pub n_threads: Option<i32>,
#[serde(default)]
pub native_tool_calling: bool,
pub tool_calling: ToolCallingMode,
#[serde(default)]
pub use_jinja: bool,
pub chat_template: ChatTemplate,
#[serde(default = "default_true")]
pub enable_thinking: bool,
/// Whether this model architecture supports vision input.
/// Derived from the featured model table, not user-configurable.
/// Derived from associated mmproj metadata, not user-configurable.
#[serde(default)]
pub vision_capable: bool,
/// Estimated tokens per image for budget planning before mtmd tokenization.
@ -106,8 +129,8 @@ impl Default for ModelSettings {
use_mlock: false,
flash_attention: None,
n_threads: None,
native_tool_calling: false,
use_jinja: false,
tool_calling: ToolCallingMode::Auto,
chat_template: ChatTemplate::Embedded,
enable_thinking: true,
vision_capable: false,
image_token_estimate: default_image_token_estimate(),
@ -116,100 +139,43 @@ impl Default for ModelSettings {
}
}
/// HuggingFace repo + filename for multimodal projection weights (vision encoder).
pub struct MmprojSpec {
pub repo: &'static str,
pub filename: &'static str,
}
impl MmprojSpec {
/// Local path for this mmproj, namespaced by repo to avoid collisions
/// between different models that use the same filename.
pub fn local_path(&self) -> std::path::PathBuf {
let repo_name = self.repo.split('/').next_back().unwrap_or(self.repo);
Paths::in_data_dir("models")
.join(repo_name)
.join(self.filename)
}
}
pub struct FeaturedModel {
/// HuggingFace spec in "author/repo-GGUF:quantization" format.
pub spec: &'static str,
/// Whether this model's GGUF template supports native tool calling via llama.cpp.
pub native_tool_calling: bool,
/// Multimodal projection weights spec. None for text-only models.
pub mmproj: Option<MmprojSpec>,
}
pub const FEATURED_MODELS: &[FeaturedModel] = &[
FeaturedModel {
spec: "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M",
native_tool_calling: false,
mmproj: None,
},
FeaturedModel {
spec: "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M",
native_tool_calling: false,
mmproj: None,
},
FeaturedModel {
spec: "bartowski/Hermes-2-Pro-Mistral-7B-GGUF:Q4_K_M",
native_tool_calling: false,
mmproj: None,
},
FeaturedModel {
spec: "bartowski/Mistral-Small-24B-Instruct-2501-GGUF:Q4_K_M",
native_tool_calling: false,
mmproj: None,
},
FeaturedModel {
spec: "unsloth/gemma-4-E4B-it-GGUF:Q4_K_M",
native_tool_calling: true,
mmproj: Some(MmprojSpec {
repo: "unsloth/gemma-4-E4B-it-GGUF",
filename: "mmproj-BF16.gguf",
}),
},
FeaturedModel {
spec: "unsloth/gemma-4-26B-A4B-it-GGUF:Q4_K_M",
native_tool_calling: true,
mmproj: Some(MmprojSpec {
repo: "unsloth/gemma-4-26B-A4B-it-GGUF",
filename: "mmproj-BF16.gguf",
}),
},
];
pub fn default_settings_for_model(model_id: &str) -> ModelSettings {
use super::hf_models::parse_model_spec;
let model_repo = model_id.split(':').next().unwrap_or(model_id);
let featured = FEATURED_MODELS.iter().find(|m| {
if let Ok((repo_id, _quant)) = parse_model_spec(m.spec) {
repo_id == model_repo
} else {
false
}
});
pub fn default_settings_for_model(_model_id: &str) -> ModelSettings {
ModelSettings {
native_tool_calling: featured.is_some_and(|m| m.native_tool_calling),
vision_capable: featured.is_some_and(|m| m.mmproj.is_some()),
..ModelSettings::default()
}
}
/// Look up the `MmprojSpec` for a featured model by its model ID.
pub fn featured_mmproj_spec(model_id: &str) -> Option<&'static MmprojSpec> {
use super::hf_models::parse_model_spec;
let model_repo = model_id.split(':').next().unwrap_or(model_id);
FEATURED_MODELS.iter().find_map(|m| {
if let Ok((repo_id, _quant)) = parse_model_spec(m.spec) {
if repo_id == model_repo {
return m.mmproj.as_ref();
}
}
None
})
/// Local path for an mmproj file, namespaced by repo to avoid collisions
/// between different models that use the same filename.
pub fn mmproj_local_path(repo_id: &str, filename: &str) -> PathBuf {
let repo_name = repo_id.split('/').next_back().unwrap_or(repo_id);
Paths::in_data_dir("models").join(repo_name).join(filename)
}
/// Check if a model ID corresponds to a featured model.
@ -263,32 +229,27 @@ pub struct LocalModelEntry {
#[serde(default)]
pub mmproj_size_bytes: u64,
#[serde(default)]
pub mmproj_checked: bool,
#[serde(default)]
pub shard_files: Vec<ShardFile>,
}
impl LocalModelEntry {
/// Populate mmproj metadata and vision settings from the featured model
/// table if this model's repo has a known vision encoder.
pub fn enrich_with_featured_mmproj(&mut self) {
if let Some(mmproj) = featured_mmproj_spec(&self.id) {
let path = mmproj.local_path();
if self.mmproj_path.as_ref() != Some(&path) {
self.mmproj_path = Some(path.clone());
self.mmproj_source_url = Some(format!(
"https://huggingface.co/{}/resolve/main/{}",
mmproj.repo, mmproj.filename
));
}
pub fn refresh_mmproj_metadata(&mut self) {
self.settings.vision_capable = self.mmproj_path.is_some();
if let Some(path) = &self.mmproj_path {
self.mmproj_checked = true;
self.settings.vision_capable = true;
if self.mmproj_size_bytes == 0 || self.settings.mmproj_size_bytes == 0 {
if let Ok(meta) = std::fs::metadata(&path) {
if let Ok(meta) = std::fs::metadata(path) {
self.mmproj_size_bytes = meta.len();
self.settings.mmproj_size_bytes = meta.len();
}
}
} else {
self.mmproj_size_bytes = 0;
self.settings.mmproj_size_bytes = 0;
}
let defaults = default_settings_for_model(&self.id);
self.settings.native_tool_calling = defaults.native_tool_calling;
}
pub fn is_downloaded(&self) -> bool {
@ -436,7 +397,7 @@ impl LocalModelRegistry {
for mut entry in featured_entries {
if !self.models.iter().any(|m| m.id == entry.id) {
entry.enrich_with_featured_mmproj();
entry.refresh_mmproj_metadata();
self.models.push(entry);
changed = true;
}
@ -455,7 +416,7 @@ impl LocalModelRegistry {
}
pub fn add_model(&mut self, mut entry: LocalModelEntry) -> Result<()> {
entry.enrich_with_featured_mmproj();
entry.refresh_mmproj_metadata();
if let Some(existing) = self.models.iter_mut().find(|m| m.id == entry.id) {
*existing = entry;
} else {

100
ui/desktop/openapi.json
View file

@ -1993,6 +1993,29 @@
}
}
},
"/local-inference/chat-templates/builtin": {
"get": {
"tags": [
"super::routes::local_inference"
],
"operationId": "list_builtin_chat_templates",
"responses": {
"200": {
"description": "llama.cpp built-in chat template names",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
}
}
}
},
"/local-inference/download": {
"post": {
"tags": [
@ -4260,6 +4283,63 @@
}
}
},
"ChatTemplate": {
"oneOf": [
{
"type": "object",
"required": [
"type"
],
"properties": {
"type": {
"type": "string",
"enum": [
"embedded"
]
}
}
},
{
"type": "object",
"required": [
"name",
"type"
],
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string",
"enum": [
"builtin"
]
}
}
},
{
"type": "object",
"required": [
"template",
"type"
],
"properties": {
"template": {
"type": "string"
},
"type": {
"type": "string",
"enum": [
"custom_inline"
]
}
}
}
],
"discriminator": {
"propertyName": "type"
}
},
"CheckProviderRequest": {
"type": "object",
"required": [
@ -6716,6 +6796,9 @@
"ModelSettings": {
"type": "object",
"properties": {
"chat_template": {
"$ref": "#/components/schemas/ChatTemplate"
},
"context_size": {
"type": "integer",
"format": "int32",
@ -6766,9 +6849,6 @@
"format": "int32",
"nullable": true
},
"native_tool_calling": {
"type": "boolean"
},
"presence_penalty": {
"type": "number",
"format": "float"
@ -6784,15 +6864,15 @@
"sampling": {
"$ref": "#/components/schemas/SamplingConfig"
},
"use_jinja": {
"type": "boolean"
"tool_calling": {
"$ref": "#/components/schemas/ToolCallingMode"
},
"use_mlock": {
"type": "boolean"
},
"vision_capable": {
"type": "boolean",
"description": "Whether this model architecture supports vision input.\nDerived from the featured model table, not user-configurable."
"description": "Whether this model architecture supports vision input.\nDerived from associated mmproj metadata, not user-configurable."
}
}
},
@ -8797,6 +8877,14 @@
}
}
},
"ToolCallingMode": {
"type": "string",
"enum": [
"auto",
"force_native",
"force_emulated"
]
},
"ToolConfirmationRequest": {
"type": "object",
"required": [

4
ui/desktop/src/api/index.ts
View file

File diff suppressed because one or more lines are too long

4
ui/desktop/src/api/sdk.gen.ts
View file

File diff suppressed because one or more lines are too long

34
ui/desktop/src/api/types.gen.ts
View file

@ -76,6 +76,16 @@ export type ChatRequest = {
user_message: Message;
};
export type ChatTemplate = {
type: 'embedded';
} | {
name: string;
type: 'builtin';
} | {
template: string;
type: 'custom_inline';
};
export type CheckProviderRequest = {
provider: string;
};
@ -863,6 +873,7 @@ export type ModelInfoResponse = {
};
export type ModelSettings = {
chat_template?: ChatTemplate;
context_size?: number | null;
enable_thinking?: boolean;
flash_attention?: boolean | null;
@ -880,16 +891,15 @@ export type ModelSettings = {
n_batch?: number | null;
n_gpu_layers?: number | null;
n_threads?: number | null;
native_tool_calling?: boolean;
presence_penalty?: number;
repeat_last_n?: number;
repeat_penalty?: number;
sampling?: SamplingConfig;
use_jinja?: boolean;
tool_calling?: ToolCallingMode;
use_mlock?: boolean;
/**
* Whether this model architecture supports vision input.
* Derived from the featured model table, not user-configurable.
* Derived from associated mmproj metadata, not user-configurable.
*/
vision_capable?: boolean;
};
@ -1544,6 +1554,8 @@ export type ToolAnnotations = {
title?: string;
};
export type ToolCallingMode = 'auto' | 'force_native' | 'force_emulated';
export type ToolConfirmationRequest = {
arguments: JsonObject;
id: string;
@ -3241,6 +3253,22 @@ export type StartTetrateSetupResponses = {
export type StartTetrateSetupResponse = StartTetrateSetupResponses[keyof StartTetrateSetupResponses];
export type ListBuiltinChatTemplatesData = {
body?: never;
path?: never;
query?: never;
url: '/local-inference/chat-templates/builtin';
};
export type ListBuiltinChatTemplatesResponses = {
/**
* llama.cpp built-in chat template names
*/
200: Array<string>;
};
export type ListBuiltinChatTemplatesResponse = ListBuiltinChatTemplatesResponses[keyof ListBuiltinChatTemplatesResponses];
export type DownloadHfModelData = {
body: DownloadModelRequest;
path?: never;

215
ui/desktop/src/components/settings/localInference/ModelSettingsPanel.tsx
View file

@ -4,9 +4,12 @@ import { Button } from '../../ui/button';
import { Switch } from '../../ui/switch';
import {
getModelSettings,
listBuiltinChatTemplates,
updateModelSettings,
type ChatTemplate,
type ModelSettings,
type SamplingConfig,
type ToolCallingMode,
} from '../../../api';
import { defineMessages, useIntl } from '../../../i18n';
@ -161,16 +164,59 @@ const i18n = defineMessages({
},
toolCalling: {
id: 'modelSettingsPanel.toolCalling',
defaultMessage: 'Tool Calling',
defaultMessage: 'Tool calling',
},
nativeToolCalling: {
id: 'modelSettingsPanel.nativeToolCalling',
defaultMessage: 'Native tool calling',
toolCallingDescription: {
id: 'modelSettingsPanel.toolCallingDescription',
defaultMessage: 'Choose how local models select native or emulated tool calling',
},
nativeToolCallingDescription: {
id: 'modelSettingsPanel.nativeToolCallingDescription',
defaultMessage:
"Use the model's built-in tool-call format instead of the shell-command emulator. Enable for large models that reliably support tool calling.",
toolCallingAuto: {
id: 'modelSettingsPanel.toolCallingAuto',
defaultMessage: 'Auto',
},
toolCallingForceNative: {
id: 'modelSettingsPanel.toolCallingForceNative',
defaultMessage: 'Force native',
},
toolCallingForceEmulated: {
id: 'modelSettingsPanel.toolCallingForceEmulated',
defaultMessage: 'Force emulated',
},
chatTemplate: {
id: 'modelSettingsPanel.chatTemplate',
defaultMessage: 'Chat template',
},
chatTemplateDescription: {
id: 'modelSettingsPanel.chatTemplateDescription',
defaultMessage: 'Use embedded GGUF metadata, a llama.cpp built-in template, or inline Jinja',
},
chatTemplateEmbedded: {
id: 'modelSettingsPanel.chatTemplateEmbedded',
defaultMessage: 'Embedded',
},
chatTemplateBuiltin: {
id: 'modelSettingsPanel.chatTemplateBuiltin',
defaultMessage: 'Built-in',
},
chatTemplateCustomInline: {
id: 'modelSettingsPanel.chatTemplateCustomInline',
defaultMessage: 'Custom inline',
},
builtinChatTemplate: {
id: 'modelSettingsPanel.builtinChatTemplate',
defaultMessage: 'Built-in template',
},
builtinChatTemplateDescription: {
id: 'modelSettingsPanel.builtinChatTemplateDescription',
defaultMessage: 'Select a llama.cpp built-in template name',
},
customChatTemplate: {
id: 'modelSettingsPanel.customChatTemplate',
defaultMessage: 'Custom chat template',
},
customChatTemplateDescription: {
id: 'modelSettingsPanel.customChatTemplateDescription',
defaultMessage: 'Paste the full Jinja chat template source',
},
});
@ -194,10 +240,12 @@ const DEFAULT_SETTINGS: ModelSettings = {
use_mlock: false,
flash_attention: null,
n_threads: null,
native_tool_calling: false,
tool_calling: 'auto',
chat_template: { type: 'embedded' },
};
type SamplingType = SamplingConfig['type'];
type ChatTemplateMode = 'embedded' | 'builtin' | 'custom_inline';
function NumberField({
label,
@ -302,16 +350,59 @@ function SelectField<T extends string>({
);
}
function TextAreaField({
label,
description,
value,
onChange,
onBlur,
}: {
label: string;
description?: string;
value: string;
onChange: (v: string) => void;
onBlur: () => void;
}) {
return (
<div className="flex flex-col gap-1">
<label className="text-xs font-medium text-text-default">{label}</label>
{description && <span className="text-xs text-text-muted">{description}</span>}
<textarea
value={value}
onChange={(e) => onChange(e.target.value)}
onBlur={onBlur}
spellCheck={false}
className="min-h-32 rounded border border-border-subtle bg-background-default px-2 py-1 font-mono text-xs text-text-default"
/>
</div>
);
}
export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
const intl = useIntl();
const [settings, setSettings] = useState<ModelSettings>(DEFAULT_SETTINGS);
const [chatTemplateDraft, setChatTemplateDraft] = useState('');
const [builtinTemplateDraft, setBuiltinTemplateDraft] = useState('chatml');
const [builtinTemplateOptions, setBuiltinTemplateOptions] = useState<string[]>(['chatml']);
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
const load = useCallback(async () => {
try {
const res = await getModelSettings({ path: { model_id: modelId } });
if (res.data) setSettings(res.data);
const [settingsResult, builtinsResult] = await Promise.allSettled([
getModelSettings({ path: { model_id: modelId } }),
listBuiltinChatTemplates(),
]);
if (builtinsResult.status === 'fulfilled' && builtinsResult.value.data?.length) {
setBuiltinTemplateOptions(builtinsResult.value.data);
}
if (settingsResult.status === 'fulfilled' && settingsResult.value.data) {
setSettings({
...settingsResult.value.data,
tool_calling: settingsResult.value.data.tool_calling ?? 'auto',
chat_template: settingsResult.value.data.chat_template ?? { type: 'embedded' },
});
}
} catch {
// use defaults
} finally {
@ -323,6 +414,18 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
load();
}, [load]);
useEffect(() => {
const chatTemplate = settings.chat_template;
if (chatTemplate?.type === 'custom_inline') {
setChatTemplateDraft(chatTemplate.template ?? '');
} else {
setChatTemplateDraft('');
}
if (chatTemplate?.type === 'builtin') {
setBuiltinTemplateDraft(chatTemplate.name ?? 'chatml');
}
}, [settings.chat_template]);
const save = async (updated: ModelSettings) => {
setSettings(updated);
setSaving(true);
@ -342,6 +445,38 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
};
const samplingType: SamplingType = settings.sampling?.type ?? 'Temperature';
const chatTemplate = settings.chat_template ?? { type: 'embedded' };
const chatTemplateMode: ChatTemplateMode =
chatTemplate.type === 'custom_inline'
? 'custom_inline'
: chatTemplate.type === 'builtin'
? 'builtin'
: 'embedded';
const setChatTemplateMode = (mode: ChatTemplateMode) => {
let next: ChatTemplate;
if (mode === 'custom_inline') {
next = { type: 'custom_inline', template: chatTemplateDraft };
} else if (mode === 'builtin') {
next = { type: 'builtin', name: builtinTemplateDraft.trim() || builtinTemplateOptions[0] || 'chatml' };
} else {
next = { type: 'embedded' };
}
updateField('chat_template', next);
};
const setBuiltinTemplateName = (name: string) => {
setBuiltinTemplateDraft(name);
if (chatTemplateMode === 'builtin') {
updateField('chat_template', { type: 'builtin', name });
}
};
const saveChatTemplateDraft = () => {
if (chatTemplateMode === 'custom_inline') {
updateField('chat_template', { type: 'custom_inline', template: chatTemplateDraft });
}
};
const setSamplingType = (type: SamplingType) => {
let sampling: SamplingConfig;
@ -366,6 +501,10 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
save({ ...settings, sampling: { ...settings.sampling!, ...partial } as SamplingConfig });
};
const visibleBuiltinTemplateOptions = builtinTemplateOptions.includes(builtinTemplateDraft)
? builtinTemplateOptions
: [builtinTemplateDraft, ...builtinTemplateOptions].filter(Boolean);
if (loading) {
return <div className="py-2 text-xs text-text-muted">{intl.formatMessage(i18n.loadingSettings)}</div>;
}
@ -585,16 +724,52 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
]}
onChange={(v) => updateField('flash_attention', v === 'auto' ? null : v === 'on')}
/>
</div>
{/* Tool Calling */}
<div className="space-y-2">
<h5 className="text-xs font-medium text-text-default">{intl.formatMessage(i18n.toolCalling)}</h5>
<ToggleField
label={intl.formatMessage(i18n.nativeToolCalling)}
description={intl.formatMessage(i18n.nativeToolCallingDescription)}
value={settings.native_tool_calling ?? false}
onChange={(v) => updateField('native_tool_calling', v)}
<SelectField<ToolCallingMode>
label={intl.formatMessage(i18n.toolCalling)}
description={intl.formatMessage(i18n.toolCallingDescription)}
value={settings.tool_calling ?? 'auto'}
options={[
{ value: 'auto', label: intl.formatMessage(i18n.toolCallingAuto) },
{ value: 'force_native', label: intl.formatMessage(i18n.toolCallingForceNative) },
{ value: 'force_emulated', label: intl.formatMessage(i18n.toolCallingForceEmulated) },
]}
onChange={(v) => updateField('tool_calling', v)}
/>
<SelectField<ChatTemplateMode>
label={intl.formatMessage(i18n.chatTemplate)}
description={intl.formatMessage(i18n.chatTemplateDescription)}
value={chatTemplateMode}
options={[
{ value: 'embedded', label: intl.formatMessage(i18n.chatTemplateEmbedded) },
{ value: 'builtin', label: intl.formatMessage(i18n.chatTemplateBuiltin) },
{
value: 'custom_inline',
label: intl.formatMessage(i18n.chatTemplateCustomInline),
},
]}
onChange={setChatTemplateMode}
/>
{chatTemplateMode === 'builtin' && (
<SelectField<string>
label={intl.formatMessage(i18n.builtinChatTemplate)}
description={intl.formatMessage(i18n.builtinChatTemplateDescription)}
value={builtinTemplateDraft}
options={visibleBuiltinTemplateOptions.map((template) => ({
value: template,
label: template,
}))}
onChange={setBuiltinTemplateName}
/>
)}
{chatTemplateMode === 'custom_inline' && (
<TextAreaField
label={intl.formatMessage(i18n.customChatTemplate)}
description={intl.formatMessage(i18n.customChatTemplateDescription)}
value={chatTemplateDraft}
onChange={setChatTemplateDraft}
onBlur={saveChatTemplateDraft}
/>
)}
</div>
</div>
);

47
ui/desktop/src/i18n/messages/en.json
View file

@ -2183,6 +2183,27 @@
"modelSettingsPanel.batchSizeDescription": {
"defaultMessage": "Prompt processing batch"
},
"modelSettingsPanel.builtinChatTemplate": {
"defaultMessage": "Built-in template"
},
"modelSettingsPanel.builtinChatTemplateDescription": {
"defaultMessage": "Select a llama.cpp built-in template name"
},
"modelSettingsPanel.chatTemplate": {
"defaultMessage": "Chat template"
},
"modelSettingsPanel.chatTemplateBuiltin": {
"defaultMessage": "Built-in"
},
"modelSettingsPanel.chatTemplateCustomInline": {
"defaultMessage": "Custom inline"
},
"modelSettingsPanel.chatTemplateDescription": {
"defaultMessage": "Use embedded GGUF metadata, a llama.cpp built-in template, or inline Jinja"
},
"modelSettingsPanel.chatTemplateEmbedded": {
"defaultMessage": "Embedded"
},
"modelSettingsPanel.contextAndGeneration": {
"defaultMessage": "Context & Generation"
},
@ -2192,6 +2213,12 @@
"modelSettingsPanel.contextSizeDescription": {
"defaultMessage": "Max context window (0 = model default)"
},
"modelSettingsPanel.customChatTemplate": {
"defaultMessage": "Custom chat template"
},
"modelSettingsPanel.customChatTemplateDescription": {
"defaultMessage": "Paste the full Jinja chat template source"
},
"modelSettingsPanel.etaLearningRate": {
"defaultMessage": "Eta (learning rate)"
},
@ -2231,12 +2258,6 @@
"modelSettingsPanel.minP": {
"defaultMessage": "Min P"
},
"modelSettingsPanel.nativeToolCalling": {
"defaultMessage": "Native tool calling"
},
"modelSettingsPanel.nativeToolCallingDescription": {
"defaultMessage": "Use the model's built-in tool-call format instead of the shell-command emulator. Enable for large models that reliably support tool calling."
},
"modelSettingsPanel.performance": {
"defaultMessage": "Performance"
},
@ -2289,7 +2310,19 @@
"defaultMessage": "CPU threads for generation"
},
"modelSettingsPanel.toolCalling": {
"defaultMessage": "Tool Calling"
"defaultMessage": "Tool calling"
},
"modelSettingsPanel.toolCallingAuto": {
"defaultMessage": "Auto"
},
"modelSettingsPanel.toolCallingDescription": {
"defaultMessage": "Choose how local models select native or emulated tool calling"
},
"modelSettingsPanel.toolCallingForceEmulated": {
"defaultMessage": "Force emulated"
},
"modelSettingsPanel.toolCallingForceNative": {
"defaultMessage": "Force native"
},
"modelSettingsPanel.topK": {
"defaultMessage": "Top K"

51
ui/desktop/src/i18n/messages/ru.json
View file

@ -2201,6 +2201,48 @@
"modelSettingsPanel.flashAttentionDescription": {
"defaultMessage": "Включить оптимизацию flash attention"
},
"modelSettingsPanel.builtinChatTemplate": {
"defaultMessage": "Встроенный шаблон"
},
"modelSettingsPanel.builtinChatTemplateDescription": {
"defaultMessage": "Выберите имя встроенного шаблона llama.cpp"
},
"modelSettingsPanel.chatTemplate": {
"defaultMessage": "Шаблон чата"
},
"modelSettingsPanel.chatTemplateBuiltin": {
"defaultMessage": "Встроенный"
},
"modelSettingsPanel.chatTemplateCustomInline": {
"defaultMessage": "Пользовательский inline"
},
"modelSettingsPanel.chatTemplateDescription": {
"defaultMessage": "Использовать встроенные метаданные GGUF, встроенный шаблон llama.cpp или inline-шаблон Jinja"
},
"modelSettingsPanel.chatTemplateEmbedded": {
"defaultMessage": "Встроенный"
},
"modelSettingsPanel.customChatTemplate": {
"defaultMessage": "Пользовательский шаблон чата"
},
"modelSettingsPanel.customChatTemplateDescription": {
"defaultMessage": "Вставьте полный исходный код шаблона Jinja"
},
"modelSettingsPanel.toolCalling": {
"defaultMessage": "Вызов инструментов"
},
"modelSettingsPanel.toolCallingAuto": {
"defaultMessage": "Авто"
},
"modelSettingsPanel.toolCallingDescription": {
"defaultMessage": "Выберите, как локальные модели используют нативный или эмулируемый вызов инструментов"
},
"modelSettingsPanel.toolCallingForceEmulated": {
"defaultMessage": "Принудительно эмулируемый"
},
"modelSettingsPanel.toolCallingForceNative": {
"defaultMessage": "Принудительно нативный"
},
"modelSettingsPanel.frequencyPenalty": {
"defaultMessage": "Штраф частоты"
},
@ -2231,12 +2273,6 @@
"modelSettingsPanel.minP": {
"defaultMessage": "Min P"
},
"modelSettingsPanel.nativeToolCalling": {
"defaultMessage": "Нативный вызов инструментов"
},
"modelSettingsPanel.nativeToolCallingDescription": {
"defaultMessage": "Использовать встроенный формат вызова инструментов модели вместо эмулятора shell-команд. Включайте для крупных моделей, надежно поддерживающих вызов tool calling."
},
"modelSettingsPanel.performance": {
"defaultMessage": "Производительность"
},
@ -2288,9 +2324,6 @@
"modelSettingsPanel.threadsDescription": {
"defaultMessage": "Потоки CPU для генерации"
},
"modelSettingsPanel.toolCalling": {
"defaultMessage": "Вызов инструментов"
},
"modelSettingsPanel.topK": {
"defaultMessage": "Top K"
},

51
ui/desktop/src/i18n/messages/tr.json
View file

@ -2279,6 +2279,48 @@
"modelSettingsPanel.flashAttentionDescription": {
"defaultMessage": "Flaş dikkati optimizasyonunu etkinleştir"
},
"modelSettingsPanel.builtinChatTemplate": {
"defaultMessage": "Yerleşik şablon"
},
"modelSettingsPanel.builtinChatTemplateDescription": {
"defaultMessage": "Bir llama.cpp yerleşik şablon adı seçin"
},
"modelSettingsPanel.chatTemplate": {
"defaultMessage": "Sohbet şablonu"
},
"modelSettingsPanel.chatTemplateBuiltin": {
"defaultMessage": "Yerleşik"
},
"modelSettingsPanel.chatTemplateCustomInline": {
"defaultMessage": "Özel satır içi"
},
"modelSettingsPanel.chatTemplateDescription": {
"defaultMessage": "Gömülü GGUF meta verilerini, llama.cpp yerleşik şablonunu veya satır içi Jinja şablonunu kullan"
},
"modelSettingsPanel.chatTemplateEmbedded": {
"defaultMessage": "Gömülü"
},
"modelSettingsPanel.customChatTemplate": {
"defaultMessage": "Özel sohbet şablonu"
},
"modelSettingsPanel.customChatTemplateDescription": {
"defaultMessage": "Tam Jinja sohbet şablonu kaynağını yapıştır"
},
"modelSettingsPanel.toolCalling": {
"defaultMessage": "Araç çağırma"
},
"modelSettingsPanel.toolCallingAuto": {
"defaultMessage": "Otomatik"
},
"modelSettingsPanel.toolCallingDescription": {
"defaultMessage": "Yerel modellerin yerel veya emüle araç çağırmayı nasıl seçeceğini belirleyin"
},
"modelSettingsPanel.toolCallingForceEmulated": {
"defaultMessage": "Emüle etmeye zorla"
},
"modelSettingsPanel.toolCallingForceNative": {
"defaultMessage": "Yereli zorla"
},
"modelSettingsPanel.frequencyPenalty": {
"defaultMessage": "Frekans cezası"
},
@ -2309,12 +2351,6 @@
"modelSettingsPanel.minP": {
"defaultMessage": "Min P"
},
"modelSettingsPanel.nativeToolCalling": {
"defaultMessage": "Yerel araç çağrısı"
},
"modelSettingsPanel.nativeToolCallingDescription": {
"defaultMessage": "Kabuk komutu emülatörü yerine modelin yerleşik araç çağrısı formatını kullanın. Araç çağırmayı güvenilir şekilde destekleyen büyük modeller için etkinleştirin."
},
"modelSettingsPanel.performance": {
"defaultMessage": "Performans"
},
@ -2366,9 +2402,6 @@
"modelSettingsPanel.threadsDescription": {
"defaultMessage": "Nesil için CPU iş parçacıkları"
},
"modelSettingsPanel.toolCalling": {
"defaultMessage": "Takım Çağırma"
},
"modelSettingsPanel.topK": {
"defaultMessage": "En İyi K"
},

51
ui/desktop/src/i18n/messages/zh-CN.json
View file

@ -2150,6 +2150,48 @@
"modelSettingsPanel.flashAttentionDescription": {
"defaultMessage": "启用 Flash Attention 优化"
},
"modelSettingsPanel.builtinChatTemplate": {
"defaultMessage": "内置模板"
},
"modelSettingsPanel.builtinChatTemplateDescription": {
"defaultMessage": "选择 llama.cpp 内置模板名称"
},
"modelSettingsPanel.chatTemplate": {
"defaultMessage": "聊天模板"
},
"modelSettingsPanel.chatTemplateBuiltin": {
"defaultMessage": "内置"
},
"modelSettingsPanel.chatTemplateCustomInline": {
"defaultMessage": "自定义内联"
},
"modelSettingsPanel.chatTemplateDescription": {
"defaultMessage": "使用嵌入式 GGUF 元数据、llama.cpp 内置模板或内联 Jinja 模板"
},
"modelSettingsPanel.chatTemplateEmbedded": {
"defaultMessage": "嵌入式"
},
"modelSettingsPanel.customChatTemplate": {
"defaultMessage": "自定义聊天模板"
},
"modelSettingsPanel.customChatTemplateDescription": {
"defaultMessage": "粘贴完整的 Jinja 聊天模板源码"
},
"modelSettingsPanel.toolCalling": {
"defaultMessage": "工具调用"
},
"modelSettingsPanel.toolCallingAuto": {
"defaultMessage": "自动"
},
"modelSettingsPanel.toolCallingDescription": {
"defaultMessage": "选择本地模型如何使用原生或模拟工具调用"
},
"modelSettingsPanel.toolCallingForceEmulated": {
"defaultMessage": "强制模拟"
},
"modelSettingsPanel.toolCallingForceNative": {
"defaultMessage": "强制原生"
},
"modelSettingsPanel.frequencyPenalty": {
"defaultMessage": "频率惩罚"
},
@ -2180,12 +2222,6 @@
"modelSettingsPanel.minP": {
"defaultMessage": "Min P"
},
"modelSettingsPanel.nativeToolCalling": {
"defaultMessage": "原生工具调用"
},
"modelSettingsPanel.nativeToolCallingDescription": {
"defaultMessage": "使用模型内置的工具调用格式，而不是 shell 命令模拟器。对可靠支持工具调用的大模型建议启用。"
},
"modelSettingsPanel.performance": {
"defaultMessage": "性能"
},
@ -2237,9 +2273,6 @@
"modelSettingsPanel.threadsDescription": {
"defaultMessage": "用于生成的 CPU 线程数"
},
"modelSettingsPanel.toolCalling": {
"defaultMessage": "工具调用"
},
"modelSettingsPanel.topK": {
"defaultMessage": "Top K"
},

15
ui/desktop/src/types/message.ts
View file

@ -97,7 +97,6 @@ export function getTextAndImageContent(message: Message): {
// Strip assistant-only markup that shouldn't appear in rendered text
if (message.role === 'assistant') {
textContent = stripToolCallMarkers(textContent);
textContent = textContent.replace(/<think>[\s\S]*?<\/think>/gi, '');
}
return { textContent, imagePaths };
@ -122,20 +121,6 @@ export function getThinkingContent(message: Message): string | null {
}
}
// Inline <think> tags in assistant text content
if (message.role === 'assistant') {
for (const content of message.content) {
if (content.type === 'text') {
const regex = /<think>([\s\S]*?)<\/think>/gi;
let match;
while ((match = regex.exec(content.text)) !== null) {
const text = match[1].trim();
if (text) parts.push(text);
}
}
}
}
return parts.length > 0 ? parts.join('') : null;
}