diff --git a/src-tauri/binaries/talkis-llm-aarch64-apple-darwin b/src-tauri/binaries/talkis-llm-aarch64-apple-darwin index b18f884..222796d 100644 Binary files a/src-tauri/binaries/talkis-llm-aarch64-apple-darwin and b/src-tauri/binaries/talkis-llm-aarch64-apple-darwin differ diff --git a/src-tauri/src/bin/talkis-llm.rs b/src-tauri/src/bin/talkis-llm.rs index 58dc575..e30db61 100644 --- a/src-tauri/src/bin/talkis-llm.rs +++ b/src-tauri/src/bin/talkis-llm.rs @@ -14,8 +14,9 @@ use llama_cpp_2::context::params::LlamaContextParams; use llama_cpp_2::llama_backend::LlamaBackend; use llama_cpp_2::llama_batch::LlamaBatch; use llama_cpp_2::model::params::LlamaModelParams; -use llama_cpp_2::model::{AddBos, LlamaModel, Special}; +use llama_cpp_2::model::{AddBos, LlamaModel}; use llama_cpp_2::sampling::LlamaSampler; +use llama_cpp_2::TokenToStringError; const SERVER_NAME: &str = "talkis-llm"; const ENGINE_NAME: &str = "llama.cpp"; @@ -350,7 +351,18 @@ fn generate( // Accumulate raw token bytes and emit only COMPLETE UTF-8 sequences. A // single token can carry a partial multibyte char (Cyrillic spans token // boundaries), so decoding tokens one-by-one yielded garbled glyphs. - if let Ok(bytes) = model.token_to_bytes(token, Special::Tokenize) { + let piece_bytes = match model.token_to_piece_bytes(token, 8, true, None) { + Err(TokenToStringError::InsufficientBufferSpace(size)) => { + let needed = size + .checked_neg() + .and_then(|value| usize::try_from(value).ok()) + .unwrap_or(32); + model.token_to_piece_bytes(token, needed, true, None) + } + result => result, + }; + + if let Ok(bytes) = piece_bytes { pending.extend_from_slice(&bytes); loop { match std::str::from_utf8(&pending) { diff --git a/src/components/LocalLlmModels.tsx b/src/components/LocalLlmModels.tsx index a2707b4..3ed93fb 100644 --- a/src/components/LocalLlmModels.tsx +++ b/src/components/LocalLlmModels.tsx @@ -5,6 +5,11 @@ import { AlertCircle, Check, Download, HardDrive, Loader2, MemoryStick, Trash2, import { AppSettings } from "../lib/store"; import { useI18n } from "../lib/i18n"; +import { + isLocalLlmEndpoint, + localLlmDeleteSettingsPatch, + selectedLocalLlmModelId, +} from "../lib/localLlmSelection"; import qwenAvatar from "../assets/adapters/qwen.png"; interface LocalLlmModel { @@ -17,12 +22,6 @@ interface LocalLlmModel { downloaded: boolean; } -interface LocalLlmStatus { - running: boolean; - model_id: string | null; - base_url: string | null; -} - interface DownloadProgress { model_id: string; status: string; @@ -56,7 +55,6 @@ export function LocalLlmModels({ }) { const { t } = useI18n(); const [models, setModels] = useState([]); - const [status, setStatus] = useState(null); const [progress, setProgress] = useState>({}); const [busy, setBusy] = useState(null); const [deleting, setDeleting] = useState(null); @@ -65,12 +63,8 @@ export function LocalLlmModels({ const refresh = async (): Promise => { try { - const [list, runtime] = await Promise.all([ - invoke("list_local_llm_models"), - invoke("get_local_llm_status"), - ]); + const list = await invoke("list_local_llm_models"); setModels(list); - setStatus(runtime); } catch { /* keep last known state */ } @@ -152,6 +146,10 @@ export function LocalLlmModels({ setError(null); try { await invoke("delete_local_llm_model", { modelId: model.id }); + const cleanupPatch = localLlmDeleteSettingsPatch(settings, model.id); + if (cleanupPatch) { + update(cleanupPatch); + } setProgress((prev) => { const next = { ...prev }; delete next[model.id]; @@ -182,10 +180,8 @@ export function LocalLlmModels({ } }; - const activeModelId = status?.running ? status.model_id : null; - const usingLocalEndpoint = Boolean( - settings.llmEndpoint?.trim().includes("127.0.0.1"), - ); + const usingLocalEndpoint = isLocalLlmEndpoint(settings.llmEndpoint); + const selectedModelId = selectedLocalLlmModelId(settings); return (
@@ -241,7 +237,7 @@ export function LocalLlmModels({ prog?.status === "downloading" || prog?.status === "starting"; const isBusy = busy === model.id; const isDeleting = deleting === model.id; - const isActive = activeModelId === model.id && usingLocalEndpoint; + const isSelected = selectedModelId === model.id && usingLocalEndpoint; // Collapsed by default like the recognition cards; force-open while a // download runs so its progress bar stays visible. const open = expandedId === model.id || isDownloading; @@ -250,12 +246,12 @@ export function LocalLlmModels({ ? t("localLlm.status.deleting") : isDownloading ? t("localLlm.status.downloading") - : isActive + : isSelected ? t("localLlm.status.selected") : model.downloaded ? t("localLlm.status.ready") : t("localLlm.status.notDownloaded"); - const statusColor = isActive + const statusColor = isSelected ? "var(--success-bright)" : model.downloaded ? "var(--success-bright)" @@ -464,7 +460,7 @@ export function LocalLlmModels({ ) ) : ( <> - {!isActive && ( + {!isSelected && (