vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-15 19:35:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'upstream' into concedo_experimental

Browse source 
# Conflicts:
#	.github/workflows/build-cache.yml
#	.github/workflows/build-cmake-pkg.yml
#	.github/workflows/build-cpu.yml
#	.github/workflows/build-cuda-windows.yml
#	.github/workflows/build-sanitize.yml
#	.github/workflows/release.yml
#	.github/workflows/winget.yml
#	CMakeLists.txt
#	README.md
#	app/llama.cpp
#	cmake/llama-config.cmake.in
#	cmake/llama.pc.in
#	common/CMakeLists.txt
#	common/build-info.h
#	docs/backend/OPENVINO.md
#	docs/backend/SYCL.md
#	docs/preset.md
#	ggml/CMakeLists.txt
#	ggml/cmake/ggml-config.cmake.in
#	ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
#	ggml/src/ggml-hip/CMakeLists.txt
#	ggml/src/ggml-openvino/ggml-decoder.cpp
#	ggml/src/ggml-openvino/ggml-decoder.h
#	ggml/src/ggml-openvino/ggml-openvino-extra.cpp
#	ggml/src/ggml-openvino/ggml-openvino-extra.h
#	ggml/src/ggml-openvino/ggml-openvino.cpp
#	ggml/src/ggml-openvino/ggml-quants.cpp
#	ggml/src/ggml-openvino/ggml-quants.h
#	ggml/src/ggml-openvino/openvino/decoder.h
#	ggml/src/ggml-openvino/openvino/node_context.h
#	ggml/src/ggml-openvino/openvino/op/cpy.cpp
#	ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp
#	ggml/src/ggml-openvino/openvino/op/get_rows.cpp
#	ggml/src/ggml-openvino/openvino/op/l2_norm.cpp
#	ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp
#	ggml/src/ggml-openvino/openvino/op/repeat.cpp
#	ggml/src/ggml-openvino/openvino/op/reshape.cpp
#	ggml/src/ggml-openvino/openvino/op/rms_norm.cpp
#	ggml/src/ggml-openvino/openvino/op/rope.cpp
#	ggml/src/ggml-openvino/openvino/op/scale.cpp
#	ggml/src/ggml-openvino/openvino/op/set_rows.cpp
#	ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp
#	ggml/src/ggml-openvino/openvino/op/view.cpp
#	ggml/src/ggml-openvino/openvino/op_table.cpp
#	ggml/src/ggml-openvino/openvino/op_table.h
#	ggml/src/ggml-openvino/openvino/translate_session.cpp
#	ggml/src/ggml-openvino/openvino/utils.cpp
#	ggml/src/ggml-openvino/openvino/utils.h
#	ggml/src/ggml-openvino/utils.cpp
#	ggml/src/ggml-openvino/utils.h
#	ggml/src/ggml-sycl/common.hpp
#	ggml/src/ggml-sycl/concat.cpp
#	ggml/src/ggml-sycl/dmmv.cpp
#	ggml/src/ggml-sycl/element_wise.cpp
#	ggml/src/ggml-sycl/element_wise.hpp
#	ggml/src/ggml-sycl/fusion.cpp
#	ggml/src/ggml-sycl/fusion.hpp
#	ggml/src/ggml-sycl/gated_delta_net.cpp
#	ggml/src/ggml-sycl/gated_delta_net.hpp
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	ggml/src/ggml-sycl/mmvq.cpp
#	ggml/src/ggml-sycl/mmvq.hpp
#	scripts/sync-ggml.last
#	src/CMakeLists.txt
#	tests/test-backend-ops.cpp
#	tests/test-chat.cpp
#	tests/test-gguf.cpp
#	tests/test-quantize-stats.cpp
#	tools/cvector-generator/cvector-generator.cpp
#	tools/mtmd/CMakeLists.txt
#	tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte
This commit is contained in:
Concedo 2026-08-14 18:10:18 +08:00
commit 886a42d446
380 changed files with 5639 additions and 4009 deletions
Download patch file Download diff file Expand all files Collapse all files

46
.github/workflows/make-release.yml vendored Normal file
View file

@ -0,0 +1,46 @@
name: Make Release
on:
workflow_dispatch:
inputs:
dry_run:
description: 'Dry run - validate without creating the tag'
required: true
type: boolean
default: true
env:
GH_TOKEN: ${{ github.token }}
permissions:
contents: write
jobs:
make-release:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Run release checks
id: checks
run: bash scripts/make-release-checks.sh ${{ github.event.inputs.dry_run == 'true' && '--dry-run' || '' }}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
- name: Create release tag
if: ${{ github.event.inputs.dry_run == 'false' }}
run: |
VERSION="${{ steps.checks.outputs.version }}"
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git tag -a "${VERSION}" -m "Release ${VERSION}"
git push origin "${VERSION}"
echo "Created and pushed tag ${VERSION}"
- name: Dry run summary
if: ${{ github.event.inputs.dry_run == 'true' }}
run: |
echo "Dry run complete - all checks passed."
echo "Would have created tag: ${{ steps.checks.outputs.version }}"

62
common/arg.cpp
View file

@ -36,6 +36,7 @@
#include <regex>
#include <set>
#include <string>
#include <system_error>
#include <thread> // for hardware_concurrency
#include <vector>
@ -561,6 +562,15 @@ void common_models_handler_apply(common_models_handler & handler, common_params
}
}
// infer the speculative type from the draft GGUF metadata when none is requested
// note: reads only the first split - sharded drafts need an explicit --spec-type
if (spec_types_is_default(params) && !params.speculative.draft.mparams.path.empty()) {
const auto types_gguf = common_speculative_types_from_gguf(params.speculative.draft.mparams.path);
if (!types_gguf.empty()) {
params.speculative.types = types_gguf;
}
}
// when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full model
const bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() ||
!plan_spec.dflash.local_path.empty() ||
@ -705,12 +715,61 @@ void common_models_handler_apply(common_models_handler & handler, common_params
// CLI argument parsing functions
//
// apply config files (if present), a later file overrides an earlier one:
// 1. system-wide: /etc/llama.cpp/config.ini (%PROGRAMDATA%\llama.cpp\config.ini on windows)
// 2. user-level: ${XDG_CONFIG_HOME:-~/.config}/llama.cpp/config.ini (%APPDATA%\llama.cpp\config.ini on windows)
static void common_params_apply_system_config(common_params & params, llama_example ex) {
std::vector<std::string> paths;
#if defined(_WIN32)
const std::string program_data = common_get_env("PROGRAMDATA");
if (!program_data.empty()) {
paths.push_back(program_data + "\\llama.cpp\\config.ini");
}
#else
paths.push_back("/etc/llama.cpp/config.ini");
#endif
try {
paths.push_back(fs_get_config_directory() + "config.ini");
} catch (const std::exception & e) {
LOG_DBG("cannot read user-level config file, skipping: %s\n", e.what());
}
std::vector<std::string> found;
for (const auto & path : paths) {
std::error_code ec;
if (std::filesystem::exists(path, ec)) {
found.push_back(path);
}
}
if (found.empty()) {
return;
}
common_preset_context ctx(ex);
ctx.ignore_unknown_keys = true; // the same config file is shared by all programs
for (const auto & path : found) {
LOG_INF("using config file: %s\n", path.c_str());
common_preset global;
common_presets presets = ctx.load_from_ini(path, global);
global.apply_to_params(params);
auto it = presets.find(COMMON_PRESET_DEFAULT_NAME);
if (it != presets.end()) {
it->second.apply_to_params(params);
}
}
}
static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) {
common_params & params = ctx_arg.params;
// setup log directly from params.verbosity: see tools/cli/cli.cpp
common_log_set_verbosity_thold(params.verbosity);
// config file applies first, so env variables and CLI arguments override it
common_params_apply_system_config(params, ctx_arg.ex);
std::unordered_map<std::string, std::pair<common_arg *, bool>> arg_to_options;
for (auto & opt : ctx_arg.options) {
for (const auto & arg : opt.args) {
@ -1391,8 +1450,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
{"--version"},
"show version and build info",
[](common_params &) {
fprintf(stderr, "version: %d (%s)\n", llama_build_number(), llama_commit());
fprintf(stderr, "built with %s for %s\n", llama_compiler(), llama_build_target());
llama_print_build_info(llama_version());
exit(0);
}
));

6
common/build-info.cpp.in
View file

@ -29,7 +29,7 @@ const char * llama_build_info(void) {
return s.c_str();
}
void llama_print_build_info(void) {
fprintf(stderr, "%s: build = %d (%s)\n", __func__, llama_build_number(), llama_commit());
fprintf(stderr, "%s: built with %s for %s\n", __func__, llama_compiler(), llama_build_target());
void llama_print_build_info(const char * llama_version) {
fprintf(stderr, "version: %s (build %d, commit %s)\n", llama_version, llama_build_number(), llama_commit());
fprintf(stderr, "built with %s for %s\n", llama_compiler(), llama_build_target());
}

3
common/build-info.h
View file

@ -2,6 +2,7 @@
#define BUILD_INFO_H
#define LLAMA_BUILD_NUMBER 999
#define LLAMA_VERSION "1.0"
#define LLAMA_COMMIT "KOBOLDCPP"
#define LLAMA_COMPILER "KCPP"
#define LLAMA_TARGET "KCPP"
@ -32,7 +33,7 @@ static inline const char * llama_build_info(void) {
return s.c_str();
}
static inline void llama_print_build_info(void) {
static inline void llama_print_build_info(const char *) {
fprintf(stderr, "%s: build = %d (%s)\n", __func__, llama_build_number(), llama_commit());
fprintf(stderr, "%s: built with %s for %s\n", __func__, llama_compiler(), llama_build_target());
}

4
common/chat-peg-parser.cpp
View file

@ -594,9 +594,7 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls(
// Full argument: name="value" or name=value
auto arg_rule = tool_arg(
tool_arg_open(eps()) +
tool_arg_name(arg_name_parser) +
literal("=") +
tool_arg_open(tool_arg_name(arg_name_parser) + literal("=")) +
arg_value_parser +
tool_arg_close(eps())
);

133
common/common.cpp
View file

@ -1025,20 +1025,21 @@ std::string fs_get_cache_directory() {
std::string cache_directory = "";
auto ensure_trailing_slash = [](std::string p) {
// Make sure to add trailing slash
if (p.back() != DIRECTORY_SEPARATOR) {
if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {
p += DIRECTORY_SEPARATOR;
}
return p;
};
if (getenv("LLAMA_CACHE")) {
cache_directory = std::getenv("LLAMA_CACHE");
} else {
cache_directory = common_get_env("LLAMA_CACHE");
if (cache_directory.empty()) {
#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
defined(__OpenBSD__) || defined(__NetBSD__)
if (std::getenv("XDG_CACHE_HOME")) {
cache_directory = std::getenv("XDG_CACHE_HOME");
} else if (std::getenv("HOME")) {
cache_directory = std::getenv("HOME") + std::string("/.cache/");
const std::string xdg_cache_home = common_get_env("XDG_CACHE_HOME");
const std::string home = common_get_env("HOME");
if (!xdg_cache_home.empty()) {
cache_directory = xdg_cache_home;
} else if (!home.empty()) {
cache_directory = home + "/.cache/";
} else {
#if defined(__linux__)
/* no $HOME is defined, fallback to getpwuid */
@ -1053,9 +1054,16 @@ std::string fs_get_cache_directory() {
#endif /* defined(__linux__) */
}
#elif defined(__APPLE__)
cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");
cache_directory = common_get_env("HOME");
if (cache_directory.empty()) {
throw std::runtime_error("Failed to find $HOME directory");
}
cache_directory += "/Library/Caches/";
#elif defined(_WIN32)
cache_directory = std::getenv("LOCALAPPDATA");
cache_directory = common_get_env("LOCALAPPDATA");
if (cache_directory.empty()) {
throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
}
#elif defined(__EMSCRIPTEN__)
GGML_ABORT("not implemented on this platform");
#else
@ -1067,6 +1075,51 @@ std::string fs_get_cache_directory() {
return ensure_trailing_slash(cache_directory);
}
std::string fs_get_config_directory() {
std::string config_directory = "";
auto ensure_trailing_slash = [](std::string p) {
if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {
p += DIRECTORY_SEPARATOR;
}
return p;
};
#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
defined(__OpenBSD__) || defined(__NetBSD__) || defined(__APPLE__)
const std::string xdg_config_home = common_get_env("XDG_CONFIG_HOME");
const std::string home = common_get_env("HOME");
if (!xdg_config_home.empty()) {
config_directory = xdg_config_home;
} else if (!home.empty()) {
config_directory = home + "/.config/";
} else {
#if defined(__linux__)
/* no $HOME is defined, fallback to getpwuid */
struct passwd *pw = getpwuid(getuid());
if ((!pw) || (!pw->pw_dir)) {
throw std::runtime_error("Failed to find $HOME directory");
}
config_directory = std::string(pw->pw_dir) + std::string("/.config/");
#else
throw std::runtime_error("Failed to find $HOME directory");
#endif
}
#elif defined(_WIN32)
config_directory = common_get_env("APPDATA");
if (config_directory.empty()) {
throw std::runtime_error("Failed to find %APPDATA% directory");
}
#elif defined(__EMSCRIPTEN__)
// caller decides what to do when there is no config directory
throw std::runtime_error("not implemented on this platform");
#else
# error Unknown architecture
#endif
config_directory = ensure_trailing_slash(config_directory);
config_directory += "llama.cpp";
return ensure_trailing_slash(config_directory);
}
std::string fs_get_cache_file(const std::string & filename) {
GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);
std::string cache_directory = fs_get_cache_directory();
@ -1228,6 +1281,8 @@ struct common_init_result::impl {
// note: the order in which model, context, etc. are declared matters because their destructors will be called bottom-to-top
common_threadpools threadpools;
llama_model_ptr model;
llama_context_ptr context;
@ -1329,6 +1384,10 @@ common_init_result::common_init_result(common_params & params, bool model_only)
}
pimpl->context.reset(lctx);
set_process_priority(params.cpuparams.priority);
pimpl->threadpools.init(lctx, params);
}
llama_model * common_init_result::model() {
@ -1677,6 +1736,10 @@ struct llama_context_params common_context_params_to_llama(const common_params &
return cparams;
}
//
// Threadpool utils
//
struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params) {
struct ggml_threadpool_params tpp;
@ -1693,6 +1756,56 @@ struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const commo
return tpp;
}
common_threadpools::~common_threadpools() {
if (!free_fn) {
return;
}
free_fn(threadpool);
free_fn(threadpool_batch);
}
void common_threadpools::init(llama_context * ctx, const common_params & params) {
GGML_ASSERT(!threadpool);
GGML_ASSERT(!threadpool_batch);
COM_INF("llama threadpool init, n_threads = %d\n", (int) params.cpuparams.n_threads);
auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
if (!cpu_dev) {
COM_WRN("%s", "no CPU backend found\n");
return;
}
auto * reg = ggml_backend_dev_backend_reg(cpu_dev);
auto * ggml_threadpool_new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new");
free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free");
struct ggml_threadpool_params tpp_batch =
ggml_threadpool_params_from_cpu_params(params.cpuparams_batch);
struct ggml_threadpool_params tpp =
ggml_threadpool_params_from_cpu_params(params.cpuparams);
if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) {
threadpool_batch = ggml_threadpool_new_fn(&tpp_batch);
if (!threadpool_batch) {
COM_WRN("batch threadpool create failed : n_threads %d\n", tpp_batch.n_threads);
return;
}
// start the non-batch threadpool in the paused state
tpp.paused = true;
}
threadpool = ggml_threadpool_new_fn(&tpp);
if (!threadpool) {
COM_WRN("threadpool create failed : n_threads %d\n", tpp.n_threads);
free_fn(threadpool_batch);
threadpool_batch = nullptr;
return;
}
llama_attach_threadpool(ctx, threadpool, threadpool_batch);
}
//
// Batch utils
//

28
common/common.h
View file

@ -882,6 +882,7 @@ bool fs_is_directory(const std::string & path);
std::string fs_get_cache_directory();
std::string fs_get_cache_file(const std::string & filename);
std::string fs_get_config_directory();
struct common_file_info {
std::string path;
@ -929,9 +930,8 @@ using common_init_result_ptr = std::unique_ptr<common_init_result>;
common_init_result_ptr common_init_from_params(common_params & params, bool model_only = false);
struct llama_model_params common_model_params_to_llama ( common_params & params);
struct llama_context_params common_context_params_to_llama(const common_params & params);
struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params);
struct llama_model_params common_model_params_to_llama ( common_params & params);
struct llama_context_params common_context_params_to_llama(const common_params & params);
// clear LoRA adapters from context, then apply new list of adapters
void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora);
@ -942,6 +942,28 @@ std::string common_get_model_endpoint();
// for testing purposes
char * common_get_model_or_exit(int, char*[]);
//
// Threadpool utils
//
struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params);
struct common_threadpools {
common_threadpools() = default;
~common_threadpools();
common_threadpools(const common_threadpools &) = delete;
common_threadpools & operator=(const common_threadpools &) = delete;
void init(llama_context * ctx, const common_params & params);
private:
ggml_threadpool * threadpool = nullptr;
ggml_threadpool * threadpool_batch = nullptr;
decltype(ggml_threadpool_free) * free_fn = nullptr;
};
//
// Context utils
//

2
common/preset.cpp
View file

@ -322,6 +322,8 @@ common_presets common_preset_context::load_from_ini(const std::string & path, co
preset.options[opt] = value;
}
LOG_DBG("accepted option: %s = %s\n", key.c_str(), preset.options[opt].c_str());
} else if (ignore_unknown_keys) {
LOG_WRN("ignoring option '%s' from %s: not supported by this program\n", key.c_str(), path.c_str());
} else {
throw std::runtime_error(string_format(
"option '%s' not recognized in preset '%s'",

4
common/preset.h
View file

@ -59,6 +59,10 @@ struct common_preset_context {
bool filter_allowed_keys = false;
std::set<std::string> allowed_keys;
// if true, options unknown to the current example are skipped instead of being an error
// used for config files shared by all binaries, where each binary only knows a subset of options
bool ignore_unknown_keys = false;
// if only_remote_allowed is true, only accept whitelisted keys
common_preset_context(llama_example ex);

86
common/speculative.cpp
View file

@ -2,6 +2,7 @@
#include "common.h"
#include "ggml.h"
#include "ggml-cpp.h"
#include "llama.h"
#include "log.h"
#include "ngram-cache.cpp"
@ -912,6 +913,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
std::vector<common_sampler_ptr> smpls;
// backend sampler chain per seq, attached to ctx_dft
std::vector<llama_sampler *> backend_chains;
int32_t n_embd_dec = 0; // draft hidden size
int32_t n_embd_enc = 0; // target_layer_ids_n * target_hidden_size
int32_t n_embd_tgt = 0; // target model hidden size
@ -985,6 +989,22 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
s.reset(common_sampler_init(model_dft, sparams));
}
// offload draft sampling to the backend
backend_chains.assign(n_seq, nullptr);
if (this->params.backend_sampling) {
for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) {
llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params());
llama_sampler_chain_add(chain, llama_sampler_init_top_k(10));
if (!llama_set_sampler(ctx_dft, seq_id, chain)) {
SPC_WRN("backend offload failed for seq_id=%d; using CPU sampler\n", (int) seq_id);
llama_sampler_free(chain);
chain = nullptr;
}
backend_chains[seq_id] = chain;
}
}
// turn on extraction of the target layers' input embeddings
for (uint32_t k = 0; k < target_layer_ids_n; ++k) {
llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true);
@ -995,6 +1015,18 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
}
~common_speculative_impl_draft_dflash() override {
auto * ctx_dft = this->params.ctx_dft;
for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) backend_chains.size(); ++seq_id) {
if (backend_chains[seq_id] == nullptr) {
continue;
}
if (ctx_dft) {
llama_set_sampler(ctx_dft, seq_id, nullptr);
}
llama_sampler_free(backend_chains[seq_id]);
}
backend_chains.clear();
llama_batch_free(batch);
llama_batch_free(batch_inject);
}
@ -2196,6 +2228,43 @@ common_speculative_type common_speculative_type_from_name(const std::string & na
return it->second;
}
std::vector<common_speculative_type> common_speculative_types_from_gguf(const std::string & path) {
struct gguf_init_params gguf_params = {
/* .no_alloc = */ true,
/* .ctx = */ nullptr,
};
gguf_context_ptr gguf_ctx(gguf_init_from_file(path.c_str(), gguf_params));
if (!gguf_ctx) {
return {};
}
const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture");
if (arch_id < 0 || gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {
return {};
}
const std::string arch = gguf_get_val_str(gguf_ctx.get(), arch_id);
if (arch != "dflash") {
const uint32_t block_count = gguf_get_val_u32(gguf_ctx.get(), gguf_find_key(gguf_ctx.get(), (arch + ".block_count").c_str()));
if (gguf_find_tensor(gguf_ctx.get(), ("blk." + std::to_string(block_count - 1) + ".nextn.eh_proj.weight").c_str()) >= 0) {
return { COMMON_SPECULATIVE_TYPE_DRAFT_MTP };
}
return {};
}
// the Markov head distinguishes draft-dspark from draft-dflash
const auto type = gguf_find_tensor(gguf_ctx.get(), "markov_w1.weight") >= 0
? COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK
: COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH;
SPC_INF("auto-detected speculative type '%s' from the draft model metadata\n", common_speculative_type_to_str(type).c_str());
return { type };
}
static uint32_t common_get_enabled_speculative_configs(const std::vector<common_speculative_type> & configs) {
uint32_t result = 0;
for (size_t i = 0; i < configs.size(); i++) {
@ -2263,6 +2332,23 @@ common_params common_base_params_to_speculative(const common_params & params) {
result.n_outputs_max = params.n_parallel;
result.n_outputs_max_per_seq = 1;
// dflash/dspark decode the whole noise block in a single pass and sample every block position on the backend
// TODO: refactor such properties to be announced by the speculative types
// something like `struct common_speculative_type_props common_speculative_type_get_props(...);`
const bool has_block_draft = std::any_of(
params.speculative.types.begin(), params.speculative.types.end(),
[](common_speculative_type t) {
return t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
});
if (has_block_draft) {
// per-seq output positions: DFlash decodes anchor + n_max masks (n_max + 1); DSpark n_max -> +1 covers both
const int32_t per_seq = std::max(1, params_spec.n_max + 1);
result.n_outputs_max = params.n_parallel * per_seq;
if (params_spec.backend_sampling) {
result.n_outputs_max_per_seq = per_seq;
}
}
return result;
}

3
common/speculative.h
View file

@ -14,6 +14,9 @@ const char * common_speculative_all_types_str();
// parse user provided types
std::vector<enum common_speculative_type> common_speculative_types_from_names(const std::vector<std::string> & names);
// infer the spec types from the GGUF metadata of a draft model; empty if unknown
std::vector<enum common_speculative_type> common_speculative_types_from_gguf(const std::string & path);
// convert string to type
enum common_speculative_type common_speculative_type_from_name(const std::string & name);

49
docs/release.md Normal file
View file

@ -0,0 +1,49 @@
# Release process
llama.cpp uses [semantic versioning](https://semver.org) (`MAJOR.MINOR.PATCH`).
## Version bump guidelines
| Change type | Version component |
|---|---|
| Breaking change to the public C API (`include/llama.h`) | `MAJOR` |
| Backward-compatible features, model support, or API addition | `MINOR` |
| Bug fix with no API change | `PATCH` |
The version is set in the three variables at the top of the root `CMakeLists.txt`:
```cmake
set(LLAMA_VERSION_MAJOR 0)
set(LLAMA_VERSION_MINOR 1)
set(LLAMA_VERSION_PATCH 0)
```
_A version bump should be included in the PR that introduces the change, or in a
dedicated bump commit merged before the release is cut._
_TODO: add PR labels (`semver: patch`, `semver: minor`, `semver: major`) to help
identify which PRs require a version bump before cutting a release._
## Making a release
Releases are created by running the [make-release](.github/workflows/make-release.yml)
which is a manual workflow.
The workflow creates an annotated git tag (e.g. `v0.1.0`) and pushes it to the
remote. No GitHub Release object is created, the tag is the release artifact.
## Building a release
By default, `LLAMA_BUILD_IS_DEV=ON` which appends a `-dev` suffix to `LLAMA_VERSION`,
marking the build as a nightly/development build. Distributors building from a
release tag must pass `-DLLAMA_BUILD_IS_DEV=OFF` to produce a clean version string
(e.g. `0.1.0` instead of `0.1.0-dev`).
## How releases reach users
Currently releases are not published to github releases, only nightly/development
builds are available there. The way users can access releases are using the following
channels:
- **llama-install.sh** — downloads pre-built binaries built from the release tag.
- **Package managers** — consume the git tag directly.
- **Build from source** — users clone the repo and check out the tag.

3
examples/test-cmake/.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
llama-build-install
install
build

13
examples/test-cmake/CMakeLists.txt Normal file
View file

@ -0,0 +1,13 @@
cmake_minimum_required(VERSION 3.14)
project(llama-simple)
set(CMAKE_CXX_STANDARD 17)
find_package(llama 0.1.0 REQUIRED)
add_executable(test-cmake test-cmake.cpp)
target_link_libraries(test-cmake PRIVATE llama)
target_compile_definitions(test-cmake PRIVATE
LLAMA_BUILD_NUMBER=${LLAMA_BUILD_NUMBER}
LLAMA_BUILD_COMMIT="${LLAMA_BUILD_COMMIT}"
)

36
examples/test-cmake/README.md Normal file
View file

@ -0,0 +1,36 @@
## cmake-test
This is just for manually testing/developing of a llama.cpp installation to
enable troubleshooting issues and exploration. The idea is that this can be used
after making changes to llama.cpp installation cmake configuration and then
verify it locally.
### Usage
The following will configure, build, and install llama.cpp
Configuring/build/install:
```console
./build-install.sh
```
The above command will create a directory named `install` in the current directory
which will have the follwing files in its lib directory:
```console
(venv) $ ls install/lib/
cmake libggml.so libllama-common.so.0 libllama.so.0.1.0 llama.cpp
libggml-base.so libggml.so.0 libllama-common.so.0.1.0 libmtmd.so pkgconfig
libggml-base.so.0 libggml.so.0.19.0 libllama.so libmtmd.so.0
libggml-base.so.0.19.0 libllama-common.so libllama.so.0 libmtmd.so.0.1.0
```
Build/run this project using the installation created above:
```console
(venv) $ ./build.sh
-- Configuring done (0.0s)
-- Generating done (0.0s)
-- Build files have been written to: /path/to/llama.cpp/examples/test-cmake/build
[100%] Built target test-cmake
[test-cmake] Using llama.cpp version 0.1.0-dev-b10335
[test-cmake] Initializing backend...
load_backend: loaded CPU backend from /path/to/llama.cpp/examples/test-cmake/install/lib/llama.cpp/libggml-cpu-alderlake.so
[test-cmake] Backend initialized.
```

19
examples/test-cmake/build-install.sh Executable file
View file

@ -0,0 +1,19 @@
#!/bin/bash
set -e
rm -rf llama-build-install install
cmake --fresh -S ../../. -B llama-build-install -DCMAKE_BUILD_TYPE=Release \
-DBUILD_SHARED_LIBS=ON \
-DGGML_BACKEND_DL=ON \
-DGGML_CPU_ALL_VARIANTS=ON \
-DLLAMA_TESTS_INSTALL=OFF \
-DCMAKE_INSTALL_PREFIX="${PWD}/install" \
-DGGML_BACKEND_DIR="${PWD}/install/lib/llama.cpp" \
-DGGML_LIB_INSTALL_DIR="${PWD}/install/lib/llama.cpp" \
-DLLAMA_LIB_INSTALL_DIR="${PWD}/install/lib/llama.cpp" \
-DLLAMA_TOOLS_INSTALL=OFF
cmake --build llama-build-install --parallel 12
cmake --install llama-build-install

7
examples/test-cmake/build.sh Executable file
View file

@ -0,0 +1,7 @@
#!/bin/bash
set -e
cmake -S . -B build -DCMAKE_PREFIX_PATH="${PWD}/install"
cmake --build build
LD_LIBRARY_PATH="${PWD}/install/lib/llama.cpp:${PWD}/install/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ./build/test-cmake

12
examples/test-cmake/test-cmake.cpp Normal file
View file

@ -0,0 +1,12 @@
#include "llama.h"
#include <cstdio>
int main(void) {
printf("[test-cmake] version: %s, build: %d (%s)\n",
llama_version(), LLAMA_BUILD_NUMBER, LLAMA_BUILD_COMMIT);
printf("[test-cmake] Initializing backend...\n");
llama_backend_init();
printf("[test-cmake] Backend initialized.\n");
llama_backend_free();
return 0;
}

88
ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp
View file

@ -1,90 +1,12 @@
#include "ggml-backend-impl.h"
#include "ggml-feats.h"
#if defined(__aarch64__)
#if defined(__linux__)
#include <sys/auxv.h>
#elif defined(__APPLE__)
#include <sys/sysctl.h>
#endif
#if !defined(HWCAP_FPHP)
#define HWCAP_FPHP (1 << 9)
#endif
#if !defined(HWCAP_ASIMDHP)
#define HWCAP_ASIMDHP (1 << 10)
#endif
#if !defined(HWCAP_ASIMDDP)
#define HWCAP_ASIMDDP (1 << 20)
#endif
#if !defined(HWCAP_SVE)
#define HWCAP_SVE (1 << 22)
#endif
#if !defined(HWCAP2_SVE2)
#define HWCAP2_SVE2 (1 << 1)
#endif
#if !defined(HWCAP2_I8MM)
#define HWCAP2_I8MM (1 << 13)
#endif
#if !defined(HWCAP2_SME)
#define HWCAP2_SME (1 << 23)
#endif
struct aarch64_features {
// has_neon not needed, aarch64 has NEON guaranteed
bool has_dotprod = false;
bool has_fp16 = false;
bool has_sve = false;
bool has_sve2 = false;
bool has_i8mm = false;
bool has_sme = false;
bool has_sme2 = false;
aarch64_features() {
#if defined(__linux__)
uint32_t hwcap = getauxval(AT_HWCAP);
uint32_t hwcap2 = getauxval(AT_HWCAP2);
has_dotprod = !!(hwcap & HWCAP_ASIMDDP);
has_fp16 = !!(hwcap & HWCAP_FPHP) && !!(hwcap & HWCAP_ASIMDHP);
has_sve = !!(hwcap & HWCAP_SVE);
has_sve2 = !!(hwcap2 & HWCAP2_SVE2);
has_i8mm = !!(hwcap2 & HWCAP2_I8MM);
has_sme = !!(hwcap2 & HWCAP2_SME);
#elif defined(__APPLE__)
int oldp = 0;
size_t size = sizeof(oldp);
if (sysctlbyname("hw.optional.arm.FEAT_DotProd", &oldp, &size, NULL, 0) == 0) {
has_dotprod = static_cast<bool>(oldp);
}
if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &oldp, &size, NULL, 0) == 0) {
has_i8mm = static_cast<bool>(oldp);
}
if (sysctlbyname("hw.optional.arm.FEAT_SME", &oldp, &size, NULL, 0) == 0) {
has_sme = static_cast<bool>(oldp);
}
if (sysctlbyname("hw.optional.arm.FEAT_SME2", &oldp, &size, NULL, 0) == 0) {
has_sme2 = static_cast<bool>(oldp);
}
// Apple apparently does not implement SVE yet
#endif
}
};
#if defined(__aarch64__) || defined(_M_ARM64)
static int ggml_backend_cpu_aarch64_score() {
int score = 1;
aarch64_features af;
const ggml_feats_arch64_runtime_t af = ggml_feats_get_arch64_runtime();
GGML_UNUSED(af);
#ifdef GGML_USE_DOTPROD
if (!af.has_dotprod) { return 0; }
@ -116,4 +38,4 @@ static int ggml_backend_cpu_aarch64_score() {
GGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_aarch64_score)
# endif // defined(__aarch64__)
# endif // defined(__aarch64__) || defined(_M_ARM64)

5
ggml/src/ggml-cpu/ggml-cpu.c
View file

@ -3630,6 +3630,11 @@ struct ggml_cplan ggml_graph_plan(
n_threads = 1;
#endif
#if defined(__wasi__)
// WASI doesn't support parallelism yet
n_threads = 1;
#endif
size_t work_size = 0;
struct ggml_cplan cplan;

2
ggml/src/ggml-cpu/ops.cpp
View file

@ -8941,7 +8941,7 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
for (int tk = 0; tk < kv_tile; tk++) {
const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;
if (kv_type == GGML_TYPE_F16) {
ggml_fp16_to_fp32_row((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);
ggml_cpu_fp16_to_fp32((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);
} else {
memcpy(V32 + tk * DV, v_data, DV * sizeof(float));
}

166
ggml/src/ggml-feats.h Normal file
View file

@ -0,0 +1,166 @@
#pragma once
#if defined(__aarch64__) || defined(_M_ARM64)
#if defined(__linux__)
#include <sys/auxv.h>
#include <sys/prctl.h>
#if !defined(HWCAP2_SVE2)
#define HWCAP2_SVE2 (1ULL << 1)
#endif
#if !defined(HWCAP_FPHP)
#define HWCAP_FPHP (1 << 9)
#endif
#if !defined(HWCAP_ASIMDHP)
#define HWCAP_ASIMDHP (1 << 10)
#endif
#if !defined(HWCAP2_I8MM)
#define HWCAP2_I8MM (1ULL << 13)
#endif
#if !defined(HWCAP_ASIMDDP)
#define HWCAP_ASIMDDP (1 << 20)
#endif
#if !defined(HWCAP_SVE)
#define HWCAP_SVE (1 << 22)
#endif
#if !defined(HWCAP2_SME)
#define HWCAP2_SME (1ULL << 23)
#endif
#if !defined(HWCAP2_SME2)
#define HWCAP2_SME2 (1ULL << 37)
#endif
#if !defined(PR_SVE_GET_VL)
#define PR_SVE_GET_VL 51
#endif
#if !defined(PR_SVE_VL_LEN_MASK)
#define PR_SVE_VL_LEN_MASK 0xffff
#endif
#elif defined(__APPLE__)
#include <sys/sysctl.h>
#elif defined(_WIN32)
#include <windows.h>
#if !defined(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE)
#define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43
#endif
#if !defined(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE)
#define PF_ARM_SVE_INSTRUCTIONS_AVAILABLE 46
#endif
#if !defined(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE)
#define PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE 47
#endif
#if !defined(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE)
#define PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE 66
#endif
#if !defined(PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE)
#define PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE 67
#endif
#if !defined(PF_ARM_SME_INSTRUCTIONS_AVAILABLE)
#define PF_ARM_SME_INSTRUCTIONS_AVAILABLE 70
#endif
#if !defined(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE)
#define PF_ARM_SME2_INSTRUCTIONS_AVAILABLE 71
#endif
#endif
typedef struct ggml_feats_arch64_runtime {
bool has_dotprod;
bool has_fp16;
bool has_sve;
bool has_sve2;
bool has_i8mm;
bool has_sme;
bool has_sme2;
int sve_cnt;
} ggml_feats_arch64_runtime_t;
static inline ggml_feats_arch64_runtime_t ggml_feats_get_arch64_runtime(void) {
ggml_feats_arch64_runtime_t runtime_feat = {};
#if defined(__linux__)
const unsigned long hwcap = getauxval(AT_HWCAP);
const unsigned long hwcap2 = getauxval(AT_HWCAP2);
runtime_feat.has_dotprod = !!(hwcap & HWCAP_ASIMDDP);
runtime_feat.has_fp16 = !!(hwcap & HWCAP_FPHP) && !!(hwcap & HWCAP_ASIMDHP);;
runtime_feat.has_sve = !!(hwcap & HWCAP_SVE);
runtime_feat.has_sve2 = !!(hwcap2 & HWCAP2_SVE2);
runtime_feat.has_i8mm = !!(hwcap2 & HWCAP2_I8MM);
runtime_feat.has_sme = !!(hwcap2 & HWCAP2_SME);
runtime_feat.has_sme2 = !!(hwcap2 & HWCAP2_SME2);
if (runtime_feat.has_sve) {
const int vl = prctl(PR_SVE_GET_VL);
if (vl >= 0) {
runtime_feat.sve_cnt = vl & PR_SVE_VL_LEN_MASK;
}
}
#elif defined(__APPLE__)
int oldp = 0;
size_t size = sizeof(oldp);
if (sysctlbyname("hw.optional.arm.FEAT_DotProd", &oldp, &size, nullptr, 0) == 0) {
runtime_feat.has_dotprod = static_cast<bool>(oldp);
}
if (sysctlbyname("hw.optional.arm.FEAT_FP16", &oldp, &size, nullptr, 0) == 0) {
runtime_feat.has_fp16 = static_cast<bool>(oldp);
}
if (sysctlbyname("hw.optional.arm.FEAT_SVE", &oldp, &size, nullptr, 0) == 0) {
runtime_feat.has_sve = static_cast<bool>(oldp);
}
if (sysctlbyname("hw.optional.arm.FEAT_SVE2", &oldp, &size, nullptr, 0) == 0) {
runtime_feat.has_sve2 = static_cast<bool>(oldp);
}
if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &oldp, &size, nullptr, 0) == 0) {
runtime_feat.has_i8mm = static_cast<bool>(oldp);
}
if (sysctlbyname("hw.optional.arm.FEAT_SME", &oldp, &size, nullptr, 0) == 0) {
runtime_feat.has_sme = static_cast<bool>(oldp);
}
if (sysctlbyname("hw.optional.arm.FEAT_SME2", &oldp, &size, nullptr, 0) == 0) {
runtime_feat.has_sme2 = static_cast<bool>(oldp);
}
// Apple does not support userspace non-streaming SVE; keep SVE vector length unknown.
runtime_feat.sve_cnt = 0;
#elif defined (_WIN32)
runtime_feat.has_dotprod = IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE) != 0;
runtime_feat.has_fp16 = IsProcessorFeaturePresent(PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE) != 0;
runtime_feat.has_sve = IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) != 0;
runtime_feat.has_sve2 = IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE) != 0;
runtime_feat.has_i8mm = IsProcessorFeaturePresent(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE) != 0;
runtime_feat.has_sme = IsProcessorFeaturePresent(PF_ARM_SME_INSTRUCTIONS_AVAILABLE) != 0;
runtime_feat.has_sme2 = IsProcessorFeaturePresent(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE) != 0;
// Windows exposes SVE feature presence, but not the runtime SVE vector length here.
runtime_feat.sve_cnt = 0;
#endif
return runtime_feat;
}
#endif // defined(__aarch64__) || defined(_M_ARM64)

10
ggml/src/ggml-metal/ggml-metal-device.cpp
View file

@ -953,6 +953,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta
nr0 = N_R0_IQ4_XS;
smem = 32*sizeof(float);
} break;
case GGML_TYPE_TQ2_0:
{
nsg = N_SG_TQ2_0;
nr0 = N_R0_TQ2_0;
} break;
default:
{
GGML_LOG_ERROR("Asserting on type %d\n", (int) tsrc0);
@ -1182,6 +1187,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m
nr0 = N_R0_IQ4_XS;
smem = 32*sizeof(float);
} break;
case GGML_TYPE_TQ2_0:
{
nsg = N_SG_TQ2_0;
nr0 = N_R0_TQ2_0;
} break;
default:
{
GGML_LOG_ERROR("Asserting on type %d\n", (int)op->src[2]->type);

3
ggml/src/ggml-metal/ggml-metal-device.m
View file

@ -1413,6 +1413,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
case GGML_TYPE_Q5_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_IQ4_NL:
case GGML_TYPE_TQ2_0:
case GGML_TYPE_I32:
return true;
default:
@ -1441,6 +1442,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
case GGML_TYPE_Q5_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
case GGML_TYPE_TQ2_0:
switch (op->type) {
case GGML_TYPE_F32:
case GGML_TYPE_F16:
@ -1476,6 +1478,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
case GGML_TYPE_Q5_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_IQ4_NL:
case GGML_TYPE_TQ2_0:
return true;
default:
return false;

3
ggml/src/ggml-metal/ggml-metal-impl.h
View file

@ -87,6 +87,9 @@
#define N_R0_IQ4_XS 2
#define N_SG_IQ4_XS 2
#define N_R0_TQ2_0 4
#define N_SG_TQ2_0 2
// function constants offsets
#define FC_FLASH_ATTN_EXT_PAD 100
#define FC_FLASH_ATTN_EXT_BLK 200

209
ggml/src/ggml-metal/ggml-metal.metal
View file

@ -468,6 +468,34 @@ void quantize_iq4_nl(device const float * src, device block_iq4_nl & dst) {
dst.d = sumq2 > 0 ? sumqx/sumq2 : d;
}
void quantize_tq2_0(device const float * src, device block_tq2_0 & dst) {
#pragma METAL fp math_mode(safe)
float amax = 0.0f; // absolute max
for (int j = 0; j < QK_K; j++) {
const float v = src[j];
amax = MAX(amax, fabs(v));
}
const float d = amax;
const float id = d ? 1.0f/d : 0.0f;
dst.d = (half) d;
for (int j = 0; j < QK_K/4; j += 32) {
for (int m = 0; m < 32; ++m) {
uint8_t q = 0;
for (int n = 0; n < 4; ++n) {
// -1, 0, 1 -> 0, 1, 2
int xi = (int)round(src[m + n*32] * id) + 1;
q += (uint8_t)((xi & 3) << (2*n));
}
dst.qs[j + m] = q;
}
src += 4*32;
}
}
template <typename type4x4>
void dequantize_q4_1(device const block_q4_1 * xb, short il, thread type4x4 & reg) {
device const uint16_t * qs = ((device const uint16_t *)xb + 2);
@ -1021,6 +1049,25 @@ void dequantize_iq4_xs(device const block_iq4_xs * xb, short il, thread type4x4
}
}
template <typename type4x4>
void dequantize_tq2_0(device const block_tq2_0 * xb, short il, thread type4x4 & reg) {
device const uint8_t * qs = xb->qs;
const float d = xb->d;
float4x4 reg_f;
// 2 bits per element, 4 elements per byte, 128 elements per 32-byte group
const short base = il * 16;
for (int k = 0; k < 16; k++) {
const int i = base + k;
const int byte = ((i >> 7) & 1) * 32 + (i & 31);
const int l = (i >> 5) & 3;
reg_f[k/4][k%4] = d * (float)(((qs[byte] >> (2*l)) & 3) - 1);
}
reg = (type4x4) reg_f;
}
enum ggml_sort_order {
GGML_SORT_ORDER_ASC,
GGML_SORT_ORDER_DESC,
@ -8001,6 +8048,7 @@ template [[host_name("kernel_cpy_f32_q4_1")]] kernel cpy_f_q_t kernel_cpy_f32_
template [[host_name("kernel_cpy_f32_q5_0")]] kernel cpy_f_q_t kernel_cpy_f32_q<QK5_0, block_q5_0, quantize_q5_0>;
template [[host_name("kernel_cpy_f32_q5_1")]] kernel cpy_f_q_t kernel_cpy_f32_q<QK5_1, block_q5_1, quantize_q5_1>;
template [[host_name("kernel_cpy_f32_iq4_nl")]] kernel cpy_f_q_t kernel_cpy_f32_q<QK4_NL, block_iq4_nl, quantize_iq4_nl>;
template [[host_name("kernel_cpy_f32_tq2_0")]] kernel cpy_f_q_t kernel_cpy_f32_q<QK_K, block_tq2_0, quantize_tq2_0>;
template<typename T4x4, typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread T4x4 &)>
kernel void kernel_cpy_q_f32(
@ -8048,6 +8096,8 @@ template [[host_name("kernel_cpy_q5_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32<
template [[host_name("kernel_cpy_q5_1_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32<float4x4, block_q5_1, 2, dequantize_q5_1>;
template [[host_name("kernel_cpy_q8_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32<float4x4, block_q8_0, 2, dequantize_q8_0>;
template [[host_name("kernel_cpy_tq2_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32<float4x4, block_tq2_0, QK_NL, dequantize_tq2_0>;
template [[host_name("kernel_cpy_q1_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_q1_0, 8, dequantize_q1_0>;
template [[host_name("kernel_cpy_q2_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_q2_0, 4, dequantize_q2_0>;
template [[host_name("kernel_cpy_q4_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_q4_0, 2, dequantize_q4_0>;
@ -8056,6 +8106,8 @@ template [[host_name("kernel_cpy_q5_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<
template [[host_name("kernel_cpy_q5_1_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_q5_1, 2, dequantize_q5_1>;
template [[host_name("kernel_cpy_q8_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_q8_0, 2, dequantize_q8_0>;
template [[host_name("kernel_cpy_tq2_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_tq2_0, QK_NL, dequantize_tq2_0>;
template<typename T>
kernel void kernel_concat(
constant ggml_metal_kargs_concat & args,
@ -9822,6 +9874,121 @@ kernel void kernel_mul_mv_mxfp4_f32(
kernel_mul_mv_mxfp4_f32_impl<N_R0_MXFP4, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
}
template<int nr0, typename args_t>
void kernel_mul_mv_tq2_0_f32_impl(
args_t args,
device const char * src0,
device const char * src1,
device char * dst,
threadgroup char * shmem,
uint3 tgpig,
ushort tiisg,
ushort sgitg) {
const short NSG = FC_mul_mv_nsg;
const int nb = args.ne00/QK_K;
const int r0 = tgpig.x;
const int r1 = tgpig.y;
const int im = tgpig.z;
const int first_row = (r0 * NSG + sgitg) * nr0;
const uint i12 = im%FC_mul_mv_ne12;
const uint i13 = im/FC_mul_mv_ne12;
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
device const float * y = (device const float *) (src1 + offset1);
device const block_tq2_0 * ax[nr0];
for (int row = 0; row < nr0; ++row) {
const uint64_t offset0 = (first_row + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
ax[row] = (device const block_tq2_0 *) ((device char *) src0 + offset0);
}
float sumf[nr0] = {0.f};
// 8 threads per block, NBLOCK blocks per pass, 2 halves per block per pass
constexpr short NBLOCK = 4;
constexpr short NB = N_SIMDWIDTH/NBLOCK; // threads per block
const short blk = tiisg / NB; // 0..NBLOCK-1, block handled by this thread
const short htg = tiisg % NB; // 0..NB-1, thread within block (0..7)
// byte and y base offsets within the block (32 elements per thread, 4 per byte)
device const float4 * yb4 = (device const float4 *)(y + 4*htg + blk*QK_K);
// hoisted per-byte coefficients (from y) and total y-sum, shared across rows
// ref: https://github.com/ggml-org/llama.cpp/pull/26980
float4 coef[4];
for (int ib = blk; ib < nb; ib += NBLOCK) {
FOR_UNROLL (short h0 = 0; h0 < 2; ++h0) {
const float4 y0 = yb4[ 0 + 32*h0];
const float4 y1 = yb4[ 8 + 32*h0];
const float4 y2 = yb4[16 + 32*h0];
const float4 y3 = yb4[24 + 32*h0];
float sumy = 0.f;
FOR_UNROLL (short j = 0; j < 4; ++j) {
coef[j] = float4(
y0[j],
y1[j] - 4.0f*y0[j],
y2[j] - 4.0f*y1[j],
y3[j] - 4.0f*y2[j]);
sumy += (y0[j] + y1[j]) + (y2[j] + y3[j]);
}
FOR_UNROLL (short row = 0; row < nr0; ++row) {
device const block_tq2_0 & xb = ax[row][ib];
device const uchar * qs = xb.qs + 4*htg + 32*h0;
float sum = -sumy;
FOR_UNROLL (short j = 0; j < 4; ++j) {
// express the 2-bit field shifts (v>>2, v>>4, v>>6) as float floor ops
const float v = (float)qs[j];
const float f0 = v;
const float f1 = floor(v*0.25f); // v>>2
const float f2 = floor(v*0.0625); // v>>4
const float f3 = floor(v*0.015625); // v>>6
sum += coef[j][0]*f0 + coef[j][1]*f1 + coef[j][2]*f2 + coef[j][3]*f3;
}
sumf[row] += xb.d * sum;
}
}
yb4 += QK_K * NBLOCK / 4;
}
device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
for (int row = 0; row < nr0; ++row) {
const float tot = simd_sum(sumf[row]);
if (tiisg == 0 && first_row + row < args.ne01) {
dst_f32[first_row + row] = tot;
}
}
}
[[host_name("kernel_mul_mv_tq2_0_f32")]]
kernel void kernel_mul_mv_tq2_0_f32(
constant ggml_metal_kargs_mul_mv & args,
device const char * src0,
device const char * src1,
device char * dst,
uint3 tgpig[[threadgroup_position_in_grid]],
ushort tiisg[[thread_index_in_simdgroup]],
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
kernel_mul_mv_tq2_0_f32_impl<N_R0_TQ2_0, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
}
template<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread float4x4 &)>
kernel void kernel_get_rows_q(
constant ggml_metal_kargs_get_rows & args,
@ -9915,6 +10082,38 @@ template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_q_t kernel_get
template [[host_name("kernel_get_rows_iq1_m")]] kernel get_rows_q_t kernel_get_rows_q<block_iq1_m, QK_NL, dequantize_iq1_m>;
template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_q_t kernel_get_rows_q<block_iq4_nl, 2, dequantize_iq4_nl>;
template [[host_name("kernel_get_rows_iq4_xs")]] kernel get_rows_q_t kernel_get_rows_q<block_iq4_xs, QK_NL, dequantize_iq4_xs>;
template [[host_name("kernel_get_rows_tq2_0")]] kernel get_rows_q_t kernel_get_rows_q<block_tq2_0, QK_NL, dequantize_tq2_0>;
template<typename TS, typename TI, short QK, typename block_q, void (*quantize_func)(device const float *, device block_q &)>
kernel void kernel_set_rows_q(
constant ggml_metal_kargs_set_rows & args,
device const void * src0,
device const void * src1,
device float * dst,
uint3 tgpig[[threadgroup_position_in_grid]],
uint tiitg[[thread_index_in_threadgroup]],
uint3 tptg [[threads_per_threadgroup]]) {
const int32_t i03 = tgpig.z;
const int32_t i02 = tgpig.y;
const int32_t i12 = i03%args.ne12;
const int32_t i11 = i02%args.ne11;
const int32_t i01 = tgpig.x*tptg.y + tiitg/tptg.x;
if (i01 >= args.ne01) {
return;
}
const int32_t i10 = i01;
const TI i1 = ((const device TI *) ((const device char *) src1 + i10*args.nb10 + i11*args.nb11 + i12*args.nb12))[0];
device block_q * dst_row = ( device block_q *) (( device char *) dst + i1*args.nb1 + i02*args.nb2 + i03*args.nb3);
const device TS * src_row = (const device TS *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03);
for (int ind = tiitg%tptg.x; ind < args.nk0; ind += tptg.x) {
quantize_func(src_row + QK*ind, dst_row[ind]);
}
}
template<typename TS, typename TI, typename block_q, void (*quantize_func)(device const float *, device block_q &)>
kernel void kernel_set_rows_q32(
@ -10011,6 +10210,11 @@ template [[host_name("kernel_set_rows_f32_i32_q5_1")]] kernel set_rows_q32_t k
template [[host_name("kernel_set_rows_f32_i64_iq4_nl")]] kernel set_rows_q32_t kernel_set_rows_q32<float, int64_t, block_iq4_nl, quantize_iq4_nl>;
template [[host_name("kernel_set_rows_f32_i32_iq4_nl")]] kernel set_rows_q32_t kernel_set_rows_q32<float, int32_t, block_iq4_nl, quantize_iq4_nl>;
typedef decltype(kernel_set_rows_q<float, int64_t, QK_K, block_tq2_0, quantize_tq2_0>) set_rows_qK_t;
template [[host_name("kernel_set_rows_f32_i64_tq2_0")]] kernel set_rows_qK_t kernel_set_rows_q<float, int64_t, QK_K, block_tq2_0, quantize_tq2_0>;
template [[host_name("kernel_set_rows_f32_i32_tq2_0")]] kernel set_rows_qK_t kernel_set_rows_q<float, int32_t, QK_K, block_tq2_0, quantize_tq2_0>;
kernel void kernel_diag_f32(
constant ggml_metal_kargs_diag & args,
device const char * src0,
@ -10786,6 +10990,7 @@ template [[host_name("kernel_mul_mm_iq1_s_f32")]] kernel mul_mm_t kernel_mul_m
template [[host_name("kernel_mul_mm_iq1_m_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq1_m, QK_NL, dequantize_iq1_m, float, float4x4, float, float2x4>;
template [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_nl, 2, dequantize_iq4_nl, float, float4x4, float, float2x4>;
template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_xs, QK_NL, dequantize_iq4_xs, float, float4x4, float, float2x4>;
template [[host_name("kernel_mul_mm_tq2_0_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_tq2_0, QK_NL, dequantize_tq2_0, float, float4x4, float, float2x4>;
template [[host_name("kernel_mul_mm_f32_f16")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, float4x4, 1, dequantize_f32, float, float4x4, half, half2x4>;
template [[host_name("kernel_mul_mm_f16_f16")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, half4x4, 1, dequantize_f16, half, half4x4, half, half2x4>;
@ -10811,6 +11016,7 @@ template [[host_name("kernel_mul_mm_iq1_s_f16")]] kernel mul_mm_t kernel_mul_m
template [[host_name("kernel_mul_mm_iq1_m_f16")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq1_m, QK_NL, dequantize_iq1_m, float, float4x4, half, half2x4>;
template [[host_name("kernel_mul_mm_iq4_nl_f16")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_nl, 2, dequantize_iq4_nl, float, float4x4, half, half2x4>;
template [[host_name("kernel_mul_mm_iq4_xs_f16")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_xs, QK_NL, dequantize_iq4_xs, float, float4x4, half, half2x4>;
template [[host_name("kernel_mul_mm_tq2_0_f16")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_tq2_0, QK_NL, dequantize_tq2_0, float, float4x4, half, half2x4>;
//
// indirect matrix-matrix multiplication
@ -10845,6 +11051,7 @@ template [[host_name("kernel_mul_mm_id_iq1_s_f32")]] kernel mul_mm_id kernel_m
template [[host_name("kernel_mul_mm_id_iq1_m_f32")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq1_m, QK_NL, dequantize_iq1_m, float, float4x4, float, float2x4>;
template [[host_name("kernel_mul_mm_id_iq4_nl_f32")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_nl, 2, dequantize_iq4_nl, float, float4x4, float, float2x4>;
template [[host_name("kernel_mul_mm_id_iq4_xs_f32")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_xs, QK_NL, dequantize_iq4_xs, float, float4x4, float, float2x4>;
template [[host_name("kernel_mul_mm_id_tq2_0_f32")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_tq2_0, QK_NL, dequantize_tq2_0, float, float4x4, float, float2x4>;
template [[host_name("kernel_mul_mm_id_f32_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, float4x4, 1, dequantize_f32, float, float4x4, half, half2x4>;
template [[host_name("kernel_mul_mm_id_f16_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, half4x4, 1, dequantize_f16, half, half4x4, half, half2x4>;
@ -10870,6 +11077,7 @@ template [[host_name("kernel_mul_mm_id_iq1_s_f16")]] kernel mul_mm_id kernel_m
template [[host_name("kernel_mul_mm_id_iq1_m_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq1_m, QK_NL, dequantize_iq1_m, float, float4x4, half, half2x4>;
template [[host_name("kernel_mul_mm_id_iq4_nl_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_nl, 2, dequantize_iq4_nl, float, float4x4, half, half2x4>;
template [[host_name("kernel_mul_mm_id_iq4_xs_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_xs, QK_NL, dequantize_iq4_xs, float, float4x4, half, half2x4>;
template [[host_name("kernel_mul_mm_id_tq2_0_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_tq2_0, QK_NL, dequantize_tq2_0, float, float4x4, half, half2x4>;
//
// matrix-vector multiplication
@ -11027,6 +11235,7 @@ template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t
template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_s_f32_impl <N_R0_IQ2_S>>>;
template [[host_name("kernel_mul_mv_id_iq4_nl_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_nl_f32_impl <N_R0_IQ4_NL>>>;
template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_xs_f32_impl <N_R0_IQ4_XS>>>;
template [[host_name("kernel_mul_mv_id_tq2_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_tq2_0_f32_impl <N_R0_TQ2_0>>>;
kernel void kernel_pool_2d_max_f32(
constant ggml_metal_kargs_pool_2d & args,

15
ggml/src/gguf.cpp
View file

@ -648,6 +648,13 @@ static struct gguf_context * gguf_init_from_reader(const struct gguf_reader & gr
GGML_ASSERT(int64_t(ctx->kv.size()) == n_kv);
const int alignment_idx = gguf_find_key(ctx, GGUF_KEY_GENERAL_ALIGNMENT);
if (alignment_idx != -1 && gguf_get_kv_type(ctx, alignment_idx) != GGUF_TYPE_UINT32) {
GGML_LOG_ERROR("%s: key '%s' must be of type %s but is %s\n",
__func__, GGUF_KEY_GENERAL_ALIGNMENT, gguf_type_name(GGUF_TYPE_UINT32),
gguf_type_name(gguf_get_kv_type(ctx, alignment_idx)));
gguf_free(ctx);
return nullptr;
}
ctx->alignment = alignment_idx == -1 ? GGUF_DEFAULT_ALIGNMENT : gguf_get_val_u32(ctx, alignment_idx);
if (ctx->alignment == 0 || (ctx->alignment & (ctx->alignment - 1)) != 0) {
@ -726,9 +733,11 @@ static struct gguf_context * gguf_init_from_reader(const struct gguf_reader & gr
}
// check that the total number of elements is representable
if (ok && ((INT64_MAX/info.t.ne[1] <= info.t.ne[0]) ||
(INT64_MAX/info.t.ne[2] <= info.t.ne[0]*info.t.ne[1]) ||
(INT64_MAX/info.t.ne[3] <= info.t.ne[0]*info.t.ne[1]*info.t.ne[2]))) {
// (a zero-element tensor is trivially representable; the guard also avoids a division by zero below)
if (ok && ggml_nelements(&info.t) > 0 &&
((INT64_MAX/info.t.ne[1] <= info.t.ne[0]) ||
(INT64_MAX/info.t.ne[2] <= info.t.ne[0]*info.t.ne[1]) ||
(INT64_MAX/info.t.ne[3] <= info.t.ne[0]*info.t.ne[1]*info.t.ne[2]))) {
GGML_LOG_ERROR("%s: total number of elements in tensor '%s' with shape "
"(%" PRIi64 ", %" PRIi64 ", %" PRIi64 ", %" PRIi64 ") is >= %" PRIi64 "\n",

3
include/llama.h
View file

@ -460,6 +460,8 @@ extern "C" {
// lora adapter
struct llama_adapter_lora;
LLAMA_API const char * llama_version(void);
// Helpers for getting default parameters
// TODO: update API to start accepting pointers to params structs (https://github.com/ggml-org/llama.cpp/discussions/9172)
LLAMA_API struct llama_model_params llama_model_default_params(void);
@ -886,6 +888,7 @@ extern "C" {
const llama_token * tokens,
size_t n_token_count);
// If tokens_out is NULL, only the token count is reported through n_token_count_out and no state is loaded
LLAMA_API size_t llama_state_seq_load_file(
struct llama_context * ctx,
const char * filepath,

11
src/llama-context.cpp
View file

@ -3120,6 +3120,17 @@ size_t llama_context::state_seq_load_file(llama_seq_id seq_id, const char * file
{
const uint32_t n_token_count = file.read_u32();
if (tokens_out == nullptr) {
const size_t n_token_max = (file.size() - file.tell()) / sizeof(llama_token);
if (n_token_count > n_token_max) {
LLAMA_LOG_ERROR("%s: token count in sequence state file exceeds the file size! %u > %zu\n", __func__, n_token_count, n_token_max);
return 0;
}
*n_token_count_out = n_token_count;
return file.tell();
}
if (n_token_count > n_token_capacity) {
LLAMA_LOG_ERROR("%s: token count in sequence state file exceeded capacity! %u > %zu\n", __func__, n_token_count, n_token_capacity);
return 0;

5
src/llama.cpp
View file

@ -31,6 +31,7 @@
#include "ggml-cpp.h"
#include "ggml-backend.h"
#include "gguf.h"
#include "build-info.h"
#include <algorithm>
#include <cassert>
@ -135,6 +136,10 @@ bool llama_supports_rpc(void) {
return ggml_backend_reg_by_name("RPC") != nullptr;
}
const char * llama_version(void) {
return LLAMA_VERSION;
}
void llama_backend_init(void) {
ggml_time_init();

13
src/models/dflash.cpp
View file

@ -14,11 +14,14 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) {
hparams.n_embd_inp_enc_impl = (uint32_t) target_layer_ids.size() * hparams.n_embd;
LLAMA_LOG_INFO("%s: DFlash extract_layers = [", __func__);
for (size_t i = 0; i < target_layer_ids.size(); ++i) {
LLAMA_LOG_INFO("%d%s", target_layer_ids[i], i + 1 < target_layer_ids.size() ? ", " : "");
std::string layers;
const char * sep = "";
for (const auto id : target_layer_ids) {
layers += sep;
layers += std::to_string(id);
sep = ", ";
}
LLAMA_LOG_INFO("]\n");
LLAMA_LOG_INFO("%s: DFlash extract_layers = [%s]\n", __func__, layers.c_str());
// DeepSeek-V4 DSpark backbone: stages are full DSV4 blocks, uniform sliding window (the draft KV ring)
ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult, false);
@ -66,7 +69,7 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) {
// DFlash has a single rope, so the SWA rope == main rope.
if (ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false) && hparams.n_swa > 0) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());
ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
}

44
tools/completion/completion.cpp
View file

@ -161,47 +161,6 @@ int llama_completion(int argc, char ** argv) {
// start measuring performance timings from here
llama_perf_context_reset(ctx);
LOG_INF("%s: llama threadpool init, n_threads = %d\n", __func__, (int) params.cpuparams.n_threads);
auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
if (!cpu_dev) {
LOG_ERR("%s: no CPU backend found\n", __func__);
return 1;
}
auto * reg = ggml_backend_dev_backend_reg(cpu_dev);
auto * ggml_threadpool_new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new");
auto * ggml_threadpool_free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free");
struct ggml_threadpool_params tpp_batch =
ggml_threadpool_params_from_cpu_params(params.cpuparams_batch);
struct ggml_threadpool_params tpp =
ggml_threadpool_params_from_cpu_params(params.cpuparams);
if (!set_process_priority(params.cpuparams.priority)) {
LOG_ERR("%s: error: failed to set process priority\n", __func__);
return 1;
}
struct ggml_threadpool * threadpool_batch = NULL;
if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) {
threadpool_batch = ggml_threadpool_new_fn(&tpp_batch);
if (!threadpool_batch) {
LOG_ERR("%s: batch threadpool create failed : n_threads %d\n", __func__, tpp_batch.n_threads);
return 1;
}
// start the non-batch threadpool in the paused state
tpp.paused = true;
}
struct ggml_threadpool * threadpool = ggml_threadpool_new_fn(&tpp);
if (!threadpool) {
LOG_ERR("%s: threadpool create failed : n_threads %d\n", __func__, tpp.n_threads);
return 1;
}
llama_attach_threadpool(ctx, threadpool, threadpool_batch);
const int n_ctx_train = llama_model_n_ctx_train(model);
const int n_ctx = llama_n_ctx(ctx);
@ -994,8 +953,5 @@ int llama_completion(int argc, char ** argv) {
llama_backend_free();
ggml_threadpool_free_fn(threadpool);
ggml_threadpool_free_fn(threadpool_batch);
return 0;
}

2
tools/gguf-split/gguf-split.cpp
View file

@ -106,7 +106,7 @@ static void split_params_parse_ex(int argc, const char ** argv, split_params & p
split_print_usage(argv[0]);
exit(0);
} else if (arg == "--version") {
fprintf(stderr, "version: %d (%s)\n", llama_build_number(), llama_commit());
fprintf(stderr, "version: %s (build %d, commit %s)\n", llama_version(), llama_build_number(), llama_commit());
fprintf(stderr, "built with %s for %s\n", llama_compiler(), llama_build_target());
exit(0);
} else if (arg == "--dry-run") {

2
tools/quantize/quantize.cpp
View file

@ -610,7 +610,7 @@ int llama_quantize(int argc, char ** argv) {
}
}
llama_print_build_info();
llama_print_build_info(llama_version());
if (params.dry_run) {
fprintf(stderr, "%s: calculating quantization size for '%s' as %s", __func__, fname_inp.c_str(), ftype_str.c_str());

212
tools/server/server-common.cpp
View file

@ -13,6 +13,8 @@
#include <sstream>
#include <fstream>
#include <limits>
#include <cstring>
#include <type_traits>
json format_error_response(const std::string & message, const enum error_type type) {
std::string type_str;
@ -58,6 +60,33 @@ json format_error_response(const std::string & message, const enum error_type ty
};
}
//
// server_slot_stats
//
json server_slot_stats::to_json() const {
json base = {
{"cache_n", n_prompt_cached},
{"prompt_n", n_prompt_processed},
{"prompt_ms", t_prompt_ms()},
{"prompt_per_token_ms", t_prompt_per_token_ms()},
{"prompt_per_second", n_prompt_tps()},
{"predicted_n", n_gen},
{"predicted_ms", t_gen_ms()},
{"predicted_per_token_ms", t_gen_per_token_ms()},
{"predicted_per_second", n_gen_tps()},
};
if (n_draft_tokens > 0) {
base["draft_n"] = n_draft_tokens;
base["draft_n_accepted"] = n_draft_accepted;
}
return base;
}
//
// random string / id
//
@ -235,6 +264,102 @@ static inline raw_buffer base64_decode(const std::string & encoded_string) {
// server_tokens implementation
//
namespace {
constexpr uint32_t SERVER_TOKENS_STATE_VERSION = 1;
uint32_t server_tokens_state_u32(size_t value) {
if (value > std::numeric_limits<uint32_t>::max()) {
throw std::runtime_error("Server tokens state is too large");
}
return value;
}
class server_tokens_state_writer {
public:
template <typename T>
void write(T value) {
static_assert(std::is_trivially_copyable<T>::value, "T must be trivially copyable");
const auto * ptr = reinterpret_cast<const char *>(&value);
data.insert(data.end(), ptr, ptr + sizeof(value));
}
template <typename T>
void write(const std::vector<T> & values) {
static_assert(std::is_trivially_copyable<T>::value, "T must be trivially copyable");
write(server_tokens_state_u32(values.size()));
if (values.empty()) {
return;
}
const auto * ptr = reinterpret_cast<const char *>(values.data());
data.insert(data.end(), ptr, ptr + values.size() * sizeof(T));
}
void write_media_chunk(const mtmd_input_chunk * chunk) {
size_t chunk_size = 0;
if (mtmd_input_chunk_save(chunk, nullptr, 0, &chunk_size) != 0 || chunk_size == 0) {
throw std::runtime_error("Cannot serialize media chunk in server tokens");
}
std::vector<char> chunk_data(server_tokens_state_u32(chunk_size));
if (mtmd_input_chunk_save(chunk, chunk_data.data(), chunk_data.size(), nullptr) != 0) {
throw std::runtime_error("Cannot serialize media chunk in server tokens");
}
write(chunk_data);
}
std::vector<char> take() {
data.resize((data.size() + sizeof(llama_token) - 1) / sizeof(llama_token) * sizeof(llama_token), 0);
return std::move(data);
}
private:
std::vector<char> data;
};
class server_tokens_state_reader {
public:
server_tokens_state_reader(const char * data, size_t size) : data(data), size(size) {}
template <typename T>
T read() {
static_assert(std::is_trivially_copyable<T>::value, "T must be trivially copyable");
if (size - pos < sizeof(T)) {
throw std::runtime_error("Unexpected end of server tokens state");
}
T value;
std::memcpy(&value, data + pos, sizeof(value));
pos += sizeof(value);
return value;
}
template <typename T>
std::vector<T> read_vector() {
static_assert(std::is_trivially_copyable<T>::value, "T must be trivially copyable");
const uint32_t n_values = read<uint32_t>();
// reject before resizing, so that a small corrupted payload cannot request a huge allocation
if (n_values > remaining() / sizeof(T)) {
throw std::runtime_error("Unexpected end of server tokens state");
}
std::vector<T> values(n_values);
if (n_values > 0) {
std::memcpy(values.data(), data + pos, values.size() * sizeof(T));
pos += values.size() * sizeof(T);
}
return values;
}
size_t remaining() const {
return size - pos;
}
private:
const char * data;
size_t size;
size_t pos = 0;
};
} // namespace
server_tokens::server_tokens(mtmd::input_chunks & mtmd_chunks, bool has_mtmd) : has_mtmd(has_mtmd) {
for (size_t i = 0; i < mtmd_chunks.size(); ++i) {
push_back(mtmd_chunks[i]);
@ -408,6 +533,73 @@ const llama_tokens & server_tokens::get_tokens() const {
return tokens;
}
std::vector<char> server_tokens::serialize() const {
static_assert(sizeof(llama_token) == sizeof(uint32_t), "unexpected llama_token size");
server_tokens_state_writer writer;
writer.write((llama_token) LLAMA_TOKEN_NULL);
writer.write(SERVER_TOKENS_STATE_VERSION);
writer.write(tokens);
std::vector<uint32_t> media_keys;
media_keys.reserve(map_idx_to_media.size());
for (const auto & item : map_idx_to_media) {
media_keys.push_back(server_tokens_state_u32(item.first));
}
writer.write(media_keys);
for (const auto & item : map_idx_to_media) {
writer.write_media_chunk(item.second.get());
}
return writer.take();
}
server_tokens server_tokens::deserialize(const llama_tokens & packed, bool has_mtmd) {
static_assert(sizeof(llama_token) == sizeof(uint32_t), "unexpected llama_token size");
if (packed.empty() || packed[0] != LLAMA_TOKEN_NULL) {
// plain token list, as written by older versions
return server_tokens(packed, has_mtmd);
}
server_tokens_state_reader reader(reinterpret_cast<const char *>(packed.data()), packed.size() * sizeof(llama_token));
reader.read<llama_token>(); // format marker
if (reader.read<uint32_t>() != SERVER_TOKENS_STATE_VERSION) {
throw std::runtime_error("Unsupported server tokens state version");
}
const llama_tokens tokens = reader.read_vector<llama_token>();
// the media start indices, followed by the media chunks in the same order
const std::vector<uint32_t> media_keys = reader.read_vector<uint32_t>();
if (!media_keys.empty() && !has_mtmd) {
throw std::runtime_error("Cannot restore media tokens without an mmproj");
}
server_tokens result(tokens, has_mtmd);
for (const uint32_t key : media_keys) {
const size_t start_idx = key;
const std::vector<char> chunk_data = reader.read_vector<char>();
if (chunk_data.empty()) {
throw std::runtime_error("Cannot load media chunk from server tokens state");
}
mtmd::input_chunk_ptr chunk(mtmd_input_chunk_load(chunk_data.data(), chunk_data.size()));
if (!chunk) {
throw std::runtime_error("Cannot load media chunk from server tokens state");
}
result.map_idx_to_media[start_idx] = std::move(chunk);
}
if (reader.remaining() >= sizeof(llama_token)) {
throw std::runtime_error("Trailing data in server tokens state");
}
return result;
}
llama_tokens server_tokens::get_text_tokens() const {
llama_tokens res;
res.reserve(tokens.size());
@ -530,14 +722,28 @@ bool server_tokens::validate(const struct llama_context * ctx) const {
const llama_model * model = llama_get_model(ctx);
const llama_vocab * vocab = llama_model_get_vocab(model);
const int32_t n_vocab = llama_vocab_n_tokens(vocab);
size_t n_media = 0;
for (size_t i = 0; i < tokens.size(); ++i) {
const auto & t = tokens[i];
if (t == LLAMA_TOKEN_NULL) {
try {
const auto & chunk = find_chunk(i);
size_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk.get());
i += n_tokens - 1; // will be +1 by the for loop
if (mtmd_input_chunk_get_type(chunk.get()) == MTMD_INPUT_CHUNK_TYPE_TEXT) {
return false;
}
const size_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk.get());
const llama_pos n_pos = mtmd_input_chunk_get_n_pos(chunk.get());
if (n_tokens == 0 || n_pos <= 0 || n_tokens > tokens.size() - i) {
return false;
}
for (size_t j = i; j < i + n_tokens; ++j) {
if (tokens[j] != LLAMA_TOKEN_NULL) {
return false;
}
}
++n_media;
i += n_tokens - 1;
} catch (const std::exception & e) {
return false;
}
@ -545,7 +751,7 @@ bool server_tokens::validate(const struct llama_context * ctx) const {
return false;
}
}
return true;
return n_media == map_idx_to_media.size();
}
server_tokens server_tokens::clone() const {

164
tools/server/server-common.h
View file

@ -201,11 +201,14 @@ public:
// for compatibility with context shift and prompt truncation
void insert(const llama_tokens & inp_tokens);
// for compatibility with speculative decoding, ctx shift, slot save/load
// for compatibility with speculative decoding, ctx shift
const llama_tokens & get_tokens() const;
llama_tokens get_text_tokens() const;
std::vector<char> serialize() const;
static server_tokens deserialize(const llama_tokens & packed, bool has_mtmd);
// for compatibility with speculative decoding
void set_token(llama_pos pos, llama_token id);
@ -213,9 +216,6 @@ public:
bool empty() const { return tokens.empty(); }
// true if the sequence actually contains image/audio chunks.
bool has_media() const { return !map_idx_to_media.empty(); }
void clear() {
map_idx_to_media.clear();
tokens.clear();
@ -230,7 +230,7 @@ public:
// split the tokens into message spans, skipping over media chunks
common_chat_msg_spans find_message_spans(const common_chat_msg_delimiters & delims) const;
// make sure all text tokens are within the vocab range
// check text token IDs and the mapping between media chunks and token ranges
bool validate(const struct llama_context * ctx) const;
server_tokens clone() const;
@ -334,6 +334,160 @@ json format_response_rerank(
std::vector<std::string> & texts,
int top_n);
//
// stats and metrics
//
// shared between server_slot and server_task_result_*
struct server_slot_stats {
uint64_t n_prompt_cached = 0;
uint64_t n_prompt_processed = 0;
uint64_t n_gen = 0;
// speculative decoding stats
// note: the per-position breakdown lives in server_slot, it is not needed in a task result
uint64_t n_draft_tokens = 0;
uint64_t n_draft_accepted = 0;
uint64_t n_draft_verif_steps = 0;
// these are absolute timestamps (in us)
// note: must be signed - they are subtracted before the later ones are set
int64_t t_start = 0;
int64_t t_prompt_last = 0;
int64_t t_gen_last = 0;
// can only move one direction: start -> prompt -> gen
void update_prompt_start() {
GGML_ASSERT(t_start == 0);
t_start = ggml_time_us();
}
void set_prompt_last(int64_t t_us) {
GGML_ASSERT(t_start > 0);
t_prompt_last = t_us;
}
void update_prompt_last() {
set_prompt_last(ggml_time_us());
}
void update_gen_last() {
GGML_ASSERT(t_prompt_last > 0);
t_gen_last = ggml_time_us();
}
// these are time durations
int64_t t_elapsed_us() const {
return ggml_time_us() - t_start;
}
double t_prompt_ms() const {
if (t_prompt_last == 0) {
return 0.0; // the prompt is not processed yet
}
return (t_prompt_last - t_start) / 1000.0;
}
int64_t t_gen_us() const {
if (t_gen_last == 0) {
return 0; // the generation is not started yet
}
// clamp to 1 us, the first token can land in the same us as t_prompt_last
return std::max<int64_t>(1, t_gen_last - t_prompt_last);
}
double t_gen_ms() const {
return t_gen_us() / 1000.0;
}
// number of decode steps spent on generation
// the first token is free, it comes from the logits of the last prompt batch
uint64_t n_gen_steps() const {
return n_gen > 0 ? n_gen - 1 : 0;
}
// other derived metrics
// note: all of them return 0.0 if the divisor is not known yet
double t_prompt_per_token_ms() const {
return n_prompt_processed > 0 ? t_prompt_ms() / n_prompt_processed : 0.0;
}
double t_gen_per_token_ms() const {
return n_gen_steps() > 0 ? t_gen_ms() / n_gen_steps() : 0.0;
}
double n_prompt_tps() const {
const double t_ms = t_prompt_ms();
return t_ms > 0.0 ? 1e3 / t_ms * n_prompt_processed : 0.0;
}
double n_gen_tps() const {
const double t_ms = t_gen_ms();
return t_ms > 0.0 ? 1e3 / t_ms * n_gen_steps() : 0.0;
}
// false if the slot never started, i.e. the task result carries no stats
bool is_set() const {
return t_start > 0;
}
json to_json() const;
};
// shared between server_context_impl and server_task_result_*
// unlike server_slot_stats, server_metrics is server-global and cumulative, not tied to a slot
struct server_metrics {
int64_t t_start = 0;
struct bucket {
uint64_t count = 0; // number of tokens
uint64_t steps = 0; // number of decode steps,
// this excludes first generated token (logits from prompt batch)
uint64_t time = 0; // in microseconds
// the rate uses the decode steps, so that "free" tokens do not inflate it
double n_per_second() const {
return time > 0 ? (double) steps / (double) time * 1e6 : 0.0;
}
void add(uint64_t n, uint64_t n_steps, uint64_t t_us) {
count += n;
steps += n_steps;
time += t_us;
}
};
// these are reset by reset_bucket(), only the rate is read from them
bucket prompt_bucket;
bucket predict_bucket;
// metrics below are cumulative since the server started
bucket prompt; // only processed tokens, cached ones are counted separately below
bucket predict;
// tokens reused from the cache need no decode, so they only have a count
uint64_t n_prompt_cached = 0;
uint64_t n_tokens_max = 0;
uint64_t n_decode = 0;
uint64_t n_busy_slots = 0;
uint64_t n_draft_tokens = 0; // Total draft tokens generated
uint64_t n_draft_accepted = 0; // Draft tokens actually accepted
uint64_t n_draft_verif_steps = 0; // Total draft token verification steps by the target model
std::vector<uint64_t> n_accepted_per_pos; // Accepted tokens per draft position
void init() {
t_start = ggml_time_us();
}
void reset_bucket() {
prompt_bucket = {};
predict_bucket = {};
}
void add_prompt(uint64_t n_tokens, uint64_t t_us) {
prompt .add(n_tokens, n_tokens, t_us);
prompt_bucket.add(n_tokens, n_tokens, t_us);
}
void add_prompt_cached(uint64_t n_tokens) {
n_prompt_cached += n_tokens;
}
};
//
// other utils
//

718
tools/server/server-context.cpp
View file

File diff suppressed because it is too large Load diff

21
tools/server/server-http.cpp
View file

@ -355,8 +355,15 @@ bool server_http_context::init(const common_params & params) {
return true;
};
auto serve_asset_cached = [](const std::string & name, bool isolation) {
return [name, isolation](const httplib::Request & req, httplib::Response & res) {
// Hashed assets never change under a given name, so they can be cached forever.
// `index.html` is the exception: its name is stable while its contents change on
// every build, and it is what names the hashed asset versions the UI loads.
static constexpr auto cache_immutable = "public, max-age=31536000, immutable";
static constexpr auto cache_revalidate = "no-cache";
// Serves an asset with ETag/304 handling, under the given caching policy.
auto serve_asset_cached = [](const std::string & name, bool isolation, const char * cache_control) {
return [name, isolation, cache_control](const httplib::Request & req, httplib::Response & res) {
if (!handle_gzip_header(req, res)) {
return true; // returns error message
}
@ -372,7 +379,7 @@ bool server_http_context::init(const common_params & params) {
res.set_header("Cross-Origin-Embedder-Policy", "require-corp");
res.set_header("Cross-Origin-Opener-Policy", "same-origin");
}
res.set_header("Cache-Control", "public, max-age=31536000, immutable");
res.set_header("Cache-Control", cache_control);
res.set_content(reinterpret_cast<const char*>(a->data), a->size, a->type.c_str());
return false;
};
@ -394,9 +401,9 @@ bool server_http_context::init(const common_params & params) {
};
};
// main index file
srv->Get(params.api_prefix + "/", serve_asset_cached("index.html", true));
srv->Get(params.api_prefix + "/index.html", serve_asset_cached("index.html", true));
// main index file -- revalidated, so a new build is picked up on the next load
srv->Get(params.api_prefix + "/", serve_asset_cached("index.html", true, cache_revalidate));
srv->Get(params.api_prefix + "/index.html", serve_asset_cached("index.html", true, cache_revalidate));
// All remaining assets registered directly from the embedded asset table.
// PWA revalidation files (sw.js, manifest, version.json) use no-cache;
@ -414,7 +421,7 @@ bool server_http_context::init(const common_params & params) {
SRV_DBG("serve nocache for %s\n", a.name.c_str());
srv->Get(params.api_prefix + "/" + a.name, serve_asset_nocache(a.name));
} else {
srv->Get(params.api_prefix + "/" + a.name, serve_asset_cached(a.name, false));
srv->Get(params.api_prefix + "/" + a.name, serve_asset_cached(a.name, false, cache_immutable));
}
}

186
tools/server/server-task.cpp
View file

@ -10,6 +10,8 @@
#include "speculative.h"
#include "server-common.h"
#include <sstream>
using json = nlohmann::ordered_json;
//
@ -236,34 +238,6 @@ common_chat_msg task_result_state::update_chat_msg(
return chat_msg;
}
//
// result_timings
//
json result_timings::to_json() const {
json base = {
{"cache_n", cache_n},
{"prompt_n", prompt_n},
{"prompt_ms", prompt_ms},
{"prompt_per_token_ms", prompt_per_token_ms},
{"prompt_per_second", prompt_per_second},
{"predicted_n", predicted_n},
{"predicted_ms", predicted_ms},
{"predicted_per_token_ms", predicted_per_token_ms},
{"predicted_per_second", predicted_per_second},
};
if (draft_n > 0) {
base["draft_n"] = draft_n;
base["draft_n_accepted"] = draft_n_accepted;
}
return base;
}
//
// result_prompt_progress
//
@ -382,7 +356,7 @@ json server_task_result_cmpl_final::to_json_non_oaicompat() {
{"stop_type", stop_type_to_str(stop)},
{"stopping_word", stopping_word},
{"tokens_cached", n_tokens_cached},
{"timings", timings.to_json()},
{"timings", stats.to_json()},
};
if (!stream && !probs_output.empty()) {
res["completion_probabilities"] = completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs);
@ -432,8 +406,8 @@ json server_task_result_cmpl_final::to_json_oaicompat() {
if (verbose) {
res["__verbose"] = to_json_non_oaicompat();
}
if (timings.prompt_n >= 0) {
res.push_back({"timings", timings.to_json()});
if (stats.is_set()) {
res.push_back({"timings", stats.to_json()});
}
return res;
@ -480,8 +454,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat() {
if (verbose) {
res["__verbose"] = to_json_non_oaicompat();
}
if (timings.prompt_n >= 0) {
res.push_back({"timings", timings.to_json()});
if (stats.is_set()) {
res.push_back({"timings", stats.to_json()});
}
return res;
@ -541,8 +515,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
});
}
if (timings.prompt_n >= 0) {
deltas.back().push_back({"timings", timings.to_json()});
if (stats.is_set()) {
deltas.back().push_back({"timings", stats.to_json()});
}
// extra fields for debugging purposes
@ -734,8 +708,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
}}
});
if (timings.prompt_n >= 0) {
server_sent_events.back().at("data").push_back({"timings", timings.to_json()});
if (stats.is_set()) {
server_sent_events.back().at("data").push_back({"timings", stats.to_json()});
}
return server_sent_events;
@ -1086,8 +1060,8 @@ json server_task_result_cmpl_partial::to_json_non_oaicompat() {
{"tokens_evaluated", n_prompt_tokens},
};
// populate the timings object when needed (usually for the last response or with timings_per_token enabled)
if (timings.prompt_n > 0) {
res.push_back({"timings", timings.to_json()});
if (stats.is_set()) {
res.push_back({"timings", stats.to_json()});
}
if (is_progress) {
res.push_back({"prompt_progress", progress.to_json()});
@ -1126,8 +1100,8 @@ json server_task_result_cmpl_partial::to_json_oaicompat() {
if (verbose) {
res["__verbose"] = to_json_non_oaicompat();
}
if (timings.prompt_n >= 0) {
res.push_back({"timings", timings.to_json()});
if (stats.is_set()) {
res.push_back({"timings", stats.to_json()});
}
if (is_progress) {
res.push_back({"prompt_progress", progress.to_json()});
@ -1180,8 +1154,8 @@ json server_task_result_cmpl_partial::to_json_oaicompat_chat() {
};
}
if (timings.prompt_n >= 0) {
last_json.push_back({"timings", timings.to_json()});
if (stats.is_set()) {
last_json.push_back({"timings", stats.to_json()});
}
if (is_progress) {
last_json.push_back({"prompt_progress", progress.to_json()});
@ -1330,8 +1304,8 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
if (!events.empty()) {
json & data = events.back().at("data");
if (timings.prompt_n >= 0) {
data.push_back({"timings", timings.to_json()});
if (stats.is_set()) {
data.push_back({"timings", stats.to_json()});
}
if (is_progress) {
data.push_back({"prompt_progress", progress.to_json()});
@ -1539,34 +1513,104 @@ json server_task_result_error::to_json() {
// server_task_result_metrics
//
json server_task_result_metrics::to_json() {
return json {
{ "idle", n_idle_slots },
{ "processing", n_processing_slots },
{ "deferred", n_tasks_deferred },
{ "t_start", t_start },
return slots_data;
}
{ "n_prompt_tokens_processed_total", n_prompt_tokens_processed_total },
{ "t_tokens_generation_total", t_tokens_generation_total },
{ "n_tokens_predicted_total", n_tokens_predicted_total },
{ "t_prompt_processing_total", t_prompt_processing_total },
{ "n_tokens_max", n_tokens_max },
{ "n_prompt_tokens_processed", n_prompt_tokens_processed },
{ "t_prompt_processing", t_prompt_processing },
{ "n_tokens_predicted", n_tokens_predicted },
{ "t_tokens_generation", t_tokens_generation },
{ "n_decode_total", n_decode_total },
{ "n_busy_slots_total", n_busy_slots_total },
{ "n_draft_tokens_total", n_draft_tokens_total },
{ "n_draft_accepted_total", n_draft_accepted_total },
{ "n_draft_verif_steps_total", n_draft_verif_steps_total },
{ "n_accepted_per_pos_total", n_accepted_per_pos_total },
{ "slots", slots_data },
// metrics definition: https://prometheus.io/docs/practices/naming/#metric-names
std::string server_task_result_metrics::to_metrics() {
const std::vector<metric_item> counters = {
{
"prompt_tokens_total",
"Number of prompt tokens processed, excluding cached tokens",
(double) metrics.prompt.count
}, {
"prompt_tokens_cached_total",
"Number of prompt tokens reused from the cache",
(double) metrics.n_prompt_cached
}, {
"prompt_seconds_total",
"Total time spent processing prompts",
metrics.prompt.time / 1.e6
}, {
"tokens_predicted_total",
"Number of generation tokens processed",
(double) metrics.predict.count
}, {
"tokens_predicted_seconds_total",
"Total time spent generating tokens",
metrics.predict.time / 1.e6
}, {
"n_decode_total",
"Total number of llama_decode() calls, excluding speculative decoding and multimodal decoding",
(double) metrics.n_decode
}, {
"n_tokens_max",
"Largest observed sequence length (prompt + generation)",
(double) metrics.n_tokens_max
}, {
"spec_decode_num_draft_tokens_total",
"Speculative: Total draft tokens generated",
(double) metrics.n_draft_tokens
}, {
"spec_decode_num_accepted_tokens_total",
"Speculative: Total draft tokens accepted by the target model",
(double) metrics.n_draft_accepted
}, {
"spec_decode_num_drafts_total",
"Speculative: Total speculative decoding verification steps",
(double) metrics.n_draft_verif_steps
},
};
const std::vector<metric_item> gauges = {
{
"prompt_tokens_seconds",
"Average prompt throughput in tokens/s",
metrics.prompt_bucket.n_per_second()
}, {
"predicted_tokens_seconds",
"Average generation throughput in tokens/s",
metrics.predict_bucket.n_per_second()
}, {
"requests_processing",
"Number of requests processing",
(double) n_processing_slots
}, {
"requests_deferred",
"Number of requests deferred",
(double) n_tasks_deferred
}, {
"n_busy_slots_per_decode",
"Average number of busy slots per llama_decode() call",
(double) metrics.n_busy_slots / std::max((double) metrics.n_decode, 1.0)
},
};
std::stringstream prometheus;
auto add_items = [&prometheus](const char * type, const std::vector<metric_item> & items) {
for (const auto & item : items) {
prometheus << "# HELP llamacpp:" << item.name << " " << item.description << "\n"
<< "# TYPE llamacpp:" << item.name << " " << type << "\n"
<< "llamacpp:" << item.name << " " << item.value << "\n";
}
};
add_items("counter", counters);
add_items("gauge", gauges);
// labeled counter: one time series per draft position
if (!metrics.n_accepted_per_pos.empty()) {
prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos_total"
" Accepted tokens per draft position\n"
<< "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos_total counter\n";
for (size_t i = 0; i < metrics.n_accepted_per_pos.size(); i++) {
prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\""
<< i << "\"} " << metrics.n_accepted_per_pos[i] << "\n";
}
}
return prometheus.str();
}
//

57
tools/server/server-task.h
View file

@ -259,26 +259,6 @@ struct server_task {
}
};
struct result_timings {
int32_t cache_n = -1;
int32_t prompt_n = -1;
double prompt_ms = 0.0;
double prompt_per_token_ms = 0.0;
double prompt_per_second = 0.0;
int32_t predicted_n = -1;
double predicted_ms = 0.0;
double predicted_per_token_ms = 0.0;
double predicted_per_second = 0.0;
// Optional speculative metrics - only included when > 0
int32_t draft_n = 0;
int32_t draft_n_accepted = 0;
json to_json() const;
};
struct result_prompt_progress {
int32_t total = 0;
int32_t cache = 0;
@ -343,7 +323,7 @@ struct server_task_result_cmpl_final : server_task_result {
bool stream;
bool include_usage;
result_timings timings;
server_slot_stats stats;
std::string prompt;
bool truncated;
@ -425,7 +405,7 @@ struct server_task_result_cmpl_partial : server_task_result {
bool is_begin = false; // whether to send 200 status to HTTP client (begin of SSE stream)
// ref: https://github.com/ggml-org/llama.cpp/pull/23884
completion_token_output prob_output;
result_timings timings;
server_slot_stats stats;
result_prompt_progress progress;
// response formatting
@ -510,38 +490,27 @@ struct server_task_result_error : server_task_result {
};
struct server_task_result_metrics : server_task_result {
// these are immediate stats, not accumulated (server_metrics is cumulative)
int n_idle_slots;
int n_processing_slots;
int n_tasks_deferred;
int64_t t_start;
// TODO: somehow reuse server_metrics in the future, instead of duplicating the fields
uint64_t n_prompt_tokens_processed_total = 0;
uint64_t t_prompt_processing_total = 0;
uint64_t n_tokens_predicted_total = 0;
uint64_t t_tokens_generation_total = 0;
uint64_t n_tokens_max = 0;
uint64_t n_prompt_tokens_processed = 0;
uint64_t t_prompt_processing = 0;
uint64_t n_tokens_predicted = 0;
uint64_t t_tokens_generation = 0;
uint64_t n_decode_total = 0;
uint64_t n_busy_slots_total = 0;
uint64_t n_draft_tokens_total = 0;
uint64_t n_draft_accepted_total = 0;
uint64_t n_draft_verif_steps_total = 0;
std::vector<uint64_t> n_accepted_per_pos_total;
server_metrics metrics;
// while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
// therefore, we use json to temporarily store the slot.to_json() result
json slots_data = json::array();
// used by /slots API
virtual json to_json() override;
// used by /metrics API
struct metric_item {
std::string name;
std::string description;
double value; // prometheus values are always float64
};
std::string to_metrics();
};
struct server_task_result_slot_save_load : server_task_result {

227
tools/server/tests/unit/test_metrics.py Normal file
View file

@ -0,0 +1,227 @@
import pytest
from utils import *
server = ServerPreset.tinyllama2()
@pytest.fixture(autouse=True)
def create_server():
global server
server = ServerPreset.tinyllama2()
server.server_metrics = True
def fetch_metrics(server: ServerProcess) -> str:
"""get /metrics as raw prometheus text"""
res = server.make_request("GET", "/metrics")
assert res.status_code == 200
assert "Process-Start-Time-Unix" in res.headers
assert isinstance(res.body, str)
return res.body
def parse_metrics(text: str) -> dict:
"""parse the prometheus text format into {name: (type, value)}"""
out = {}
types = {}
for line in text.splitlines():
if line.startswith("# TYPE "):
_, _, name, kind = line.split(" ", 3)
types[name] = kind
elif line.startswith("llamacpp:") and "{" not in line:
name, value = line.split(" ", 1)
assert name in types, f"{name} has no # TYPE line"
out[name] = (types[name], float(value))
return out
def test_metrics_disabled():
global server
server.server_metrics = False
server.start()
res = server.make_request("GET", "/metrics")
assert res.status_code == 501 # ERROR_TYPE_NOT_SUPPORTED
def test_metrics_prometheus_format():
global server
server.start()
server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 8})
text = fetch_metrics(server)
metrics = parse_metrics(text)
expected_counters = [
"llamacpp:prompt_tokens_total",
"llamacpp:prompt_tokens_cached_total",
"llamacpp:prompt_seconds_total",
"llamacpp:tokens_predicted_total",
"llamacpp:tokens_predicted_seconds_total",
"llamacpp:n_decode_total",
"llamacpp:n_tokens_max",
"llamacpp:spec_decode_num_draft_tokens_total",
"llamacpp:spec_decode_num_accepted_tokens_total",
"llamacpp:spec_decode_num_drafts_total",
]
expected_gauges = [
"llamacpp:prompt_tokens_seconds",
"llamacpp:predicted_tokens_seconds",
"llamacpp:requests_processing",
"llamacpp:requests_deferred",
"llamacpp:n_busy_slots_per_decode",
]
for name in expected_counters:
assert metrics[name][0] == "counter"
for name in expected_gauges:
assert metrics[name][0] == "gauge"
# every metric must carry a help line
for name in expected_counters + expected_gauges:
assert f"# HELP {name} " in text
assert metrics["llamacpp:n_decode_total"][1] > 0
assert metrics["llamacpp:requests_processing"][1] == 0
def test_metrics_prompt_processed_and_cached():
global server
server.n_slots = 1 # keep the prompt cache on a single slot
server.start()
prompt = "the quick brown fox jumps over the lazy dog"
n_processed = 0
n_cached = 0
for _ in range(2):
res = server.make_request("POST", "/completion", data={"prompt": prompt, "n_predict": 4})
assert res.status_code == 200
n_processed += res.body["timings"]["prompt_n"]
n_cached += res.body["timings"]["cache_n"]
# the second request must reuse the prompt of the first one
assert n_cached > 0
metrics = parse_metrics(fetch_metrics(server))
# cached tokens are counted apart, they cost no decode
assert metrics["llamacpp:prompt_tokens_total"][1] == n_processed
assert metrics["llamacpp:prompt_tokens_cached_total"][1] == n_cached
def test_metrics_predicted_total_matches_requests():
global server
server.start()
n_predicted = 0
for n_predict in [1, 4, 16]:
res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": n_predict})
assert res.status_code == 200
n_predicted += res.body["timings"]["predicted_n"]
metrics = parse_metrics(fetch_metrics(server))
assert metrics["llamacpp:tokens_predicted_total"][1] == n_predicted
def test_metrics_generation_rate_excludes_first_token():
global server
server.start()
# the first token comes from the logits of the last prompt batch, so it costs no decode step
res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 1})
timings = res.body["timings"]
assert timings["predicted_n"] == 1
assert timings["predicted_per_second"] == 0.0
assert timings["predicted_per_token_ms"] == 0.0
res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 16})
timings = res.body["timings"]
assert timings["predicted_n"] == 16
# the rate is over 15 decode steps, not 16 tokens
expected = 1e3 / timings["predicted_ms"] * 15
assert abs(timings["predicted_per_second"] - expected) < 1e-6
@pytest.mark.parametrize("n_predict", [1, 8])
def test_metrics_timings_are_finite(n_predict: int):
global server
server.start()
res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": n_predict})
timings = res.body["timings"]
# a null here means the server produced inf or nan
for key, value in timings.items():
assert value is not None, f"{key} is null"
assert value >= 0, f"{key} is negative"
assert timings["prompt_ms"] > 0
assert timings["prompt_per_token_ms"] > 0
def test_metrics_timings_on_prompt_progress():
global server
server.start()
# a long prompt so that it is split over several batches (n_batch = 32)
prompt = "the quick brown fox jumps over the lazy dog " * 8
chunks = list(server.make_stream_request("POST", "/completion", data={
"prompt": prompt,
"n_predict": 4,
"stream": True,
"timings_per_token": True,
"return_progress": True,
}))
progress = [c for c in chunks if "prompt_progress" in c]
assert len(progress) > 1 # the prompt did not fit in a single batch
# the very first update is sent before any prompt token is decoded
first = progress[0]["timings"]
assert first["prompt_n"] == 0
assert first["prompt_ms"] == 0.0
assert first["predicted_n"] == 0
assert first["predicted_ms"] == 0.0
# timings must never go backwards, nor report bogus values
prompt_ms = 0.0
for chunk in progress:
timings = chunk["timings"]
for key, value in timings.items():
assert value is not None, f"{key} is null"
assert value >= 0, f"{key} is negative"
assert timings["prompt_ms"] >= prompt_ms
prompt_ms = timings["prompt_ms"]
assert prompt_ms > 0
def test_metrics_slots_idle_after_completion():
global server
server.server_slots = True
server.start()
server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 8})
res = server.make_request("GET", "/slots")
assert res.status_code == 200
for slot in res.body:
assert slot["is_processing"] is False
if "next_token" in slot:
# the budget of the finished task must not leak into the idle slot
assert slot["next_token"][0]["n_remain"] == -1
assert slot["next_token"][0]["n_decoded"] == 0
def test_metrics_embedding_prompt_is_counted():
global server
server = ServerPreset.bert_bge_small()
server.server_metrics = True
server.start()
res = server.make_request("POST", "/v1/embeddings", data={"input": ["hello world", "goodbye world"]})
assert res.status_code == 200
# embedding tasks never sample a token, but their prompt still costs a decode
metrics = parse_metrics(fetch_metrics(server))
assert metrics["llamacpp:prompt_tokens_total"][1] > 0
assert metrics["llamacpp:n_decode_total"][1] > 0
assert metrics["llamacpp:tokens_predicted_total"][1] == 0

377
tools/server/tests/unit/test_slot_save.py
View file

@ -2,6 +2,10 @@ import pytest
from utils import *
import base64
import requests
import struct
# sequence state file: magic(4) version(4) payload_size(4), then payload_size llama_token words
STATE_FILE_HEADER_SIZE = 12
server = ServerPreset.tinyllama2()
@ -72,6 +76,60 @@ def test_slot_save_restore():
assert res.body["timings"]["prompt_n"] == 1
def test_slot_restore_legacy_token_list():
global server
server.start()
res = server.make_request("POST", "/completion", data={
"prompt": "What is the capital of France?",
"id_slot": 1,
"cache_prompt": True,
})
assert res.status_code == 200
res = server.make_request("POST", "/slots/1?action=save", data={
"filename": "slot_legacy.bin",
})
assert res.status_code == 200
assert res.body["n_saved"] == 84
# rewrite the token payload into a plain token list, as written by servers that predate the packed server_tokens format
path = os.path.join("tmp", "slot_legacy.bin")
with open(path, "rb") as f:
data = bytearray(f.read())
# the payload written by this server starts with a packed header: LLAMA_TOKEN_NULL(4) version(4) n_tokens(4)
packed_header_size = 12
payload_size = struct.unpack_from("=I", data, STATE_FILE_HEADER_SIZE - 4)[0]
payload_end = STATE_FILE_HEADER_SIZE + payload_size * 4
n_tokens = struct.unpack_from("=I", data, STATE_FILE_HEADER_SIZE + 8)[0]
assert n_tokens == 84
tokens_start = STATE_FILE_HEADER_SIZE + packed_header_size
data = data[:STATE_FILE_HEADER_SIZE] + data[tokens_start:tokens_start + n_tokens * 4] + data[payload_end:]
struct.pack_into("=I", data, STATE_FILE_HEADER_SIZE - 4, n_tokens)
with open(path, "wb") as f:
f.write(data)
# the plain token list must restore, and the restored KV must be reusable
res = server.make_request("POST", "/slots/0?action=restore", data={
"filename": "slot_legacy.bin",
})
assert res.status_code == 200
assert res.body["n_restored"] == 84
res = server.make_request("POST", "/completion", data={
"prompt": "What is the capital of Germany?",
"id_slot": 0,
"cache_prompt": True,
})
assert res.status_code == 200
assert res.body["timings"]["prompt_n"] == 6 # only the different part is processed
def test_slot_erase():
global server
server.start()
@ -103,14 +161,12 @@ def test_slot_erase():
#
# Multimodal server (mmproj loaded) slot save/restore.
#
# Regression coverage for issue #21133: slot save/restore/erase must be gated on
# the slot's CONTENT (does it actually hold image/audio tokens) rather than the
# model's CAPABILITY (is an mmproj loaded). A pure-text slot on a multimodal
# server must save/restore/erase normally; a slot that actually holds an image
# must be rejected with ERROR_TYPE_NOT_SUPPORTED (HTTP 501).
# A pure-text slot on a multimodal server and a slot containing images must both support save/restore.
# Erase remains gated on the slot's content.
#
IMG_URL_CAT = "https://huggingface.co/ggml-org/tinygemma3-GGUF/resolve/main/test/91_cat.png"
IMG_URL_TRUCK = "https://huggingface.co/ggml-org/tinygemma3-GGUF/resolve/main/test/11_truck.png"
def _get_img_base64(url: str) -> str:
@ -121,8 +177,7 @@ def _get_img_base64(url: str) -> str:
@pytest.fixture
def mmproj_server():
# tinygemma3 is a small multimodal model: the mmproj is provided by the HF
# registry API and auto-downloaded on first run.
# tinygemma3 is a small multimodal model: the mmproj is provided by the HF registry API and auto-downloaded on first run.
os.environ['LLAMA_MEDIA_MARKER'] = '<__media__>'
mm_server = ServerPreset.tinygemma3()
mm_server.slot_save_path = "./tmp"
@ -159,10 +214,7 @@ def test_slot_save_restore_text_only_on_multimodal(mmproj_server):
assert res.status_code == 200
assert res.body["n_restored"] == n_saved
# The restored slot is usable for a follow-up completion. We do NOT assert
# prefix reuse here: tinygemma3 is a SWA model, which forces full prompt
# re-processing after a restore (a model property, not the save/restore gate
# under test).
# Prefix reuse is not checked with the default SWA cache.
res = server.make_request("POST", "/completion", data={
"prompt": "The quick brown fox jumps over the lazy dog.",
"id_slot": 0,
@ -171,54 +223,307 @@ def test_slot_save_restore_text_only_on_multimodal(mmproj_server):
assert res.status_code == 200
def test_slot_save_rejected_when_slot_holds_image(mmproj_server):
def test_slot_save_restore_with_image(mmproj_server):
server = mmproj_server
# Use the full SWA cache so the restored image prefix can be reused.
server.swa_full = True
server.start()
# Process a prompt that actually contains an image on slot 1.
prompt_cat = {
"prompt_string": "What is this: <__media__>\n",
"multimodal_data": [_get_img_base64(IMG_URL_CAT)],
}
res = server.make_request("POST", "/completions", data={
"temperature": 0.0,
"top_k": 1,
"id_slot": 1,
"cache_prompt": True,
"prompt": prompt_cat,
})
assert res.status_code == 200
content_cat = res.body["content"]
prompt_n_full = res.body["timings"]["prompt_n"]
assert res.body["timings"]["cache_n"] == 0
assert prompt_n_full > 32 # text plus image tokens are all processed
res = server.make_request("POST", "/slots/1?action=save", data={
"filename": "mm_slot_image.bin",
})
assert res.status_code == 200
n_saved = res.body["n_saved"]
n_written = res.body["n_written"]
assert n_saved > 0
assert n_written > 0
res = server.make_request("POST", "/slots/1?action=erase")
assert res.status_code == 200
res = server.make_request("POST", "/slots/0?action=restore", data={
"filename": "mm_slot_image.bin",
})
assert res.status_code == 200
assert res.body["n_restored"] == n_saved
assert res.body["n_read"] == n_written
# a different image must not reuse the restored image tokens; only the text prefix before the image is common
res = server.make_request("POST", "/completions", data={
"temperature": 0.0,
"top_k": 1,
"id_slot": 0,
"cache_prompt": True,
"prompt": {
"prompt_string": "What is this: <__media__>\n",
"multimodal_data": [ _get_img_base64(IMG_URL_CAT) ],
"multimodal_data": [_get_img_base64(IMG_URL_TRUCK)],
},
})
assert res.status_code == 200
cache_n = res.body["timings"]["cache_n"]
assert cache_n < 16
assert res.body["timings"]["prompt_n"] == prompt_n_full - cache_n
# restore again and resend the same image: the image tokens must be reused and greedy sampling must reproduce the original content
res = server.make_request("POST", "/slots/0?action=restore", data={
"filename": "mm_slot_image.bin",
})
assert res.status_code == 200
assert res.body["n_restored"] == n_saved
res = server.make_request("POST", "/completions", data={
"temperature": 0.0,
"top_k": 1,
"id_slot": 0,
"cache_prompt": True,
"prompt": prompt_cat,
})
assert res.status_code == 200
assert res.body["timings"]["cache_n"] == prompt_n_full - 1
assert res.body["timings"]["prompt_n"] == 1
assert res.body["content"] == content_cat
def test_slot_save_restore_with_two_images(mmproj_server):
server = mmproj_server
server.swa_full = True
server.n_ctx = 2048 # two images need more than the default 512 per slot
server.start()
prompt = {
"prompt_string": "A: <__media__> B: <__media__>\n",
"multimodal_data": [_get_img_base64(IMG_URL_CAT), _get_img_base64(IMG_URL_TRUCK)],
}
res = server.make_request("POST", "/completions", data={
"temperature": 0.0,
"top_k": 1,
"id_slot": 1,
"cache_prompt": True,
"prompt": prompt,
})
assert res.status_code == 200
content = res.body["content"]
prompt_n_full = res.body["timings"]["prompt_n"]
assert prompt_n_full > 64
res = server.make_request("POST", "/slots/1?action=save", data={
"filename": "mm_slot_two_images.bin",
})
assert res.status_code == 200
n_saved = res.body["n_saved"]
res = server.make_request("POST", "/slots/0?action=restore", data={
"filename": "mm_slot_two_images.bin",
})
assert res.status_code == 200
assert res.body["n_restored"] == n_saved
res = server.make_request("POST", "/completions", data={
"temperature": 0.0,
"top_k": 1,
"id_slot": 0,
"cache_prompt": True,
"prompt": prompt,
})
assert res.status_code == 200
assert res.body["timings"]["cache_n"] == prompt_n_full - 1
assert res.body["timings"]["prompt_n"] == 1
assert res.body["content"] == content
def test_slot_save_restore_with_image_across_restart(mmproj_server):
server = mmproj_server
server.swa_full = True
server.start()
prompt_cat = {
"prompt_string": "What is this: <__media__>\n",
"multimodal_data": [_get_img_base64(IMG_URL_CAT)],
}
res = server.make_request("POST", "/completions", data={
"temperature": 0.0,
"top_k": 1,
"id_slot": 0,
"cache_prompt": True,
"prompt": prompt_cat,
})
assert res.status_code == 200
content = res.body["content"]
prompt_n_full = res.body["timings"]["prompt_n"]
res = server.make_request("POST", "/slots/0?action=save", data={
"filename": "mm_slot_restart.bin",
})
assert res.status_code == 200
n_saved = res.body["n_saved"]
# restart the server with the same model and mmproj: the saved file must restore in the new process and the image KV must be reused
server.stop()
server.start()
res = server.make_request("POST", "/slots/0?action=restore", data={
"filename": "mm_slot_restart.bin",
})
assert res.status_code == 200
assert res.body["n_restored"] == n_saved
res = server.make_request("POST", "/completions", data={
"temperature": 0.0,
"top_k": 1,
"id_slot": 0,
"cache_prompt": True,
"prompt": prompt_cat,
})
assert res.status_code == 200
assert res.body["timings"]["cache_n"] == prompt_n_full - 1
assert res.body["timings"]["prompt_n"] == 1
assert res.body["content"] == content
def test_slot_save_restore_image_payload_larger_than_context(mmproj_server):
server = mmproj_server
server.swa_full = True
server.start()
# the slot context, as the server computed it (n_ctx split across the slots)
res = server.make_request("GET", "/props")
assert res.status_code == 200
n_ctx_slot = res.body["default_generation_settings"]["n_ctx"]
# a filler token, used to grow the prompt up to the slot context
res = server.make_request("POST", "/tokenize", data={"content": " hello" * 8})
assert res.status_code == 200
assert len(res.body["tokens"]) == 8
res = server.make_request("POST", "/completions", data={
"temperature": 0.0,
"top_k": 1,
"id_slot": 0,
"cache_prompt": True,
"prompt": {
"prompt_string": "What is this: <__media__>\n",
"multimodal_data": [_get_img_base64(IMG_URL_CAT)],
},
})
assert res.status_code == 200
# Saving a slot that holds image tokens must be rejected (HTTP 501,
# not_supported_error).
res = server.make_request("POST", "/slots/1?action=save", data={
"filename": "mm_slot_image.bin",
prompt_cat = {
"prompt_string": "What is this: <__media__>\n" + " hello" * (n_ctx_slot - res.body["timings"]["prompt_n"] - 8),
"multimodal_data": [_get_img_base64(IMG_URL_CAT)],
}
res = server.make_request("POST", "/completions", data={
"temperature": 0.0,
"top_k": 1,
"id_slot": 0,
"cache_prompt": True,
"prompt": prompt_cat,
})
assert res.status_code != 200
assert res.body["error"]["type"] == "not_supported_error"
assert res.status_code == 200
prompt_n_full = res.body["timings"]["cache_n"] + res.body["timings"]["prompt_n"]
res = server.make_request("POST", "/slots/0?action=save", data={
"filename": "mm_slot_large_payload.bin",
})
assert res.status_code == 200
path = os.path.join("tmp", "mm_slot_large_payload.bin")
with open(path, "rb") as f:
data = bytearray(f.read())
payload_size = struct.unpack_from("=I", data, STATE_FILE_HEADER_SIZE - 4)[0]
assert payload_size > n_ctx_slot # the scenario under test: the payload does not fit in n_ctx
# drop the image from the slot, then restore it from the file
res = server.make_request("POST", "/completion", data={
"prompt": "The quick brown fox",
"id_slot": 0,
"cache_prompt": True,
})
assert res.status_code == 200
res = server.make_request("POST", "/slots/0?action=restore", data={
"filename": "mm_slot_large_payload.bin",
})
assert res.status_code == 200
res = server.make_request("POST", "/completions", data={
"temperature": 0.0,
"top_k": 1,
"id_slot": 0,
"cache_prompt": True,
"prompt": prompt_cat,
})
assert res.status_code == 200
assert res.body["timings"]["cache_n"] == prompt_n_full - 1
assert res.body["timings"]["prompt_n"] == 1
def test_slot_erase_text_only_on_multimodal(mmproj_server):
def test_slot_restore_media_file_without_mmproj(mmproj_server):
server = mmproj_server
server.start()
res = server.make_request("POST", "/completion", data={
"prompt": "The quick brown fox jumps over the lazy dog.",
"id_slot": 1,
res = server.make_request("POST", "/completions", data={
"temperature": 0.0,
"top_k": 1,
"id_slot": 0,
"cache_prompt": True,
"prompt": {
"prompt_string": "What is this: <__media__>\n",
"multimodal_data": [_get_img_base64(IMG_URL_CAT)],
},
})
assert res.status_code == 200
prompt_n = res.body["timings"]["prompt_n"]
assert prompt_n > 0 # all tokens are processed
# Erasing a pure-text slot must succeed even though an mmproj is loaded.
res = server.make_request("POST", "/slots/1?action=erase")
assert res.status_code == 200
# Re-running the same prompt should process all tokens again.
res = server.make_request("POST", "/completion", data={
"prompt": "The quick brown fox jumps over the lazy dog.",
"id_slot": 1,
"cache_prompt": True,
res = server.make_request("POST", "/slots/0?action=save", data={
"filename": "mm_slot_no_mmproj.bin",
})
assert res.status_code == 200
assert res.body["timings"]["prompt_n"] == prompt_n # all tokens are processed again
# restart the same model without the mmproj: restoring the media file must fail gracefully and leave the slot usable
server.stop()
server.no_mmproj = True
server.start()
res = server.make_request("POST", "/slots/0?action=restore", data={
"filename": "mm_slot_no_mmproj.bin",
})
assert res.status_code == 400
assert "Cannot restore media tokens without an mmproj" in res.body["error"]["message"]
# A failed restore must leave the slot empty and usable.
res = server.make_request("POST", "/completions", data={
"temperature": 0.0,
"top_k": 1,
"id_slot": 1,
"cache_prompt": True,
"prompt": "The quick brown fox",
})
assert res.status_code == 200
content = res.body["content"]
res = server.make_request("POST", "/completions", data={
"temperature": 0.0,
"top_k": 1,
"id_slot": 0,
"cache_prompt": True,
"prompt": "The quick brown fox",
})
assert res.status_code == 200
assert res.body["timings"]["cache_n"] == 0
assert res.body["content"] == content

6
tools/server/tests/utils.py
View file

@ -86,6 +86,7 @@ class ServerProcess:
server_reranking: bool | None = False
server_metrics: bool | None = False
kv_unified: bool | None = False
swa_full: bool | None = False
server_slots: bool | None = False
pooling: str | None = None
api_key: str | None = None
@ -106,6 +107,7 @@ class ServerProcess:
chat_template_file: str | None = None
server_path: str | None = None
mmproj_url: str | None = None
no_mmproj: bool | None = None
media_path: str | None = None
sleep_idle_seconds: int | None = None
cache_ram: int | None = None
@ -198,6 +200,8 @@ class ServerProcess:
server_args.append("--metrics")
if self.kv_unified:
server_args.append("--kv-unified")
if self.swa_full:
server_args.append("--swa-full")
if self.server_slots:
server_args.append("--slots")
else:
@ -259,6 +263,8 @@ class ServerProcess:
server_args.extend(["--chat-template-file", self.chat_template_file])
if self.mmproj_url:
server_args.extend(["--mmproj-url", self.mmproj_url])
if self.no_mmproj:
server_args.append("--no-mmproj")
if self.media_path:
server_args.extend(["--media-path", self.media_path])
if self.sleep_idle_seconds is not None:

2
tools/ui/pwa-assets-dark.config.ts
View file

@ -1,5 +1,5 @@
import { writeThemeFavicons } from './scripts/favicon-colorize';
import { FAVICON_COLORS, PWA_ASSET_GENERATOR } from './src/lib/constants/pwa';
import { FAVICON_COLORS, PWA_ASSET_GENERATOR } from './src/lib/constants/pwa.constants';
import { defineConfig } from '@vite-pwa/assets-generator/config';
writeThemeFavicons(FAVICON_COLORS.LIGHT, FAVICON_COLORS.DARK, {

2
tools/ui/pwa-assets.config.ts
View file

@ -4,7 +4,7 @@ import {
PWA_ASSET_GENERATOR,
PWA_GENERATOR_DEVICES,
THEME_COLORS
} from './src/lib/constants/pwa';
} from './src/lib/constants/pwa.constants';
import { SplashOrientation } from './src/lib/enums/splash.enums';
import {
combinePresetAndAppleSplashScreens,

2
tools/ui/scripts/vite-plugin-build-info.ts
View file

@ -1,4 +1,4 @@
import { BUILD_CONFIG } from '../src/lib/constants/pwa';
import { BUILD_CONFIG } from '../src/lib/constants/pwa.constants';
import { existsSync, writeFileSync } from 'node:fs';
import { resolve } from 'path';
import type { Plugin } from 'vite';

2
tools/ui/scripts/vite-plugin-relativize-base.ts
View file

@ -1,4 +1,4 @@
import { BUILD_CONFIG } from '../src/lib/constants/pwa';
import { BUILD_CONFIG } from '../src/lib/constants/pwa.constants';
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
import { resolve } from 'path';
import type { Plugin } from 'vite';

9
tools/ui/scripts/vite-plugin-splash-screen.ts
View file

@ -1,5 +1,10 @@
import { NEWLINE, TAB } from '../src/lib/constants/code';
import { APPLE_DEVICES, BUILD_CONFIG, REGEX_PATTERNS, SPLASH_LINK } from '../src/lib/constants/pwa';
import {
APPLE_DEVICES,
BUILD_CONFIG,
REGEX_PATTERNS,
SPLASH_LINK
} from '../src/lib/constants/pwa.constants';
import { NEWLINE, TAB } from '../src/lib/constants/special-characters.constants';
import { SplashOrientation } from '../src/lib/enums/splash.enums';
import type { SplashDimensions } from '../src/lib/types';
import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';

2
tools/ui/src/app.d.ts vendored
View file

@ -32,8 +32,8 @@ import type {
ApiRouterModelsStatusResponse,
ApiRouterModelsUnloadRequest,
ApiRouterModelsUnloadResponse,
// Chat types
ChatAttachmentDisplayItem,
// Chat types
ChatMessagePromptProgress,
ChatMessageSiblingInfo,
ChatMessageTimings,

1
tools/ui/src/app.html
View file

@ -2,6 +2,7 @@
<html lang="en">
<head>
<meta charset="utf-8" />
<link rel="icon" href="favicon.ico" sizes="48x48" />
<link rel="icon" href="favicon.svg" sizes="any" type="image/svg+xml" />

2
tools/ui/src/lib/components/app/actions/ActionIconCopyToClipboard.svelte
View file

@ -1,7 +1,7 @@
<script lang="ts">
import ActionIcon from './ActionIcon.svelte';
import { Copy } from '@lucide/svelte';
import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes';
import { ICON_CLASS_DEFAULT } from '$lib/constants';
import { copyToClipboard } from '$lib/utils';
export let ariaLabel: string = 'Copy to clipboard';

2
tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemMcpResource.svelte
View file

@ -3,7 +3,7 @@
import { X } from '@lucide/svelte';
import { ActionIcon } from '$lib/components/app';
import * as Tooltip from '$lib/components/ui/tooltip';
import { mcpStore } from '$lib/stores/mcp.svelte';
import { mcpStore } from '$lib/stores';
import type { MCPResourceAttachment } from '$lib/types';
import { getResourceDisplayName, getResourceIcon } from '$lib/utils';

2
tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemThumbnailFile.svelte
View file

@ -1,7 +1,7 @@
<script lang="ts">
import { Music, Video, X } from '@lucide/svelte';
import { ActionIcon } from '$lib/components/app';
import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes';
import { ICON_CLASS_DEFAULT } from '$lib/constants';
import { AttachmentType } from '$lib/enums';
import {
formatFileSize,

5
tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview.svelte → tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreview.svelte
View file

@ -5,7 +5,8 @@
ChatAttachmentsPreviewNavButtons,
ChatAttachmentsPreviewThumbnailStrip
} from '$lib/components/app';
import { modelsStore } from '$lib/stores/models.svelte';
import { UI_DATA_ATTRS } from '$lib/constants';
import { modelsStore } from '$lib/stores';
import {
createBase64DataUrl,
formatFileSize,
@ -90,7 +91,7 @@
const index = currentIndex;
setTimeout(() => {
const thumbnail = document.querySelector(`[data-thumbnail-index="${index}"]`);
const thumbnail = document.querySelector(`[${UI_DATA_ATTRS.THUMBNAIL_INDEX}="${index}"]`);
thumbnail?.scrollIntoView({ behavior: 'smooth', block: 'nearest', inline: 'center' });
}, 0);

2
tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemPdf.svelte
View file

@ -3,7 +3,7 @@
import { SyntaxHighlightedCode } from '$lib/components/app';
import * as Alert from '$lib/components/ui/alert';
import { Button } from '$lib/components/ui/button';
import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes';
import { ICON_CLASS_DEFAULT } from '$lib/constants';
import { PdfViewMode } from '$lib/enums';
import type { ChatAttachmentDisplayItem } from '$lib/types';
import { getLanguageFromFilename } from '$lib/utils';

4
tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewThumbnailStrip.svelte
View file

@ -1,7 +1,7 @@
<script lang="ts">
import { FileText, Music, Video } from '@lucide/svelte';
import { HorizontalScrollCarousel } from '$lib/components/app/misc';
import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes';
import { ICON_CLASS_DEFAULT, UI_DATA_ATTRS } from '$lib/constants';
interface PreviewItem {
id: string;
@ -36,7 +36,7 @@
<HorizontalScrollCarousel class="max-w-full">
{#each items as item, index (item.id)}
<button
data-thumbnail-index={index}
{...{ [UI_DATA_ATTRS.THUMBNAIL_INDEX]: index }}
class={[
'relative flex-shrink-0 cursor-pointer overflow-hidden rounded border-2 bg-black/80 backdrop-blur-sm transition-all hover:opacity-90',
index === currentIndex ? 'border-white' : 'border-transparent opacity-60',

112
tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte
View file

@ -3,12 +3,11 @@
import {
ChatAttachmentsList,
ChatFormActions,
ChatFormContenteditable,
ChatFormFileInputInvisible,
ChatFormCurrentWorkingDirectory,
ChatFormInput,
ChatFormInputFileInputInvisible,
ChatFormMcpResourcesList,
ChatFormPickers,
ChatFormTextarea,
ChatFormWorkingDirectory,
DialogMcpResourcesBrowser
} from '$lib/components/app';
import {
@ -26,19 +25,16 @@
SpecialFileType
} from '$lib/enums';
import { useChatFormPickers } from '$lib/hooks/use-chat-form-pickers.svelte';
import { chatStore } from '$lib/stores/chat.svelte';
import {
activeConversation,
activeMessages,
chatStore,
conversationsStore,
pendingCwd
} from '$lib/stores/conversations.svelte';
import { mcpStore } from '$lib/stores/mcp.svelte';
import { mcpHasResourceAttachments } from '$lib/stores/mcp-resources.svelte';
import { modelOptions, selectedModelId } from '$lib/stores/models.svelte';
import { isRouterMode } from '$lib/stores/server.svelte';
import { config } from '$lib/stores/settings.svelte';
import { toolsStore } from '$lib/stores/tools.svelte';
mcpResourceStore,
mcpStore,
modelsStore,
serverStore,
settingsStore,
toolsStore
} from '$lib/stores';
import type {
FileMentionEntry,
GetPromptResult,
@ -113,7 +109,7 @@
}: Props = $props();
// Component References
// Shared handle of the two input renderers (textarea + contenteditable).
// Shared handle of the two input renderers (plain textarea + rich chat form input).
type ChatInputHandle = {
focus(): void;
resetHeight(): void;
@ -124,16 +120,16 @@
let audioRecorder: AudioRecorder | undefined;
let chatFormActionsRef: ChatFormActions | undefined = $state(undefined);
let fileInputRef: ChatFormFileInputInvisible | undefined = $state(undefined);
let fileInputRef: ChatFormInputFileInputInvisible | undefined = $state(undefined);
let pickersRef: { handleKeydown: (event: KeyboardEvent) => boolean } | undefined =
$state(undefined);
let inputRef: ChatInputHandle | undefined = $state(undefined);
// Render-mode gate: the plain textarea by default, the contenteditable
// Render-mode gate: the plain textarea by default, the rich chat form input
// while the buffer carries a `file://` mention link or a complete code
// span (badges and code chips need a DOM the textarea cannot provide).
// Demotes back once neither remains.
let useContenteditable = $state(false);
let useRichInput = $state(false);
// Audio Recording State
let isRecording = $state(false);
@ -143,7 +139,7 @@
// float above the box.
let mentionAnchor: HTMLDivElement | null = $state(null);
let cwd = $derived(activeConversation()?.cwd ?? pendingCwd());
let cwd = $derived(conversationsStore.activeConversation?.cwd ?? conversationsStore.pendingCwd);
const pickers = useChatFormPickers({
focusInput: refocusInput,
@ -184,7 +180,7 @@
let isResourceDialogOpen = $state(false);
let preSelectedResourceUri = $state<string | undefined>(undefined);
let currentConfig = $derived(config());
let currentConfig = $derived(settingsStore.config);
let pasteLongTextToFileLength = $derived.by(() => {
const n = Number(currentConfig.pasteLongTextToFileLen);
@ -192,18 +188,18 @@
return Number.isNaN(n) ? Number(SETTING_CONFIG_DEFAULT.pasteLongTextToFileLen) : n;
});
let isRouter = $derived(isRouterMode());
let isRouter = $derived(serverStore.isRouterMode);
let conversationModel = $derived(
chatStore.getConversationModel(activeMessages() as DatabaseMessage[])
chatStore.getConversationModel(conversationsStore.activeMessages as DatabaseMessage[])
);
let activeModelId = $derived.by(() => {
const options = modelOptions();
const options = modelsStore.models;
if (!isRouter) {
return options.length > 0 ? options[0].model : null;
}
const selectedId = selectedModelId();
const selectedId = modelsStore.selectedModelId;
if (selectedId) {
const model = options.find((m) => m.id === selectedId);
@ -220,7 +216,9 @@
return null;
});
let hasModelSelected = $derived(!isRouter || !!conversationModel || !!selectedModelId());
let hasModelSelected = $derived(
!isRouter || !!conversationModel || !!modelsStore.selectedModelId
);
let hasLoadingAttachments = $derived(uploadedFiles.some((f) => f.isLoading));
let hasAttachments = $derived(
(attachments && attachments.length > 0) || (uploadedFiles && uploadedFiles.length > 0)
@ -243,16 +241,15 @@
}
$effect(() => {
const wantContenteditable =
containsFileMentionLink(value ?? '') || containsCodeSpan(value ?? '');
const wantRichInput = containsFileMentionLink(value ?? '') || containsCodeSpan(value ?? '');
if (useContenteditable === wantContenteditable) return;
if (useRichInput === wantRichInput) return;
if (!caretOffsetPinned) {
pendingCaretOffset = inputRef?.getCaretOffset() ?? (value ?? '').length;
}
useContenteditable = wantContenteditable;
useRichInput = wantRichInput;
queueCaretRestore();
});
@ -316,7 +313,7 @@
// Caret inside a fenced code block (closed, or still open
// while being typed): Enter adds a line, never submits. The
// contenteditable consumes this case locally; this gate
// rich chat form input consumes this case locally; this gate
// covers the plain textarea, where skipping submit lets the
// native newline through.
if (!isModifier && isOffsetInCodeBlock(value ?? '', inputRef?.getCaretOffset() ?? 0)) {
@ -509,9 +506,9 @@
value = built.newValue;
onValueChange?.(built.newValue);
// Already in contenteditable mode: no renderer flip, so the swap
// Already in rich chat form input mode: no renderer flip, so the swap
// effect's caret restore never runs.
if (useContenteditable) {
if (useRichInput) {
queueCaretRestore();
}
}
@ -545,7 +542,7 @@
}
</script>
<ChatFormFileInputInvisible bind:this={fileInputRef} onFileSelect={handleFileSelect} />
<ChatFormInputFileInputInvisible bind:this={fileInputRef} onFileSelect={handleFileSelect} />
<form
class="relative grid {className}"
@ -604,37 +601,22 @@
<div
class="flex-column relative min-h-12 items-center rounded-4xl md:rounded-3xl py-2 pb-2.25 shadow-sm transition-all focus-within:shadow-md md:py-3!"
>
{#if useContenteditable}
<ChatFormContenteditable
class="px-5 py-1.5 md:pt-0 mb-0.5"
bind:this={inputRef}
bind:value
onKeydown={handleKeydown}
onInput={() => {
pickers.handleInput();
onValueChange?.(value);
}}
onPaste={handlePaste}
{disabled}
{placeholder}
/>
{:else}
<ChatFormTextarea
class="px-5 py-1.5 md:pt-0"
bind:this={inputRef}
bind:value
onKeydown={handleKeydown}
onInput={() => {
pickers.handleInput();
onValueChange?.(value);
}}
onPaste={handlePaste}
{disabled}
{placeholder}
/>
{/if}
<ChatFormInput
class="px-5 py-1.5 md:pt-0"
bind:this={inputRef}
bind:value
onKeydown={handleKeydown}
onInput={() => {
pickers.handleInput();
onValueChange?.(value);
}}
onPaste={handlePaste}
{disabled}
{placeholder}
{useRichInput}
/>
{#if mcpHasResourceAttachments()}
{#if mcpResourceStore.hasAttachments}
<ChatFormMcpResourcesList
class="mb-3"
onResourceClick={(uri) => {
@ -668,7 +650,7 @@
<ContextGaugePopup />
{#if toolsStore.hasEnabledCwdTools}
<ChatFormWorkingDirectory
<ChatFormCurrentWorkingDirectory
directory={cwd}
isOpen={pickers.isWorkingDirectoryPickerOpen}
bind:query={pickers.workingDirectoryQuery}

3
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddButton.svelte
View file

@ -2,8 +2,7 @@
import { Plus } from '@lucide/svelte';
import { Button } from '$lib/components/ui/button';
import * as Tooltip from '$lib/components/ui/tooltip';
import { ATTACHMENT_TOOLTIP_TEXT } from '$lib/constants';
import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes';
import { ATTACHMENT_TOOLTIP_TEXT, ICON_CLASS_DEFAULT } from '$lib/constants';
interface Props {
disabled?: boolean;

62
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte
View file

@ -12,40 +12,19 @@
import {
ATTACHMENT_FILE_ITEMS,
ATTACHMENT_TOOLTIP_TEXT,
ICON_CLASS_DEFAULT,
TOOLTIP_DELAY_DURATION
} from '$lib/constants';
import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes';
import { getChatFormActionsContext } from '$lib/contexts';
import { useAttachmentMenu } from '$lib/hooks/use-attachment-menu.svelte';
interface Props {
class?: string;
disabled?: boolean;
hasAudioModality?: boolean;
hasVideoModality?: boolean;
hasVisionModality?: boolean;
hasMcpPromptsSupport?: boolean;
hasMcpResourcesSupport?: boolean;
onFileUpload?: () => void;
onSystemPromptClick?: () => void;
onMcpPromptClick?: () => void;
onMcpSettingsClick?: () => void;
onMcpResourcesClick?: () => void;
}
let {
class: className = '',
disabled = false,
hasAudioModality = false,
hasMcpPromptsSupport = false,
hasMcpResourcesSupport = false,
hasVideoModality = false,
hasVisionModality = false,
onFileUpload,
onMcpPromptClick,
onMcpResourcesClick,
onMcpSettingsClick,
onSystemPromptClick
}: Props = $props();
let { class: className = '' }: Props = $props();
const chatFormActions = getChatFormActionsContext();
let dropdownOpen = $state(false);
// The system message action moves focus to the message editor, so the menu
@ -54,18 +33,23 @@
function handleMcpSettingsClick() {
dropdownOpen = false;
onMcpSettingsClick?.();
chatFormActions.onMcpSettingsClick?.();
}
const attachmentMenu = useAttachmentMenu(
() => ({
hasAudioModality,
hasMcpPromptsSupport,
hasMcpResourcesSupport,
hasVideoModality,
hasVisionModality
hasAudioModality: chatFormActions.hasAudioModality,
hasMcpPromptsSupport: chatFormActions.hasMcpPromptsSupport,
hasMcpResourcesSupport: chatFormActions.hasMcpResourcesSupport,
hasVideoModality: chatFormActions.hasVideoModality,
hasVisionModality: chatFormActions.hasVisionModality
}),
() => ({
onFileUpload: chatFormActions.onFileUpload,
onMcpPromptClick: chatFormActions.onMcpPromptClick,
onMcpResourcesClick: chatFormActions.onMcpResourcesClick,
onSystemPromptClick: chatFormActions.onSystemPromptClick
}),
() => ({ onFileUpload, onMcpPromptClick, onMcpResourcesClick, onSystemPromptClick }),
() => {
dropdownOpen = false;
}
@ -85,7 +69,7 @@
buttonVariants({ variant: 'secondary' }),
'file-upload-button h-8 w-8 cursor-pointer rounded-full p-0'
)}
{disabled}
disabled={chatFormActions.disabled}
>
<span class="sr-only">{ATTACHMENT_TOOLTIP_TEXT}</span>
@ -162,7 +146,7 @@
class="flex cursor-pointer items-center gap-2"
onclick={() => {
suppressCloseAutoFocus = true;
onSystemPromptClick?.();
chatFormActions.onSystemPromptClick?.();
}}
>
<MessageSquare class={ICON_CLASS_DEFAULT} />
@ -174,12 +158,12 @@
<ChatFormActionAddMcpServersSubmenu onMcpSettingsClick={handleMcpSettingsClick} />
{#if hasMcpPromptsSupport}
{#if chatFormActions.hasMcpPromptsSupport}
<DropdownMenu.Separator />
<DropdownMenu.Item
class="flex cursor-pointer items-center gap-2"
onclick={onMcpPromptClick}
onclick={chatFormActions.onMcpPromptClick}
>
<Zap class={ICON_CLASS_DEFAULT} />
@ -187,10 +171,10 @@
</DropdownMenu.Item>
{/if}
{#if hasMcpResourcesSupport}
{#if chatFormActions.hasMcpResourcesSupport}
<DropdownMenu.Item
class="flex cursor-pointer items-center gap-2"
onclick={onMcpResourcesClick}
onclick={chatFormActions.onMcpResourcesClick}
>
<FolderOpen class={ICON_CLASS_DEFAULT} />

6
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte
View file

@ -4,11 +4,9 @@
import { DropdownMenuSearchable, McpLogo, McpServerIdentity } from '$lib/components/app';
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
import { Switch } from '$lib/components/ui/switch';
import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes';
import { ROUTES } from '$lib/constants/routes';
import { ICON_CLASS_DEFAULT, ROUTES } from '$lib/constants';
import { HealthCheckStatus } from '$lib/enums';
import { conversationsStore } from '$lib/stores/conversations.svelte';
import { mcpStore } from '$lib/stores/mcp.svelte';
import { conversationsStore, mcpStore } from '$lib/stores';
import type { MCPServerSettingsEntry } from '$lib/types';
interface Props {

2
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddReasoningSubmenu.svelte
View file

@ -2,7 +2,7 @@
import { Check, Info, Lightbulb, LightbulbOff } from '@lucide/svelte';
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
import * as Tooltip from '$lib/components/ui/tooltip';
import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes';
import { ICON_CLASS_DEFAULT } from '$lib/constants';
import { useReasoningMenu } from '$lib/hooks/use-reasoning-menu.svelte';
const reasoning = useReasoningMenu();

62
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte
View file

@ -14,47 +14,28 @@
import * as Sheet from '$lib/components/ui/sheet';
import { Switch } from '$lib/components/ui/switch';
import * as Tooltip from '$lib/components/ui/tooltip';
import { TOOLTIP_DELAY_DURATION } from '$lib/constants';
import { ATTACHMENT_FILE_ITEMS } from '$lib/constants/attachment-menu';
import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes';
import {
ATTACHMENT_FILE_ITEMS,
ICON_CLASS_DEFAULT,
TOOLTIP_DELAY_DURATION
} from '$lib/constants';
import { getChatFormActionsContext } from '$lib/contexts';
import { HealthCheckStatus } from '$lib/enums';
import { AttachmentAction } from '$lib/enums/attachment.enums';
import { useAttachmentMenu } from '$lib/hooks/use-attachment-menu.svelte';
import { useReasoningMenu } from '$lib/hooks/use-reasoning-menu.svelte';
import { useToolsPanel } from '$lib/hooks/use-tools-panel.svelte';
import { conversationsStore } from '$lib/stores/conversations.svelte';
import { mcpStore } from '$lib/stores/mcp.svelte';
import { conversationsStore, mcpStore } from '$lib/stores';
import type { Snippet } from 'svelte';
interface Props {
class?: string;
disabled?: boolean;
hasAudioModality?: boolean;
hasVideoModality?: boolean;
hasVisionModality?: boolean;
hasMcpPromptsSupport?: boolean;
hasMcpResourcesSupport?: boolean;
onFileUpload?: () => void;
onSystemPromptClick?: () => void;
onMcpPromptClick?: () => void;
onMcpResourcesClick?: () => void;
trigger: Snippet<[{ disabled: boolean; onclick?: () => void }]>;
}
let {
class: className = '',
disabled = false,
hasAudioModality = false,
hasMcpPromptsSupport = false,
hasMcpResourcesSupport = false,
hasVideoModality = false,
hasVisionModality = false,
onFileUpload,
onMcpPromptClick,
onMcpResourcesClick,
onSystemPromptClick,
trigger
}: Props = $props();
let { class: className = '', trigger }: Props = $props();
const chatFormActions = getChatFormActionsContext();
let sheetOpen = $state(false);
let reasoningExpanded = $state(false);
@ -64,13 +45,18 @@
const attachmentMenu = useAttachmentMenu(
() => ({
hasAudioModality,
hasMcpPromptsSupport,
hasMcpResourcesSupport,
hasVideoModality,
hasVisionModality
hasAudioModality: chatFormActions.hasAudioModality,
hasMcpPromptsSupport: chatFormActions.hasMcpPromptsSupport,
hasMcpResourcesSupport: chatFormActions.hasMcpResourcesSupport,
hasVideoModality: chatFormActions.hasVideoModality,
hasVisionModality: chatFormActions.hasVisionModality
}),
() => ({
onFileUpload: chatFormActions.onFileUpload,
onMcpPromptClick: chatFormActions.onMcpPromptClick,
onMcpResourcesClick: chatFormActions.onMcpResourcesClick,
onSystemPromptClick: chatFormActions.onSystemPromptClick
}),
() => ({ onFileUpload, onMcpPromptClick, onMcpResourcesClick, onSystemPromptClick }),
() => {
sheetOpen = false;
}
@ -90,7 +76,7 @@
<div class="flex items-center gap-1 {className}">
<Sheet.Root bind:open={sheetOpen}>
{@render trigger({ disabled, onclick: () => (sheetOpen = true) })}
{@render trigger({ disabled: chatFormActions.disabled, onclick: () => (sheetOpen = true) })}
<Sheet.Content side="bottom" class="max-h-[85vh] gap-0 overflow-y-auto">
<Sheet.Header>
@ -349,7 +335,7 @@
<span>System Message</span>
</button>
{#if hasMcpPromptsSupport}
{#if chatFormActions.hasMcpPromptsSupport}
<button
type="button"
class={sheetItemClass}
@ -361,7 +347,7 @@
</button>
{/if}
{#if hasMcpResourcesSupport}
{#if chatFormActions.hasMcpResourcesSupport}
<button
type="button"
class={sheetItemClass}

6
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddToolsSubmenu.svelte
View file

@ -4,11 +4,9 @@
import * as Collapsible from '$lib/components/ui/collapsible';
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
import * as Tooltip from '$lib/components/ui/tooltip';
import { CLI_FLAGS } from '$lib/constants';
import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes';
import { CLI_FLAGS, ICON_CLASS_DEFAULT } from '$lib/constants';
import { useToolsPanel } from '$lib/hooks/use-tools-panel.svelte';
import { mcpStore } from '$lib/stores/mcp.svelte';
import { toolsStore } from '$lib/stores/tools.svelte';
import { mcpStore, toolsStore } from '$lib/stores';
const toolsPanel = useToolsPanel();
const hasMcpServersAvailable = $derived(mcpStore.getServers().length > 0);

57
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte
View file

@ -2,66 +2,15 @@
import ChatFormActionAddButton from './ChatFormActionAddButton.svelte';
import ChatFormActionAddDropdown from './ChatFormActionAddDropdown.svelte';
import ChatFormActionAddSheet from './ChatFormActionAddSheet.svelte';
import { isMobile } from '$lib/stores/viewport.svelte';
interface Props {
disabled?: boolean;
hasAudioModality?: boolean;
hasVideoModality?: boolean;
hasMcpPromptsSupport?: boolean;
hasMcpResourcesSupport?: boolean;
hasVisionModality?: boolean;
onFileUpload?: () => void;
onMcpPromptClick?: () => void;
onMcpResourcesClick?: () => void;
onMcpSettingsClick?: () => void;
onSystemPromptClick?: () => void;
}
let {
disabled = false,
hasAudioModality = false,
hasMcpPromptsSupport = false,
hasMcpResourcesSupport = false,
hasVideoModality = false,
hasVisionModality = false,
onFileUpload,
onMcpPromptClick,
onMcpResourcesClick,
onMcpSettingsClick,
onSystemPromptClick
}: Props = $props();
import { isMobile } from '$lib/stores';
</script>
{#if isMobile.current}
<ChatFormActionAddSheet
{disabled}
{hasAudioModality}
{hasVideoModality}
{hasVisionModality}
{hasMcpPromptsSupport}
{hasMcpResourcesSupport}
{onFileUpload}
{onSystemPromptClick}
{onMcpPromptClick}
{onMcpResourcesClick}
>
<ChatFormActionAddSheet>
{#snippet trigger({ disabled, onclick })}
<ChatFormActionAddButton {disabled} {onclick} />
{/snippet}
</ChatFormActionAddSheet>
{:else}
<ChatFormActionAddDropdown
{disabled}
{hasAudioModality}
{hasVideoModality}
{hasVisionModality}
{hasMcpPromptsSupport}
{hasMcpResourcesSupport}
{onFileUpload}
{onMcpPromptClick}
{onMcpResourcesClick}
{onMcpSettingsClick}
{onSystemPromptClick}
/>
<ChatFormActionAddDropdown />
{/if}

31
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte
View file

@ -1,15 +1,6 @@
<script lang="ts">
import { ModelsSelectorDropdown, ModelsSelectorSheet } from '$lib/components/app';
import { chatStore } from '$lib/stores/chat.svelte';
import { activeMessages } from '$lib/stores/conversations.svelte';
import {
modelOptions,
modelsStore,
selectedModelId,
selectedModelName
} from '$lib/stores/models.svelte';
import { isRouterMode, serverError } from '$lib/stores/server.svelte';
import { isMobile } from '$lib/stores/viewport.svelte';
import { chatStore, conversationsStore, isMobile, modelsStore, serverStore } from '$lib/stores';
interface Props {
disabled?: boolean;
@ -35,17 +26,17 @@
useGlobalSelection = false
}: Props = $props();
let isRouter = $derived(isRouterMode());
let isOffline = $derived(!!serverError());
let isRouter = $derived(serverStore.isRouterMode);
let isOffline = $derived(!!serverStore.error);
let conversationModel = $derived(
chatStore.getConversationModel(activeMessages() as DatabaseMessage[])
chatStore.getConversationModel(conversationsStore.activeMessages as DatabaseMessage[])
);
let lastSyncedConversationModel: string | null = null;
let selectorModel = $derived.by(() => {
const storeModel = selectedModelName();
const storeModel = modelsStore.selectedModelName;
if (storeModel && storeModel !== conversationModel) {
return storeModel;
@ -60,7 +51,7 @@
$effect(() => {
if (conversationModel && conversationModel !== lastSyncedConversationModel) {
if (modelOptions().some((m) => m.model === conversationModel)) {
if (modelsStore.models.some((m) => m.model === conversationModel)) {
modelsStore.selectedModelName = conversationModel;
modelsStore.selectModelByName(conversationModel);
} else {
@ -73,24 +64,24 @@
isRouter &&
!modelsStore.selectedModelId &&
modelsStore.loadedModelIds.length > 0 &&
activeMessages().length > 0 &&
conversationsStore.activeMessages.length > 0 &&
!conversationModel
) {
lastSyncedConversationModel = null;
const first = modelOptions().find((m) => modelsStore.loadedModelIds.includes(m.model));
const first = modelsStore.models.find((m) => modelsStore.loadedModelIds.includes(m.model));
if (first) modelsStore.selectModelById(first.id);
}
});
let activeModelId = $derived.by(() => {
const options = modelOptions();
const options = modelsStore.models;
if (!isRouter) {
return options.length > 0 ? options[0].model : null;
}
const selectedId = selectedModelId();
const selectedId = modelsStore.selectedModelId;
if (selectedId) {
const model = options.find((m) => m.id === selectedId);
@ -140,7 +131,7 @@
});
$effect(() => {
hasModelSelected = !isRouter || !!conversationModel || !!selectedModelId();
hasModelSelected = !isRouter || !!conversationModel || !!modelsStore.selectedModelId;
});
$effect(() => {

2
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte
View file

@ -2,7 +2,7 @@
import { Mic, Square } from '@lucide/svelte';
import { Button } from '$lib/components/ui/button';
import * as Tooltip from '$lib/components/ui/tooltip';
import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes';
import { ICON_CLASS_DEFAULT } from '$lib/constants';
interface Props {
class?: string;

71
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte
View file

@ -10,18 +10,11 @@
ChatFormContextGauge
} from '$lib/components/app';
import { Button } from '$lib/components/ui/button';
import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes';
import { ROUTES } from '$lib/constants/routes';
import { ICON_CLASS_DEFAULT, ROUTES } from '$lib/constants';
import { setChatFormActionsContext } from '$lib/contexts';
import { FileTypeCategory, MessageRole } from '$lib/enums';
import { ChatService } from '$lib/services';
import {
activeProcessingState,
isChatStreaming,
isLoading as chatIsLoading
} from '$lib/stores/chat.svelte';
import { activeMessages, conversationsStore } from '$lib/stores/conversations.svelte';
import { mcpStore } from '$lib/stores/mcp.svelte';
import { config } from '$lib/stores/settings.svelte';
import { chatStore, conversationsStore, mcpStore, settingsStore } from '$lib/stores';
import { getFileTypeCategory } from '$lib/utils';
interface Props {
@ -62,7 +55,7 @@
uploadedFiles = []
}: Props = $props();
let currentConfig = $derived(config());
let currentConfig = $derived(settingsStore.config);
let hasMcpPromptsSupport = $derived.by(() => {
const perChatOverrides = conversationsStore.getAllMcpServerOverrides();
@ -104,7 +97,7 @@
let hasProcessedTokens = $derived.by(() => {
if (!page.params.id) return false;
const messages = activeMessages() as DatabaseMessage[];
const messages = conversationsStore.activeMessages as DatabaseMessage[];
let totalHistoricalTokens = 0;
@ -126,9 +119,9 @@
if (totalHistoricalTokens > 0) return true;
if (!chatIsLoading() && !isChatStreaming()) return false;
if (!chatStore.isLoading && !chatStore.isStreaming()) return false;
const processingState = activeProcessingState();
const processingState = chatStore.activeProcessingState;
if (!processingState) return false;
@ -140,6 +133,42 @@
return livePromptTokens > 0 || liveOutputTokens > 0;
});
setChatFormActionsContext({
get disabled() {
return disabled;
},
get hasAudioModality() {
return hasAudioModality;
},
get hasMcpPromptsSupport() {
return hasMcpPromptsSupport;
},
get hasMcpResourcesSupport() {
return hasMcpResourcesSupport;
},
get hasVideoModality() {
return hasVideoModality;
},
get hasVisionModality() {
return hasVisionModality;
},
get onFileUpload() {
return onFileUpload;
},
get onMcpPromptClick() {
return onMcpPromptClick;
},
get onMcpResourcesClick() {
return onMcpResourcesClick;
},
get onMcpSettingsClick() {
return () => goto(ROUTES.MCP_SERVERS);
},
get onSystemPromptClick() {
return onSystemPromptClick;
}
});
</script>
<div
@ -148,19 +177,7 @@
>
{#if showAddButton}
<div class="mr-auto flex items-center gap-2">
<ChatFormActionsAdd
{disabled}
{hasAudioModality}
{hasVideoModality}
{hasVisionModality}
{hasMcpPromptsSupport}
{hasMcpResourcesSupport}
{onFileUpload}
{onSystemPromptClick}
{onMcpPromptClick}
{onMcpResourcesClick}
onMcpSettingsClick={() => goto(ROUTES.MCP_SERVERS)}
/>
<ChatFormActionsAdd />
</div>
{/if}

14
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte
View file

@ -1,32 +1,32 @@
<script lang="ts">
import ContextGaugeDial from './ContextGaugeDial.svelte';
import { useContextGauge } from '$lib/hooks/use-context-gauge.svelte';
import { chatStore, isChatStreaming, isLoading } from '$lib/stores/chat.svelte';
import {
chatStore,
conversationsStore,
gaugeTriggerClick,
gaugeTriggerEnter,
gaugeTriggerKeydown,
gaugeTriggerLeave,
gaugeTriggerPointerDown
} from '$lib/stores/context-gauge-popup.svelte';
import { activeConversation, activeMessages } from '$lib/stores/conversations.svelte';
} from '$lib/stores';
import { untrack } from 'svelte';
const gauge = useContextGauge();
$effect(() => {
const conv = activeConversation();
const conv = conversationsStore.activeConversation;
untrack(() => chatStore.setActiveProcessingConversation(conv?.id ?? null));
});
$effect(() => {
const conv = activeConversation();
const messages = activeMessages() as DatabaseMessage[];
const conv = conversationsStore.activeConversation;
const messages = conversationsStore.activeMessages as DatabaseMessage[];
if (!conv) return;
if (isLoading() || isChatStreaming()) return;
if (chatStore.isLoading || chatStore.isStreaming()) return;
if (messages.length === 0) {
untrack(() => chatStore.clearProcessingState(conv.id));

2
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDial.svelte
View file

@ -1,6 +1,6 @@
<script lang="ts">
import type { ColorLevel } from './context-gauge';
import { colorLevelTextClass } from './context-gauge';
import type { ColorLevel } from '$lib/enums';
interface Props {
percent: number | null;

9
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte
View file

@ -3,12 +3,7 @@
import ContextGaugeDetails from './ContextGaugeDetails.svelte';
import ContextGaugeLoadModel from './ContextGaugeLoadModel.svelte';
import { useContextGauge } from '$lib/hooks/use-context-gauge.svelte';
import {
gaugeCardEnter,
gaugeCardLeave,
gaugePopup,
gaugePopupClose
} from '$lib/stores/context-gauge-popup.svelte';
import { gaugeCardEnter, gaugeCardLeave, gaugePopup, gaugePopupClose } from '$lib/stores';
import { formatParameters } from '$lib/utils/formatters';
const gauge = useContextGauge();
@ -92,7 +87,7 @@
<span class={colorLevelTextClass(gauge.colorLevel)}>{gauge.contextPercent}%</span> used
</span>
<span>
{formatParameters((gauge.contextTotal ?? 0) - gauge.contextUsed)} remaining
{formatParameters(gauge.contextAvailable ?? 0)} remaining
</span>
</div>
{:else}

22
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/context-gauge.ts
View file

@ -1,25 +1,25 @@
export type ColorLevel = 'ok' | 'warning' | 'critical' | 'neutral';
import { ColorLevel } from '$lib/enums';
const WARNING_THRESHOLD = 80;
const CRITICAL_THRESHOLD = 95;
export function colorLevelFromPercent(percent: number | null): ColorLevel {
if (percent === null) return 'neutral';
if (percent === null) return ColorLevel.NEUTRAL;
if (percent >= CRITICAL_THRESHOLD) return 'critical';
if (percent >= CRITICAL_THRESHOLD) return ColorLevel.CRITICAL;
if (percent >= WARNING_THRESHOLD) return 'warning';
if (percent >= WARNING_THRESHOLD) return ColorLevel.WARNING;
return 'ok';
return ColorLevel.OK;
}
export function colorLevelTextClass(level: ColorLevel): string {
switch (level) {
case 'critical':
case ColorLevel.CRITICAL:
return 'text-red-400';
case 'warning':
case ColorLevel.WARNING:
return 'text-amber-400';
case 'ok':
case ColorLevel.OK:
return 'text-muted-foreground';
default:
return 'text-muted-foreground';
@ -28,11 +28,11 @@ export function colorLevelTextClass(level: ColorLevel): string {
export function colorLevelBgClass(level: ColorLevel): string {
switch (level) {
case 'critical':
case ColorLevel.CRITICAL:
return 'bg-red-500';
case 'warning':
case ColorLevel.WARNING:
return 'bg-amber-500';
case 'ok':
case ColorLevel.OK:
return 'bg-green-500';
default:
return 'bg-muted';

39
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormWorkingDirectory.svelte → tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectory.svelte
View file

@ -1,29 +1,20 @@
<script lang="ts">
import ChatFormWorkingDirectoryChip from './ChatFormWorkingDirectoryChip.svelte';
import ChatFormWorkingDirectoryResultsList from './ChatFormWorkingDirectoryResultsList.svelte';
import ChatFormCurrentWorkingDirectoryChip from './ChatFormCurrentWorkingDirectoryChip.svelte';
import ChatFormCurrentWorkingDirectoryResultsList from './ChatFormCurrentWorkingDirectoryResultsList.svelte';
import { FolderOpen } from '@lucide/svelte';
import SearchInput from '$lib/components/app/forms/SearchInput.svelte';
import * as Popover from '$lib/components/ui/popover';
import {
DEFAULT_MOBILE_BREAKPOINT,
HOME_TILDE,
MAX_RESULTS_SHOWN,
NATIVE_LIMIT,
NATIVE_MAX_DEPTH,
SEARCH_DEBOUNCE_MS,
SEARCH_LIMIT,
SEARCH_MAX_DEPTH
} from '$lib/constants';
import { DEFAULT_MOBILE_BREAKPOINT, HOME_TILDE, SEARCH, UI_DATA_ATTRS } from '$lib/constants';
import { BuiltInTool, GlobSearchType, KeyboardKey } from '$lib/enums';
import { useDebouncedSearch } from '$lib/hooks/use-debounced-search.svelte';
import { usePickerNavigation } from '$lib/hooks/use-picker-navigation.svelte';
import { useScrollActiveRow } from '$lib/hooks/use-scroll-active-row.svelte';
import { ToolsService } from '$lib/services/tools.service';
import { toolsStore } from '$lib/stores/tools.svelte';
import { toolsStore } from '$lib/stores';
import type { GlobEntry } from '$lib/types';
import {
abbreviateHome,
buildCaseInsensitiveGlob,
type GlobEntry,
joinPath,
lastPathSegment,
runGlobSearchWithChildren
@ -129,7 +120,7 @@
});
useScrollActiveRow({
dataIndex: 'result',
dataAttr: UI_DATA_ATTRS.RESULT_INDEX,
getContainer: () => listContainer,
getCount: () => queryResults.length,
getIndex: () => nav.hoveredIndex,
@ -142,7 +133,7 @@
// children too, so path navigation does not require a trailing slash.
const search = useDebouncedSearch({
canRun: () => isOpen && fileSearchEnabled,
debounceMs: SEARCH_DEBOUNCE_MS,
debounceMs: SEARCH.DEBOUNCE_MS,
getQuery: () => query.trim(),
run: async (q, signal, isCurrent) => {
const trimmed = q.trim();
@ -162,8 +153,8 @@
const res = await runGlobSearchWithChildren(
trimmed,
homeBase ?? HOME_TILDE,
SEARCH_MAX_DEPTH,
SEARCH_LIMIT,
SEARCH.MAX_DEPTH,
SEARCH.LIMIT,
signal,
{ type: GlobSearchType.DIR }
);
@ -179,7 +170,7 @@
}
searchScope = res.exactDir ?? res.args.path;
queryResults = res.entries.map((e) => e.path).slice(0, MAX_RESULTS_SHOWN);
queryResults = res.entries.map((e) => e.path).slice(0, SEARCH.MAX_RESULTS_SHOWN);
if (queryResults.length > 0) {
nav.reset(0);
@ -223,8 +214,8 @@
try {
const res = await ToolsService.executeToolRaw(BuiltInTool.FILE_GLOB_SEARCH, {
include: buildCaseInsensitiveGlob(name),
limit: NATIVE_LIMIT,
max_depth: NATIVE_MAX_DEPTH,
limit: SEARCH.NATIVE_LIMIT,
max_depth: SEARCH.NATIVE_MAX_DEPTH,
path: homeBase ?? HOME_TILDE,
type: GlobSearchType.DIR
});
@ -259,7 +250,7 @@
// user cancelled - silently ignore; other errors are logged
if (err instanceof DOMException && err.name === 'AbortError') return;
console.error('[ChatFormWorkingDirectory] showDirectoryPicker failed:', err);
console.error('[ChatFormCurrentWorkingDirectory] showDirectoryPicker failed:', err);
}
}
@ -340,7 +331,7 @@
onclick={onOpen}
{disabled}
>
<ChatFormWorkingDirectoryChip
<ChatFormCurrentWorkingDirectoryChip
{directory}
{homeBase}
{disabled}
@ -381,7 +372,7 @@
{#if !fileSearchEnabled}
<div class="px-2 py-1.5 text-sm text-muted-foreground">{searchUnavailableMessage}</div>
{:else if query.trim() && (search.isSearching || queryResults.length > 0 || searchError)}
<ChatFormWorkingDirectoryResultsList
<ChatFormCurrentWorkingDirectoryResultsList
results={queryResults}
hoveredIndex={nav.hoveredIndex}
isSearching={search.isSearching}

0
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormWorkingDirectoryChip.svelte → tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectoryChip.svelte
View file

3
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormWorkingDirectoryResultsList.svelte → tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectoryResultsList.svelte
View file

@ -1,6 +1,7 @@
<script lang="ts">
import { Folder } from '@lucide/svelte';
import { cn } from '$lib/components/ui/utils';
import { UI_DATA_ATTRS } from '$lib/constants';
import { highlightMatch } from '$lib/utils';
import { fly } from 'svelte/transition';
@ -46,7 +47,7 @@
{#each results as path, index (path)}
<button
type="button"
data-result-index={index}
{...{ [UI_DATA_ATTRS.RESULT_INDEX]: index }}
data-highlighted={index === hoveredIndex ? '' : undefined}
class={cn(
'relative flex w-full cursor-pointer items-center gap-2 rounded-sm px-2 py-1.5 text-sm outline-hidden select-none data-highlighted:bg-accent data-highlighted:text-accent-foreground'

78
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInput.svelte Normal file
View file

@ -0,0 +1,78 @@
<script lang="ts">
import ChatFormInputBasic from './ChatFormInputBasic.svelte';
import ChatFormInputRich from './ChatFormInputRich.svelte';
interface Props {
class?: string;
disabled?: boolean;
onInput?: () => void;
onKeydown?: (event: KeyboardEvent) => void;
onPaste?: (event: ClipboardEvent) => void;
placeholder?: string;
value?: string;
useRichInput?: boolean;
}
let {
class: className = '',
disabled = false,
onInput,
onKeydown,
onPaste,
placeholder = 'Ask anything...',
useRichInput = false,
value = $bindable('')
}: Props = $props();
let basicRef: ChatFormInputBasic | undefined = $state();
let richRef: ChatFormInputRich | undefined = $state();
// The two renderers share one imperative handle (focus/caret/height), so
// the parent can drive whichever variant is mounted through this one.
export function getElement() {
return useRichInput ? richRef?.getElement() : basicRef?.getElement();
}
export function focus() {
if (useRichInput) richRef?.focus();
else basicRef?.focus();
}
export function resetHeight() {
if (useRichInput) richRef?.resetHeight();
else basicRef?.resetHeight();
}
export function getCaretOffset(): number {
return useRichInput ? (richRef?.getCaretOffset() ?? 0) : (basicRef?.getCaretOffset() ?? 0);
}
export function setCaretOffset(offset: number) {
if (useRichInput) richRef?.setCaretOffset(offset);
else basicRef?.setCaretOffset(offset);
}
</script>
{#if useRichInput}
<ChatFormInputRich
bind:this={richRef}
class={className}
{disabled}
{onInput}
{onKeydown}
{onPaste}
{placeholder}
bind:value
/>
{:else}
<ChatFormInputBasic
bind:this={basicRef}
class={className}
{disabled}
{onInput}
{onKeydown}
{onPaste}
{placeholder}
bind:value
/>
{/if}

4
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte → tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputBasic.svelte
View file

@ -1,5 +1,5 @@
<script lang="ts">
import { isMobile } from '$lib/stores/viewport.svelte';
import { isMobile } from '$lib/stores';
import { autoResizeTextarea } from '$lib/utils';
import { onMount } from 'svelte';
@ -48,7 +48,7 @@
}
}
// Plain-text caret offsets, shared with the contenteditable variant so
// Plain-text caret offsets, shared with the rich chat form input variant so
// the picker/paste flows can address either renderer through one handle.
export function getCaretOffset(): number {
if (!textareaElement) return 0;

0
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormFileInputInvisible.svelte → tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputFileInputInvisible.svelte
View file

69
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContenteditable.svelte → tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputRich.svelte
View file

@ -1,8 +1,9 @@
<script lang="ts">
import { TRIM_LEADING_PADDING_REGEX, TRIM_TRAILING_PADDING_REGEX } from '$lib/constants';
import { ColorMode } from '$lib/enums';
import { isMobile } from '$lib/stores/viewport.svelte';
import type { ContentToken, SourceHistoryEntry } from '$lib/utils';
import { CODE_BLOCK, CODE_TOKEN_ATTR, UI_DATA_ATTRS } from '$lib/constants';
import { BooleanString, ChatFormInputRichTokenKind, ColorMode } from '$lib/enums';
import { isMobile } from '$lib/stores';
import type { ChatFormInputRichToken } from '$lib/types';
import type { SourceHistoryEntry } from '$lib/utils';
import {
badgeAwareWordJump,
buildFragment,
@ -52,7 +53,7 @@
// browser's native undo stack.
const history = new SourceHistory();
// Browsers disagree on what an empty contenteditable contains (`<br>`,
// Browsers disagree on what an empty rich chat form input contains (`<br>`,
// `<div><br></div>`, or nothing), so emptiness is decided by the
// serialized source, not the DOM shape.
function syncEmptyState(serialized?: string) {
@ -60,15 +61,15 @@
const source = serialized ?? serializeContent(rootElement);
rootElement.dataset.empty = source.length === 0 ? 'true' : 'false';
rootElement.dataset.empty = source.length === 0 ? BooleanString.TRUE : BooleanString.FALSE;
}
function renderTokens(tokens: ContentToken[]) {
function renderTokens(tokens: ChatFormInputRichToken[]) {
if (!rootElement) return;
const caret = rangeToTextOffset(rootElement, safeRange());
// eslint-disable-next-line svelte/no-dom-manipulating -- the token layer is owned imperatively; Svelte renders only the contenteditable host, never its children
// eslint-disable-next-line svelte/no-dom-manipulating -- the token layer is owned imperatively; Svelte renders only the rich chat form input host, never its children
rootElement.replaceChildren(buildFragment(tokens));
syncCodeBlockHatches(rootElement);
@ -105,8 +106,8 @@
const prefix = open[0];
const language = open[1].trim().split(/\s+/)[0] ?? '';
const content = segment.slice(prefix.length, -3);
const leading = content.match(TRIM_LEADING_PADDING_REGEX)?.[0] ?? '';
const trailing = content.match(TRIM_TRAILING_PADDING_REGEX)?.[0] ?? '';
const leading = content.match(CODE_BLOCK.TRIM_LEADING_PADDING_REGEX)?.[0] ?? '';
const trailing = content.match(CODE_BLOCK.TRIM_TRAILING_PADDING_REGEX)?.[0] ?? '';
const core = content.slice(leading.length, content.length - trailing.length);
// autoDetect off: re-guessing the language on every keystroke
// costs ~38ms a call and flickers while typing
@ -126,7 +127,9 @@
}
function highlightCodeBlocks(root: HTMLElement) {
for (const el of root.querySelectorAll<HTMLElement>('code[data-code-token="block"]')) {
for (const el of root.querySelectorAll<HTMLElement>(
`code[${CODE_TOKEN_ATTR}="${ChatFormInputRichTokenKind.CODE_BLOCK}"]`
)) {
highlightCodeBlockElement(el);
}
}
@ -150,7 +153,10 @@
}
while (node && node !== rootElement) {
if (node instanceof HTMLElement && node.dataset.codeToken === 'block') {
if (
node instanceof HTMLElement &&
node.getAttribute(CODE_TOKEN_ATTR) === ChatFormInputRichTokenKind.CODE_BLOCK
) {
const caret = rangeToTextOffset(rootElement, range);
if (highlightCodeBlockElement(node)) {
@ -188,11 +194,13 @@
* (deduped via the data attribute) swapped on mode change.
*/
function loadHighlightTheme(isDark: boolean) {
document.querySelectorAll('style[data-highlight-theme-preview]').forEach((s) => s.remove());
document
.querySelectorAll(`style[${UI_DATA_ATTRS.HIGHLIGHT_THEME_PREVIEW}]`)
.forEach((s) => s.remove());
const style = document.createElement('style');
style.setAttribute('data-highlight-theme-preview', 'true');
style.setAttribute(UI_DATA_ATTRS.HIGHLIGHT_THEME_PREVIEW, BooleanString.TRUE);
style.textContent = isDark ? githubDarkCss : githubLightCss;
document.head.appendChild(style);
@ -310,7 +318,7 @@
source[source.length - 2] !== '\n' &&
last?.nodeType === Node.TEXT_NODE
) {
// eslint-disable-next-line svelte/no-dom-manipulating -- the token layer is owned imperatively; Svelte renders only the contenteditable host, never its children
// eslint-disable-next-line svelte/no-dom-manipulating -- the token layer is owned imperatively; Svelte renders only the rich chat form input host, never its children
rootElement.appendChild(document.createTextNode('\n'));
restoreCaret(source.length);
resizeHeight();
@ -403,7 +411,10 @@
let node: Node | null = container.parentNode;
while (node && node !== rootElement) {
if (node instanceof HTMLElement && node.dataset.codeToken === 'block') {
if (
node instanceof HTMLElement &&
node.getAttribute(CODE_TOKEN_ATTR) === ChatFormInputRichTokenKind.CODE_BLOCK
) {
const tail = document.createRange();
tail.setStart(container, offset);
@ -461,7 +472,11 @@
const first = rootElement.firstChild;
if (!(first instanceof HTMLElement) || first.dataset.codeToken !== 'block') return false;
if (
!(first instanceof HTMLElement) ||
first.getAttribute(CODE_TOKEN_ATTR) !== ChatFormInputRichTokenKind.CODE_BLOCK
)
return false;
const range = safeRange();
@ -482,7 +497,7 @@
if (firstLineEnd !== -1 && caret > firstLineEnd) return false;
}
// eslint-disable-next-line svelte/no-dom-manipulating -- the token layer is owned imperatively; Svelte renders only the contenteditable host, never its children
// eslint-disable-next-line svelte/no-dom-manipulating -- the token layer is owned imperatively; Svelte renders only the rich chat form input host, never its children
rootElement.prepend(document.createElement('br'));
restoreCaret(0, extend);
@ -506,7 +521,11 @@
const second = first.nextSibling;
if (!(second instanceof HTMLElement) || second.dataset.codeToken !== 'block') return;
if (
!(second instanceof HTMLElement) ||
second.getAttribute(CODE_TOKEN_ATTR) !== ChatFormInputRichTokenKind.CODE_BLOCK
)
return;
const range = safeRange();
const onHatch =
@ -786,7 +805,7 @@
}
</script>
<div class="flex-1 {className}">
<div class="flex-1 {className} mb-0.5">
<div
bind:this={rootElement}
contenteditable={!disabled}
@ -797,7 +816,7 @@
data-placeholder={placeholder}
tabindex={disabled ? -1 : 0}
class={[
'chat-form-contenteditable text-md min-h-12 w-full overflow-y-auto whitespace-pre-wrap wrap-break-word border-0 bg-transparent p-0 leading-6 outline-none focus-visible:ring-0 focus-visible:ring-offset-0',
'chat-form-input-rich text-md min-h-12 w-full overflow-y-auto whitespace-pre-wrap wrap-break-word border-0 bg-transparent p-0 leading-6 outline-none focus-visible:ring-0 focus-visible:ring-offset-0',
disabled && 'cursor-not-allowed'
]}
style="max-height: var(--max-message-height);"
@ -814,18 +833,18 @@
<style>
/* pre-wrap is load-bearing: without it Chromium collapses \n in
text nodes and converts them to spaces while typing */
.chat-form-contenteditable {
.chat-form-input-rich {
white-space: pre-wrap;
}
.chat-form-contenteditable:global([data-empty='true'])::before {
.chat-form-input-rich:global([data-empty='true'])::before {
content: attr(data-placeholder);
color: var(--muted-foreground);
pointer-events: none;
}
/* Inline code - mirrors markdown-content.css */
.chat-form-contenteditable :global(code[data-code-token='inline']) {
.chat-form-input-rich :global(code[data-code-token='code_inline']) {
background: var(--muted);
color: var(--muted-foreground);
padding: 0.125rem 0.375rem;
@ -834,7 +853,7 @@
}
/* Fenced code block - mirrors .code-block-wrapper in markdown-content.css */
.chat-form-contenteditable :global(code[data-code-token='block']) {
.chat-form-input-rich :global(code[data-code-token='code_block']) {
display: block;
margin: 0.25rem 0;
padding: 0.75rem 1rem;

10
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte
View file

@ -3,11 +3,7 @@
ChatAttachmentsListItemMcpResource,
HorizontalScrollCarousel
} from '$lib/components/app';
import { mcpStore } from '$lib/stores/mcp.svelte';
import {
mcpHasResourceAttachments,
mcpResourceAttachments
} from '$lib/stores/mcp-resources.svelte';
import { mcpResourceStore, mcpStore } from '$lib/stores';
interface Props {
class?: string;
@ -16,8 +12,8 @@
let { class: className, onResourceClick }: Props = $props();
const attachments = $derived(mcpResourceAttachments());
const hasAttachments = $derived(mcpHasResourceAttachments());
const attachments = $derived(mcpResourceStore.attachments);
const hasAttachments = $derived(mcpResourceStore.hasAttachments);
function handleRemove(attachmentId: string) {
mcpStore.removeResourceAttachment(attachmentId);

2
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerItemHeader.svelte
View file

@ -1,5 +1,5 @@
<script lang="ts">
import { mcpStore } from '$lib/stores/mcp.svelte';
import { mcpStore } from '$lib/stores';
import type { MCPServerSettingsEntry } from '$lib/types';
import type { Snippet } from 'svelte';

4
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerList.svelte
View file

@ -1,7 +1,7 @@
<script lang="ts" generics="T">
import { SearchInput } from '$lib/components/app';
import ScrollArea from '$lib/components/ui/scroll-area/scroll-area.svelte';
import { CHAT_FORM_POPOVER_MAX_HEIGHT } from '$lib/constants';
import { CHAT_FORM_POPOVER_MAX_HEIGHT, UI_DATA_ATTRS } from '$lib/constants';
import { useScrollActiveRow } from '$lib/hooks/use-scroll-active-row.svelte';
import type { Snippet } from 'svelte';
@ -55,7 +55,7 @@
// selectedIndex/items.length are untracked so hover and result replacement
// never re-fire the scroll; keyboard nav is the only path that bumps the trigger.
useScrollActiveRow({
dataIndex: 'picker',
dataAttr: UI_DATA_ATTRS.PICKER_INDEX,
getContainer: () => listContainer,
getCount: () => items.length,
getIndex: () => selectedIndex,

3
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItem.svelte
View file

@ -1,4 +1,5 @@
<script lang="ts">
import { UI_DATA_ATTRS } from '$lib/constants';
import type { Snippet } from 'svelte';
interface Props {
@ -24,7 +25,7 @@
<button
type="button"
data-picker-index={dataIndex}
{...{ [UI_DATA_ATTRS.PICKER_INDEX]: dataIndex }}
{disabled}
{onclick}
{onmouseenter}

0
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormCommandPicker.svelte → tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerCommand.svelte
View file

3
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte
View file

@ -9,8 +9,7 @@
} from '$lib/components/app/chat';
import Badge from '$lib/components/ui/badge/badge.svelte';
import { KeyboardKey } from '$lib/enums';
import { conversationsStore } from '$lib/stores/conversations.svelte';
import { mcpStore } from '$lib/stores/mcp.svelte';
import { conversationsStore, mcpStore } from '$lib/stores';
import type { GetPromptResult, MCPPromptInfo, MCPServerSettingsEntry } from '$lib/types';
import { debounce, uuid } from '$lib/utils';
import { SvelteMap } from 'svelte/reactivity';

20
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormMentionPicker.svelte → tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMention.svelte
View file

@ -4,19 +4,13 @@
import HighlightedMatch from '$lib/components/app/forms/HighlightedMatch.svelte';
import * as Popover from '$lib/components/ui/popover';
import * as Tooltip from '$lib/components/ui/tooltip';
import {
FILE_GLOB_SEARCH_PICKERS_DEFAULT_SEARCH_DEPTH,
HOME_TILDE,
SEARCH_DEBOUNCE_MS
} from '$lib/constants';
import { FILE_GLOB_SEARCH_PICKERS, HOME_TILDE, SEARCH } from '$lib/constants';
import { BuiltInTool, FileMentionEntryType, GlobSearchType, KeyboardKey } from '$lib/enums';
import { useDebouncedSearch } from '$lib/hooks/use-debounced-search.svelte';
import { usePickerNavigation } from '$lib/hooks/use-picker-navigation.svelte';
import { config } from '$lib/stores/settings.svelte';
import { toolsStore } from '$lib/stores/tools.svelte';
import { isMobile } from '$lib/stores/viewport.svelte';
import type { FileMentionEntry } from '$lib/types';
import { abbreviateHome, type GlobEntryResult, runGlobSearchWithChildren } from '$lib/utils';
import { isMobile, settingsStore, toolsStore } from '$lib/stores';
import type { FileMentionEntry, GlobEntryResult } from '$lib/types';
import { abbreviateHome, runGlobSearchWithChildren } from '$lib/utils';
/**
* Floating file/folder mention picker. The chat input is the search
@ -68,9 +62,9 @@
// Coerce the depth setting to a positive integer; an invalid value
// would otherwise reach the server as max_depth 0 = unlimited.
const searchDepth = $derived.by(() => {
const n = Number(config().mentionSearchMaxDepth);
const n = Number(settingsStore.config.mentionSearchMaxDepth);
return Number.isInteger(n) && n > 0 ? n : FILE_GLOB_SEARCH_PICKERS_DEFAULT_SEARCH_DEPTH;
return Number.isInteger(n) && n > 0 ? n : FILE_GLOB_SEARCH_PICKERS.DEFAULT_SEARCH_DEPTH;
});
const home = $derived(toolsStore.serverHome);
@ -80,7 +74,7 @@
const search = useDebouncedSearch({
canRun: () => isOpen && fileSearchEnabled,
debounceMs: SEARCH_DEBOUNCE_MS,
debounceMs: SEARCH.DEBOUNCE_MS,
getQuery: () => trimmedQuery,
run: async (query, signal, isCurrent) => {
try {

12
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickers.svelte
View file

@ -1,7 +1,7 @@
<script lang="ts">
import ChatFormCommandPicker from './ChatFormCommandPicker.svelte';
import ChatFormMentionPicker from './ChatFormMentionPicker.svelte';
import ChatFormPickerCommand from './ChatFormPickerCommand.svelte';
import ChatFormPickerMcpPrompts from './ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte';
import ChatFormPickerMention from './ChatFormPickerMention.svelte';
import type {
ChatFormCommand,
FileMentionEntry,
@ -55,9 +55,9 @@
scopePath
}: Props = $props();
let commandPickerRef: ChatFormCommandPicker | undefined = $state(undefined);
let commandPickerRef: ChatFormPickerCommand | undefined = $state(undefined);
let promptPickerRef: ChatFormPickerMcpPrompts | undefined = $state(undefined);
let mentionPickerRef: ChatFormMentionPicker | undefined = $state(undefined);
let mentionPickerRef: ChatFormPickerMention | undefined = $state(undefined);
/** Delegate keyboard events to the active picker child; true if handled. */
export function handleKeydown(event: KeyboardEvent): boolean {
@ -77,7 +77,7 @@
}
</script>
<ChatFormCommandPicker
<ChatFormPickerCommand
bind:this={commandPickerRef}
isOpen={isCommandPickerOpen ?? false}
query={commandQuery ?? ''}
@ -96,7 +96,7 @@
{onPromptLoadError}
/>
<ChatFormMentionPicker
<ChatFormPickerMention
bind:this={mentionPickerRef}
isOpen={isMentionPickerOpen ?? false}
query={mentionQuery ?? ''}

112
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte
View file

@ -7,21 +7,22 @@
ChatMessageSystem,
ChatMessageUser
} from '$lib/components/app/chat';
import { SYSTEM_MESSAGE_PLACEHOLDER } from '$lib/constants';
import { REASONING_TAGS } from '$lib/constants/agentic';
import { ROUTES } from '$lib/constants/routes';
import { getChatActionsContext, setMessageEditContext } from '$lib/contexts';
import { REASONING_TAGS, ROUTES, SYSTEM_MESSAGE_PLACEHOLDER } from '$lib/constants';
import { setChatMessageActionsContext, setChatMessageEditContext } from '$lib/contexts';
import { AgenticSectionType, AttachmentType, MessageRole } from '$lib/enums';
import { DatabaseService } from '$lib/services/database.service';
import { chatStore, pendingEditMessageId } from '$lib/stores/chat.svelte';
import { conversationsStore } from '$lib/stores/conversations.svelte';
import { isMobile } from '$lib/stores/viewport.svelte';
import type { DatabaseMessageExtraMcpPrompt } from '$lib/types';
import { chatStore, conversationsStore, isMobile } from '$lib/stores';
import type {
ChatMessageActions,
ChatMessageDeletionInfo,
DatabaseMessageExtraMcpPrompt
} from '$lib/types';
import { deriveAgenticSections } from '$lib/utils';
import { parseFilesToMessageExtras } from '$lib/utils/browser-only';
interface Props {
class?: string;
chatActions: ChatMessageActions;
message: DatabaseMessage;
toolMessages?: DatabaseMessage[];
isLastAssistantMessage?: boolean;
@ -31,6 +32,7 @@
}
let {
chatActions,
class: className = '',
isLastAssistantMessage = false,
isLastUserMessage = false,
@ -40,14 +42,7 @@
toolMessages = []
}: Props = $props();
const chatActions = getChatActionsContext();
let deletionInfo = $state<{
totalCount: number;
userMessages: number;
assistantMessages: number;
messageTypes: string[];
} | null>(null);
let deletionInfo = $state<ChatMessageDeletionInfo | null>(null);
// The system message placeholder must never surface as editable content; keeping
// it in the derived (not just in handleEdit) guards against prop invalidation
// reverting the override while editing
@ -116,7 +111,7 @@
let showSaveOnlyOption = $derived(message.role === MessageRole.USER);
let showBranchAfterEditOption = $derived(message.role === MessageRole.ASSISTANT);
setMessageEditContext({
setChatMessageEditContext({
cancel: handleCancelEdit,
get editedContent() {
return editedContent;
@ -170,6 +165,30 @@
startEdit: handleEdit
});
setChatMessageActionsContext({
confirmDelete: handleConfirmDelete,
copy: handleCopy,
get deletionInfo() {
return deletionInfo;
},
get forkConversation() {
const isForkableUser = message.role === MessageRole.USER && !mcpPromptExtra;
return isForkableUser || message.role === MessageRole.ASSISTANT
? handleForkConversation
: undefined;
},
navigateToSibling: handleNavigateToSibling,
requestDelete: handleDelete,
setShowDeleteDialog: handleShowDeleteDialogChange,
get showDeleteDialog() {
return showDeleteDialog;
},
get siblingInfo() {
return siblingInfo;
}
});
let mcpPromptExtra = $derived.by(() => {
if (message.role !== MessageRole.USER) return null;
@ -187,7 +206,7 @@
});
$effect(() => {
const pendingId = pendingEditMessageId();
const pendingId = chatStore.pendingEditMessageId;
if (pendingId && pendingId === message.id && !isEditing) {
handleEdit();
@ -364,73 +383,22 @@
<div class="chat-message" class:chat-message--synthetic={isSynthetic}>
{#if message.role === MessageRole.SYSTEM}
<ChatMessageSystem
bind:textareaElement
class={className}
{deletionInfo}
{message}
onConfirmDelete={handleConfirmDelete}
onCopy={handleCopy}
onDelete={handleDelete}
onEdit={handleEdit}
onNavigateToSibling={handleNavigateToSibling}
onShowDeleteDialogChange={handleShowDeleteDialogChange}
{showDeleteDialog}
{siblingInfo}
/>
<ChatMessageSystem bind:textareaElement class={className} {message} />
{:else if mcpPromptExtra}
<ChatMessageMcpPrompt
class={className}
{deletionInfo}
{message}
mcpPrompt={mcpPromptExtra}
onConfirmDelete={handleConfirmDelete}
onCopy={handleCopy}
onDelete={handleDelete}
onEdit={handleEdit}
onNavigateToSibling={handleNavigateToSibling}
onShowDeleteDialogChange={handleShowDeleteDialogChange}
{showDeleteDialog}
{siblingInfo}
/>
<ChatMessageMcpPrompt class={className} {message} mcpPrompt={mcpPromptExtra} />
{:else if isSynthetic}
<ChatMessageSynthetic {message} class={className} />
{:else if message.role === MessageRole.USER}
<ChatMessageUser
class={className}
{deletionInfo}
{isLastUserMessage}
{message}
{nextAssistantMessage}
onConfirmDelete={handleConfirmDelete}
onCopy={handleCopy}
onDelete={handleDelete}
onEdit={handleEdit}
onForkConversation={handleForkConversation}
onNavigateToSibling={handleNavigateToSibling}
onShowDeleteDialogChange={handleShowDeleteDialogChange}
{showDeleteDialog}
{siblingInfo}
/>
<ChatMessageUser class={className} {isLastUserMessage} {message} {nextAssistantMessage} />
{:else}
<ChatMessageAssistant
bind:textareaElement
class={className}
{deletionInfo}
{isLastAssistantMessage}
{message}
{toolMessages}
onConfirmDelete={handleConfirmDelete}
onContinue={handleContinue}
onCopy={handleCopy}
onDelete={handleDelete}
onEdit={handleEdit}
onForkConversation={handleForkConversation}
onNavigateToSibling={handleNavigateToSibling}
onRegenerate={handleRegenerate}
onShowDeleteDialogChange={handleShowDeleteDialogChange}
{showDeleteDialog}
{siblingInfo}
/>
{/if}
</div>

58
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte
View file

@ -8,76 +8,48 @@
ChatMessageAssistantStatistics,
ChatMessageEditForm
} from '$lib/components/app';
import { getMessageEditContext } from '$lib/contexts';
import { getChatMessageEditContext } from '$lib/contexts';
import { MessageRole } from '$lib/enums';
import { useProcessingState } from '$lib/hooks/use-processing-state.svelte';
import { chatStore, isChatStreaming, isLoading } from '$lib/stores/chat.svelte';
import { modelsStore } from '$lib/stores/models.svelte';
import { isRouterMode } from '$lib/stores/server.svelte';
import { config } from '$lib/stores/settings.svelte';
import { chatStore, modelsStore, serverStore, settingsStore } from '$lib/stores';
import { modelLoadProgressText } from '$lib/utils';
import { hasAgenticContent } from '$lib/utils';
interface Props {
class?: string;
deletionInfo: {
totalCount: number;
userMessages: number;
assistantMessages: number;
messageTypes: string[];
} | null;
isLastAssistantMessage?: boolean;
message: DatabaseMessage;
toolMessages?: DatabaseMessage[];
onCopy: () => void;
onConfirmDelete: () => void;
onContinue?: () => void;
onDelete: () => void;
onEdit?: () => void;
onForkConversation?: (options: { name: string; includeAttachments: boolean }) => void;
onNavigateToSibling?: (siblingId: string) => void;
onRegenerate: (modelOverride?: string) => void;
onShowDeleteDialogChange: (show: boolean) => void;
showDeleteDialog: boolean;
siblingInfo?: ChatMessageSiblingInfo | null;
textareaElement?: HTMLTextAreaElement;
}
let {
class: className = '',
deletionInfo,
isLastAssistantMessage = false,
message,
onConfirmDelete,
onContinue,
onCopy,
onDelete,
onEdit,
onForkConversation,
onNavigateToSibling,
onRegenerate,
onShowDeleteDialogChange,
showDeleteDialog,
siblingInfo = null,
textareaElement = $bindable(),
toolMessages = []
}: Props = $props();
// Get edit context
const editCtx = getMessageEditContext();
const editCtx = getChatMessageEditContext();
const isAgentic = $derived(hasAgenticContent(message, toolMessages));
const processingState = useProcessingState();
let currentConfig = $derived(config());
let isRouter = $derived(isRouterMode());
let currentConfig = $derived(settingsStore.config);
let isRouter = $derived(serverStore.isRouterMode);
let showRawOutput = $state(false);
let displayedModel = $derived(message.model ?? null);
let isCurrentlyLoading = $derived(isLoading());
let isStreaming = $derived(isChatStreaming());
let isCurrentlyLoading = $derived(chatStore.isLoading);
let isStreaming = $derived(chatStore.isStreaming());
let hasNoContent = $derived(!message?.content?.trim());
let isActivelyProcessing = $derived(isCurrentlyLoading || isStreaming);
@ -175,7 +147,7 @@
<ChatMessageAgenticContent
{message}
{toolMessages}
isStreaming={isChatStreaming()}
isStreaming={chatStore.isStreaming()}
{isLastAssistantMessage}
/>
{/if}
@ -190,14 +162,14 @@
<div class="inline-flex flex-wrap items-start gap-2 text-xs text-muted-foreground">
<ChatMessageAssistantModel
{displayedModel}
isLoading={isLoading()}
isLoading={chatStore.isLoading}
{isRouter}
{onRegenerate}
/>
<ChatMessageAssistantStatistics
{message}
isLoading={isLoading()}
isLoading={chatStore.isLoading}
{processingState}
showMessageStats={currentConfig.showMessageStats}
/>
@ -210,18 +182,8 @@
role={MessageRole.ASSISTANT}
justify="start"
actionsPosition="left"
{siblingInfo}
{showDeleteDialog}
{deletionInfo}
{onCopy}
{onEdit}
{onRegenerate}
onContinue={currentConfig.enableContinueGeneration ? onContinue : undefined}
{onForkConversation}
{onDelete}
{onConfirmDelete}
{onNavigateToSibling}
{onShowDeleteDialogChange}
showRawOutputSwitch={currentConfig.showRawOutputSwitch}
rawOutputEnabled={showRawOutput}
onRawOutputToggle={(enabled) => (showRawOutput = enabled)}

2
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantModel.svelte
View file

@ -1,7 +1,7 @@
<script lang="ts">
import { ModelBadge, ModelsSelectorDropdown } from '$lib/components/app';
import { ServerModelStatus } from '$lib/enums';
import { modelsStore } from '$lib/stores/models.svelte';
import { modelsStore } from '$lib/stores';
import { copyToClipboard } from '$lib/utils';
interface Props {

2
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantProcessingInfo.svelte
View file

@ -10,7 +10,7 @@
let { modelLoadingText, position, processingState }: Props = $props();
const marginClass = position === 'top' ? 'mt-6' : 'mt-4';
const marginClass = $derived(position === 'top' ? 'mt-6' : 'mt-4');
</script>
<div class="{marginClass} w-full max-w-3xl" in:fade>

20
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantStatistics.svelte
View file

@ -2,6 +2,7 @@
import { ChatMessageStatistics } from '$lib/components/app';
import { ChatMessageStatisticsMode } from '$lib/enums';
import type { UseProcessingStateReturn } from '$lib/hooks/use-processing-state.svelte';
import { agenticStore } from '$lib/stores';
interface Props {
message: DatabaseMessage;
@ -11,9 +12,26 @@
}
let { isLoading, message, processingState, showMessageStats }: Props = $props();
// A running agentic flow stamps per-turn timings on its root message at each
// turn boundary and the cumulative agentic totals only on exit; while it runs,
// show the session's live totals on the root message instead.
const liveLlm = $derived(agenticStore.getLiveLlmTotals(message.convId));
const isLiveFlowRoot = $derived(
liveLlm !== null && agenticStore.getFlowRootMessageId(message.convId) === message.id
);
</script>
{#if showMessageStats && message.timings && message.timings.predicted_n && message.timings.predicted_ms}
{#if showMessageStats && isLiveFlowRoot && liveLlm}
<ChatMessageStatistics
mode={ChatMessageStatisticsMode.GENERATION}
isLive
promptTokens={liveLlm.prompt_n}
promptMs={liveLlm.prompt_ms}
predictedTokens={liveLlm.predicted_n}
predictedMs={liveLlm.predicted_ms}
/>
{:else if showMessageStats && message.timings && message.timings.predicted_n && message.timings.predicted_ms}
{@const agentic = message.timings.agentic}
<ChatMessageStatistics
mode={ChatMessageStatisticsMode.GENERATION}

48
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPrompt.svelte
View file

@ -4,7 +4,7 @@
ChatMessageEditForm,
ChatMessageMcpPromptContent
} from '$lib/components/app';
import { getMessageEditContext } from '$lib/contexts';
import { getChatMessageEditContext } from '$lib/contexts';
import { McpPromptVariant, MessageRole } from '$lib/enums';
import type { DatabaseMessageExtraMcpPrompt } from '$lib/types';
@ -12,39 +12,12 @@
class?: string;
message: DatabaseMessage;
mcpPrompt: DatabaseMessageExtraMcpPrompt;
siblingInfo?: ChatMessageSiblingInfo | null;
showDeleteDialog: boolean;
deletionInfo: {
totalCount: number;
userMessages: number;
assistantMessages: number;
messageTypes: string[];
} | null;
onCopy: () => void;
onEdit: () => void;
onDelete: () => void;
onConfirmDelete: () => void;
onNavigateToSibling?: (siblingId: string) => void;
onShowDeleteDialogChange: (show: boolean) => void;
}
let {
class: className = '',
deletionInfo,
mcpPrompt,
message,
onConfirmDelete,
onCopy,
onDelete,
onEdit,
onNavigateToSibling,
onShowDeleteDialogChange,
showDeleteDialog,
siblingInfo = null
}: Props = $props();
let { class: className = '', mcpPrompt, message }: Props = $props();
// Get edit context
const editCtx = getMessageEditContext();
const editCtx = getChatMessageEditContext();
</script>
<div
@ -63,20 +36,7 @@
{#if message.timestamp}
<div class="max-w-[80%]">
<ChatMessageActionIcons
actionsPosition="right"
{deletionInfo}
justify="end"
{onConfirmDelete}
{onCopy}
{onDelete}
{onEdit}
{onNavigateToSibling}
{onShowDeleteDialogChange}
{siblingInfo}
{showDeleteDialog}
role={MessageRole.USER}
/>
<ChatMessageActionIcons actionsPosition="right" justify="end" role={MessageRole.USER} />
</div>
{/if}
{/if}

2
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPromptContent.svelte
View file

@ -3,7 +3,7 @@
import { Card } from '$lib/components/ui/card';
import * as Tooltip from '$lib/components/ui/tooltip';
import { McpPromptVariant } from '$lib/enums';
import { mcpStore } from '$lib/stores/mcp.svelte';
import { mcpStore } from '$lib/stores';
import type { DatabaseMessageExtraMcpPrompt } from '$lib/types';
import { SvelteMap } from 'svelte/reactivity';

52
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte
View file

@ -4,47 +4,20 @@
import { Button } from '$lib/components/ui/button';
import { Card } from '$lib/components/ui/card';
import { INPUT_CLASSES } from '$lib/constants';
import { getMessageEditContext } from '$lib/contexts';
import { getChatMessageEditContext } from '$lib/contexts';
import { KeyboardKey, MessageRole } from '$lib/enums';
import { config } from '$lib/stores/settings.svelte';
import { settingsStore } from '$lib/stores';
import { autoResizeTextarea, isIMEComposing } from '$lib/utils';
interface Props {
class?: string;
message: DatabaseMessage;
siblingInfo?: ChatMessageSiblingInfo | null;
showDeleteDialog: boolean;
deletionInfo: {
totalCount: number;
userMessages: number;
assistantMessages: number;
messageTypes: string[];
} | null;
onCopy: () => void;
onEdit: () => void;
onDelete: () => void;
onConfirmDelete: () => void;
onNavigateToSibling?: (siblingId: string) => void;
onShowDeleteDialogChange: (show: boolean) => void;
textareaElement?: HTMLTextAreaElement;
}
let {
class: className = '',
deletionInfo,
message,
onConfirmDelete,
onCopy,
onDelete,
onEdit,
onNavigateToSibling,
onShowDeleteDialogChange,
showDeleteDialog,
siblingInfo = null,
textareaElement = $bindable()
}: Props = $props();
let { class: className = '', message, textareaElement = $bindable() }: Props = $props();
const editCtx = getMessageEditContext();
const editCtx = getChatMessageEditContext();
function handleEditKeydown(event: KeyboardEvent) {
if (event.key === KeyboardKey.ENTER && !event.shiftKey && !isIMEComposing(event)) {
@ -64,7 +37,7 @@
let contentHeight = $state(0);
const MAX_HEIGHT = 200; // pixels
const currentConfig = config();
const currentConfig = settingsStore.config;
let showExpandButton = $derived(contentHeight > MAX_HEIGHT);
@ -218,20 +191,7 @@
{#if message.timestamp}
<div class="max-w-[80%]">
<ChatMessageActionIcons
actionsPosition="right"
{deletionInfo}
justify="end"
{onConfirmDelete}
{onCopy}
{onDelete}
{onEdit}
{onNavigateToSibling}
{onShowDeleteDialogChange}
{siblingInfo}
{showDeleteDialog}
role={MessageRole.USER}
/>
<ChatMessageActionIcons actionsPosition="right" justify="end" role={MessageRole.USER} />
</div>
{/if}
{/if}

9
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte
View file

@ -12,13 +12,8 @@
import ChatMessageToolCallBlockSearchResults from './ChatMessageToolCallBlockSearchResults.svelte';
import ChatMessageToolCallBlockWriteFile from './ChatMessageToolCallBlockWriteFile.svelte';
import { BuiltInTool } from '$lib/enums';
import type { DatabaseMessageExtra } from '$lib/types';
import {
type AgenticSection,
extractSearchQuery,
extractSearchResults,
isWebSearchToolName
} from '$lib/utils';
import type { AgenticSection, DatabaseMessageExtra } from '$lib/types';
import { extractSearchQuery, extractSearchResults, isWebSearchToolName } from '$lib/utils';
interface Props {
section: AgenticSection;

8
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte
View file

@ -7,15 +7,13 @@
import { Loader2 } from '@lucide/svelte';
import { MarkdownContent, SyntaxHighlightedCode } from '$lib/components/app';
import { MAX_HEIGHT_CODE_BLOCK } from '$lib/constants';
import { getBuiltinToolUi } from '$lib/constants/built-in-tools';
import { AttachmentType, FileTypeText, MimeTypeAudio, ToolResultKind } from '$lib/enums';
import type { DatabaseMessageExtra } from '$lib/types';
import type { AgenticSection, DatabaseMessageExtra, ToolResultLine } from '$lib/types';
import {
type AgenticSection,
classifyToolResult,
formatJsonPretty,
parseToolResultWithMedia,
type ToolResultLine
getBuiltinToolUi,
parseToolResultWithMedia
} from '$lib/utils';
import { createBase64DataUrl } from '$lib/utils/data-url';

5
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte
View file

@ -3,8 +3,9 @@
import ToolCallBlock from './ToolCallBlock.svelte';
import { XCircle } from '@lucide/svelte';
import { MAX_HEIGHT_CODE_BLOCK, RESULT_STAT_SEPARATOR } from '$lib/constants';
import { toolsStore } from '$lib/stores/tools.svelte';
import { abbreviateHome, type AgenticSection, computeLineDiff, prefixFor } from '$lib/utils';
import { toolsStore } from '$lib/stores';
import type { AgenticSection } from '$lib/types';
import { abbreviateHome, computeLineDiff, prefixFor } from '$lib/utils';
interface Props {
section: AgenticSection;

17
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte
View file

@ -10,21 +10,18 @@
import ToolCallBlock from './ToolCallBlock.svelte';
import { AlertTriangle, Check, Loader2, XCircle } from '@lucide/svelte';
import { CollapsibleTerminalBlock } from '$lib/components/app';
import { SETTINGS_KEYS } from '$lib/constants';
import { TOOL_RUNTIME_SCROLL_AT_BOTTOM_THRESHOLD_PX } from '$lib/constants/auto-scroll';
import { config } from '$lib/stores/settings.svelte';
import { toolsStore } from '$lib/stores/tools.svelte';
import type { DatabaseMessageExtra } from '$lib/types';
import { SETTINGS_KEYS, TOOL_RUNTIME_SCROLL_AT_BOTTOM_THRESHOLD_PX } from '$lib/constants';
import { AttachmentType } from '$lib/enums';
import { settingsStore, toolsStore } from '$lib/stores';
import type { AgenticSection, DatabaseMessageExtra, ToolResultLine } from '$lib/types';
import {
abbreviateHome,
type AgenticSection,
type ExecShellExitStatus,
highlightCode,
isExitCodeSummaryLine,
parseExecShellCommandError,
parseExecShellCommandExitStatus,
parseToolResultWithMedia,
type ToolResultLine
parseToolResultWithMedia
} from '$lib/utils';
interface Props {
@ -94,7 +91,7 @@
);
const useFullHeightCodeBlocks = $derived(
Boolean(config()[SETTINGS_KEYS.FULL_HEIGHT_CODE_BLOCKS])
Boolean(settingsStore.config[SETTINGS_KEYS.FULL_HEIGHT_CODE_BLOCKS])
);
const autoScroll = $derived(isLive && !useFullHeightCodeBlocks);
@ -223,7 +220,7 @@
>
{#each outputLines as line, i (i)}
<div class="font-mono text-[11px] leading-relaxed whitespace-pre-wrap">{line.text}</div>
{#if line.media}
{#if line.media?.type === AttachmentType.IMAGE}
<img
src={line.media.base64Url}
alt={line.media.name}

5
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockFileGlobSearch.svelte
View file

@ -2,8 +2,9 @@
import { parseFileGlobSearchMeta } from './parsers/file-glob-search';
import ToolCallBlock from './ToolCallBlock.svelte';
import { XCircle } from '@lucide/svelte';
import { toolsStore } from '$lib/stores/tools.svelte';
import { abbreviateHome, type AgenticSection } from '$lib/utils';
import { toolsStore } from '$lib/stores';
import type { AgenticSection } from '$lib/types';
import { abbreviateHome } from '$lib/utils';
interface Props {
section: AgenticSection;

2
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockGetDatetime.svelte
View file

@ -1,7 +1,7 @@
<script lang="ts">
import { Clock, Loader2 } from '@lucide/svelte';
import { AgenticSectionType } from '$lib/enums';
import type { AgenticSection } from '$lib/utils';
import type { AgenticSection } from '$lib/types';
interface Props {
section: AgenticSection;

Some files were not shown because too many files have changed in this diff Show more