vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-27 09:13:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

server: refactor sleep handling, allow access /metrics during sleep (#27376)

Browse source 
* add cached responses

* refactor on_sleeping_state

* allow accessing metrics during sleep

* metrics task should not reset timer

* updated docs

* fix

* fix get_res_model_info

* add test

* fix a race condition

* split metrics and slots tasks / results

* should_reset_buckets
This commit is contained in:
Xuan-Son Nguyen 2026-08-19 20:48:09 +02:00 • committed by GitHub
parent ee0ea03adf
commit 947fd9bb2b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 422 additions and 181 deletions
Download patch file Download diff file Expand all files Collapse all files

30
tools/server/README-dev.md
View file

@ -291,6 +291,36 @@ The flow for downloading a new model:
- If a stop request comes in, the router asks the child process to stop (same mechanism as running a model in child process)
- Otherwise, upon completion, we call `load_models()` to refresh the list of models
### Sleep mode
Sleep mode was initially introduced in PR [#18228](https://github.com/ggml-org/llama.cpp/pull/18228). The main idea is to have:
- `server_queue` keeping track of the idle timeout
- When the timeout is detected, `server_queue` signals to `server_context_impl` that it should go into sleep
- `server_context_impl` frees all `llama_context` and `mtmd_context`
Compared to simply exiting the whole process, this approach allows accessing some read-only endpoints during sleep, while also handling wakeup-on-request. Any inference request will wake the server up.
Call stack on entering sleeping:
- `server_queue::start_loop` (main thread) sees no task for `idle_sleep_ms` --> `sleeping = true`
- `cb0(true)` --> `server_routes::update_cached_responses`
- snapshots `/props`, `/models` and metrics; the model is still alive here
- `cb1(true)` --> `server_context_impl::handle_sleeping_state`
- `callback_state(SERVER_STATE_SLEEPING)` --> reported to router in child mode
- `destroy()` --> frees `llama_context` and `mtmd_context`
- `condition_tasks.wait` until `req_stop_sleeping`
Call stack on waking up:
- `server_res_generator` constructor (HTTP thread) --> `server_queue::wait_until_no_sleep`
- sets `req_stop_sleeping = true`, then waits until `sleeping == false`
- `server_queue::start_loop` (main thread) wakes up
- `cb1(false)` --> `server_context_impl::handle_sleeping_state`
- `load_model()`, which then emits `callback_state(SERVER_STATE_READY)`
- `cb0(false)` --> `server_routes::update_cached_responses`
- nothing to do, the cache is only read during sleep
- `sleeping = false` --> `notify_all` unblocks the HTTP thread, the request is handled as usual
Endpoints created with `create_response(true)` (`/health`, `/props`, `/models`, `/metrics`) skip `wait_until_no_sleep`, so they answer from the cached responses instead of waking the server.
### Notable Related PRs
- Initial server implementation: https://github.com/ggml-org/llama.cpp/pull/1443

1
tools/server/README.md
View file

@ -2071,6 +2071,7 @@ Note that the following endpoints are exempt from being considered as incoming t
- `GET /health`
- `GET /props`
- `GET /models`
- `GET /metrics`
## More examples

388
tools/server/server-context.cpp
View file

@ -818,6 +818,14 @@ public:
}
}
server_metrics get_metrics() const {
return metrics;
}
void reset_metrics_bucket() {
metrics.reset_bucket();
}
private:
// note: accessing these fields outside of this class is not thread-safe
// use server_context methods instead
@ -898,6 +906,10 @@ private:
void handle_sleeping_state(bool new_state) {
GGML_ASSERT(sleeping != new_state);
if (new_state) {
if (callback_state) {
callback_state(SERVER_STATE_SLEEPING, {});
// note: for sleeping == false, event is emitted by load_model()
}
SRV_INF("%s", "server is entering sleeping state\n");
destroy();
} else {
@ -2290,8 +2302,8 @@ private:
// returns false to decline the task, it is offered again after the decode is done
bool process_single_task(server_task && task, bool is_yielding) {
// while yielding, an encode / decode is running and only accessing metrics is safe
if (is_yielding && task.type != SERVER_TASK_TYPE_METRICS) {
// while yielding, an encode / decode is running and only reading the server state is safe
if (is_yielding && task.type != SERVER_TASK_TYPE_METRICS && task.type != SERVER_TASK_TYPE_SLOT_GET) {
SRV_DBG("decoding, decline task, id_task = %d\n", task.id);
return false;
}
@ -2417,28 +2429,17 @@ private:
} break;
case SERVER_TASK_TYPE_METRICS:
{
json slots_data = json::array();
int n_idle_slots = 0;
int n_processing_slots = 0;
for (server_slot & slot : slots) {
json slot_data = slot.to_json(slots_debug == 0);
if (slot.is_processing()) {
n_processing_slots++;
} else {
n_idle_slots++;
}
slots_data.push_back(slot_data);
}
SRV_DBG("n_idle_slots = %d, n_processing_slots = %d\n", n_idle_slots, n_processing_slots);
SRV_DBG("n_processing_slots = %d\n", n_processing_slots);
auto res = std::make_unique<server_task_result_metrics>();
res->id = task.id;
res->slots_data = std::move(slots_data);
res->n_idle_slots = n_idle_slots;
res->n_processing_slots = n_processing_slots;
res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size();
res->metrics = metrics;
@ -2446,6 +2447,28 @@ private:
if (task.metrics_reset_bucket) {
metrics.reset_bucket();
}
queue_results.send(std::move(res));
} break;
case SERVER_TASK_TYPE_SLOT_GET:
{
json slots_data = json::array();
int n_idle_slots = 0;
for (server_slot & slot : slots) {
if (!slot.is_processing()) {
n_idle_slots++;
}
slots_data.push_back(slot.to_json(slots_debug == 0));
}
SRV_DBG("n_idle_slots = %d\n", n_idle_slots);
auto res = std::make_unique<server_task_result_slots>();
res->id = task.id;
res->slots_data = std::move(slots_data);
res->n_idle_slots = n_idle_slots;
queue_results.send(std::move(res));
} break;
case SERVER_TASK_TYPE_SLOT_SAVE:
@ -4142,12 +4165,6 @@ struct server_res_generator : server_res_spipe {
void server_context::set_state_callback(server_state_callback_t callback) {
impl->callback_state = std::move(callback);
impl->queue_tasks.on_sleeping_state([this](bool sleeping) {
if (sleeping) {
impl->callback_state(SERVER_STATE_SLEEPING, {});
}
// for sleeping == false, event is emitted by load_model()
});
}
//
@ -4431,6 +4448,119 @@ server_routes::server_routes(const common_params & params, server_context & ctx_
queue_tasks(ctx_server.impl->queue_tasks),
queue_results(ctx_server.impl->queue_results) {
init_routes();
// note: this must be registered before load_model()
// so that on sleep phase, the callback is called before ctx is destroyed
queue_tasks.on_sleeping_state([this](bool is_sleeping) {
update_cached_responses(is_sleeping);
});
}
static json get_res_model_info(const server_context_meta & meta) {
// note: do NOT use ctx_server here, otherwise it's not possible to use this during sleep
return {
{"id", meta.model_name},
{"aliases", meta.model_aliases},
{"tags", meta.model_tags},
{"object", "model"},
{"created", std::time(0)},
{"owned_by", "llamacpp"},
{"meta", {
{"vocab_type", meta.model_vocab_type},
{"n_vocab", meta.model_vocab_n_tokens},
{"n_ctx", meta.slot_n_ctx},
{"n_ctx_train", meta.model_n_ctx_train},
{"n_embd", meta.model_n_embd_inp},
{"n_params", meta.model_n_params},
{"size", meta.model_size},
{"ftype", meta.model_ftype},
}},
};
}
static json get_res_models(const server_context_meta & meta) {
// note: do NOT use ctx_server here, otherwise it's not possible to use this during sleep
return {
{"models", {
{
{"name", meta.model_name},
{"model", meta.model_name},
{"modified_at", ""},
{"size", ""},
{"digest", ""}, // dummy value, llama.cpp does not support managing model file's hash
{"type", "model"},
{"description", ""},
{"tags", {""}},
{"capabilities", meta.has_mtmd ? json({"completion","multimodal"}) : json({"completion"})},
{"parameters", ""},
{"details", {
{"parent_model", ""},
{"format", "gguf"},
{"family", ""},
{"families", {""}},
{"parameter_size", ""},
{"quantization_level", ""}
}}
}
}},
{"object", "list"},
{"data", {
get_res_model_info(meta),
}}
};
}
static json get_res_props(const server_context_meta & meta, const common_params & params, bool is_sleeping) {
// note: do NOT use ctx_server here, otherwise it's not possible to use this during sleep
task_params tparams;
tparams.sampling = params.sampling;
json default_generation_settings_for_props = json {
{ "params", tparams.to_json(true) },
{ "n_ctx", meta.slot_n_ctx },
};
std::string tmpl_default = common_chat_templates_source(meta.chat_params.tmpls.get(), "");
std::string tmpl_tools = common_chat_templates_source(meta.chat_params.tmpls.get(), "tool_use");
json props = {
{ "default_generation_settings", default_generation_settings_for_props },
{ "total_slots", params.n_parallel },
{ "model_alias", meta.model_name },
{ "model_ftype", meta.model_ftype },
{ "model_path", meta.model_path },
{ "modalities", json {
{"vision", meta.has_inp_image},
{"video", meta.has_inp_video},
{"audio", meta.has_inp_audio},
} },
{ "media_marker", get_media_marker() },
{ "endpoint_slots", params.endpoint_slots },
{ "endpoint_props", params.endpoint_props },
{ "endpoint_metrics", params.endpoint_metrics },
{ "ui", params.ui },
{ "ui_settings", meta.json_ui_settings },
{ "chat_template", tmpl_default },
{ "chat_template_caps", meta.chat_template_caps },
{ "bos_token", meta.bos_token_str },
{ "eos_token", meta.eos_token_str },
{ "build_info", meta.build_info },
{ "is_sleeping", is_sleeping },
{ "cors_proxy_enabled", params.ui_mcp_proxy },
};
if (params.use_jinja) {
if (!tmpl_tools.empty()) {
props["chat_template_tool_use"] = tmpl_tools;
}
}
return props;
}
json server_routes::get_model_info() const {
return get_res_model_info(*meta);
}
void server_routes::init_routes() {
@ -4451,41 +4581,64 @@ void server_routes::init_routes() {
};
this->get_metrics = [this](const server_http_req & req) {
auto res = create_response();
auto res = create_response(true);
if (!params.endpoint_metrics) {
res->error(format_error_response("This server does not support metrics endpoint. Start it with `--metrics`", ERROR_TYPE_NOT_SUPPORTED));
return res;
}
// request slots data using task queue
{
server_task task(SERVER_TASK_TYPE_METRICS);
task.id = res->rd.get_new_id();
// render response using cached_metrics
auto use_cached_metrics = [&]() {
std::unique_lock<std::mutex> lock(mutex_cache);
res->headers["Process-Start-Time-Unix"] = std::to_string(cached_metrics.t_start);
server_task_result_metrics tmp;
tmp.metrics = cached_metrics;
res->content_type = "text/plain; version=0.0.4";
res->status = 200;
res->data = tmp.to_metrics();
// the gauges are averaged over the window between two scrapes
task.metrics_reset_bucket = true;
res->rd.post_task(std::move(task), true); // high-priority task
cached_metrics.reset_bucket();
should_reset_buckets = true;
};
if (queue_tasks.is_sleeping()) {
use_cached_metrics();
} else {
// request slots data using task queue
{
server_task task(SERVER_TASK_TYPE_METRICS);
task.id = res->rd.get_new_id();
// the gauges are averaged over the window between two scrapes
task.metrics_reset_bucket = true;
res->rd.post_task(std::move(task), true); // high-priority task
}
// a task posted right before sleeping is never processed, do not wait for it
auto result = res->rd.next([&]{
return req.should_stop() || queue_tasks.is_sleeping();
});
if (!result) {
if (!req.should_stop()) {
use_cached_metrics();
}
return res;
}
if (result->is_error()) {
res->error(result->to_json());
return res;
}
auto res_task = dynamic_cast<server_task_result_metrics*>(result.get());
GGML_ASSERT(res_task != nullptr);
res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->metrics.t_start);
res->content_type = "text/plain; version=0.0.4";
res->status = 200;
res->data = res_task->to_metrics();
}
// get the result
auto result = res->rd.next(req.should_stop);
if (!result) {
// connection was closed
GGML_ASSERT(req.should_stop());
return res;
}
if (result->is_error()) {
res->error(result->to_json());
return res;
}
auto res_task = dynamic_cast<server_task_result_metrics*>(result.get());
GGML_ASSERT(res_task != nullptr);
res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->metrics.t_start);
res->content_type = "text/plain; version=0.0.4";
res->status = 200;
res->data = res_task->to_metrics();
return res;
};
@ -4498,7 +4651,7 @@ void server_routes::init_routes() {
// request slots data using task queue
{
server_task task(SERVER_TASK_TYPE_METRICS);
server_task task(SERVER_TASK_TYPE_SLOT_GET);
task.id = res->rd.get_new_id();
res->rd.post_task(std::move(task), true); // high-priority task
}
@ -4516,7 +4669,7 @@ void server_routes::init_routes() {
return res;
}
auto * res_task = dynamic_cast<server_task_result_metrics*>(result.get());
auto * res_task = dynamic_cast<server_task_result_slots*>(result.get());
GGML_ASSERT(res_task != nullptr);
// optionally return "fail_on_no_slot" error
@ -4566,53 +4719,13 @@ void server_routes::init_routes() {
this->get_props = [this](const server_http_req &) {
auto res = create_response(true);
// this endpoint can be accessed during sleeping
// the next LOC is to avoid someone accidentally use ctx_server
bool ctx_server; // do NOT delete this line
GGML_UNUSED(ctx_server);
task_params tparams;
tparams.sampling = params.sampling;
json default_generation_settings_for_props = json {
{ "params", tparams.to_json(true) },
{ "n_ctx", meta->slot_n_ctx },
};
std::string tmpl_default = common_chat_templates_source(meta->chat_params.tmpls.get(), "");
std::string tmpl_tools = common_chat_templates_source(meta->chat_params.tmpls.get(), "tool_use");
json props = {
{ "default_generation_settings", default_generation_settings_for_props },
{ "total_slots", params.n_parallel },
{ "model_alias", meta->model_name },
{ "model_ftype", meta->model_ftype },
{ "model_path", meta->model_path },
{ "modalities", json {
{"vision", meta->has_inp_image},
{"video", meta->has_inp_video},
{"audio", meta->has_inp_audio},
} },
{ "media_marker", get_media_marker() },
{ "endpoint_slots", params.endpoint_slots },
{ "endpoint_props", params.endpoint_props },
{ "endpoint_metrics", params.endpoint_metrics },
{ "ui", params.ui },
{ "ui_settings", meta->json_ui_settings },
{ "chat_template", tmpl_default },
{ "chat_template_caps", meta->chat_template_caps },
{ "bos_token", meta->bos_token_str },
{ "eos_token", meta->eos_token_str },
{ "build_info", meta->build_info },
{ "is_sleeping", queue_tasks.is_sleeping() },
{ "cors_proxy_enabled", params.ui_mcp_proxy },
};
if (params.use_jinja) {
if (!tmpl_tools.empty()) {
props["chat_template_tool_use"] = tmpl_tools;
}
// note: do NOT use ctx_server here, this endpoint must be accessible during sleep
if (queue_tasks.is_sleeping()) {
std::unique_lock<std::mutex> lock(mutex_cache);
res->ok(cached_props);
} else {
res->ok(get_res_props(*meta, params, false));
}
res->ok(props);
return res;
};
@ -4874,42 +4987,13 @@ void server_routes::init_routes() {
this->get_models = [this](const server_http_req &) {
auto res = create_response(true);
// this endpoint can be accessed during sleeping
// the next LOC is to avoid someone accidentally use ctx_server
bool ctx_server; // do NOT delete this line
GGML_UNUSED(ctx_server);
json models = {
{"models", {
{
{"name", meta->model_name},
{"model", meta->model_name},
{"modified_at", ""},
{"size", ""},
{"digest", ""}, // dummy value, llama.cpp does not support managing model file's hash
{"type", "model"},
{"description", ""},
{"tags", {""}},
{"capabilities", meta->has_mtmd ? json({"completion","multimodal"}) : json({"completion"})},
{"parameters", ""},
{"details", {
{"parent_model", ""},
{"format", "gguf"},
{"family", ""},
{"families", {""}},
{"parameter_size", ""},
{"quantization_level", ""}
}}
}
}},
{"object", "list"},
{"data", {
get_model_info(),
}}
};
res->ok(models);
// note: do NOT use ctx_server here, this endpoint must be accessible during sleep
if (queue_tasks.is_sleeping()) {
std::unique_lock<std::mutex> lock(mutex_cache);
res->ok(cached_models);
} else {
res->ok(get_res_models(*meta));
}
return res;
};
@ -5119,27 +5203,6 @@ void server_routes::init_routes() {
};
}
json server_routes::get_model_info() const {
return json {
{"id", meta->model_name},
{"aliases", meta->model_aliases},
{"tags", meta->model_tags},
{"object", "model"},
{"created", std::time(0)},
{"owned_by", "llamacpp"},
{"meta", {
{"vocab_type", meta->model_vocab_type},
{"n_vocab", meta->model_vocab_n_tokens},
{"n_ctx", meta->slot_n_ctx},
{"n_ctx_train", meta->model_n_ctx_train},
{"n_embd", meta->model_n_embd_inp},
{"n_params", meta->model_n_params},
{"size", meta->model_size},
{"ftype", meta->model_ftype},
}},
};
}
std::unique_ptr<server_res_generator> server_routes::handle_slots_save(const server_http_req & req, int id_slot) {
auto res = create_response();
const json request_data = json::parse(req.body);
@ -5388,3 +5451,24 @@ std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const l
res->ok(response);
return res;
}
void server_routes::update_cached_responses(bool is_sleeping) {
// caller is task_queue, so ctx_server can be accessed without holding locks
std::unique_lock<std::mutex> lock(mutex_cache);
if (is_sleeping) {
cached_models = get_res_models(*meta);
cached_props = get_res_props(*meta, params, true);
cached_metrics = ctx_server.get_metrics();
should_reset_buckets = false;
SRV_DBG("%s\n", "cached responses updated");
} else if (should_reset_buckets) {
// a scrape during sleep already reported these buckets
ctx_server.reset_metrics_bucket();
should_reset_buckets = false;
}
}

13
tools/server/server-context.h
View file

@ -8,6 +8,7 @@
#include <cstddef>
#include <memory>
#include <mutex>
#include <set>
struct server_context_impl; // private implementation
@ -174,9 +175,19 @@ private:
std::unique_ptr<const server_context_meta> meta;
const common_params & params;
const server_context_impl & ctx_server;
server_context_impl & ctx_server;
server_queue & queue_tasks;
server_response & queue_results;
std::unique_ptr<server_res_generator> create_response(bool bypass_sleep = false);
// cached responses, to be used during sleep
std::mutex mutex_cache;
json cached_models = nullptr;
json cached_props = nullptr;
server_metrics cached_metrics;
// set when a scrape during sleep already reported the throughput buckets
bool should_reset_buckets = false;
// call right before sleep to update the cached responses
void update_cached_responses(bool is_sleeping);
};

33
tools/server/server-queue.cpp
View file

@ -3,6 +3,7 @@
#include "log.h"
#include <algorithm>
#include <chrono>
#include <thread>
@ -20,6 +21,10 @@
// server_queue
//
static bool task_resets_idle_timer(server_task_type type) {
return type != SERVER_TASK_TYPE_METRICS;
}
int server_queue::post(server_task && task, bool front) {
std::unique_lock<std::mutex> lock(mutex_tasks);
GGML_ASSERT(task.id != -1);
@ -27,20 +32,24 @@ int server_queue::post(server_task && task, bool front) {
if (task.type == SERVER_TASK_TYPE_CANCEL) {
cleanup_pending_task(task.id_target);
}
const int task_id = task.id;
const int task_id = task.id;
const bool reset_timer = task_resets_idle_timer(task.type);
QUE_DBG("new task, id = %d, front = %d\n", task_id, front);
if (front) {
queue_tasks.push_front(std::move(task));
} else {
queue_tasks.push_back(std::move(task));
}
time_last_task = ggml_time_ms();
if (reset_timer) {
time_last_task = ggml_time_ms();
}
condition_tasks.notify_one();
return task_id;
}
int server_queue::post(std::vector<server_task> && tasks, bool front) {
std::unique_lock<std::mutex> lock(mutex_tasks);
bool reset_timer = false;
for (auto & task : tasks) {
if (task.id == -1) {
task.id = id++;
@ -49,6 +58,7 @@ int server_queue::post(std::vector<server_task> && tasks, bool front) {
if (task.type == SERVER_TASK_TYPE_CANCEL) {
cleanup_pending_task(task.id_target);
}
reset_timer |= task_resets_idle_timer(task.type);
QUE_DBG("new task, id = %d/%d, front = %d\n", task.id, (int) tasks.size(), front);
if (front) {
queue_tasks.push_front(std::move(task));
@ -56,7 +66,9 @@ int server_queue::post(std::vector<server_task> && tasks, bool front) {
queue_tasks.push_back(std::move(task));
}
}
time_last_task = ggml_time_ms();
if (reset_timer) {
time_last_task = ggml_time_ms();
}
condition_tasks.notify_one();
return 0;
}
@ -294,11 +306,14 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
QUE_DBG("%s", "update slots\n");
// this will run the main inference process for all slots
const int64_t t_update_slots = ggml_time_ms();
callback_update_slots();
{
// update_slots() may take a while to finish, we need to make sure it's not counted as idle
// shift instead of reset, so that non-task_resets_idle_timer tasks do not delay the sleep
std::unique_lock<std::mutex> lock(mutex_tasks);
time_last_task = ggml_time_ms();
const int64_t now = ggml_time_ms();
time_last_task = std::min(now, time_last_task + (now - t_update_slots));
}
QUE_DBG("%s", "waiting for new tasks\n");
@ -312,7 +327,10 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
if (should_sleep()) {
QUE_INF("%s", "entering sleeping state\n");
sleeping = true;
callback_sleeping_state(true);
// Call order cb0 -> cb1 -> cb{N}
for (auto & cb : callback_sleeping_state) {
cb(true);
}
req_stop_sleeping = false;
// wait until we are requested to exit sleeping state
condition_tasks.wait(lock, [&]{
@ -323,7 +341,10 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
}
QUE_INF("%s", "exiting sleeping state\n");
req_stop_sleeping = false;
callback_sleeping_state(false);
// Call order cb{N} -> cb1 -> cb0
for (size_t i = callback_sleeping_state.size(); i > 0; i--) {
callback_sleeping_state[i - 1](false);
}
sleeping = false;
time_last_task = ggml_time_ms();
condition_tasks.notify_all(); // notify wait_until_no_sleep()

19
tools/server/server-queue.h
View file

@ -44,7 +44,7 @@ private:
// callback functions
std::function<bool(server_task &&, bool)> callback_new_task;
std::function<void(void)> callback_update_slots;
std::function<void(bool)> callback_sleeping_state;
std::vector<std::function<void(bool)>> callback_sleeping_state;
public:
~server_queue() { worker_stop(); }
@ -86,6 +86,7 @@ public:
*
* Sleeping procedure (disabled if idle_sleep_ms < 0):
* - If there is no task after idle_sleep_ms, enter sleeping state
* note: metrics tasks are processed as usual, but do not reset the idle timer
* - Call callback_sleeping_state(true)
* - Wait until req_stop_sleeping is set to true
* - Call callback_sleeping_state(false)
@ -127,18 +128,12 @@ public:
}
// Register callback for sleeping state change; multiple callbacks are allowed
// note: when entering sleeping state, the callback is called AFTER sleeping is set to true
// when leaving sleeping state, the callback is called BEFORE sleeping is set to false
// for example: register order cb0, cb1, cb2
// entering sleep: queue.sleeping = true --> cb0(true) --> cb1(true) --> cb2(true)
// leaving sleep: cb2(false) --> cb1(false) --> cb0(false) --> queue.sleeping = false
// note: caller will hold mutex_tasks while calling the callbacks
void on_sleeping_state(std::function<void(bool)> callback) {
if (callback_sleeping_state) {
auto prev_callback = std::move(callback_sleeping_state);
callback_sleeping_state = [prev_callback, callback](bool sleeping) {
prev_callback(sleeping);
callback(sleeping);
};
} else {
callback_sleeping_state = std::move(callback);
}
callback_sleeping_state.push_back(std::move(callback));
}
private:

7
tools/server/server-task.cpp
View file

@ -1512,10 +1512,15 @@ json server_task_result_error::to_json() {
//
// server_task_result_metrics
//
json server_task_result_metrics::to_json() {
json server_task_result_slots::to_json() {
return slots_data;
}
json server_task_result_metrics::to_json() {
// not used, /metrics renders prometheus text via to_metrics()
return json{};
}
// metrics definition: https://prometheus.io/docs/practices/naming/#metric-names
std::string server_task_result_metrics::to_metrics() {
const std::vector<metric_item> counters = {

24
tools/server/server-task.h
View file

@ -22,6 +22,7 @@ enum server_task_type {
SERVER_TASK_TYPE_CONTROL,
SERVER_TASK_TYPE_NEXT_RESPONSE,
SERVER_TASK_TYPE_METRICS,
SERVER_TASK_TYPE_SLOT_GET,
SERVER_TASK_TYPE_SLOT_SAVE,
SERVER_TASK_TYPE_SLOT_RESTORE,
SERVER_TASK_TYPE_SLOT_ERASE,
@ -489,22 +490,16 @@ struct server_task_result_error : server_task_result {
virtual json to_json() override;
};
// used by /metrics API
struct server_task_result_metrics : server_task_result {
// these are immediate stats, not accumulated (server_metrics is cumulative)
int n_idle_slots;
int n_processing_slots;
int n_tasks_deferred;
int n_processing_slots = 0;
int n_tasks_deferred = 0;
server_metrics metrics;
// while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
// therefore, we use json to temporarily store the slot.to_json() result
json slots_data = json::array();
// used by /slots API
virtual json to_json() override;
// used by /metrics API
struct metric_item {
std::string name;
std::string description;
@ -513,6 +508,17 @@ struct server_task_result_metrics : server_task_result {
std::string to_metrics();
};
// used by /slots API
struct server_task_result_slots : server_task_result {
int n_idle_slots = 0;
// while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
// therefore, we use json to temporarily store the slot.to_json() result
json slots_data = json::array();
virtual json to_json() override;
};
struct server_task_result_slot_save_load : server_task_result {
std::string filename;
bool is_save; // true = save, false = load

88
tools/server/tests/unit/test_sleep.py
View file

@ -11,6 +11,35 @@ def create_server():
server = ServerPreset.tinyllama2()
def is_sleeping(server: ServerProcess) -> bool:
res = server.make_request("GET", "/props")
assert res.status_code == 200
return res.body["is_sleeping"]
def wait_for_sleep(server: ServerProcess, timeout: float = 10.0):
start = time.time()
while time.time() - start < timeout:
if is_sleeping(server):
return
time.sleep(0.1)
raise TimeoutError("server did not go to sleep")
def fetch_metrics(server: ServerProcess) -> str:
res = server.make_request("GET", "/metrics")
assert res.status_code == 200
assert isinstance(res.body, str)
return res.body
def get_metric(text: str, name: str) -> float:
prefix = f"llamacpp:{name} "
values = [ln for ln in text.splitlines() if ln.startswith(prefix)]
assert len(values) == 1, f"{name} not found in metrics"
return float(values[0][len(prefix):])
def test_server_sleep():
global server
server.sleep_idle_seconds = 1
@ -25,6 +54,10 @@ def test_server_sleep():
res = server.make_request("GET", "/props")
assert res.status_code == 200
assert res.body["is_sleeping"] == True
res = server.make_request("GET", "/models")
assert res.status_code == 200
assert len(res.body["data"]) == 1
assert res.body["data"][0]["id"] == server.model_alias
# make a generation request to wake up the server
res = server.make_request("POST", "/completion", data={
@ -37,3 +70,58 @@ def test_server_sleep():
res = server.make_request("GET", "/props")
assert res.status_code == 200
assert res.body["is_sleeping"] == False
def test_server_sleep_read_only_endpoints():
global server
server.sleep_idle_seconds = 1
server.server_metrics = True
server.start()
res = server.make_request("POST", "/completion", data={
"n_predict": 4,
"prompt": "Hello",
})
assert res.status_code == 200
# the first scrape resets the throughput buckets, so that the second one reports
# the same zero rates as the snapshot taken on entering sleep
fetch_metrics(server)
metrics_awake = fetch_metrics(server)
assert get_metric(metrics_awake, "tokens_predicted_total") > 0
wait_for_sleep(server)
# during sleep, metrics are served from the snapshot taken right before sleeping
assert fetch_metrics(server) == metrics_awake
# scraping /metrics must not wake the server up
assert is_sleeping(server)
def test_server_sleep_metrics_buckets():
global server
server.sleep_idle_seconds = 1
server.server_metrics = True
server.start()
res = server.make_request("POST", "/completion", data={
"n_predict": 8,
"prompt": "Hello",
})
assert res.status_code == 200
wait_for_sleep(server)
# the first scrape reports the throughput of the last generation
assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") > 0
# nothing runs while sleeping, so the next scrapes report an empty window
assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") == 0
assert is_sleeping(server)
# waking up must not report the buckets again
res = server.make_request("POST", "/tokenize", data={"content": "Hello"})
assert res.status_code == 200
assert is_sleeping(server) == False
assert get_metric(fetch_metrics(server), "predicted_tokens_seconds") == 0