mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-08-25 08:14:24 +00:00
server: refactor + correctness fixes for metrics (#26920)
* server: refactor metrics * move most fields to server_slot_stats * cont * rm result_timings * tie stats to batch * cont * nits: move place in code * exclude first generated token * more accurate batch metrics tracking * n_predict --> n_gen * metrics_on_prediction * metrics_flush_idle * metrics: seperate cache/processed prompt tokens * refactor server_task_result_metrics * add test * nits * fix flush before reset() * cont * rm dead code * nits
This commit is contained in:
Xuan-Son Nguyen
GitHub
parent
e79e4bf660
commit
decaf508bb
6 changed files with 813 additions and 475 deletions
|
|
@ -60,6 +60,33 @@ json format_error_response(const std::string & message, const enum error_type ty
|
|||
};
|
||||
}
|
||||
|
||||
//
|
||||
// server_slot_stats
|
||||
//
|
||||
|
||||
json server_slot_stats::to_json() const {
|
||||
json base = {
|
||||
{"cache_n", n_prompt_cached},
|
||||
|
||||
{"prompt_n", n_prompt_processed},
|
||||
{"prompt_ms", t_prompt_ms()},
|
||||
{"prompt_per_token_ms", t_prompt_per_token_ms()},
|
||||
{"prompt_per_second", n_prompt_tps()},
|
||||
|
||||
{"predicted_n", n_gen},
|
||||
{"predicted_ms", t_gen_ms()},
|
||||
{"predicted_per_token_ms", t_gen_per_token_ms()},
|
||||
{"predicted_per_second", n_gen_tps()},
|
||||
};
|
||||
|
||||
if (n_draft_tokens > 0) {
|
||||
base["draft_n"] = n_draft_tokens;
|
||||
base["draft_n_accepted"] = n_draft_accepted;
|
||||
}
|
||||
|
||||
return base;
|
||||
}
|
||||
|
||||
//
|
||||
// random string / id
|
||||
//
|
||||
|
|
|
|||
|
|
@ -334,6 +334,160 @@ json format_response_rerank(
|
|||
std::vector<std::string> & texts,
|
||||
int top_n);
|
||||
|
||||
//
|
||||
// stats and metrics
|
||||
//
|
||||
|
||||
// shared between server_slot and server_task_result_*
|
||||
struct server_slot_stats {
|
||||
uint64_t n_prompt_cached = 0;
|
||||
uint64_t n_prompt_processed = 0;
|
||||
uint64_t n_gen = 0;
|
||||
|
||||
// speculative decoding stats
|
||||
// note: the per-position breakdown lives in server_slot, it is not needed in a task result
|
||||
uint64_t n_draft_tokens = 0;
|
||||
uint64_t n_draft_accepted = 0;
|
||||
uint64_t n_draft_verif_steps = 0;
|
||||
|
||||
// these are absolute timestamps (in us)
|
||||
// note: must be signed - they are subtracted before the later ones are set
|
||||
int64_t t_start = 0;
|
||||
int64_t t_prompt_last = 0;
|
||||
int64_t t_gen_last = 0;
|
||||
|
||||
// can only move one direction: start -> prompt -> gen
|
||||
void update_prompt_start() {
|
||||
GGML_ASSERT(t_start == 0);
|
||||
t_start = ggml_time_us();
|
||||
}
|
||||
void set_prompt_last(int64_t t_us) {
|
||||
GGML_ASSERT(t_start > 0);
|
||||
t_prompt_last = t_us;
|
||||
}
|
||||
void update_prompt_last() {
|
||||
set_prompt_last(ggml_time_us());
|
||||
}
|
||||
void update_gen_last() {
|
||||
GGML_ASSERT(t_prompt_last > 0);
|
||||
t_gen_last = ggml_time_us();
|
||||
}
|
||||
|
||||
// these are time durations
|
||||
int64_t t_elapsed_us() const {
|
||||
return ggml_time_us() - t_start;
|
||||
}
|
||||
double t_prompt_ms() const {
|
||||
if (t_prompt_last == 0) {
|
||||
return 0.0; // the prompt is not processed yet
|
||||
}
|
||||
return (t_prompt_last - t_start) / 1000.0;
|
||||
}
|
||||
int64_t t_gen_us() const {
|
||||
if (t_gen_last == 0) {
|
||||
return 0; // the generation is not started yet
|
||||
}
|
||||
// clamp to 1 us, the first token can land in the same us as t_prompt_last
|
||||
return std::max<int64_t>(1, t_gen_last - t_prompt_last);
|
||||
}
|
||||
double t_gen_ms() const {
|
||||
return t_gen_us() / 1000.0;
|
||||
}
|
||||
|
||||
// number of decode steps spent on generation
|
||||
// the first token is free, it comes from the logits of the last prompt batch
|
||||
uint64_t n_gen_steps() const {
|
||||
return n_gen > 0 ? n_gen - 1 : 0;
|
||||
}
|
||||
|
||||
// other derived metrics
|
||||
// note: all of them return 0.0 if the divisor is not known yet
|
||||
double t_prompt_per_token_ms() const {
|
||||
return n_prompt_processed > 0 ? t_prompt_ms() / n_prompt_processed : 0.0;
|
||||
}
|
||||
double t_gen_per_token_ms() const {
|
||||
return n_gen_steps() > 0 ? t_gen_ms() / n_gen_steps() : 0.0;
|
||||
}
|
||||
double n_prompt_tps() const {
|
||||
const double t_ms = t_prompt_ms();
|
||||
return t_ms > 0.0 ? 1e3 / t_ms * n_prompt_processed : 0.0;
|
||||
}
|
||||
double n_gen_tps() const {
|
||||
const double t_ms = t_gen_ms();
|
||||
return t_ms > 0.0 ? 1e3 / t_ms * n_gen_steps() : 0.0;
|
||||
}
|
||||
|
||||
// false if the slot never started, i.e. the task result carries no stats
|
||||
bool is_set() const {
|
||||
return t_start > 0;
|
||||
}
|
||||
|
||||
json to_json() const;
|
||||
};
|
||||
|
||||
// shared between server_context_impl and server_task_result_*
|
||||
// unlike server_slot_stats, server_metrics is server-global and cumulative, not tied to a slot
|
||||
struct server_metrics {
|
||||
int64_t t_start = 0;
|
||||
|
||||
struct bucket {
|
||||
uint64_t count = 0; // number of tokens
|
||||
uint64_t steps = 0; // number of decode steps,
|
||||
// this excludes first generated token (logits from prompt batch)
|
||||
uint64_t time = 0; // in microseconds
|
||||
|
||||
// the rate uses the decode steps, so that "free" tokens do not inflate it
|
||||
double n_per_second() const {
|
||||
return time > 0 ? (double) steps / (double) time * 1e6 : 0.0;
|
||||
}
|
||||
|
||||
void add(uint64_t n, uint64_t n_steps, uint64_t t_us) {
|
||||
count += n;
|
||||
steps += n_steps;
|
||||
time += t_us;
|
||||
}
|
||||
};
|
||||
|
||||
// these are reset by reset_bucket(), only the rate is read from them
|
||||
bucket prompt_bucket;
|
||||
bucket predict_bucket;
|
||||
|
||||
// metrics below are cumulative since the server started
|
||||
bucket prompt; // only processed tokens, cached ones are counted separately below
|
||||
bucket predict;
|
||||
|
||||
// tokens reused from the cache need no decode, so they only have a count
|
||||
uint64_t n_prompt_cached = 0;
|
||||
|
||||
uint64_t n_tokens_max = 0;
|
||||
|
||||
uint64_t n_decode = 0;
|
||||
uint64_t n_busy_slots = 0;
|
||||
|
||||
uint64_t n_draft_tokens = 0; // Total draft tokens generated
|
||||
uint64_t n_draft_accepted = 0; // Draft tokens actually accepted
|
||||
uint64_t n_draft_verif_steps = 0; // Total draft token verification steps by the target model
|
||||
std::vector<uint64_t> n_accepted_per_pos; // Accepted tokens per draft position
|
||||
|
||||
void init() {
|
||||
t_start = ggml_time_us();
|
||||
}
|
||||
|
||||
void reset_bucket() {
|
||||
prompt_bucket = {};
|
||||
predict_bucket = {};
|
||||
}
|
||||
|
||||
void add_prompt(uint64_t n_tokens, uint64_t t_us) {
|
||||
prompt .add(n_tokens, n_tokens, t_us);
|
||||
prompt_bucket.add(n_tokens, n_tokens, t_us);
|
||||
}
|
||||
|
||||
void add_prompt_cached(uint64_t n_tokens) {
|
||||
n_prompt_cached += n_tokens;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// other utils
|
||||
//
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -10,6 +10,8 @@
|
|||
#include "speculative.h"
|
||||
#include "server-common.h"
|
||||
|
||||
#include <sstream>
|
||||
|
||||
using json = nlohmann::ordered_json;
|
||||
|
||||
//
|
||||
|
|
@ -236,34 +238,6 @@ common_chat_msg task_result_state::update_chat_msg(
|
|||
return chat_msg;
|
||||
}
|
||||
|
||||
//
|
||||
|
||||
// result_timings
|
||||
//
|
||||
|
||||
json result_timings::to_json() const {
|
||||
json base = {
|
||||
{"cache_n", cache_n},
|
||||
|
||||
{"prompt_n", prompt_n},
|
||||
{"prompt_ms", prompt_ms},
|
||||
{"prompt_per_token_ms", prompt_per_token_ms},
|
||||
{"prompt_per_second", prompt_per_second},
|
||||
|
||||
{"predicted_n", predicted_n},
|
||||
{"predicted_ms", predicted_ms},
|
||||
{"predicted_per_token_ms", predicted_per_token_ms},
|
||||
{"predicted_per_second", predicted_per_second},
|
||||
};
|
||||
|
||||
if (draft_n > 0) {
|
||||
base["draft_n"] = draft_n;
|
||||
base["draft_n_accepted"] = draft_n_accepted;
|
||||
}
|
||||
|
||||
return base;
|
||||
}
|
||||
|
||||
//
|
||||
// result_prompt_progress
|
||||
//
|
||||
|
|
@ -382,7 +356,7 @@ json server_task_result_cmpl_final::to_json_non_oaicompat() {
|
|||
{"stop_type", stop_type_to_str(stop)},
|
||||
{"stopping_word", stopping_word},
|
||||
{"tokens_cached", n_tokens_cached},
|
||||
{"timings", timings.to_json()},
|
||||
{"timings", stats.to_json()},
|
||||
};
|
||||
if (!stream && !probs_output.empty()) {
|
||||
res["completion_probabilities"] = completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs);
|
||||
|
|
@ -432,8 +406,8 @@ json server_task_result_cmpl_final::to_json_oaicompat() {
|
|||
if (verbose) {
|
||||
res["__verbose"] = to_json_non_oaicompat();
|
||||
}
|
||||
if (timings.prompt_n >= 0) {
|
||||
res.push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
res.push_back({"timings", stats.to_json()});
|
||||
}
|
||||
|
||||
return res;
|
||||
|
|
@ -480,8 +454,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat() {
|
|||
if (verbose) {
|
||||
res["__verbose"] = to_json_non_oaicompat();
|
||||
}
|
||||
if (timings.prompt_n >= 0) {
|
||||
res.push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
res.push_back({"timings", stats.to_json()});
|
||||
}
|
||||
|
||||
return res;
|
||||
|
|
@ -541,8 +515,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
|
|||
});
|
||||
}
|
||||
|
||||
if (timings.prompt_n >= 0) {
|
||||
deltas.back().push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
deltas.back().push_back({"timings", stats.to_json()});
|
||||
}
|
||||
|
||||
// extra fields for debugging purposes
|
||||
|
|
@ -734,8 +708,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
|
|||
}}
|
||||
});
|
||||
|
||||
if (timings.prompt_n >= 0) {
|
||||
server_sent_events.back().at("data").push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
server_sent_events.back().at("data").push_back({"timings", stats.to_json()});
|
||||
}
|
||||
|
||||
return server_sent_events;
|
||||
|
|
@ -1086,8 +1060,8 @@ json server_task_result_cmpl_partial::to_json_non_oaicompat() {
|
|||
{"tokens_evaluated", n_prompt_tokens},
|
||||
};
|
||||
// populate the timings object when needed (usually for the last response or with timings_per_token enabled)
|
||||
if (timings.prompt_n > 0) {
|
||||
res.push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
res.push_back({"timings", stats.to_json()});
|
||||
}
|
||||
if (is_progress) {
|
||||
res.push_back({"prompt_progress", progress.to_json()});
|
||||
|
|
@ -1126,8 +1100,8 @@ json server_task_result_cmpl_partial::to_json_oaicompat() {
|
|||
if (verbose) {
|
||||
res["__verbose"] = to_json_non_oaicompat();
|
||||
}
|
||||
if (timings.prompt_n >= 0) {
|
||||
res.push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
res.push_back({"timings", stats.to_json()});
|
||||
}
|
||||
if (is_progress) {
|
||||
res.push_back({"prompt_progress", progress.to_json()});
|
||||
|
|
@ -1180,8 +1154,8 @@ json server_task_result_cmpl_partial::to_json_oaicompat_chat() {
|
|||
};
|
||||
}
|
||||
|
||||
if (timings.prompt_n >= 0) {
|
||||
last_json.push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
last_json.push_back({"timings", stats.to_json()});
|
||||
}
|
||||
if (is_progress) {
|
||||
last_json.push_back({"prompt_progress", progress.to_json()});
|
||||
|
|
@ -1330,8 +1304,8 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
|
|||
|
||||
if (!events.empty()) {
|
||||
json & data = events.back().at("data");
|
||||
if (timings.prompt_n >= 0) {
|
||||
data.push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
data.push_back({"timings", stats.to_json()});
|
||||
}
|
||||
if (is_progress) {
|
||||
data.push_back({"prompt_progress", progress.to_json()});
|
||||
|
|
@ -1539,34 +1513,104 @@ json server_task_result_error::to_json() {
|
|||
// server_task_result_metrics
|
||||
//
|
||||
json server_task_result_metrics::to_json() {
|
||||
return json {
|
||||
{ "idle", n_idle_slots },
|
||||
{ "processing", n_processing_slots },
|
||||
{ "deferred", n_tasks_deferred },
|
||||
{ "t_start", t_start },
|
||||
return slots_data;
|
||||
}
|
||||
|
||||
{ "n_prompt_tokens_processed_total", n_prompt_tokens_processed_total },
|
||||
{ "t_tokens_generation_total", t_tokens_generation_total },
|
||||
{ "n_tokens_predicted_total", n_tokens_predicted_total },
|
||||
{ "t_prompt_processing_total", t_prompt_processing_total },
|
||||
|
||||
{ "n_tokens_max", n_tokens_max },
|
||||
|
||||
{ "n_prompt_tokens_processed", n_prompt_tokens_processed },
|
||||
{ "t_prompt_processing", t_prompt_processing },
|
||||
{ "n_tokens_predicted", n_tokens_predicted },
|
||||
{ "t_tokens_generation", t_tokens_generation },
|
||||
|
||||
{ "n_decode_total", n_decode_total },
|
||||
{ "n_busy_slots_total", n_busy_slots_total },
|
||||
|
||||
{ "n_draft_tokens_total", n_draft_tokens_total },
|
||||
{ "n_draft_accepted_total", n_draft_accepted_total },
|
||||
{ "n_draft_verif_steps_total", n_draft_verif_steps_total },
|
||||
{ "n_accepted_per_pos_total", n_accepted_per_pos_total },
|
||||
|
||||
{ "slots", slots_data },
|
||||
// metrics definition: https://prometheus.io/docs/practices/naming/#metric-names
|
||||
std::string server_task_result_metrics::to_metrics() {
|
||||
const std::vector<metric_item> counters = {
|
||||
{
|
||||
"prompt_tokens_total",
|
||||
"Number of prompt tokens processed, excluding cached tokens",
|
||||
(double) metrics.prompt.count
|
||||
}, {
|
||||
"prompt_tokens_cached_total",
|
||||
"Number of prompt tokens reused from the cache",
|
||||
(double) metrics.n_prompt_cached
|
||||
}, {
|
||||
"prompt_seconds_total",
|
||||
"Total time spent processing prompts",
|
||||
metrics.prompt.time / 1.e6
|
||||
}, {
|
||||
"tokens_predicted_total",
|
||||
"Number of generation tokens processed",
|
||||
(double) metrics.predict.count
|
||||
}, {
|
||||
"tokens_predicted_seconds_total",
|
||||
"Total time spent generating tokens",
|
||||
metrics.predict.time / 1.e6
|
||||
}, {
|
||||
"n_decode_total",
|
||||
"Total number of llama_decode() calls, excluding speculative decoding and multimodal decoding",
|
||||
(double) metrics.n_decode
|
||||
}, {
|
||||
"n_tokens_max",
|
||||
"Largest observed sequence length (prompt + generation)",
|
||||
(double) metrics.n_tokens_max
|
||||
}, {
|
||||
"spec_decode_num_draft_tokens_total",
|
||||
"Speculative: Total draft tokens generated",
|
||||
(double) metrics.n_draft_tokens
|
||||
}, {
|
||||
"spec_decode_num_accepted_tokens_total",
|
||||
"Speculative: Total draft tokens accepted by the target model",
|
||||
(double) metrics.n_draft_accepted
|
||||
}, {
|
||||
"spec_decode_num_drafts_total",
|
||||
"Speculative: Total speculative decoding verification steps",
|
||||
(double) metrics.n_draft_verif_steps
|
||||
},
|
||||
};
|
||||
|
||||
const std::vector<metric_item> gauges = {
|
||||
{
|
||||
"prompt_tokens_seconds",
|
||||
"Average prompt throughput in tokens/s",
|
||||
metrics.prompt_bucket.n_per_second()
|
||||
}, {
|
||||
"predicted_tokens_seconds",
|
||||
"Average generation throughput in tokens/s",
|
||||
metrics.predict_bucket.n_per_second()
|
||||
}, {
|
||||
"requests_processing",
|
||||
"Number of requests processing",
|
||||
(double) n_processing_slots
|
||||
}, {
|
||||
"requests_deferred",
|
||||
"Number of requests deferred",
|
||||
(double) n_tasks_deferred
|
||||
}, {
|
||||
"n_busy_slots_per_decode",
|
||||
"Average number of busy slots per llama_decode() call",
|
||||
(double) metrics.n_busy_slots / std::max((double) metrics.n_decode, 1.0)
|
||||
},
|
||||
};
|
||||
|
||||
std::stringstream prometheus;
|
||||
|
||||
auto add_items = [&prometheus](const char * type, const std::vector<metric_item> & items) {
|
||||
for (const auto & item : items) {
|
||||
prometheus << "# HELP llamacpp:" << item.name << " " << item.description << "\n"
|
||||
<< "# TYPE llamacpp:" << item.name << " " << type << "\n"
|
||||
<< "llamacpp:" << item.name << " " << item.value << "\n";
|
||||
}
|
||||
};
|
||||
|
||||
add_items("counter", counters);
|
||||
add_items("gauge", gauges);
|
||||
|
||||
// labeled counter: one time series per draft position
|
||||
if (!metrics.n_accepted_per_pos.empty()) {
|
||||
prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos_total"
|
||||
" Accepted tokens per draft position\n"
|
||||
<< "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos_total counter\n";
|
||||
for (size_t i = 0; i < metrics.n_accepted_per_pos.size(); i++) {
|
||||
prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\""
|
||||
<< i << "\"} " << metrics.n_accepted_per_pos[i] << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
return prometheus.str();
|
||||
}
|
||||
|
||||
//
|
||||
|
|
|
|||
|
|
@ -259,26 +259,6 @@ struct server_task {
|
|||
}
|
||||
};
|
||||
|
||||
struct result_timings {
|
||||
int32_t cache_n = -1;
|
||||
|
||||
int32_t prompt_n = -1;
|
||||
double prompt_ms = 0.0;
|
||||
double prompt_per_token_ms = 0.0;
|
||||
double prompt_per_second = 0.0;
|
||||
|
||||
int32_t predicted_n = -1;
|
||||
double predicted_ms = 0.0;
|
||||
double predicted_per_token_ms = 0.0;
|
||||
double predicted_per_second = 0.0;
|
||||
|
||||
// Optional speculative metrics - only included when > 0
|
||||
int32_t draft_n = 0;
|
||||
int32_t draft_n_accepted = 0;
|
||||
|
||||
json to_json() const;
|
||||
};
|
||||
|
||||
struct result_prompt_progress {
|
||||
int32_t total = 0;
|
||||
int32_t cache = 0;
|
||||
|
|
@ -343,7 +323,7 @@ struct server_task_result_cmpl_final : server_task_result {
|
|||
|
||||
bool stream;
|
||||
bool include_usage;
|
||||
result_timings timings;
|
||||
server_slot_stats stats;
|
||||
std::string prompt;
|
||||
|
||||
bool truncated;
|
||||
|
|
@ -425,7 +405,7 @@ struct server_task_result_cmpl_partial : server_task_result {
|
|||
bool is_begin = false; // whether to send 200 status to HTTP client (begin of SSE stream)
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/23884
|
||||
completion_token_output prob_output;
|
||||
result_timings timings;
|
||||
server_slot_stats stats;
|
||||
result_prompt_progress progress;
|
||||
|
||||
// response formatting
|
||||
|
|
@ -510,38 +490,27 @@ struct server_task_result_error : server_task_result {
|
|||
};
|
||||
|
||||
struct server_task_result_metrics : server_task_result {
|
||||
// these are immediate stats, not accumulated (server_metrics is cumulative)
|
||||
int n_idle_slots;
|
||||
int n_processing_slots;
|
||||
int n_tasks_deferred;
|
||||
int64_t t_start;
|
||||
|
||||
// TODO: somehow reuse server_metrics in the future, instead of duplicating the fields
|
||||
uint64_t n_prompt_tokens_processed_total = 0;
|
||||
uint64_t t_prompt_processing_total = 0;
|
||||
uint64_t n_tokens_predicted_total = 0;
|
||||
uint64_t t_tokens_generation_total = 0;
|
||||
|
||||
uint64_t n_tokens_max = 0;
|
||||
|
||||
uint64_t n_prompt_tokens_processed = 0;
|
||||
uint64_t t_prompt_processing = 0;
|
||||
|
||||
uint64_t n_tokens_predicted = 0;
|
||||
uint64_t t_tokens_generation = 0;
|
||||
|
||||
uint64_t n_decode_total = 0;
|
||||
uint64_t n_busy_slots_total = 0;
|
||||
|
||||
uint64_t n_draft_tokens_total = 0;
|
||||
uint64_t n_draft_accepted_total = 0;
|
||||
uint64_t n_draft_verif_steps_total = 0;
|
||||
std::vector<uint64_t> n_accepted_per_pos_total;
|
||||
server_metrics metrics;
|
||||
|
||||
// while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
|
||||
// therefore, we use json to temporarily store the slot.to_json() result
|
||||
json slots_data = json::array();
|
||||
|
||||
// used by /slots API
|
||||
virtual json to_json() override;
|
||||
|
||||
// used by /metrics API
|
||||
struct metric_item {
|
||||
std::string name;
|
||||
std::string description;
|
||||
double value; // prometheus values are always float64
|
||||
};
|
||||
std::string to_metrics();
|
||||
};
|
||||
|
||||
struct server_task_result_slot_save_load : server_task_result {
|
||||
|
|
|
|||
227
tools/server/tests/unit/test_metrics.py
Normal file
227
tools/server/tests/unit/test_metrics.py
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
import pytest
|
||||
from utils import *
|
||||
|
||||
server = ServerPreset.tinyllama2()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def create_server():
|
||||
global server
|
||||
server = ServerPreset.tinyllama2()
|
||||
server.server_metrics = True
|
||||
|
||||
|
||||
def fetch_metrics(server: ServerProcess) -> str:
|
||||
"""get /metrics as raw prometheus text"""
|
||||
res = server.make_request("GET", "/metrics")
|
||||
assert res.status_code == 200
|
||||
assert "Process-Start-Time-Unix" in res.headers
|
||||
assert isinstance(res.body, str)
|
||||
return res.body
|
||||
|
||||
|
||||
def parse_metrics(text: str) -> dict:
|
||||
"""parse the prometheus text format into {name: (type, value)}"""
|
||||
out = {}
|
||||
types = {}
|
||||
for line in text.splitlines():
|
||||
if line.startswith("# TYPE "):
|
||||
_, _, name, kind = line.split(" ", 3)
|
||||
types[name] = kind
|
||||
elif line.startswith("llamacpp:") and "{" not in line:
|
||||
name, value = line.split(" ", 1)
|
||||
assert name in types, f"{name} has no # TYPE line"
|
||||
out[name] = (types[name], float(value))
|
||||
return out
|
||||
|
||||
|
||||
def test_metrics_disabled():
|
||||
global server
|
||||
server.server_metrics = False
|
||||
server.start()
|
||||
res = server.make_request("GET", "/metrics")
|
||||
assert res.status_code == 501 # ERROR_TYPE_NOT_SUPPORTED
|
||||
|
||||
|
||||
def test_metrics_prometheus_format():
|
||||
global server
|
||||
server.start()
|
||||
server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 8})
|
||||
|
||||
text = fetch_metrics(server)
|
||||
metrics = parse_metrics(text)
|
||||
|
||||
expected_counters = [
|
||||
"llamacpp:prompt_tokens_total",
|
||||
"llamacpp:prompt_tokens_cached_total",
|
||||
"llamacpp:prompt_seconds_total",
|
||||
"llamacpp:tokens_predicted_total",
|
||||
"llamacpp:tokens_predicted_seconds_total",
|
||||
"llamacpp:n_decode_total",
|
||||
"llamacpp:n_tokens_max",
|
||||
"llamacpp:spec_decode_num_draft_tokens_total",
|
||||
"llamacpp:spec_decode_num_accepted_tokens_total",
|
||||
"llamacpp:spec_decode_num_drafts_total",
|
||||
]
|
||||
expected_gauges = [
|
||||
"llamacpp:prompt_tokens_seconds",
|
||||
"llamacpp:predicted_tokens_seconds",
|
||||
"llamacpp:requests_processing",
|
||||
"llamacpp:requests_deferred",
|
||||
"llamacpp:n_busy_slots_per_decode",
|
||||
]
|
||||
|
||||
for name in expected_counters:
|
||||
assert metrics[name][0] == "counter"
|
||||
for name in expected_gauges:
|
||||
assert metrics[name][0] == "gauge"
|
||||
|
||||
# every metric must carry a help line
|
||||
for name in expected_counters + expected_gauges:
|
||||
assert f"# HELP {name} " in text
|
||||
|
||||
assert metrics["llamacpp:n_decode_total"][1] > 0
|
||||
assert metrics["llamacpp:requests_processing"][1] == 0
|
||||
|
||||
|
||||
def test_metrics_prompt_processed_and_cached():
|
||||
global server
|
||||
server.n_slots = 1 # keep the prompt cache on a single slot
|
||||
server.start()
|
||||
|
||||
prompt = "the quick brown fox jumps over the lazy dog"
|
||||
|
||||
n_processed = 0
|
||||
n_cached = 0
|
||||
for _ in range(2):
|
||||
res = server.make_request("POST", "/completion", data={"prompt": prompt, "n_predict": 4})
|
||||
assert res.status_code == 200
|
||||
n_processed += res.body["timings"]["prompt_n"]
|
||||
n_cached += res.body["timings"]["cache_n"]
|
||||
|
||||
# the second request must reuse the prompt of the first one
|
||||
assert n_cached > 0
|
||||
|
||||
metrics = parse_metrics(fetch_metrics(server))
|
||||
|
||||
# cached tokens are counted apart, they cost no decode
|
||||
assert metrics["llamacpp:prompt_tokens_total"][1] == n_processed
|
||||
assert metrics["llamacpp:prompt_tokens_cached_total"][1] == n_cached
|
||||
|
||||
|
||||
def test_metrics_predicted_total_matches_requests():
|
||||
global server
|
||||
server.start()
|
||||
|
||||
n_predicted = 0
|
||||
for n_predict in [1, 4, 16]:
|
||||
res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": n_predict})
|
||||
assert res.status_code == 200
|
||||
n_predicted += res.body["timings"]["predicted_n"]
|
||||
|
||||
metrics = parse_metrics(fetch_metrics(server))
|
||||
assert metrics["llamacpp:tokens_predicted_total"][1] == n_predicted
|
||||
|
||||
|
||||
def test_metrics_generation_rate_excludes_first_token():
|
||||
global server
|
||||
server.start()
|
||||
|
||||
# the first token comes from the logits of the last prompt batch, so it costs no decode step
|
||||
res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 1})
|
||||
timings = res.body["timings"]
|
||||
assert timings["predicted_n"] == 1
|
||||
assert timings["predicted_per_second"] == 0.0
|
||||
assert timings["predicted_per_token_ms"] == 0.0
|
||||
|
||||
res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 16})
|
||||
timings = res.body["timings"]
|
||||
assert timings["predicted_n"] == 16
|
||||
# the rate is over 15 decode steps, not 16 tokens
|
||||
expected = 1e3 / timings["predicted_ms"] * 15
|
||||
assert abs(timings["predicted_per_second"] - expected) < 1e-6
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_predict", [1, 8])
|
||||
def test_metrics_timings_are_finite(n_predict: int):
|
||||
global server
|
||||
server.start()
|
||||
res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": n_predict})
|
||||
timings = res.body["timings"]
|
||||
|
||||
# a null here means the server produced inf or nan
|
||||
for key, value in timings.items():
|
||||
assert value is not None, f"{key} is null"
|
||||
assert value >= 0, f"{key} is negative"
|
||||
|
||||
assert timings["prompt_ms"] > 0
|
||||
assert timings["prompt_per_token_ms"] > 0
|
||||
|
||||
|
||||
def test_metrics_timings_on_prompt_progress():
|
||||
global server
|
||||
server.start()
|
||||
|
||||
# a long prompt so that it is split over several batches (n_batch = 32)
|
||||
prompt = "the quick brown fox jumps over the lazy dog " * 8
|
||||
chunks = list(server.make_stream_request("POST", "/completion", data={
|
||||
"prompt": prompt,
|
||||
"n_predict": 4,
|
||||
"stream": True,
|
||||
"timings_per_token": True,
|
||||
"return_progress": True,
|
||||
}))
|
||||
|
||||
progress = [c for c in chunks if "prompt_progress" in c]
|
||||
assert len(progress) > 1 # the prompt did not fit in a single batch
|
||||
|
||||
# the very first update is sent before any prompt token is decoded
|
||||
first = progress[0]["timings"]
|
||||
assert first["prompt_n"] == 0
|
||||
assert first["prompt_ms"] == 0.0
|
||||
assert first["predicted_n"] == 0
|
||||
assert first["predicted_ms"] == 0.0
|
||||
|
||||
# timings must never go backwards, nor report bogus values
|
||||
prompt_ms = 0.0
|
||||
for chunk in progress:
|
||||
timings = chunk["timings"]
|
||||
for key, value in timings.items():
|
||||
assert value is not None, f"{key} is null"
|
||||
assert value >= 0, f"{key} is negative"
|
||||
assert timings["prompt_ms"] >= prompt_ms
|
||||
prompt_ms = timings["prompt_ms"]
|
||||
|
||||
assert prompt_ms > 0
|
||||
|
||||
|
||||
def test_metrics_slots_idle_after_completion():
|
||||
global server
|
||||
server.server_slots = True
|
||||
server.start()
|
||||
server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 8})
|
||||
|
||||
res = server.make_request("GET", "/slots")
|
||||
assert res.status_code == 200
|
||||
for slot in res.body:
|
||||
assert slot["is_processing"] is False
|
||||
if "next_token" in slot:
|
||||
# the budget of the finished task must not leak into the idle slot
|
||||
assert slot["next_token"][0]["n_remain"] == -1
|
||||
assert slot["next_token"][0]["n_decoded"] == 0
|
||||
|
||||
|
||||
def test_metrics_embedding_prompt_is_counted():
|
||||
global server
|
||||
server = ServerPreset.bert_bge_small()
|
||||
server.server_metrics = True
|
||||
server.start()
|
||||
|
||||
res = server.make_request("POST", "/v1/embeddings", data={"input": ["hello world", "goodbye world"]})
|
||||
assert res.status_code == 200
|
||||
|
||||
# embedding tasks never sample a token, but their prompt still costs a decode
|
||||
metrics = parse_metrics(fetch_metrics(server))
|
||||
assert metrics["llamacpp:prompt_tokens_total"][1] > 0
|
||||
assert metrics["llamacpp:n_decode_total"][1] > 0
|
||||
assert metrics["llamacpp:tokens_predicted_total"][1] == 0
|
||||
Loading…
Add table
Add a link
Reference in a new issue