mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-08-24 07:45:49 +00:00
server: refactor + correctness fixes for metrics (#26920)
* server: refactor metrics * move most fields to server_slot_stats * cont * rm result_timings * tie stats to batch * cont * nits: move place in code * exclude first generated token * more accurate batch metrics tracking * n_predict --> n_gen * metrics_on_prediction * metrics_flush_idle * metrics: seperate cache/processed prompt tokens * refactor server_task_result_metrics * add test * nits * fix flush before reset() * cont * rm dead code * nits
This commit is contained in:
Xuan-Son Nguyen
GitHub
parent
e79e4bf660
commit
decaf508bb
6 changed files with 813 additions and 475 deletions
|
|
@ -10,6 +10,8 @@
|
|||
#include "speculative.h"
|
||||
#include "server-common.h"
|
||||
|
||||
#include <sstream>
|
||||
|
||||
using json = nlohmann::ordered_json;
|
||||
|
||||
//
|
||||
|
|
@ -236,34 +238,6 @@ common_chat_msg task_result_state::update_chat_msg(
|
|||
return chat_msg;
|
||||
}
|
||||
|
||||
//
|
||||
|
||||
// result_timings
|
||||
//
|
||||
|
||||
json result_timings::to_json() const {
|
||||
json base = {
|
||||
{"cache_n", cache_n},
|
||||
|
||||
{"prompt_n", prompt_n},
|
||||
{"prompt_ms", prompt_ms},
|
||||
{"prompt_per_token_ms", prompt_per_token_ms},
|
||||
{"prompt_per_second", prompt_per_second},
|
||||
|
||||
{"predicted_n", predicted_n},
|
||||
{"predicted_ms", predicted_ms},
|
||||
{"predicted_per_token_ms", predicted_per_token_ms},
|
||||
{"predicted_per_second", predicted_per_second},
|
||||
};
|
||||
|
||||
if (draft_n > 0) {
|
||||
base["draft_n"] = draft_n;
|
||||
base["draft_n_accepted"] = draft_n_accepted;
|
||||
}
|
||||
|
||||
return base;
|
||||
}
|
||||
|
||||
//
|
||||
// result_prompt_progress
|
||||
//
|
||||
|
|
@ -382,7 +356,7 @@ json server_task_result_cmpl_final::to_json_non_oaicompat() {
|
|||
{"stop_type", stop_type_to_str(stop)},
|
||||
{"stopping_word", stopping_word},
|
||||
{"tokens_cached", n_tokens_cached},
|
||||
{"timings", timings.to_json()},
|
||||
{"timings", stats.to_json()},
|
||||
};
|
||||
if (!stream && !probs_output.empty()) {
|
||||
res["completion_probabilities"] = completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs);
|
||||
|
|
@ -432,8 +406,8 @@ json server_task_result_cmpl_final::to_json_oaicompat() {
|
|||
if (verbose) {
|
||||
res["__verbose"] = to_json_non_oaicompat();
|
||||
}
|
||||
if (timings.prompt_n >= 0) {
|
||||
res.push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
res.push_back({"timings", stats.to_json()});
|
||||
}
|
||||
|
||||
return res;
|
||||
|
|
@ -480,8 +454,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat() {
|
|||
if (verbose) {
|
||||
res["__verbose"] = to_json_non_oaicompat();
|
||||
}
|
||||
if (timings.prompt_n >= 0) {
|
||||
res.push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
res.push_back({"timings", stats.to_json()});
|
||||
}
|
||||
|
||||
return res;
|
||||
|
|
@ -541,8 +515,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
|
|||
});
|
||||
}
|
||||
|
||||
if (timings.prompt_n >= 0) {
|
||||
deltas.back().push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
deltas.back().push_back({"timings", stats.to_json()});
|
||||
}
|
||||
|
||||
// extra fields for debugging purposes
|
||||
|
|
@ -734,8 +708,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
|
|||
}}
|
||||
});
|
||||
|
||||
if (timings.prompt_n >= 0) {
|
||||
server_sent_events.back().at("data").push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
server_sent_events.back().at("data").push_back({"timings", stats.to_json()});
|
||||
}
|
||||
|
||||
return server_sent_events;
|
||||
|
|
@ -1086,8 +1060,8 @@ json server_task_result_cmpl_partial::to_json_non_oaicompat() {
|
|||
{"tokens_evaluated", n_prompt_tokens},
|
||||
};
|
||||
// populate the timings object when needed (usually for the last response or with timings_per_token enabled)
|
||||
if (timings.prompt_n > 0) {
|
||||
res.push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
res.push_back({"timings", stats.to_json()});
|
||||
}
|
||||
if (is_progress) {
|
||||
res.push_back({"prompt_progress", progress.to_json()});
|
||||
|
|
@ -1126,8 +1100,8 @@ json server_task_result_cmpl_partial::to_json_oaicompat() {
|
|||
if (verbose) {
|
||||
res["__verbose"] = to_json_non_oaicompat();
|
||||
}
|
||||
if (timings.prompt_n >= 0) {
|
||||
res.push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
res.push_back({"timings", stats.to_json()});
|
||||
}
|
||||
if (is_progress) {
|
||||
res.push_back({"prompt_progress", progress.to_json()});
|
||||
|
|
@ -1180,8 +1154,8 @@ json server_task_result_cmpl_partial::to_json_oaicompat_chat() {
|
|||
};
|
||||
}
|
||||
|
||||
if (timings.prompt_n >= 0) {
|
||||
last_json.push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
last_json.push_back({"timings", stats.to_json()});
|
||||
}
|
||||
if (is_progress) {
|
||||
last_json.push_back({"prompt_progress", progress.to_json()});
|
||||
|
|
@ -1330,8 +1304,8 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
|
|||
|
||||
if (!events.empty()) {
|
||||
json & data = events.back().at("data");
|
||||
if (timings.prompt_n >= 0) {
|
||||
data.push_back({"timings", timings.to_json()});
|
||||
if (stats.is_set()) {
|
||||
data.push_back({"timings", stats.to_json()});
|
||||
}
|
||||
if (is_progress) {
|
||||
data.push_back({"prompt_progress", progress.to_json()});
|
||||
|
|
@ -1539,34 +1513,104 @@ json server_task_result_error::to_json() {
|
|||
// server_task_result_metrics
|
||||
//
|
||||
json server_task_result_metrics::to_json() {
|
||||
return json {
|
||||
{ "idle", n_idle_slots },
|
||||
{ "processing", n_processing_slots },
|
||||
{ "deferred", n_tasks_deferred },
|
||||
{ "t_start", t_start },
|
||||
return slots_data;
|
||||
}
|
||||
|
||||
{ "n_prompt_tokens_processed_total", n_prompt_tokens_processed_total },
|
||||
{ "t_tokens_generation_total", t_tokens_generation_total },
|
||||
{ "n_tokens_predicted_total", n_tokens_predicted_total },
|
||||
{ "t_prompt_processing_total", t_prompt_processing_total },
|
||||
|
||||
{ "n_tokens_max", n_tokens_max },
|
||||
|
||||
{ "n_prompt_tokens_processed", n_prompt_tokens_processed },
|
||||
{ "t_prompt_processing", t_prompt_processing },
|
||||
{ "n_tokens_predicted", n_tokens_predicted },
|
||||
{ "t_tokens_generation", t_tokens_generation },
|
||||
|
||||
{ "n_decode_total", n_decode_total },
|
||||
{ "n_busy_slots_total", n_busy_slots_total },
|
||||
|
||||
{ "n_draft_tokens_total", n_draft_tokens_total },
|
||||
{ "n_draft_accepted_total", n_draft_accepted_total },
|
||||
{ "n_draft_verif_steps_total", n_draft_verif_steps_total },
|
||||
{ "n_accepted_per_pos_total", n_accepted_per_pos_total },
|
||||
|
||||
{ "slots", slots_data },
|
||||
// metrics definition: https://prometheus.io/docs/practices/naming/#metric-names
|
||||
std::string server_task_result_metrics::to_metrics() {
|
||||
const std::vector<metric_item> counters = {
|
||||
{
|
||||
"prompt_tokens_total",
|
||||
"Number of prompt tokens processed, excluding cached tokens",
|
||||
(double) metrics.prompt.count
|
||||
}, {
|
||||
"prompt_tokens_cached_total",
|
||||
"Number of prompt tokens reused from the cache",
|
||||
(double) metrics.n_prompt_cached
|
||||
}, {
|
||||
"prompt_seconds_total",
|
||||
"Total time spent processing prompts",
|
||||
metrics.prompt.time / 1.e6
|
||||
}, {
|
||||
"tokens_predicted_total",
|
||||
"Number of generation tokens processed",
|
||||
(double) metrics.predict.count
|
||||
}, {
|
||||
"tokens_predicted_seconds_total",
|
||||
"Total time spent generating tokens",
|
||||
metrics.predict.time / 1.e6
|
||||
}, {
|
||||
"n_decode_total",
|
||||
"Total number of llama_decode() calls, excluding speculative decoding and multimodal decoding",
|
||||
(double) metrics.n_decode
|
||||
}, {
|
||||
"n_tokens_max",
|
||||
"Largest observed sequence length (prompt + generation)",
|
||||
(double) metrics.n_tokens_max
|
||||
}, {
|
||||
"spec_decode_num_draft_tokens_total",
|
||||
"Speculative: Total draft tokens generated",
|
||||
(double) metrics.n_draft_tokens
|
||||
}, {
|
||||
"spec_decode_num_accepted_tokens_total",
|
||||
"Speculative: Total draft tokens accepted by the target model",
|
||||
(double) metrics.n_draft_accepted
|
||||
}, {
|
||||
"spec_decode_num_drafts_total",
|
||||
"Speculative: Total speculative decoding verification steps",
|
||||
(double) metrics.n_draft_verif_steps
|
||||
},
|
||||
};
|
||||
|
||||
const std::vector<metric_item> gauges = {
|
||||
{
|
||||
"prompt_tokens_seconds",
|
||||
"Average prompt throughput in tokens/s",
|
||||
metrics.prompt_bucket.n_per_second()
|
||||
}, {
|
||||
"predicted_tokens_seconds",
|
||||
"Average generation throughput in tokens/s",
|
||||
metrics.predict_bucket.n_per_second()
|
||||
}, {
|
||||
"requests_processing",
|
||||
"Number of requests processing",
|
||||
(double) n_processing_slots
|
||||
}, {
|
||||
"requests_deferred",
|
||||
"Number of requests deferred",
|
||||
(double) n_tasks_deferred
|
||||
}, {
|
||||
"n_busy_slots_per_decode",
|
||||
"Average number of busy slots per llama_decode() call",
|
||||
(double) metrics.n_busy_slots / std::max((double) metrics.n_decode, 1.0)
|
||||
},
|
||||
};
|
||||
|
||||
std::stringstream prometheus;
|
||||
|
||||
auto add_items = [&prometheus](const char * type, const std::vector<metric_item> & items) {
|
||||
for (const auto & item : items) {
|
||||
prometheus << "# HELP llamacpp:" << item.name << " " << item.description << "\n"
|
||||
<< "# TYPE llamacpp:" << item.name << " " << type << "\n"
|
||||
<< "llamacpp:" << item.name << " " << item.value << "\n";
|
||||
}
|
||||
};
|
||||
|
||||
add_items("counter", counters);
|
||||
add_items("gauge", gauges);
|
||||
|
||||
// labeled counter: one time series per draft position
|
||||
if (!metrics.n_accepted_per_pos.empty()) {
|
||||
prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos_total"
|
||||
" Accepted tokens per draft position\n"
|
||||
<< "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos_total counter\n";
|
||||
for (size_t i = 0; i < metrics.n_accepted_per_pos.size(); i++) {
|
||||
prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\""
|
||||
<< i << "\"} " << metrics.n_accepted_per_pos[i] << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
return prometheus.str();
|
||||
}
|
||||
|
||||
//
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue