From b5315e16e0c3d19707d4a3e3a9f727f1141d2282 Mon Sep 17 00:00:00 2001 From: Pascal Date: Fri, 3 Jul 2026 12:47:04 +0200 Subject: [PATCH] server + ui: ping silent SSE streams every 1s and kick only after 3s so slow prefill never drops healthy connections (#25241) * server + ui: ping silent SSE streams every 1s and kick only after 3s so slow prefill never drops healthy connections * server + ui: sse_ping_interval becomes a per-request body field Address review from ngxson: the global default returns to 30 so API clients see no behavior change, and the WebUI sends sse_ping_interval: 1 in the request body since it owns the 3s visibility-kick contract and declares the cadence it needs. Positive values keep the existing > 0 gate, -1 keeps its disabled semantics. * server: move sse_ping_interval into the request schema Address review from ngxson: the field is now a typed field_num with hard limits (-1, INT32_MAX) bound to task_params, seeded from the CLI default alongside the other inherited parameters. The raw json_value read and its redundant comment are gone, and schema evaluation brings type and range validation for free. --- tools/server/README.md | 2 ++ tools/server/server-context.cpp | 9 ++++++--- tools/server/server-schema.cpp | 5 +++++ tools/server/server-task.h | 2 ++ tools/ui/src/lib/constants/stream.ts | 2 +- tools/ui/src/lib/services/chat.service.ts | 1 + tools/ui/src/lib/types/api.d.ts | 1 + 7 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tools/server/README.md b/tools/server/README.md index e88bc5f28..501b66123 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -521,6 +521,8 @@ These words will not be included in the completion, so make sure to add them to `return_progress`: Include prompt processing progress in `stream` mode. The progress will be contained inside `prompt_progress` with 4 values: `total`, `cache`, `processed`, and `time_ms`. The overall progress is `processed/total`, while the actual timed progress is `(processed-cache)/(total-cache)`. The `time_ms` field contains the elapsed time in milliseconds since prompt processing started. Default: `false` +`sse_ping_interval`: Interval in seconds between SSE comment pings emitted while the stream stays silent, keeping the connection observable during long prompt processing. Overrides the server `--sse-ping-interval` setting for this request, `-1` disables pings. Default: server setting + `post_sampling_probs`: Returns the probabilities of top `n_probs` tokens after applying sampling chain. `response_fields`: A list of response fields, for example: `"response_fields": ["content", "generation_settings/n_predict"]`. If the specified field is missing, it will simply be omitted from the response without triggering an error. Note that fields with a slash will be unnested; for example, `generation_settings/n_predict` will move the field `n_predict` from the `generation_settings` object to the root of the response and give it a new name. diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 20e93258f..bb3b91ab5 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -4089,6 +4089,8 @@ std::unique_ptr server_routes::handle_completions_impl( auto & rd = res->rd; auto & params = this->params; + int32_t sse_ping_interval = params.sse_ping_interval; + try { std::vector tasks; @@ -4139,6 +4141,7 @@ std::unique_ptr server_routes::handle_completions_impl( task.params.message_spans = task.tokens.find_message_spans(delimiters); task.id_slot = json_value(data, "id_slot", -1); + sse_ping_interval = task.params.sse_ping_interval; // OAI-compat task.params.res_type = res_type; @@ -4228,7 +4231,7 @@ std::unique_ptr server_routes::handle_completions_impl( } res->status = 200; res->content_type = "text/event-stream"; - res->next = [res_this = res.get(), res_type, &req, ¶ms](std::string & output) -> bool { + res->next = [res_this = res.get(), res_type, sse_ping_interval, &req](std::string & output) -> bool { static auto format_error = [](task_response_type res_type, const json & res_json) { if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { return format_anthropic_sse({ @@ -4277,10 +4280,10 @@ std::unique_ptr server_routes::handle_completions_impl( // receive subsequent results bool timeout = false; int64_t start_time = ggml_time_ms(); - auto result = rd.next([&timeout, &start_time, ¶ms, &effective_should_stop]() { + auto result = rd.next([&timeout, &start_time, sse_ping_interval, &effective_should_stop]() { if (effective_should_stop()) { return true; // should_stop condition met - } else if (params.sse_ping_interval > 0 && ggml_time_ms() - start_time > (int64_t)params.sse_ping_interval * 1000) { + } else if (sse_ping_interval > 0 && ggml_time_ms() - start_time > (int64_t)sse_ping_interval * 1000) { timeout = true; return true; // timeout } diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp index 07a842bd6..5713cc831 100644 --- a/tools/server/server-schema.cpp +++ b/tools/server/server-schema.cpp @@ -37,6 +37,10 @@ std::vector> make_llama_cmpl_schema(const common_params & add((new field_bool("return_progress", params.return_progress)) ->set_desc("Include prompt processing progress events in stream mode")); + add((new field_num("sse_ping_interval", params.sse_ping_interval)) + ->set_hard_limits(-1, INT32_MAX) + ->set_desc("Interval in seconds between SSE comment pings emitted while the stream stays silent, -1 disables pings")); + add((new field_num("n_predict", params.n_predict)) ->set_hard_limits(-1, INT32_MAX) ->add_alias("max_completion_tokens") @@ -504,6 +508,7 @@ task_params eval_llama_cmpl_schema( params.n_cache_reuse = params_base.n_cache_reuse; params.cache_prompt = params_base.cache_prompt; params.antiprompt = params_base.antiprompt; + params.sse_ping_interval = params_base.sse_ping_interval; // enabling this will output extra debug information in the HTTP responses from the server params.verbose = params_base.verbosity > 9; diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 293bdf053..49f62d386 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -54,6 +54,8 @@ struct task_params { bool return_tokens = false; bool return_progress = false; + int32_t sse_ping_interval = 30; // seconds between SSE comment pings while the stream stays silent, -1 disables + int32_t n_keep = 0; // number of tokens to keep from initial prompt int32_t n_discard = 0; // number of tokens after n_keep that may be discarded when shifting context, 0 defaults to half int32_t n_predict = -1; // new tokens to predict diff --git a/tools/ui/src/lib/constants/stream.ts b/tools/ui/src/lib/constants/stream.ts index 3d042451f..67951ee95 100644 --- a/tools/ui/src/lib/constants/stream.ts +++ b/tools/ui/src/lib/constants/stream.ts @@ -1,3 +1,3 @@ // grace window after a visibilitychange before we kick a reader whose socket likely died // while the tab was hidden. covers brief background pauses without thrashing live streams -export const STREAM_VISIBILITY_KICK_MS = 1000; +export const STREAM_VISIBILITY_KICK_MS = 3000; diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 7dfee3773..dbd0a94f1 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -255,6 +255,7 @@ export class ChatService { }), stream, return_progress: stream ? true : undefined, + sse_ping_interval: stream ? 1 : undefined, tools: tools && tools.length > 0 ? tools : undefined }; diff --git a/tools/ui/src/lib/types/api.d.ts b/tools/ui/src/lib/types/api.d.ts index ec695ac61..5421f8b7f 100644 --- a/tools/ui/src/lib/types/api.d.ts +++ b/tools/ui/src/lib/types/api.d.ts @@ -265,6 +265,7 @@ export interface ApiChatCompletionRequest { stream?: boolean; model?: string; return_progress?: boolean; + sse_ping_interval?: number; tools?: ApiChatCompletionTool[]; // Reasoning parameters reasoning_format?: string;