Honor the configured AI timeout while waiting for the first stream chunk

The OpenAI-compatible stream reader bounded every chunk wait at 12s.
Local backends (LM Studio, llama.cpp) legitimately spend minutes on
prompt processing before the first SSE chunk, so raising the provider
timeout in Settings changed nothing and Pulse dropped the stream with
'AI response timed out' (discussion #1571). The wait for first bytes now
uses the configured request timeout; the 12s bound still applies to
inter-chunk gaps once the stream is flowing.
This commit is contained in:
rcourtman 2026-07-14 09:44:22 +01:00
parent b3ae6f72b6
commit 1c06484512
2 changed files with 26 additions and 10 deletions

View file

@ -48,6 +48,11 @@ type OpenAIClient struct {
// body chunks. Bound that gap separately from the full turn timeout so chat
// fallback can move before the drawer looks dead.
streamChunkTimeout time.Duration
// Local OpenAI-compatible backends (LM Studio, llama.cpp, vLLM on CPU) can
// legitimately spend minutes on prompt processing before the first SSE
// chunk, so the wait for first bytes honors the configured request timeout
// instead of the inter-chunk gap bound (issue discussion #1571).
streamFirstChunkTimeout time.Duration
}
// NewOpenAIClient creates a new OpenAI API client
@ -72,14 +77,15 @@ func NewOpenAICompatibleClient(providerName, apiKey, model, baseURL string, time
timeout = 300 * time.Second // Default 5 minutes
}
return &OpenAIClient{
providerName: providerName,
apiKey: apiKey,
model: model,
baseURL: baseURL,
requestTimeout: timeout,
client: &http.Client{Timeout: timeout},
streamClient: newOpenAIStreamHTTPClient(timeout),
streamChunkTimeout: boundedOpenAIStreamChunkTimeout(timeout),
providerName: providerName,
apiKey: apiKey,
model: model,
baseURL: baseURL,
requestTimeout: timeout,
client: &http.Client{Timeout: timeout},
streamClient: newOpenAIStreamHTTPClient(timeout),
streamChunkTimeout: boundedOpenAIStreamChunkTimeout(timeout),
streamFirstChunkTimeout: timeout,
}
}
@ -1126,9 +1132,15 @@ func (c *OpenAIClient) ChatStream(ctx context.Context, req ChatRequest, callback
return false, nil
}
receivedFirstChunk := false
for {
n, err := readOpenAIStreamChunk(streamCtx, reader, buf, c.streamChunkTimeout)
chunkTimeout := c.streamChunkTimeout
if !receivedFirstChunk {
chunkTimeout = c.streamFirstChunkTimeout
}
n, err := readOpenAIStreamChunk(streamCtx, reader, buf, chunkTimeout)
if n > 0 {
receivedFirstChunk = true
pendingData += string(buf[:n])
lines := strings.Split(pendingData, "\n")

View file

@ -187,12 +187,14 @@ func TestNewOpenAIClient_BoundsStreamResponseHeaderTimeout(t *testing.T) {
require.True(t, ok)
assert.Equal(t, openaiStreamResponseHeaderTimeout, transport.ResponseHeaderTimeout)
assert.Equal(t, openaiStreamChunkTimeout, client.streamChunkTimeout)
assert.Equal(t, 300*time.Second, client.streamFirstChunkTimeout)
shortTimeoutClient := NewOpenAIClient("sk-test", "gpt-4", "https://api.openai.com/v1", 2*time.Second)
shortTransport, ok := shortTimeoutClient.streamClient.Transport.(*http.Transport)
require.True(t, ok)
assert.Equal(t, 2*time.Second, shortTransport.ResponseHeaderTimeout)
assert.Equal(t, 2*time.Second, shortTimeoutClient.streamChunkTimeout)
assert.Equal(t, 2*time.Second, shortTimeoutClient.streamFirstChunkTimeout)
}
func TestNewOpenAICompatibleClient_NormalizesProviderBasePaths(t *testing.T) {
@ -247,7 +249,9 @@ func TestNewOpenAICompatibleClient_NormalizesProviderBasePaths(t *testing.T) {
func TestOpenAIClient_ChatStream_TimesOutWaitingForFirstStreamChunk(t *testing.T) {
body := newBlockingReadCloser()
client := NewOpenAIClient("sk-test", "gpt-4", "https://example.invalid/v1", time.Second)
client.streamChunkTimeout = 10 * time.Millisecond
// The wait for first bytes honors the full request timeout (local backends
// can spend minutes on prompt processing), not the inter-chunk gap bound.
client.streamFirstChunkTimeout = 10 * time.Millisecond
client.streamClient = &http.Client{
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
return &http.Response{