mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-08-26 09:23:10 +00:00
Honor the configured AI timeout while waiting for the first stream chunk
The OpenAI-compatible stream reader bounded every chunk wait at 12s. Local backends (LM Studio, llama.cpp) legitimately spend minutes on prompt processing before the first SSE chunk, so raising the provider timeout in Settings changed nothing and Pulse dropped the stream with 'AI response timed out' (discussion #1571). The wait for first bytes now uses the configured request timeout; the 12s bound still applies to inter-chunk gaps once the stream is flowing.
This commit is contained in:
parent
b3ae6f72b6
commit
1c06484512
2 changed files with 26 additions and 10 deletions
|
|
@ -48,6 +48,11 @@ type OpenAIClient struct {
|
|||
// body chunks. Bound that gap separately from the full turn timeout so chat
|
||||
// fallback can move before the drawer looks dead.
|
||||
streamChunkTimeout time.Duration
|
||||
// Local OpenAI-compatible backends (LM Studio, llama.cpp, vLLM on CPU) can
|
||||
// legitimately spend minutes on prompt processing before the first SSE
|
||||
// chunk, so the wait for first bytes honors the configured request timeout
|
||||
// instead of the inter-chunk gap bound (issue discussion #1571).
|
||||
streamFirstChunkTimeout time.Duration
|
||||
}
|
||||
|
||||
// NewOpenAIClient creates a new OpenAI API client
|
||||
|
|
@ -72,14 +77,15 @@ func NewOpenAICompatibleClient(providerName, apiKey, model, baseURL string, time
|
|||
timeout = 300 * time.Second // Default 5 minutes
|
||||
}
|
||||
return &OpenAIClient{
|
||||
providerName: providerName,
|
||||
apiKey: apiKey,
|
||||
model: model,
|
||||
baseURL: baseURL,
|
||||
requestTimeout: timeout,
|
||||
client: &http.Client{Timeout: timeout},
|
||||
streamClient: newOpenAIStreamHTTPClient(timeout),
|
||||
streamChunkTimeout: boundedOpenAIStreamChunkTimeout(timeout),
|
||||
providerName: providerName,
|
||||
apiKey: apiKey,
|
||||
model: model,
|
||||
baseURL: baseURL,
|
||||
requestTimeout: timeout,
|
||||
client: &http.Client{Timeout: timeout},
|
||||
streamClient: newOpenAIStreamHTTPClient(timeout),
|
||||
streamChunkTimeout: boundedOpenAIStreamChunkTimeout(timeout),
|
||||
streamFirstChunkTimeout: timeout,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1126,9 +1132,15 @@ func (c *OpenAIClient) ChatStream(ctx context.Context, req ChatRequest, callback
|
|||
return false, nil
|
||||
}
|
||||
|
||||
receivedFirstChunk := false
|
||||
for {
|
||||
n, err := readOpenAIStreamChunk(streamCtx, reader, buf, c.streamChunkTimeout)
|
||||
chunkTimeout := c.streamChunkTimeout
|
||||
if !receivedFirstChunk {
|
||||
chunkTimeout = c.streamFirstChunkTimeout
|
||||
}
|
||||
n, err := readOpenAIStreamChunk(streamCtx, reader, buf, chunkTimeout)
|
||||
if n > 0 {
|
||||
receivedFirstChunk = true
|
||||
pendingData += string(buf[:n])
|
||||
lines := strings.Split(pendingData, "\n")
|
||||
|
||||
|
|
|
|||
|
|
@ -187,12 +187,14 @@ func TestNewOpenAIClient_BoundsStreamResponseHeaderTimeout(t *testing.T) {
|
|||
require.True(t, ok)
|
||||
assert.Equal(t, openaiStreamResponseHeaderTimeout, transport.ResponseHeaderTimeout)
|
||||
assert.Equal(t, openaiStreamChunkTimeout, client.streamChunkTimeout)
|
||||
assert.Equal(t, 300*time.Second, client.streamFirstChunkTimeout)
|
||||
|
||||
shortTimeoutClient := NewOpenAIClient("sk-test", "gpt-4", "https://api.openai.com/v1", 2*time.Second)
|
||||
shortTransport, ok := shortTimeoutClient.streamClient.Transport.(*http.Transport)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, 2*time.Second, shortTransport.ResponseHeaderTimeout)
|
||||
assert.Equal(t, 2*time.Second, shortTimeoutClient.streamChunkTimeout)
|
||||
assert.Equal(t, 2*time.Second, shortTimeoutClient.streamFirstChunkTimeout)
|
||||
}
|
||||
|
||||
func TestNewOpenAICompatibleClient_NormalizesProviderBasePaths(t *testing.T) {
|
||||
|
|
@ -247,7 +249,9 @@ func TestNewOpenAICompatibleClient_NormalizesProviderBasePaths(t *testing.T) {
|
|||
func TestOpenAIClient_ChatStream_TimesOutWaitingForFirstStreamChunk(t *testing.T) {
|
||||
body := newBlockingReadCloser()
|
||||
client := NewOpenAIClient("sk-test", "gpt-4", "https://example.invalid/v1", time.Second)
|
||||
client.streamChunkTimeout = 10 * time.Millisecond
|
||||
// The wait for first bytes honors the full request timeout (local backends
|
||||
// can spend minutes on prompt processing), not the inter-chunk gap bound.
|
||||
client.streamFirstChunkTimeout = 10 * time.Millisecond
|
||||
client.streamClient = &http.Client{
|
||||
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue