mirror of
https://github.com/alibaba/open-code-review.git
synced 2026-07-15 12:19:10 +00:00
feat: 四层模型端点，适配 anthropic
This commit is contained in:
kite
parent
b680c29ba4
commit
8a93673268
9 changed files with 895 additions and 270 deletions
|
|
@ -222,7 +222,7 @@ ocr review --commit abc123
|
|||
| 参数 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
| `PLAN_MODE_LINE_THRESHOLD` | 50 | 触发 Plan 阶段的文件变更行数阈值 |
|
||||
| `TOKEN_WARNING_THRESHOLD` | 50000 | 触发记忆压缩的 token 数量阈值 |
|
||||
| `MAX_TOKENS` | 58888 | 大模型请求的最大 token 数限制，内部阈值为该值的 80%（用于警告和记忆压缩） |
|
||||
| `MAX_TOOL_REQUEST_TIMES` | 20 | 每个文件的最大工具调用次数 |
|
||||
| `TOOL_REQUEST_WAIT_TIME_MS` | 10000 | 工具调用等待时间（毫秒） |
|
||||
| `MAX_SUBTASK_EXECUTION_TIME_MINUTES` | 5 | 单个子任务最长执行时间（分钟） |
|
||||
|
|
|
|||
|
|
@ -8,54 +8,6 @@ import (
|
|||
"strconv"
|
||||
)
|
||||
|
||||
// Environment variable names that override config file values.
|
||||
const (
|
||||
envLLMURL = "OCR_LLM_URL"
|
||||
envLLMToken = "OCR_LLM_TOKEN"
|
||||
envLLMModel = "OCR_LLM_MODEL"
|
||||
envLanguage = "OCR_LANGUAGE"
|
||||
)
|
||||
|
||||
func envOrDefault(env, fallback string) string {
|
||||
if v := os.Getenv(env); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
// LoadMergedConfig loads config from path, then overrides with environment variables if set.
|
||||
// Returns nil, nil if file does not exist and no env vars are set.
|
||||
func LoadMergedConfig(path string) (*Config, error) {
|
||||
cfg, err := LoadAppConfig(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
hasEnv := os.Getenv(envLLMURL) != "" || os.Getenv(envLLMToken) != "" ||
|
||||
os.Getenv(envLLMModel) != "" || os.Getenv(envLanguage) != ""
|
||||
if !hasEnv && cfg == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if cfg == nil {
|
||||
cfg = &Config{}
|
||||
}
|
||||
|
||||
if v := os.Getenv(envLLMURL); v != "" {
|
||||
cfg.Llm.URL = v
|
||||
}
|
||||
if v := os.Getenv(envLLMToken); v != "" {
|
||||
cfg.Llm.AuthToken = v
|
||||
}
|
||||
if v := os.Getenv(envLLMModel); v != "" {
|
||||
cfg.Llm.Model = v
|
||||
}
|
||||
if v := os.Getenv(envLanguage); v != "" {
|
||||
cfg.Language = v
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// Default config file location: ~/.open-code-review/config.json
|
||||
func defaultConfigPath() string {
|
||||
home, err := os.UserHomeDir()
|
||||
|
|
@ -122,10 +74,11 @@ type Config struct {
|
|||
}
|
||||
|
||||
type LlmConfig struct {
|
||||
Provider string `json:"provider,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
AuthToken string `json:"auth_token,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
AuthToken string `json:"auth_token,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
UseAnthropic *bool `json:"use_anthropic,omitempty"` // nil = default true; false = OpenAI protocol
|
||||
}
|
||||
|
||||
// TelemetryConfig holds telemetry-specific settings.
|
||||
|
|
@ -177,6 +130,12 @@ func setConfigValue(cfg *Config, key, value string) error {
|
|||
cfg.Llm.AuthToken = value
|
||||
case "llm.model", "llm.Model":
|
||||
cfg.Llm.Model = value
|
||||
case "llm.use_anthropic", "llm.UseAnthropic":
|
||||
b, err := strconv.ParseBool(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid boolean for llm.use_anthropic: %w", err)
|
||||
}
|
||||
cfg.Llm.UseAnthropic = &b
|
||||
case "language", "Language":
|
||||
cfg.Language = value
|
||||
case "telemetry.enabled", "telemetry.Enabled":
|
||||
|
|
@ -200,7 +159,7 @@ func setConfigValue(cfg *Config, key, value string) error {
|
|||
cfg.ensureTelemetry()
|
||||
cfg.Telemetry.ContentLog = b
|
||||
default:
|
||||
return fmt.Errorf("unknown config key: %s\nSupported keys: llm.provider, llm.url, llm.auth_token, llm.model, language, telemetry.enabled, telemetry.exporter, telemetry.otlp_endpoint, telemetry.content_logging", key)
|
||||
return fmt.Errorf("unknown config key: %s\nSupported keys: llm.provider, llm.url, llm.auth_token, llm.model, llm.use_anthropic, language, telemetry.enabled, telemetry.exporter, telemetry.otlp_endpoint, telemetry.content_logging", key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/open-code-review/open-code-review/internal/config/template"
|
||||
"github.com/open-code-review/open-code-review/internal/config/testconnection"
|
||||
"github.com/open-code-review/open-code-review/internal/llm"
|
||||
)
|
||||
|
|
@ -23,43 +25,55 @@ func runLLM(args []string) error {
|
|||
}
|
||||
|
||||
func runLLMTest() error {
|
||||
cfg, err := LoadMergedConfig(defaultConfigPath())
|
||||
appCfg, err := LoadAppConfig(defaultConfigPath())
|
||||
if err != nil {
|
||||
return fmt.Errorf("load config: %w", err)
|
||||
}
|
||||
if cfg == nil || cfg.Llm.URL == "" || cfg.Llm.AuthToken == "" {
|
||||
return fmt.Errorf("llm.url and llm.auth_token are required in %s, or set OCR_LLM_URL and OCR_LLM_TOKEN environment variables", defaultConfigPath())
|
||||
|
||||
ep, err := llm.ResolveEndpoint(defaultConfigPath())
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve LLM endpoint: %w", err)
|
||||
}
|
||||
|
||||
task, err := testconnection.LoadDefault()
|
||||
if err != nil {
|
||||
return fmt.Errorf("load test task config: %w", err)
|
||||
}
|
||||
task.ApplyLanguage(cfg.Language)
|
||||
if appCfg != nil {
|
||||
task.ApplyLanguage(appCfg.Language)
|
||||
}
|
||||
|
||||
timeout := 30 * time.Second
|
||||
if task.Timeout > 0 {
|
||||
timeout = time.Duration(task.Timeout) * time.Second
|
||||
}
|
||||
|
||||
llmClient := llm.NewClient(llm.ClientConfig{
|
||||
URL: cfg.Llm.URL,
|
||||
APIKey: cfg.Llm.AuthToken,
|
||||
Model: cfg.Llm.Model,
|
||||
Timeout: timeout,
|
||||
})
|
||||
tpl, err := template.LoadDefault()
|
||||
if err != nil {
|
||||
return fmt.Errorf("load default template: %w", err)
|
||||
}
|
||||
|
||||
llmClient := llm.NewLLMClient(ep)
|
||||
|
||||
messages := make([]llm.Message, 0, len(task.Messages))
|
||||
for _, m := range task.Messages {
|
||||
messages = append(messages, llm.Message{Role: m.Role, Content: m.Content})
|
||||
}
|
||||
|
||||
resp, err := llmClient.GeneralRequest(messages, cfg.Llm.Model, nil)
|
||||
resp, err := func() (*llm.ChatResponse, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
return llmClient.CompletionsWithCtx(ctx, llm.ChatRequest{
|
||||
Model: ep.Model,
|
||||
Messages: messages,
|
||||
MaxTokens: tpl.MaxTokens,
|
||||
})
|
||||
}()
|
||||
if err != nil {
|
||||
return fmt.Errorf("llm request failed: %w", err)
|
||||
}
|
||||
|
||||
model := cfg.Llm.Model
|
||||
model := ep.Model
|
||||
if resp.Model != "" {
|
||||
model = resp.Model
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,9 +9,9 @@ import (
|
|||
|
||||
"github.com/open-code-review/open-code-review/internal/agent"
|
||||
"github.com/open-code-review/open-code-review/internal/config/rules"
|
||||
"github.com/open-code-review/open-code-review/internal/diff"
|
||||
"github.com/open-code-review/open-code-review/internal/config/template"
|
||||
"github.com/open-code-review/open-code-review/internal/config/toolsconfig"
|
||||
"github.com/open-code-review/open-code-review/internal/diff"
|
||||
"github.com/open-code-review/open-code-review/internal/llm"
|
||||
"github.com/open-code-review/open-code-review/internal/stdout"
|
||||
"github.com/open-code-review/open-code-review/internal/telemetry"
|
||||
|
|
@ -63,20 +63,21 @@ func runReview(args []string) error {
|
|||
return fmt.Errorf("resolve repo: %w", err)
|
||||
}
|
||||
|
||||
cfg, err := LoadMergedConfig(defaultConfigPath())
|
||||
appCfg, err := LoadAppConfig(defaultConfigPath())
|
||||
if err != nil {
|
||||
return fmt.Errorf("load app config: %w", err)
|
||||
}
|
||||
if cfg == nil || cfg.Llm.URL == "" || cfg.Llm.AuthToken == "" {
|
||||
return fmt.Errorf("llm.url and llm.auth_token are required in $HOME/.open-code-review/config.json, or set OCR_LLM_URL and OCR_LLM_TOKEN environment variables")
|
||||
if appCfg != nil {
|
||||
tpl.ApplyLanguage(appCfg.Language)
|
||||
}
|
||||
model := cfg.Llm.Model
|
||||
tpl.ApplyLanguage(cfg.Language)
|
||||
llmClient := llm.NewClient(llm.ClientConfig{
|
||||
URL: cfg.Llm.URL,
|
||||
APIKey: cfg.Llm.AuthToken,
|
||||
Model: model,
|
||||
})
|
||||
|
||||
ep, err := llm.ResolveEndpoint(defaultConfigPath())
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve LLM endpoint: %w", err)
|
||||
}
|
||||
|
||||
llmClient := llm.NewLLMClient(ep)
|
||||
model := ep.Model
|
||||
|
||||
collector := tool.NewCommentCollector()
|
||||
mode := tool.ParseReviewMode(opts.from, opts.to, opts.commit)
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ type Args struct {
|
|||
SystemRule *rules.SystemRule
|
||||
|
||||
// LLM client for model inference.
|
||||
LLMClient *llm.Client
|
||||
LLMClient llm.LLMClient
|
||||
|
||||
// Tool registry mapping tool aliases to implementations.
|
||||
Tools tool.Registry
|
||||
|
|
@ -434,14 +434,16 @@ func (a *Agent) executeSubtask(ctx context.Context, d model.Diff) error {
|
|||
}
|
||||
|
||||
tokenCount := countMessagesTokens(messages)
|
||||
if tokenCount > a.args.Template.TokenWarningThreshold {
|
||||
msg := fmt.Sprintf("prompt tokens (%d) exceed threshold (%d)", tokenCount, a.args.Template.TokenWarningThreshold)
|
||||
maxAllowed := a.args.Template.MaxTokens
|
||||
tokenLimit := maxAllowed * 4 / 5 // 80% of MaxTokens
|
||||
if tokenCount > tokenLimit {
|
||||
msg := fmt.Sprintf("prompt tokens (%d) exceed %d%% of max_tokens(%d)", tokenCount, 80, maxAllowed)
|
||||
fmt.Fprintf(stdout.Writer(), "[ocr] WARNING: %s for %s\n", msg, newPath)
|
||||
a.recordWarning("token_threshold_exceeded", newPath, msg)
|
||||
telemetry.Event(ctx, "token.threshold.exceeded",
|
||||
telemetry.AnyToAttr("file.path", newPath),
|
||||
telemetry.AnyToAttr("tokens", tokenCount),
|
||||
telemetry.AnyToAttr("threshold", a.args.Template.TokenWarningThreshold))
|
||||
telemetry.AnyToAttr("max_tokens", maxAllowed))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -484,23 +486,20 @@ func (a *Agent) resolveSystemRule(path string) string {
|
|||
return a.args.SystemRule.Resolve(path)
|
||||
}
|
||||
|
||||
// filterLargeDiffs drops diffs whose diff content alone consumes more than 80% of the token threshold.
|
||||
// This prevents obviously oversized files from triggering API errors in the plan phase.
|
||||
// filterLargeDiffs drops diffs whose diff content alone consumes more than 80% of MaxTokens.
|
||||
func (a *Agent) filterLargeDiffs(diffs []model.Diff) []model.Diff {
|
||||
threshold := a.args.Template.TokenWarningThreshold
|
||||
if threshold <= 0 {
|
||||
limit := a.args.Template.MaxTokens * 4 / 5
|
||||
if limit <= 0 {
|
||||
return diffs
|
||||
}
|
||||
|
||||
limit := float64(threshold) * 0.8
|
||||
var kept []model.Diff
|
||||
skipped := 0
|
||||
|
||||
for _, d := range diffs {
|
||||
tokens := llm.CountTokens(d.Diff)
|
||||
if float64(tokens) > limit {
|
||||
fmt.Fprintf(stdout.Writer(), "[ocr] Skipping %s (~%d tokens exceeds 80%% of threshold %d)\n",
|
||||
d.NewPath, tokens, threshold)
|
||||
if tokens > limit {
|
||||
fmt.Fprintf(stdout.Writer(), "[ocr] Skipping %s (~%d tokens exceeds 80%% of max_tokens(%d))\n",
|
||||
d.NewPath, tokens, a.args.Template.MaxTokens)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
|
@ -508,7 +507,7 @@ func (a *Agent) filterLargeDiffs(diffs []model.Diff) []model.Diff {
|
|||
}
|
||||
|
||||
if skipped > 0 {
|
||||
fmt.Fprintf(stdout.Writer(), "[ocr] Pre-filtered %d file(s) exceeding 80%% token threshold\n", skipped)
|
||||
fmt.Fprintf(stdout.Writer(), "[ocr] Pre-filtered %d file(s) exceeding 80%% of max_tokens\n", skipped)
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
|
@ -571,7 +570,11 @@ func (a *Agent) executePlanPhase(ctx context.Context, newPath, rawDiff, changeFi
|
|||
rec := fs.AppendTaskRecord(session.PlanTask, messages)
|
||||
startTime := time.Now()
|
||||
|
||||
resp, err := a.args.LLMClient.GeneralRequestWithCtx(ctx, messages, a.args.Model, nil)
|
||||
resp, err := a.args.LLMClient.CompletionsWithCtx(ctx, llm.ChatRequest{
|
||||
Model: a.args.Model,
|
||||
Messages: messages,
|
||||
MaxTokens: a.args.Template.MaxTokens,
|
||||
})
|
||||
if err != nil {
|
||||
rec.SetError(err, time.Since(startTime))
|
||||
return "", fmt.Errorf("plan request: %w", err)
|
||||
|
|
@ -644,9 +647,10 @@ func (a *Agent) performLlmCodeReview(ctx context.Context, messages []llm.Message
|
|||
startTime := time.Now()
|
||||
|
||||
resp, err := a.args.LLMClient.Completions(llm.ChatRequest{
|
||||
Model: a.args.Model,
|
||||
Messages: messages,
|
||||
Tools: a.args.MainToolDefs,
|
||||
Model: a.args.Model,
|
||||
Messages: messages,
|
||||
Tools: a.args.MainToolDefs,
|
||||
MaxTokens: a.args.Template.MaxTokens,
|
||||
})
|
||||
duration := time.Since(startTime)
|
||||
if err != nil {
|
||||
|
|
@ -809,9 +813,10 @@ func (a *Agent) collectPendingComments() []model.LlmComment {
|
|||
|
||||
// addNextMessage adds assistant + tool response messages to the conversation history.
|
||||
func (a *Agent) addNextMessage(assistantContent string, toolCalls []llm.ToolCall, results []tool.ToolCallResult, messages *[]llm.Message, filePath string) bool {
|
||||
// Check if context compression is needed
|
||||
// Check if context compression is needed (80% of MaxTokens)
|
||||
tokenCount := countMessagesTokens(*messages)
|
||||
if tokenCount > a.args.Template.TokenWarningThreshold {
|
||||
limit := a.args.Template.MaxTokens * 4 / 5
|
||||
if tokenCount > limit {
|
||||
*messages = a.compressAndRecord(*messages, filePath)
|
||||
}
|
||||
|
||||
|
|
@ -827,7 +832,7 @@ func (a *Agent) addNextMessage(assistantContent string, toolCalls []llm.ToolCall
|
|||
*messages = append(*messages, llm.NewToolResultMessage(r.ToolCallID, r.Result))
|
||||
}
|
||||
|
||||
return countMessagesTokens(*messages) < a.args.Template.TokenWarningThreshold
|
||||
return countMessagesTokens(*messages) < limit
|
||||
}
|
||||
|
||||
func countMessagesTokens(msgs []llm.Message) int {
|
||||
|
|
@ -874,7 +879,11 @@ func (a *Agent) compressAndRecord(msgs []llm.Message, filePath string) []llm.Mes
|
|||
}
|
||||
|
||||
startTime := time.Now()
|
||||
resp, err := a.args.LLMClient.GeneralRequest(compressionMsgs, a.args.Model, nil)
|
||||
resp, err := a.args.LLMClient.Completions(llm.ChatRequest{
|
||||
Model: a.args.Model,
|
||||
Messages: compressionMsgs,
|
||||
MaxTokens: a.args.Template.MaxTokens,
|
||||
})
|
||||
duration := time.Since(startTime)
|
||||
|
||||
fs := a.session.GetOrCreateFileSession(filePath)
|
||||
|
|
|
|||
|
|
@ -38,9 +38,9 @@
|
|||
],
|
||||
"timeout": 120
|
||||
},
|
||||
"TOKEN_WARNING_THRESHOLD": 50000,
|
||||
"TOOL_REQUEST_WAIT_TIME_MS": 10000,
|
||||
"MAX_TOOL_REQUEST_TIMES": 20,
|
||||
"MAX_SUBTASK_EXECUTION_TIME_MINUTES": 5,
|
||||
"PLAN_MODE_LINE_THRESHOLD": 50
|
||||
"PLAN_MODE_LINE_THRESHOLD": 50,
|
||||
"MAX_TOKENS": 58888
|
||||
}
|
||||
|
|
@ -13,7 +13,7 @@ type Template struct {
|
|||
MainTask LlmConversation `json:"MAIN_TASK"`
|
||||
PlanTask *LlmConversation `json:"PLAN_TASK,omitempty"`
|
||||
MemoryCompressionTask LlmConversation `json:"MEMORY_COMPRESSION_TASK"`
|
||||
TokenWarningThreshold int `json:"TOKEN_WARNING_THRESHOLD"`
|
||||
MaxTokens int `json:"MAX_TOKENS"`
|
||||
ToolRequestWaitTimeMs int `json:"TOOL_REQUEST_WAIT_TIME_MS"`
|
||||
MaxToolRequestTimes int `json:"MAX_TOOL_REQUEST_TIMES"`
|
||||
MaxSubtaskExecMinutes int `json:"MAX_SUBTASK_EXECUTION_TIME_MINUTES"`
|
||||
|
|
@ -60,8 +60,8 @@ func (t *Template) ApplyLanguage(lang string) {
|
|||
applyLanguage(&t.MemoryCompressionTask, instruction)
|
||||
}
|
||||
func (t *Template) Validate() error {
|
||||
if t.TokenWarningThreshold <= 0 {
|
||||
return fmt.Errorf("token_warning_threshold must be positive")
|
||||
if t.MaxTokens <= 0 {
|
||||
return fmt.Errorf("max_tokens must be positive")
|
||||
}
|
||||
if t.MaxToolRequestTimes <= 0 {
|
||||
return fmt.Errorf("max_tool_request_times must be positive")
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
// Package llm provides an OpenAI-compatible LLM client interface.
|
||||
// OpenCodeReview supports any service that implements the OpenAI Chat Completion API schema,
|
||||
// including OpenAI, Claude (via Anthropic's OpenAI-compatible endpoint), local models, etc.
|
||||
// Package llm provides LLM client interfaces supporting multiple protocols.
|
||||
// Supported protocols: Anthropic Messages API, OpenAI Chat Completions API.
|
||||
package llm
|
||||
|
||||
import (
|
||||
|
|
@ -23,6 +22,15 @@ import (
|
|||
|
||||
const maxRetries = 10 // Maximum number of retry attempts with exponential backoff.
|
||||
|
||||
// LLMClient is the unified interface for all LLM protocol implementations.
|
||||
type LLMClient interface {
|
||||
Completions(req ChatRequest) (*ChatResponse, error)
|
||||
CompletionsWithCtx(ctx context.Context, req ChatRequest) (*ChatResponse, error)
|
||||
StreamCompletion(req ChatRequest, cb func(chunk []byte) error) error
|
||||
}
|
||||
|
||||
// --- Shared data types ---
|
||||
|
||||
// Message represents a single message in a chat conversation.
|
||||
// Content can be either plain string (for system/user/assistant/tool messages)
|
||||
// or an array of content blocks (used by Claude for multi-part content).
|
||||
|
|
@ -30,17 +38,17 @@ const maxRetries = 10 // Maximum number of retry attempts with exponential backo
|
|||
type Message struct {
|
||||
Role string `json:"role"`
|
||||
Content any `json:"content"` // string or []ContentBlock
|
||||
ToolCallID string `json:"tool_call_id,omitempty"` // OpenAI tool result identifier
|
||||
ToolCallID string `json:"tool_call_id,omitempty"` // OpenAI tool call identifier
|
||||
ToolCalls []ToolCall `json:"tool_calls,omitempty"` // assistant tool invocations
|
||||
}
|
||||
|
||||
// ContentBlock represents a single block within a multi-part message content.
|
||||
// Used by Claude's Messages API for tool results and multimodal content.
|
||||
type ContentBlock struct {
|
||||
Type string `json:"type"` // "text" or "tool_result"
|
||||
Text string `json:"text,omitempty"` // for type="text"
|
||||
ToolUseID string `json:"tool_use_id,omitempty"` // for type="tool_result"
|
||||
Content []ContentBlock `json:"content,omitempty"` // nested text blocks inside tool_result
|
||||
Type string `json:"type"` // "text" or "tool_result"
|
||||
Text string `json:"text,omitempty"` // for type="text"
|
||||
ToolUseID string `json:"tool_use_id,omitempty"` // for type="tool_result"
|
||||
Content []ContentBlock `json:"content,omitempty"` // nested text blocks inside tool_result
|
||||
}
|
||||
|
||||
// NewTextMessage creates a message with simple string content.
|
||||
|
|
@ -96,7 +104,6 @@ func extractBlockText(block ContentBlock) string {
|
|||
return sb.String()
|
||||
}
|
||||
|
||||
|
||||
// Choice holds a single choice from the response.
|
||||
type Choice struct {
|
||||
Message ResponseMessage `json:"message"`
|
||||
|
|
@ -169,24 +176,108 @@ type FunctionDef struct {
|
|||
|
||||
// ClientConfig holds configuration for connecting to an LLM service.
|
||||
type ClientConfig struct {
|
||||
URL string // Full API endpoint URL (e.g., "https://api.openai.com/v1/chat/completions")
|
||||
APIKey string // Bearer token
|
||||
URL string // Full API endpoint URL
|
||||
APIKey string // Bearer token / API key
|
||||
Model string // Default model override
|
||||
Timeout time.Duration // Request timeout
|
||||
}
|
||||
|
||||
// Client sends requests to an OpenAI-compatible chat completion API.
|
||||
type Client struct {
|
||||
// --- Factory ---
|
||||
|
||||
// NewLLMClient creates the appropriate client based on the resolved endpoint protocol.
|
||||
// protocol: "anthropic" -> AnthropicClient, anything else -> OpenAIClient.
|
||||
func NewLLMClient(ep ResolvedEndpoint) LLMClient {
|
||||
cfg := ClientConfig{
|
||||
URL: ep.URL,
|
||||
APIKey: ep.Token,
|
||||
Model: ep.Model,
|
||||
}
|
||||
if ep.Protocol == "anthropic" {
|
||||
return NewAnthropicClient(cfg)
|
||||
}
|
||||
return NewOpenAIClient(cfg)
|
||||
}
|
||||
|
||||
// --- Token counting with tiktoken ---
|
||||
|
||||
// modelTokenizerCache caches initialized tiktoken encoders keyed by encoding name.
|
||||
type modelTokenizerCache struct {
|
||||
mu sync.RWMutex
|
||||
cache map[string]*tiktoken.Tiktoken
|
||||
}
|
||||
|
||||
func newModelTokenizerCache() *modelTokenizerCache {
|
||||
return &modelTokenizerCache{cache: make(map[string]*tiktoken.Tiktoken)}
|
||||
}
|
||||
|
||||
func (c *modelTokenizerCache) getOrLoad(encName string) (*tiktoken.Tiktoken, error) {
|
||||
c.mu.RLock()
|
||||
if tke, ok := c.cache[encName]; ok {
|
||||
c.mu.RUnlock()
|
||||
return tke, nil
|
||||
}
|
||||
c.mu.RUnlock()
|
||||
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if tke, ok := c.cache[encName]; ok {
|
||||
return tke, nil
|
||||
}
|
||||
enc, err := tiktoken.GetEncoding(encName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get tiktoken encoding %q: %w", encName, err)
|
||||
}
|
||||
c.cache[encName] = enc
|
||||
return enc, nil
|
||||
}
|
||||
|
||||
var defaultTokenizer = newModelTokenizerCache()
|
||||
|
||||
func countTokensWithEncoding(text string, encName string) int {
|
||||
tke, err := defaultTokenizer.getOrLoad(encName)
|
||||
if err != nil {
|
||||
return len([]byte(text)) / 4
|
||||
}
|
||||
return len(tke.Encode(text, nil, nil))
|
||||
}
|
||||
|
||||
func CountTokens(text string) int {
|
||||
return CountTokensForModel(text, "")
|
||||
}
|
||||
|
||||
func CountTokensForModel(text string, modelName string) int {
|
||||
if text == "" {
|
||||
return 0
|
||||
}
|
||||
encName := encodingForModel(modelName)
|
||||
return countTokensWithEncoding(text, encName)
|
||||
}
|
||||
|
||||
func encodingForModel(modelName string) string {
|
||||
lower := strings.ToLower(modelName)
|
||||
switch {
|
||||
case strings.Contains(lower, "o1") || strings.Contains(lower, "o3") || strings.Contains(lower, "o4"):
|
||||
return "o200k_base"
|
||||
default:
|
||||
return "cl100k_base"
|
||||
}
|
||||
}
|
||||
|
||||
// --- OpenAIClient ---
|
||||
|
||||
// OpenAIClient sends requests to an OpenAI-compatible chat completion API.
|
||||
type OpenAIClient struct {
|
||||
cfg ClientConfig
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewClient creates a new LLM client from configuration.
|
||||
func NewClient(cfg ClientConfig) *Client {
|
||||
// NewOpenAIClient creates a new OpenAI-compatible LLM client.
|
||||
func NewOpenAIClient(cfg ClientConfig) *OpenAIClient {
|
||||
if cfg.Timeout <= 0 {
|
||||
cfg.Timeout = 5 * time.Minute
|
||||
}
|
||||
return &Client{
|
||||
return &OpenAIClient{
|
||||
cfg: cfg,
|
||||
client: &http.Client{
|
||||
Timeout: cfg.Timeout,
|
||||
|
|
@ -194,6 +285,11 @@ func NewClient(cfg ClientConfig) *Client {
|
|||
}
|
||||
}
|
||||
|
||||
// NewClient is kept as an alias for backward compatibility during transition.
|
||||
func NewClient(cfg ClientConfig) *OpenAIClient {
|
||||
return NewOpenAIClient(cfg)
|
||||
}
|
||||
|
||||
// ChatRequest represents the payload for a chat completion call.
|
||||
type ChatRequest struct {
|
||||
Model string `json:"model"`
|
||||
|
|
@ -201,15 +297,16 @@ type ChatRequest struct {
|
|||
Tools []ToolDef `json:"tools,omitempty"`
|
||||
Stream bool `json:"stream,omitempty"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
}
|
||||
|
||||
// Completions sends a chat completion request and returns the parsed response.
|
||||
func (c *Client) Completions(req ChatRequest) (*ChatResponse, error) {
|
||||
func (c *OpenAIClient) Completions(req ChatRequest) (*ChatResponse, error) {
|
||||
return c.CompletionsWithCtx(context.Background(), req)
|
||||
}
|
||||
|
||||
// CompletionsWithCtx sends a chat completion request with context support for cancellation and timeout.
|
||||
func (c *Client) CompletionsWithCtx(ctx context.Context, req ChatRequest) (*ChatResponse, error) {
|
||||
func (c *OpenAIClient) CompletionsWithCtx(ctx context.Context, req ChatRequest) (*ChatResponse, error) {
|
||||
model := req.Model
|
||||
if model == "" {
|
||||
model = c.cfg.Model
|
||||
|
|
@ -227,13 +324,13 @@ func (c *Client) CompletionsWithCtx(ctx context.Context, req ChatRequest) (*Chat
|
|||
return result, err
|
||||
}
|
||||
|
||||
// GeneralRequest sends a simple chat request without or with optional tool calls (for plan phase, compression, etc.).
|
||||
func (c *Client) GeneralRequest(messages []Message, model string, tools []ToolDef) (*ChatResponse, error) {
|
||||
// GeneralRequest sends a simple chat request without or with optional tool calls.
|
||||
func (c *OpenAIClient) GeneralRequest(messages []Message, model string, tools []ToolDef) (*ChatResponse, error) {
|
||||
return c.GeneralRequestWithCtx(context.Background(), messages, model, tools)
|
||||
}
|
||||
|
||||
// GeneralRequestWithCtx sends a simple chat request with context support.
|
||||
func (c *Client) GeneralRequestWithCtx(ctx context.Context, messages []Message, model string, tools []ToolDef) (*ChatResponse, error) {
|
||||
func (c *OpenAIClient) GeneralRequestWithCtx(ctx context.Context, messages []Message, model string, tools []ToolDef) (*ChatResponse, error) {
|
||||
return c.CompletionsWithCtx(ctx, ChatRequest{
|
||||
Model: model,
|
||||
Messages: messages,
|
||||
|
|
@ -241,85 +338,8 @@ func (c *Client) GeneralRequestWithCtx(ctx context.Context, messages []Message,
|
|||
})
|
||||
}
|
||||
|
||||
// --- Token counting with tiktoken ---
|
||||
|
||||
// modelTokenizerCache caches initialized tiktoken encoders keyed by encoding name.
|
||||
type modelTokenizerCache struct {
|
||||
mu sync.RWMutex
|
||||
cache map[string]*tiktoken.Tiktoken
|
||||
}
|
||||
|
||||
func newModelTokenizerCache() *modelTokenizerCache {
|
||||
return &modelTokenizerCache{cache: make(map[string]*tiktoken.Tiktoken)}
|
||||
}
|
||||
|
||||
func (c *modelTokenizerCache) getOrLoad(encName string) (*tiktoken.Tiktoken, error) {
|
||||
// Fast path: read-only check
|
||||
c.mu.RLock()
|
||||
if tke, ok := c.cache[encName]; ok {
|
||||
c.mu.RUnlock()
|
||||
return tke, nil
|
||||
}
|
||||
c.mu.RUnlock()
|
||||
|
||||
// Slow path: load under write lock
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if tke, ok := c.cache[encName]; ok {
|
||||
return tke, nil // another goroutine loaded it already
|
||||
}
|
||||
enc, err := tiktoken.GetEncoding(encName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get tiktoken encoding %q: %w", encName, err)
|
||||
}
|
||||
c.cache[encName] = enc
|
||||
return enc, nil
|
||||
}
|
||||
|
||||
var defaultTokenizer = newModelTokenizerCache()
|
||||
|
||||
// countTokensWithEncoding counts tokens using the specified tiktoken encoding.
|
||||
// It lazily caches the tokenizer under the hood. If loading fails, falls back
|
||||
// to byte estimation (len(text)/4).
|
||||
func countTokensWithEncoding(text string, encName string) int {
|
||||
tke, err := defaultTokenizer.getOrLoad(encName)
|
||||
if err != nil {
|
||||
// Encoding unavailable — fall back to byte estimation.
|
||||
return len([]byte(text)) / 4
|
||||
}
|
||||
return len(tke.Encode(text, nil, nil))
|
||||
}
|
||||
|
||||
// CountTokens returns the number of tokens in text using the default tiktoken
|
||||
// encoding (cl100k_base). For model-specific counting, use CountTokensForModel.
|
||||
func CountTokens(text string) int {
|
||||
return CountTokensForModel(text, "")
|
||||
}
|
||||
|
||||
// CountTokensForModel returns the number of tokens in text using a tiktoken
|
||||
// encoding selected based on the given model name. Falls back to cl100k_base.
|
||||
func CountTokensForModel(text string, modelName string) int {
|
||||
if text == "" {
|
||||
return 0
|
||||
}
|
||||
encName := encodingForModel(modelName)
|
||||
return countTokensWithEncoding(text, encName)
|
||||
}
|
||||
|
||||
// encodingForModel selects the tiktoken encoding best suited for the given model name.
|
||||
func encodingForModel(modelName string) string {
|
||||
lower := strings.ToLower(modelName)
|
||||
switch {
|
||||
case strings.Contains(lower, "o1") || strings.Contains(lower, "o3") || strings.Contains(lower, "o4"):
|
||||
return "o200k_base"
|
||||
default:
|
||||
return "cl100k_base"
|
||||
}
|
||||
}
|
||||
|
||||
// StreamCompletion initiates a streaming chat completion. The callback is invoked per chunk.
|
||||
func (c *Client) StreamCompletion(req ChatRequest, cb func(chunk []byte) error) error {
|
||||
func (c *OpenAIClient) StreamCompletion(req ChatRequest, cb func(chunk []byte) error) error {
|
||||
req.Stream = true
|
||||
|
||||
model := req.Model
|
||||
|
|
@ -375,23 +395,446 @@ func (c *Client) StreamCompletion(req ChatRequest, cb func(chunk []byte) error)
|
|||
})
|
||||
}
|
||||
|
||||
// doRequest builds and sends a non-streaming completion request, returning the parsed response.
|
||||
func (c *OpenAIClient) doRequest(model string, req ChatRequest) (*ChatResponse, error) {
|
||||
return c.doRequestCtx(context.Background(), model, req)
|
||||
}
|
||||
|
||||
// doRequestCtx builds and sends a non-streaming completion request with context support.
|
||||
func (c *OpenAIClient) doRequestCtx(ctx context.Context, model string, req ChatRequest) (*ChatResponse, error) {
|
||||
if model == "" {
|
||||
model = c.cfg.Model
|
||||
}
|
||||
req.Model = model
|
||||
payload, _ := json.Marshal(req)
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.cfg.URL, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
httpReq.Header.Set("Authorization", "Bearer "+c.cfg.APIKey)
|
||||
|
||||
resp, err := c.client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response body: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
detail := extractErrorMessage(bodyBytes)
|
||||
return nil, fmt.Errorf("API error %d: %s", resp.StatusCode, detail)
|
||||
}
|
||||
|
||||
var apiResp struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
Choices []Choice `json:"choices"`
|
||||
}
|
||||
if err := json.Unmarshal(bodyBytes, &apiResp); err != nil {
|
||||
return nil, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
|
||||
return &ChatResponse{
|
||||
ID: apiResp.ID,
|
||||
Model: apiResp.Model,
|
||||
Choices: apiResp.Choices,
|
||||
Headers: resp.Header,
|
||||
Usage: resolveUsage(bodyBytes),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// --- AnthropicClient ---
|
||||
|
||||
const anthropicVersion = "2023-06-01"
|
||||
|
||||
// AnthropicClient implements the Anthropic Messages API.
|
||||
type AnthropicClient struct {
|
||||
cfg ClientConfig
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewAnthropicClient creates a new Anthropic Messages API client.
|
||||
func NewAnthropicClient(cfg ClientConfig) *AnthropicClient {
|
||||
if cfg.Timeout <= 0 {
|
||||
cfg.Timeout = 5 * time.Minute
|
||||
}
|
||||
if !strings.HasSuffix(cfg.URL, "/v1/messages") && !strings.HasSuffix(cfg.URL, "/v1/messages/") {
|
||||
baseURL := strings.TrimRight(cfg.URL, "/")
|
||||
if !strings.HasSuffix(baseURL, "/v1/messages") {
|
||||
cfg.URL = baseURL + "/v1/messages"
|
||||
}
|
||||
}
|
||||
return &AnthropicClient{
|
||||
cfg: cfg,
|
||||
client: &http.Client{
|
||||
Timeout: cfg.Timeout,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Completions sends a chat completion request and returns the parsed response.
|
||||
func (c *AnthropicClient) Completions(req ChatRequest) (*ChatResponse, error) {
|
||||
return c.CompletionsWithCtx(context.Background(), req)
|
||||
}
|
||||
|
||||
// CompletionsWithCtx sends a chat completion request with context support.
|
||||
func (c *AnthropicClient) CompletionsWithCtx(ctx context.Context, req ChatRequest) (*ChatResponse, error) {
|
||||
model := req.Model
|
||||
if model == "" {
|
||||
model = c.cfg.Model
|
||||
}
|
||||
|
||||
var result *ChatResponse
|
||||
err := c.withRetryCtx(ctx, func() error {
|
||||
resp, err := c.doRequestCtx(ctx, model, req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result = resp
|
||||
return nil
|
||||
})
|
||||
return result, err
|
||||
}
|
||||
|
||||
// StreamCompletion initiates a streaming chat completion using SSE. The callback
|
||||
// is invoked per chunk with raw JSON data stripped of the "data: " prefix.
|
||||
func (c *AnthropicClient) StreamCompletion(req ChatRequest, cb func(chunk []byte) error) error {
|
||||
req.Stream = true
|
||||
|
||||
model := req.Model
|
||||
if model == "" {
|
||||
model = c.cfg.Model
|
||||
}
|
||||
|
||||
return c.withRetry(func() error {
|
||||
body := c.buildRequestBody(model, req)
|
||||
body.Stream = true
|
||||
|
||||
payload, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal request body: %w", err)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequest(http.MethodPost, c.cfg.URL, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
httpReq.Header.Set("x-api-key", c.cfg.APIKey)
|
||||
httpReq.Header.Set("anthropic-version", anthropicVersion)
|
||||
|
||||
resp, err := c.client.Do(httpReq)
|
||||
if err != nil {
|
||||
return fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if isRetryableStatus(resp.StatusCode) {
|
||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("API error %d: %s", resp.StatusCode, string(bodyBytes))
|
||||
}
|
||||
if resp.StatusCode >= 400 {
|
||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("API error %d: %s (non-retryable)", resp.StatusCode, string(bodyBytes))
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(resp.Body)
|
||||
var eventType string
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
// Capture event type line: "event: message_delta"
|
||||
if strings.HasPrefix(line, "event: ") {
|
||||
eventType = strings.TrimPrefix(line, "event: ")
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip empty lines and non-data lines
|
||||
if !strings.HasPrefix(line, "data: ") {
|
||||
continue
|
||||
}
|
||||
|
||||
data := strings.TrimPrefix(line, "data: ")
|
||||
if data == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// message_stop signals end of stream
|
||||
if eventType == "message_stop" {
|
||||
break
|
||||
}
|
||||
|
||||
if err := cb([]byte(data)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return scanner.Err()
|
||||
})
|
||||
}
|
||||
|
||||
// anthropicRequest is the request body for Anthropic Messages API.
|
||||
type anthropicRequest struct {
|
||||
Model string `json:"model"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
System string `json:"system,omitempty"`
|
||||
Messages []anthroMessage `json:"messages"`
|
||||
Tools []anthroTool `json:"tools,omitempty"`
|
||||
Stream bool `json:"stream,omitempty"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
}
|
||||
|
||||
type anthroMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content any `json:"content"` // string or []interface{}
|
||||
}
|
||||
|
||||
// anthropicToolUseBlock represents a tool_use content block in Anthropic's Messages API.
|
||||
type anthropicToolUseBlock struct {
|
||||
Type string `json:"type"` // "tool_use"
|
||||
ID string `json:"id"` // tool use ID
|
||||
Name string `json:"name"` // function name
|
||||
Input map[string]any `json:"input"` // function arguments (parsed as object)
|
||||
}
|
||||
|
||||
type anthroTool struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
InputSchema map[string]any `json:"input_schema"`
|
||||
}
|
||||
|
||||
// doRequestCtx builds and sends an Anthropic Messages API request.
|
||||
func (c *AnthropicClient) doRequestCtx(ctx context.Context, model string, req ChatRequest) (*ChatResponse, error) {
|
||||
if model == "" {
|
||||
model = c.cfg.Model
|
||||
}
|
||||
|
||||
body := c.buildRequestBody(model, req)
|
||||
payload, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request body: %w", err)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.cfg.URL, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
httpReq.Header.Set("x-api-key", c.cfg.APIKey)
|
||||
httpReq.Header.Set("anthropic-version", anthropicVersion)
|
||||
|
||||
resp, err := c.client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response body: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
detail := extractErrorMessage(bodyBytes)
|
||||
return nil, fmt.Errorf("API error %d: %s", resp.StatusCode, detail)
|
||||
}
|
||||
|
||||
chatResp, err := c.parseResponse(bodyBytes, resp.Header)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
return chatResp, nil
|
||||
}
|
||||
|
||||
// buildRequestBody converts the shared ChatRequest into Anthropic format.
|
||||
func (c *AnthropicClient) buildRequestBody(model string, req ChatRequest) anthropicRequest {
|
||||
messages := make([]anthroMessage, 0, len(req.Messages))
|
||||
var systemMsg string
|
||||
|
||||
var pendingToolResults []Message // collect consecutive tool messages
|
||||
|
||||
flushToolResults := func() {
|
||||
if len(pendingToolResults) == 0 {
|
||||
return
|
||||
}
|
||||
// Merge all pending tool results into a single user message
|
||||
var blocks []interface{}
|
||||
for _, tr := range pendingToolResults {
|
||||
blocks = append(blocks, ContentBlock{
|
||||
Type: "tool_result",
|
||||
ToolUseID: tr.ToolCallID,
|
||||
Content: []ContentBlock{{
|
||||
Type: "text",
|
||||
Text: fmt.Sprintf("%v", tr.Content),
|
||||
}},
|
||||
})
|
||||
}
|
||||
messages = append(messages, anthroMessage{Role: "user", Content: blocks})
|
||||
pendingToolResults = nil
|
||||
}
|
||||
|
||||
for _, msg := range req.Messages {
|
||||
switch msg.Role {
|
||||
case "system":
|
||||
if s, ok := msg.Content.(string); ok {
|
||||
systemMsg = s
|
||||
}
|
||||
flushToolResults()
|
||||
case "tool":
|
||||
pendingToolResults = append(pendingToolResults, msg)
|
||||
case "assistant":
|
||||
flushToolResults()
|
||||
// Build Anthropic content blocks from text + tool calls
|
||||
var blocks []interface{}
|
||||
if s, ok := msg.Content.(string); ok && s != "" {
|
||||
blocks = append(blocks, ContentBlock{Type: "text", Text: s})
|
||||
}
|
||||
for _, tc := range msg.ToolCalls {
|
||||
argsMap := map[string]any{}
|
||||
if tc.Function.Arguments != "" {
|
||||
if err := json.Unmarshal([]byte(tc.Function.Arguments), &argsMap); err != nil {
|
||||
fmt.Fprintf(stdout.Writer(), "[llm] WARNING: failed to parse tool call arguments JSON for %q: %v\n", tc.ID, err)
|
||||
}
|
||||
}
|
||||
blocks = append(blocks, anthropicToolUseBlock{
|
||||
Type: "tool_use",
|
||||
ID: tc.ID,
|
||||
Name: tc.Function.Name,
|
||||
Input: argsMap,
|
||||
})
|
||||
}
|
||||
if len(blocks) > 0 {
|
||||
messages = append(messages, anthroMessage{Role: "assistant", Content: blocks})
|
||||
} else {
|
||||
s, _ := msg.Content.(string)
|
||||
messages = append(messages, anthroMessage{Role: "assistant", Content: s})
|
||||
}
|
||||
default:
|
||||
// user or other roles: flush tool results first
|
||||
flushToolResults()
|
||||
content := msg.Content
|
||||
if blkArr, ok := content.([]ContentBlock); ok {
|
||||
converted := make([]ContentBlock, len(blkArr))
|
||||
for i, b := range blkArr {
|
||||
converted[i] = ContentBlock{
|
||||
Type: b.Type,
|
||||
Text: b.Text,
|
||||
ToolUseID: b.ToolUseID,
|
||||
Content: b.Content,
|
||||
}
|
||||
}
|
||||
content = converted
|
||||
}
|
||||
messages = append(messages, anthroMessage{Role: msg.Role, Content: content})
|
||||
}
|
||||
}
|
||||
flushToolResults() // flush any remaining tool results at the end
|
||||
|
||||
tools := make([]anthroTool, 0, len(req.Tools))
|
||||
for _, t := range req.Tools {
|
||||
tools = append(tools, anthroTool{
|
||||
Name: t.Function.Name,
|
||||
Description: t.Function.Description,
|
||||
InputSchema: t.Function.Parameters,
|
||||
})
|
||||
}
|
||||
|
||||
maxTokens := req.MaxTokens
|
||||
if maxTokens <= 0 {
|
||||
maxTokens = 8192 // Anthropic default
|
||||
}
|
||||
|
||||
return anthropicRequest{
|
||||
Model: model,
|
||||
MaxTokens: maxTokens,
|
||||
System: systemMsg,
|
||||
Messages: messages,
|
||||
Tools: tools,
|
||||
Stream: false,
|
||||
Temperature: req.Temperature,
|
||||
}
|
||||
}
|
||||
|
||||
// parseResponse converts Anthropic JSON response into ChatResponse.
|
||||
func (c *AnthropicClient) parseResponse(body []byte, headers http.Header) (*ChatResponse, error) {
|
||||
type contentBlockResp struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text,omitempty"`
|
||||
ID string `json:"id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Input any `json:"input,omitempty"`
|
||||
}
|
||||
|
||||
var resp struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
Type string `json:"type"`
|
||||
Role string `json:"role"`
|
||||
Content []contentBlockResp `json:"content"`
|
||||
Usage *UsageInfo `json:"usage"`
|
||||
StopReason string `json:"stop_reason,omitempty"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Build the response message from content blocks.
|
||||
var textParts []string
|
||||
var toolCalls []ToolCall
|
||||
|
||||
for _, block := range resp.Content {
|
||||
switch block.Type {
|
||||
case "text":
|
||||
textParts = append(textParts, block.Text)
|
||||
case "tool_use":
|
||||
argsJSON, _ := json.Marshal(block.Input)
|
||||
toolCalls = append(toolCalls, ToolCall{
|
||||
ID: block.ID,
|
||||
Type: "function",
|
||||
Function: FunctionCall{
|
||||
Name: block.Name,
|
||||
Arguments: string(argsJSON),
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var contentStr *string
|
||||
if len(textParts) > 0 {
|
||||
s := strings.Join(textParts, "\n")
|
||||
contentStr = &s
|
||||
}
|
||||
|
||||
finishReason := resp.StopReason
|
||||
if finishReason == "" {
|
||||
finishReason = "stop"
|
||||
}
|
||||
|
||||
return &ChatResponse{
|
||||
ID: resp.ID,
|
||||
Model: resp.Model,
|
||||
Choices: []Choice{{
|
||||
Message: ResponseMessage{
|
||||
Role: resp.Role,
|
||||
Content: contentStr,
|
||||
ToolCalls: toolCalls,
|
||||
},
|
||||
FinishReason: finishReason,
|
||||
}},
|
||||
Headers: headers,
|
||||
Usage: resp.Usage,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// --- Retry logic ---
|
||||
|
||||
// stripThinkTags removes reasoning wrapper tags from content.
|
||||
func stripThinkTags(s string) string {
|
||||
// Construct tag strings from individual bytes.
|
||||
openBytes := []byte{0x3c, 't', 'h', 'i', 'n', 'k', 0x3e}
|
||||
closeBytes := []byte{0x3c, 0x2f, 't', 'h', 'i', 'n', 'k', 0x3e}
|
||||
s = strings.ReplaceAll(s, string(openBytes), "")
|
||||
s = strings.ReplaceAll(s, string(closeBytes), "")
|
||||
return s
|
||||
}
|
||||
|
||||
func (c *Client) withRetry(fn func() error) error {
|
||||
return c.withRetryCtx(context.Background(), fn)
|
||||
}
|
||||
|
||||
func (c *Client) withRetryCtx(ctx context.Context, fn func() error) error {
|
||||
func retryWithCtx(ctx context.Context, fn func() error) error {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||
select {
|
||||
|
|
@ -416,6 +859,22 @@ func (c *Client) withRetryCtx(ctx context.Context, fn func() error) error {
|
|||
return fmt.Errorf("request failed after %d retries: %w", maxRetries, lastErr)
|
||||
}
|
||||
|
||||
func (c *OpenAIClient) withRetry(fn func() error) error {
|
||||
return retryWithCtx(context.Background(), fn)
|
||||
}
|
||||
|
||||
func (c *OpenAIClient) withRetryCtx(ctx context.Context, fn func() error) error {
|
||||
return retryWithCtx(ctx, fn)
|
||||
}
|
||||
|
||||
func (c *AnthropicClient) withRetry(fn func() error) error {
|
||||
return retryWithCtx(context.Background(), fn)
|
||||
}
|
||||
|
||||
func (c *AnthropicClient) withRetryCtx(ctx context.Context, fn func() error) error {
|
||||
return retryWithCtx(ctx, fn)
|
||||
}
|
||||
|
||||
// isRetryable determines whether an error is transient and worth retrying.
|
||||
func isRetryable(err error) bool {
|
||||
msg := err.Error()
|
||||
|
|
@ -465,58 +924,14 @@ func sleepWithBackoff(attempt int) {
|
|||
time.Sleep(delay)
|
||||
}
|
||||
|
||||
|
||||
// doRequest builds and sends a non-streaming completion request, returning the parsed response.
|
||||
func (c *Client) doRequest(model string, req ChatRequest) (*ChatResponse, error) {
|
||||
return c.doRequestCtx(context.Background(), model, req)
|
||||
}
|
||||
|
||||
// doRequestCtx builds and sends a non-streaming completion request with context support.
|
||||
func (c *Client) doRequestCtx(ctx context.Context, model string, req ChatRequest) (*ChatResponse, error) {
|
||||
if model == "" {
|
||||
model = c.cfg.Model
|
||||
}
|
||||
req.Model = model
|
||||
payload, _ := json.Marshal(req)
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.cfg.URL, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
httpReq.Header.Set("Authorization", "Bearer "+c.cfg.APIKey)
|
||||
|
||||
resp, err := c.client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response body: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
detail := extractErrorMessage(bodyBytes)
|
||||
return nil, fmt.Errorf("API error %d: %s", resp.StatusCode, detail)
|
||||
}
|
||||
|
||||
var apiResp struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
Choices []Choice `json:"choices"`
|
||||
}
|
||||
if err := json.Unmarshal(bodyBytes, &apiResp); err != nil {
|
||||
return nil, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
|
||||
return &ChatResponse{
|
||||
ID: apiResp.ID,
|
||||
Model: apiResp.Model,
|
||||
Choices: apiResp.Choices,
|
||||
Headers: resp.Header,
|
||||
Usage: resolveUsage(bodyBytes),
|
||||
}, nil
|
||||
// stripThinkTags removes reasoning wrapper tags from content.
|
||||
func stripThinkTags(s string) string {
|
||||
// Construct tag strings from individual bytes.
|
||||
openBytes := []byte{0x3c, 't', 'h', 'i', 'n', 'k', 0x3e}
|
||||
closeBytes := []byte{0x3c, 0x2f, 't', 'h', 'i', 'n', 'k', 0x3e}
|
||||
s = strings.ReplaceAll(s, string(openBytes), "")
|
||||
s = strings.ReplaceAll(s, string(closeBytes), "")
|
||||
return s
|
||||
}
|
||||
|
||||
// extractErrorMessage attempts to pull a human-readable error message from
|
||||
|
|
|
|||
227
internal/llm/resolver.go
Normal file
227
internal/llm/resolver.go
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
package llm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ResolvedEndpoint holds the resolved LLM endpoint configuration.
|
||||
type ResolvedEndpoint struct {
|
||||
URL string
|
||||
Token string
|
||||
Model string
|
||||
Protocol string // "anthropic" or "openai"
|
||||
}
|
||||
|
||||
// Environment variable names for OCR-specific configuration.
|
||||
const (
|
||||
envOCRLLMURL = "OCR_LLM_URL"
|
||||
envOCRLLMToken = "OCR_LLM_TOKEN"
|
||||
envOCRLLMModel = "OCR_LLM_MODEL"
|
||||
envOCRUseAnthropic = "OCR_USE_ANTHROPIC"
|
||||
)
|
||||
|
||||
// Environment variable names from Claude Code configuration.
|
||||
const (
|
||||
envCCBaseURL = "ANTHROPIC_BASE_URL"
|
||||
envCCToken = "ANTHROPIC_AUTH_TOKEN"
|
||||
envCCModel = "ANTHROPIC_MODEL"
|
||||
)
|
||||
|
||||
// ResolveEndpoint reads from 4 strategy sources in priority order.
|
||||
// Each strategy requires all three fields (URL, Token, Model) to be non-empty.
|
||||
// Returns the first valid strategy's result.
|
||||
func ResolveEndpoint(configPath string) (ResolvedEndpoint, error) {
|
||||
strategies := []struct {
|
||||
name string
|
||||
fn func() (ResolvedEndpoint, bool, error)
|
||||
}{
|
||||
{"OCR environment", tryOCREnv},
|
||||
{"OCR config file", func() (ResolvedEndpoint, bool, error) { return tryOCRConfig(configPath) }},
|
||||
{"Claude Code environment", tryCCEnv},
|
||||
{"Shell rc file", tryShellRC},
|
||||
}
|
||||
|
||||
for _, s := range strategies {
|
||||
ep, ok, err := s.fn()
|
||||
if err != nil {
|
||||
return ResolvedEndpoint{}, fmt.Errorf("resolve %s: %w", s.name, err)
|
||||
}
|
||||
if ok && ep.URL != "" && ep.Token != "" && ep.Model != "" {
|
||||
return ep, nil
|
||||
}
|
||||
}
|
||||
|
||||
return ResolvedEndpoint{}, fmt.Errorf("no valid LLM endpoint configured; one of OCR_LLM_URL/OCR_LLM_TOKEN/OCR_LLM_MODEL, ~/.open-code-review/config.json, or ANTHROPIC_BASE_URL/ANTHROPIC_AUTH_TOKEN/ANTHROPIC_MODEL must be set")
|
||||
}
|
||||
|
||||
// tryOCREnv reads OCR-specific environment variables.
|
||||
func tryOCREnv() (ResolvedEndpoint, bool, error) {
|
||||
url := os.Getenv(envOCRLLMURL)
|
||||
token := os.Getenv(envOCRLLMToken)
|
||||
model := os.Getenv(envOCRLLMModel)
|
||||
if url == "" || token == "" || model == "" {
|
||||
return ResolvedEndpoint{}, false, nil
|
||||
}
|
||||
|
||||
useAnthropic := true // default true
|
||||
if v := os.Getenv(envOCRUseAnthropic); v != "" {
|
||||
lower := strings.ToLower(v)
|
||||
useAnthropic = lower == "true" || lower == "1" || lower == "yes"
|
||||
}
|
||||
|
||||
protocol := "anthropic"
|
||||
if !useAnthropic {
|
||||
protocol = "openai"
|
||||
}
|
||||
|
||||
return ResolvedEndpoint{URL: url, Token: token, Model: model, Protocol: protocol}, true, nil
|
||||
}
|
||||
|
||||
// llmFileConfig represents the llm section in config.json.
|
||||
type llmFileConfig struct {
|
||||
URL string `json:"url,omitempty"`
|
||||
AuthToken string `json:"auth_token,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
UseAnthropic *bool `json:"use_anthropic,omitempty"` // pointer to distinguish unset from false
|
||||
}
|
||||
|
||||
type configFile struct {
|
||||
Llm llmFileConfig `json:"llm,omitempty"`
|
||||
}
|
||||
|
||||
// tryOCRConfig reads the OCR config file.
|
||||
func tryOCRConfig(path string) (ResolvedEndpoint, bool, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return ResolvedEndpoint{}, false, nil
|
||||
}
|
||||
return ResolvedEndpoint{}, false, err
|
||||
}
|
||||
|
||||
var cfg configFile
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return ResolvedEndpoint{}, false, fmt.Errorf("parse config: %w", err)
|
||||
}
|
||||
|
||||
if cfg.Llm.URL == "" || cfg.Llm.AuthToken == "" || cfg.Llm.Model == "" {
|
||||
return ResolvedEndpoint{}, false, nil
|
||||
}
|
||||
|
||||
useAnthropic := true // default true
|
||||
if cfg.Llm.UseAnthropic != nil {
|
||||
useAnthropic = *cfg.Llm.UseAnthropic
|
||||
}
|
||||
|
||||
protocol := "anthropic"
|
||||
if !useAnthropic {
|
||||
protocol = "openai"
|
||||
}
|
||||
|
||||
return ResolvedEndpoint{URL: cfg.Llm.URL, Token: cfg.Llm.AuthToken, Model: cfg.Llm.Model, Protocol: protocol}, true, nil
|
||||
}
|
||||
|
||||
// tryCCEnv reads Claude Code environment variables.
|
||||
func tryCCEnv() (ResolvedEndpoint, bool, error) {
|
||||
baseURL := os.Getenv(envCCBaseURL)
|
||||
token := os.Getenv(envCCToken)
|
||||
model := os.Getenv(envCCModel)
|
||||
if baseURL == "" || token == "" || model == "" {
|
||||
return ResolvedEndpoint{}, false, nil
|
||||
}
|
||||
|
||||
url := ensureMessagesSuffix(baseURL)
|
||||
|
||||
return ResolvedEndpoint{URL: url, Token: token, Model: model, Protocol: "anthropic"}, true, nil
|
||||
}
|
||||
|
||||
// tryShellRC parses ~/.zshrc and ~/.bashrc for ANTHROPIC_* exports.
|
||||
func tryShellRC() (ResolvedEndpoint, bool, error) {
|
||||
files := shellRCFiles()
|
||||
for _, f := range files {
|
||||
ep, ok, err := parseShellRC(f)
|
||||
if err != nil || ok {
|
||||
return ep, ok, err
|
||||
}
|
||||
}
|
||||
return ResolvedEndpoint{}, false, nil
|
||||
}
|
||||
|
||||
func shellRCFiles() []string {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
candidates := []string{
|
||||
filepath.Join(home, ".zshrc"),
|
||||
filepath.Join(home, ".bashrc"),
|
||||
filepath.Join(home, ".bash_profile"),
|
||||
filepath.Join(home, ".profile"),
|
||||
}
|
||||
var valid []string
|
||||
for _, f := range candidates {
|
||||
if _, err := os.Stat(f); err == nil {
|
||||
valid = append(valid, f)
|
||||
}
|
||||
}
|
||||
return valid
|
||||
}
|
||||
|
||||
var exportRe = regexp.MustCompile(`^export\s+(ANTHROPIC_\w+)\s*=\s*(?:"([^"]*)"|'([^']*)'|(.+))\s*$`)
|
||||
|
||||
func parseShellRC(path string) (ResolvedEndpoint, bool, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return ResolvedEndpoint{}, false, nil
|
||||
}
|
||||
|
||||
var baseURL, token, model string
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
matches := exportRe.FindStringSubmatch(line)
|
||||
if matches == nil {
|
||||
continue
|
||||
}
|
||||
key := matches[1]
|
||||
value := matches[2]
|
||||
if value == "" {
|
||||
value = matches[3]
|
||||
}
|
||||
if value == "" {
|
||||
value = matches[4]
|
||||
}
|
||||
value = strings.TrimSpace(value)
|
||||
|
||||
switch key {
|
||||
case "ANTHROPIC_BASE_URL":
|
||||
baseURL = value
|
||||
case "ANTHROPIC_AUTH_TOKEN":
|
||||
token = value
|
||||
case "ANTHROPIC_MODEL":
|
||||
model = value
|
||||
}
|
||||
}
|
||||
|
||||
if baseURL == "" || token == "" || model == "" {
|
||||
return ResolvedEndpoint{}, false, nil
|
||||
}
|
||||
|
||||
url := ensureMessagesSuffix(baseURL)
|
||||
|
||||
return ResolvedEndpoint{URL: url, Token: token, Model: model, Protocol: "anthropic"}, true, nil
|
||||
}
|
||||
|
||||
// ensureMessagesSuffix appends /v1/messages to base URLs that lack a versioned path.
|
||||
func ensureMessagesSuffix(rawURL string) string {
|
||||
u := strings.TrimRight(rawURL, "/")
|
||||
if strings.Contains(u, "/v1/") {
|
||||
// Already has versioned path — don't modify.
|
||||
return rawURL
|
||||
}
|
||||
return u + "/v1/messages"
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue