mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 15:59:30 +00:00
fix(usage): preserve provider-billed zero totals (#101177)
* fix(usage): preserve provider-billed zero totals * fix(usage): harden provider-billed cost provenance * fix(openrouter): retry delayed generation metadata * fix(openrouter): satisfy retry lint * refactor(openrouter): consume streamed billed cost * chore: keep release notes out of contributor PR --------- Co-authored-by: snowzlmbot <293528334+snowzlmbot@users.noreply.github.com> Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
snowzlmbot
GitHub
parent
70c2e837cd
commit
53580e13a4
15 changed files with 292 additions and 422 deletions
|
|
@ -36,35 +36,6 @@ import {
|
|||
} from "./provider-catalog.js";
|
||||
import { resolveThinkingProfile } from "./provider-policy-api.js";
|
||||
|
||||
function createOpenRouterDoneStream(params: { responseId: string; totalCost: number }) {
|
||||
const stream = createAssistantMessageEventStream();
|
||||
queueMicrotask(() => {
|
||||
stream.push({
|
||||
type: "done",
|
||||
reason: "stop",
|
||||
message: {
|
||||
role: "assistant",
|
||||
api: "openai-completions",
|
||||
provider: "openrouter",
|
||||
model: "openrouter/auto",
|
||||
content: [{ type: "text", text: "ok" }],
|
||||
responseId: params.responseId,
|
||||
stopReason: "stop",
|
||||
timestamp: Date.now(),
|
||||
usage: {
|
||||
input: 1,
|
||||
output: 1,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 2,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: params.totalCost },
|
||||
},
|
||||
} as never,
|
||||
});
|
||||
});
|
||||
return stream;
|
||||
}
|
||||
|
||||
function createOpenRouterDoneStreamWithoutGeneration() {
|
||||
const stream = createAssistantMessageEventStream();
|
||||
queueMicrotask(() => {
|
||||
|
|
@ -85,35 +56,6 @@ function createOpenRouterDoneStreamWithoutGeneration() {
|
|||
return stream;
|
||||
}
|
||||
|
||||
function createOpenRouterAbortedStream() {
|
||||
const stream = createAssistantMessageEventStream();
|
||||
queueMicrotask(() => {
|
||||
stream.push({
|
||||
type: "error",
|
||||
reason: "aborted",
|
||||
error: {
|
||||
role: "assistant",
|
||||
api: "openai-completions",
|
||||
provider: "openrouter",
|
||||
model: "openrouter/auto",
|
||||
content: [],
|
||||
responseId: "gen-aborted",
|
||||
stopReason: "aborted",
|
||||
timestamp: Date.now(),
|
||||
usage: {
|
||||
input: 1,
|
||||
output: 1,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 2,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0.001 },
|
||||
},
|
||||
} as never,
|
||||
});
|
||||
});
|
||||
return stream;
|
||||
}
|
||||
|
||||
type OpenRouterManifest = {
|
||||
providerAuthChoices?: Array<{
|
||||
provider?: string;
|
||||
|
|
@ -808,172 +750,6 @@ describe("openrouter provider hooks", () => {
|
|||
expect(headers.get("x-openrouter-title")).toBe("OpenClaw");
|
||||
});
|
||||
|
||||
it("reconciles OpenRouter streamed usage with generation metadata cost", async () => {
|
||||
const provider = await registerSingleProviderPlugin(openrouterPlugin);
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
expect(url).toBe("https://openrouter.ai/api/v1/generation?id=gen-cost-1");
|
||||
return new Response(JSON.stringify({ data: { total_cost: 0.0042 } }), {
|
||||
headers: { "Content-Type": "application/json" },
|
||||
status: 200,
|
||||
});
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const baseStreamFn = vi.fn(() =>
|
||||
createOpenRouterDoneStream({ responseId: "gen-cost-1", totalCost: 0.001 }),
|
||||
);
|
||||
|
||||
try {
|
||||
const wrapped = provider.wrapStreamFn?.({
|
||||
provider: "openrouter",
|
||||
modelId: "openrouter/auto",
|
||||
streamFn: baseStreamFn,
|
||||
} as never);
|
||||
if (!wrapped) {
|
||||
throw new Error("expected OpenRouter wrapper");
|
||||
}
|
||||
const stream = await wrapped(
|
||||
{
|
||||
provider: "openrouter",
|
||||
api: "openai-completions",
|
||||
id: "openrouter/auto",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
compat: {},
|
||||
} as never,
|
||||
{ messages: [] } as never,
|
||||
{ apiKey: "or-test-key" } as never,
|
||||
);
|
||||
const message = await stream.result();
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledOnce();
|
||||
expect(message.usage.cost.total).toBe(0.0042);
|
||||
} finally {
|
||||
vi.unstubAllGlobals();
|
||||
}
|
||||
});
|
||||
|
||||
it("falls back to streamed cost estimate when generation metadata response is oversized", async () => {
|
||||
const provider = await registerSingleProviderPlugin(openrouterPlugin);
|
||||
// Body exceeds the 16 MiB cap; readProviderJsonResponse must reject it and
|
||||
// applyOpenRouterBilledCost must fall back to the streamed estimate.
|
||||
const oversizedBody = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array(16 * 1024 * 1024 + 1).fill(0x78));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
expect(url).toBe("https://openrouter.ai/api/v1/generation?id=gen-oversized-1");
|
||||
return new Response(oversizedBody, {
|
||||
headers: { "Content-Type": "application/json" },
|
||||
status: 200,
|
||||
});
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const baseStreamFn = vi.fn(() =>
|
||||
createOpenRouterDoneStream({ responseId: "gen-oversized-1", totalCost: 0.001 }),
|
||||
);
|
||||
|
||||
try {
|
||||
const wrapped = provider.wrapStreamFn?.({
|
||||
provider: "openrouter",
|
||||
modelId: "openrouter/auto",
|
||||
streamFn: baseStreamFn,
|
||||
} as never);
|
||||
if (!wrapped) {
|
||||
throw new Error("expected OpenRouter wrapper");
|
||||
}
|
||||
const stream = await wrapped(
|
||||
{
|
||||
provider: "openrouter",
|
||||
api: "openai-completions",
|
||||
id: "openrouter/auto",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
compat: {},
|
||||
} as never,
|
||||
{ messages: [] } as never,
|
||||
{ apiKey: "or-test-key" } as never,
|
||||
);
|
||||
const message = await stream.result();
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledOnce();
|
||||
expect(message.usage.cost.total).toBe(0.001);
|
||||
} finally {
|
||||
vi.unstubAllGlobals();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not fetch generation metadata for custom OpenRouter-compatible routes", async () => {
|
||||
const provider = await registerSingleProviderPlugin(openrouterPlugin);
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const baseStreamFn = vi.fn(() =>
|
||||
createOpenRouterDoneStream({ responseId: "gen-custom-route", totalCost: 0.001 }),
|
||||
);
|
||||
|
||||
try {
|
||||
const wrapped = provider.wrapStreamFn?.({
|
||||
provider: "openrouter",
|
||||
modelId: "openrouter/auto",
|
||||
streamFn: baseStreamFn,
|
||||
} as never);
|
||||
if (!wrapped) {
|
||||
throw new Error("expected OpenRouter wrapper");
|
||||
}
|
||||
const stream = await wrapped(
|
||||
{
|
||||
provider: "openrouter",
|
||||
api: "openai-completions",
|
||||
id: "openrouter/auto",
|
||||
baseUrl: "https://proxy.example.test/api/v1",
|
||||
compat: {},
|
||||
} as never,
|
||||
{ messages: [] } as never,
|
||||
{ apiKey: "or-test-key" } as never,
|
||||
);
|
||||
const message = await stream.result();
|
||||
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(message.usage.cost.total).toBe(0.001);
|
||||
} finally {
|
||||
vi.unstubAllGlobals();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not fetch generation metadata for aborted stream errors", async () => {
|
||||
const provider = await registerSingleProviderPlugin(openrouterPlugin);
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const baseStreamFn = vi.fn(() => createOpenRouterAbortedStream());
|
||||
|
||||
try {
|
||||
const wrapped = provider.wrapStreamFn?.({
|
||||
provider: "openrouter",
|
||||
modelId: "openrouter/auto",
|
||||
streamFn: baseStreamFn,
|
||||
} as never);
|
||||
if (!wrapped) {
|
||||
throw new Error("expected OpenRouter wrapper");
|
||||
}
|
||||
const stream = await wrapped(
|
||||
{
|
||||
provider: "openrouter",
|
||||
api: "openai-completions",
|
||||
id: "openrouter/auto",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
compat: {},
|
||||
} as never,
|
||||
{ messages: [] } as never,
|
||||
{ apiKey: "or-test-key" } as never,
|
||||
);
|
||||
const message = await stream.result();
|
||||
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(message.stopReason).toBe("aborted");
|
||||
} finally {
|
||||
vi.unstubAllGlobals();
|
||||
}
|
||||
});
|
||||
|
||||
it("merges resolved OpenRouter model params into transport params", async () => {
|
||||
const provider = await registerSingleProviderPlugin(openrouterPlugin);
|
||||
const patch = provider.extraParamsForTransport?.({
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
// Openrouter tests cover openrouter plugin behavior.
|
||||
import OpenAI from "openai";
|
||||
import { AuthStorage, ModelRegistry } from "openclaw/plugin-sdk/agent-sessions";
|
||||
import { streamSimple } from "openclaw/plugin-sdk/llm";
|
||||
import {
|
||||
registerProviderPlugin,
|
||||
requireRegisteredProvider,
|
||||
|
|
@ -156,6 +157,51 @@ describeLive("openrouter plugin live", () => {
|
|||
await expectWeatherToolCall(client, autoModel.id);
|
||||
await expectWeatherToolCall(client, normalized.id);
|
||||
}, 30_000);
|
||||
|
||||
it("preserves a billed zero from the live free route", async () => {
|
||||
const { providers } = await registerOpenRouterPlugin();
|
||||
const provider = requireRegisteredProvider(providers, "openrouter");
|
||||
const resolved = provider.resolveDynamicModel?.({
|
||||
provider: "openrouter",
|
||||
modelId: "openrouter/free",
|
||||
modelRegistry: new ModelRegistryCtor(AuthStorage.inMemory()),
|
||||
});
|
||||
if (!resolved) {
|
||||
throw new Error("openrouter provider did not resolve openrouter/free");
|
||||
}
|
||||
const model =
|
||||
provider.normalizeResolvedModel?.({
|
||||
provider: "openrouter",
|
||||
modelId: resolved.id,
|
||||
model: resolved,
|
||||
}) ?? resolved;
|
||||
const wrapped = provider.wrapStreamFn?.({
|
||||
provider: "openrouter",
|
||||
modelId: model.id,
|
||||
streamFn: streamSimple,
|
||||
} as never);
|
||||
if (!wrapped) {
|
||||
throw new Error("openrouter provider did not expose its stream wrapper");
|
||||
}
|
||||
|
||||
const stream = await wrapped(
|
||||
model,
|
||||
{
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: "Reply with exactly OK.",
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
],
|
||||
},
|
||||
{ apiKey: OPENROUTER_API_KEY, maxTokens: 16 },
|
||||
);
|
||||
const message = await stream.result();
|
||||
|
||||
expect(message.usage.cost.total).toBe(0);
|
||||
expect(message.usage.cost.totalOrigin).toBe("provider-billed");
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
describeCatalogLive("openrouter plugin live model catalog", () => {
|
||||
|
|
|
|||
|
|
@ -1,16 +1,6 @@
|
|||
// Openrouter plugin module implements stream behavior.
|
||||
import type { StreamFn } from "openclaw/plugin-sdk/agent-core";
|
||||
import {
|
||||
createAssistantMessageEventStream,
|
||||
type AssistantMessage,
|
||||
type AssistantMessageEvent,
|
||||
} from "openclaw/plugin-sdk/llm";
|
||||
import type { ProviderWrapStreamFnContext } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import {
|
||||
assertOkOrThrowHttpError,
|
||||
fetchWithTimeoutGuarded,
|
||||
readProviderJsonResponse,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import { OPENROUTER_THINKING_STREAM_HOOKS } from "openclaw/plugin-sdk/provider-stream-family";
|
||||
import { createPayloadPatchStreamWrapper } from "openclaw/plugin-sdk/provider-stream-shared";
|
||||
import { createSubsystemLogger } from "openclaw/plugin-sdk/runtime-env";
|
||||
|
|
@ -22,13 +12,6 @@ import {
|
|||
} from "./provider-catalog.js";
|
||||
|
||||
const log = createSubsystemLogger("openrouter-stream");
|
||||
const OPENROUTER_GENERATION_LOOKUP_TIMEOUT_MS = 2_000;
|
||||
|
||||
type OpenRouterGenerationResponse = {
|
||||
data?: {
|
||||
total_cost?: unknown;
|
||||
};
|
||||
};
|
||||
|
||||
function readString(value: unknown): string | undefined {
|
||||
return typeof value === "string" ? value.trim() : undefined;
|
||||
|
|
@ -74,133 +57,6 @@ function shouldPatchOpenRouterRoutingPayload(model: Parameters<StreamFn>[0]): bo
|
|||
return (api === undefined || api === "openai-completions") && isVerifiedOpenRouterRoute(model);
|
||||
}
|
||||
|
||||
function resolveOpenRouterGenerationUrl(
|
||||
model: Parameters<StreamFn>[0],
|
||||
responseId: string,
|
||||
): string {
|
||||
const baseUrl = readString(model.baseUrl) || OPENROUTER_BASE_URL;
|
||||
const url = new URL("generation", `${baseUrl.replace(/\/$/, "")}/`);
|
||||
url.searchParams.set("id", responseId);
|
||||
return url.href;
|
||||
}
|
||||
|
||||
function readOpenRouterTotalCost(payload: OpenRouterGenerationResponse): number | undefined {
|
||||
const totalCost = payload.data?.total_cost;
|
||||
if (typeof totalCost !== "number" || !Number.isFinite(totalCost) || totalCost < 0) {
|
||||
return undefined;
|
||||
}
|
||||
return totalCost;
|
||||
}
|
||||
|
||||
function isDoneEvent(
|
||||
event: AssistantMessageEvent,
|
||||
): event is Extract<AssistantMessageEvent, { type: "done" }> {
|
||||
return event.type === "done";
|
||||
}
|
||||
|
||||
async function fetchOpenRouterGenerationTotalCost(params: {
|
||||
apiKey: string;
|
||||
model: Parameters<StreamFn>[0];
|
||||
responseId: string;
|
||||
}): Promise<number | undefined> {
|
||||
const url = resolveOpenRouterGenerationUrl(params.model, params.responseId);
|
||||
const { response, release } = await fetchWithTimeoutGuarded(
|
||||
url,
|
||||
{
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
"HTTP-Referer": "https://openclaw.ai",
|
||||
"X-OpenRouter-Title": "OpenClaw",
|
||||
},
|
||||
},
|
||||
OPENROUTER_GENERATION_LOOKUP_TIMEOUT_MS,
|
||||
fetch,
|
||||
{ auditContext: "openrouter-generation-cost" },
|
||||
);
|
||||
try {
|
||||
await assertOkOrThrowHttpError(response, "OpenRouter generation metadata request failed");
|
||||
return readOpenRouterTotalCost(
|
||||
await readProviderJsonResponse<OpenRouterGenerationResponse>(
|
||||
response,
|
||||
"openrouter.generation-cost",
|
||||
),
|
||||
);
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
async function applyOpenRouterBilledCost(params: {
|
||||
apiKey: string | undefined;
|
||||
message: AssistantMessage;
|
||||
model: Parameters<StreamFn>[0];
|
||||
}): Promise<void> {
|
||||
const apiKey = readString(params.apiKey);
|
||||
const responseId = readString((params.message as { responseId?: unknown }).responseId);
|
||||
if (!apiKey || !responseId || !params.message.usage?.cost) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const totalCost = await fetchOpenRouterGenerationTotalCost({
|
||||
apiKey,
|
||||
model: params.model,
|
||||
responseId,
|
||||
});
|
||||
if (totalCost !== undefined) {
|
||||
params.message.usage.cost.total = totalCost;
|
||||
}
|
||||
} catch (error) {
|
||||
log.debug?.(
|
||||
`kept streamed OpenRouter cost estimate because generation metadata lookup failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function createOpenRouterBilledCostWrapper(
|
||||
baseStreamFn: StreamFn | undefined,
|
||||
): StreamFn | undefined {
|
||||
if (!baseStreamFn) {
|
||||
return baseStreamFn;
|
||||
}
|
||||
return async (model, context, options) => {
|
||||
const source = await baseStreamFn(model, context, options);
|
||||
if (!isVerifiedOpenRouterRoute(model)) {
|
||||
return source;
|
||||
}
|
||||
const output = createAssistantMessageEventStream();
|
||||
const stream = output as unknown as { push(event: unknown): void; end(): void };
|
||||
void (async () => {
|
||||
try {
|
||||
for await (const event of source as AsyncIterable<AssistantMessageEvent>) {
|
||||
if (isDoneEvent(event)) {
|
||||
await applyOpenRouterBilledCost({
|
||||
apiKey: options?.apiKey,
|
||||
message: event.message,
|
||||
model,
|
||||
});
|
||||
}
|
||||
stream.push(event);
|
||||
}
|
||||
} catch (error) {
|
||||
stream.push({
|
||||
type: "error",
|
||||
reason: "error",
|
||||
error: {
|
||||
role: "assistant",
|
||||
content: [],
|
||||
stopReason: "error",
|
||||
errorMessage: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
stream.end();
|
||||
}
|
||||
})();
|
||||
return output as ReturnType<StreamFn>;
|
||||
};
|
||||
}
|
||||
|
||||
function mergeOpenRouterAuthHeaders(options: Parameters<StreamFn>[2]): Parameters<StreamFn>[2] {
|
||||
const apiKey = readString(options?.apiKey);
|
||||
if (!apiKey) {
|
||||
|
|
@ -458,11 +314,9 @@ export function wrapOpenRouterProviderStream(
|
|||
: ctx.streamFn;
|
||||
const wrapStreamFn = OPENROUTER_THINKING_STREAM_HOOKS.wrapStreamFn ?? undefined;
|
||||
if (!wrapStreamFn) {
|
||||
return createOpenRouterBilledCostWrapper(
|
||||
createOpenRouterAnthropicPrefillWrapper(
|
||||
createOpenRouterAuthHeaderWrapper(
|
||||
createOpenRouterDeepSeekV4ReplayWrapper(routedStreamFn, ctx.thinkingLevel),
|
||||
),
|
||||
return createOpenRouterAnthropicPrefillWrapper(
|
||||
createOpenRouterAuthHeaderWrapper(
|
||||
createOpenRouterDeepSeekV4ReplayWrapper(routedStreamFn, ctx.thinkingLevel),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
|
@ -474,11 +328,9 @@ export function wrapOpenRouterProviderStream(
|
|||
? undefined
|
||||
: ctx.thinkingLevel,
|
||||
}) ?? undefined;
|
||||
return createOpenRouterBilledCostWrapper(
|
||||
createOpenRouterAnthropicPrefillWrapper(
|
||||
createOpenRouterAuthHeaderWrapper(
|
||||
createOpenRouterDeepSeekV4ReplayWrapper(wrappedStreamFn, ctx.thinkingLevel),
|
||||
),
|
||||
return createOpenRouterAnthropicPrefillWrapper(
|
||||
createOpenRouterAuthHeaderWrapper(
|
||||
createOpenRouterDeepSeekV4ReplayWrapper(wrappedStreamFn, ctx.thinkingLevel),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,15 @@ export function calculateCost<TApi extends Api>(model: Model<TApi>, usage: Usage
|
|||
return usage.cost;
|
||||
}
|
||||
|
||||
/** Replaces the catalog estimate when the provider reports an authoritative billed total. */
|
||||
export function applyProviderReportedUsageCost(usage: Usage, reportedCost: unknown): void {
|
||||
if (typeof reportedCost !== "number" || !Number.isFinite(reportedCost) || reportedCost < 0) {
|
||||
return;
|
||||
}
|
||||
usage.cost.total = reportedCost;
|
||||
usage.cost.totalOrigin = "provider-billed";
|
||||
}
|
||||
|
||||
const EXTENDED_THINKING_LEVELS: ModelThinkingLevel[] = [
|
||||
"off",
|
||||
"minimal",
|
||||
|
|
|
|||
|
|
@ -12,8 +12,12 @@ type OpenAICompatibleDelta = DeepPartial<ChatCompletionChunk["choices"][number][
|
|||
type OpenAICompatibleChoice = Omit<DeepPartial<ChatCompletionChunk["choices"][number]>, "delta"> & {
|
||||
delta?: OpenAICompatibleDelta;
|
||||
};
|
||||
type OpenAICompatibleChatCompletionChunk = Omit<DeepPartial<ChatCompletionChunk>, "choices"> & {
|
||||
type OpenAICompatibleChatCompletionChunk = Omit<
|
||||
DeepPartial<ChatCompletionChunk>,
|
||||
"choices" | "usage"
|
||||
> & {
|
||||
choices?: OpenAICompatibleChoice[];
|
||||
usage?: DeepPartial<ChatCompletionChunk["usage"]> & { cost?: unknown };
|
||||
};
|
||||
type FirstEventSimpleStreamOptions = SimpleStreamOptions & {
|
||||
firstEventTimeoutMs?: number;
|
||||
|
|
@ -142,7 +146,12 @@ function makeToolCallChunk(
|
|||
|
||||
function makeFinishChunk(
|
||||
finishReason: string,
|
||||
usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number },
|
||||
usage?: {
|
||||
prompt_tokens: number;
|
||||
completion_tokens: number;
|
||||
total_tokens: number;
|
||||
cost?: unknown;
|
||||
},
|
||||
): OpenAICompatibleChatCompletionChunk {
|
||||
return {
|
||||
id: "chatcmpl-test",
|
||||
|
|
@ -212,6 +221,52 @@ describe("OpenAI-compatible completions params", () => {
|
|||
}
|
||||
});
|
||||
|
||||
it("preserves a valid provider-reported usage cost", async () => {
|
||||
mockChunksRef.chunks = [
|
||||
makeTextChunk("ok"),
|
||||
makeFinishChunk("stop", {
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 5,
|
||||
total_tokens: 15,
|
||||
cost: 0,
|
||||
}),
|
||||
];
|
||||
const pricedModel = {
|
||||
...model,
|
||||
cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 },
|
||||
} satisfies Model<"openai-completions">;
|
||||
|
||||
const result = await streamOpenAICompletions(pricedModel, context, {
|
||||
apiKey: "sk-test",
|
||||
}).result();
|
||||
|
||||
expect(result.usage.cost.total).toBe(0);
|
||||
expect(result.usage.cost.totalOrigin).toBe("provider-billed");
|
||||
});
|
||||
|
||||
it("keeps the catalog estimate for an invalid provider-reported usage cost", async () => {
|
||||
mockChunksRef.chunks = [
|
||||
makeTextChunk("ok"),
|
||||
makeFinishChunk("stop", {
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 5,
|
||||
total_tokens: 15,
|
||||
cost: -1,
|
||||
}),
|
||||
];
|
||||
const pricedModel = {
|
||||
...model,
|
||||
cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 },
|
||||
} satisfies Model<"openai-completions">;
|
||||
|
||||
const result = await streamOpenAICompletions(pricedModel, context, {
|
||||
apiKey: "sk-test",
|
||||
}).result();
|
||||
|
||||
expect(result.usage.cost.total).toBeCloseTo(0.00002);
|
||||
expect(result.usage.cost.totalOrigin).toBeUndefined();
|
||||
});
|
||||
|
||||
it("fails when streaming headers arrive but no first SSE event follows", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -13,7 +13,11 @@ import type {
|
|||
} from "openai/resources/chat/completions.js";
|
||||
import { getEnvApiKey } from "../env-api-keys.js";
|
||||
import { getAiTransportHost } from "../host.js";
|
||||
import { calculateCost, clampThinkingLevel } from "../model-utils.js";
|
||||
import {
|
||||
applyProviderReportedUsageCost,
|
||||
calculateCost,
|
||||
clampThinkingLevel,
|
||||
} from "../model-utils.js";
|
||||
import type {
|
||||
AssistantMessage,
|
||||
CacheRetention,
|
||||
|
|
@ -1276,6 +1280,7 @@ function parseChunkUsage(
|
|||
completion_tokens?: number;
|
||||
prompt_cache_hit_tokens?: number;
|
||||
prompt_tokens_details?: { cached_tokens?: number; cache_write_tokens?: number };
|
||||
cost?: unknown;
|
||||
},
|
||||
model: Model<"openai-completions">,
|
||||
): AssistantMessage["usage"] {
|
||||
|
|
@ -1304,6 +1309,7 @@ function parseChunkUsage(
|
|||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
calculateCost(model, usage);
|
||||
applyProviderReportedUsageCost(usage, rawUsage.cost);
|
||||
return usage;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -274,6 +274,8 @@ export interface Usage {
|
|||
cacheRead: number;
|
||||
cacheWrite: number;
|
||||
total: number;
|
||||
/** Provenance for the recorded total cost; provider-billed totals are authoritative. */
|
||||
totalOrigin?: "provider-billed";
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -659,6 +659,7 @@ describe("sanitizeSessionHistory", () => {
|
|||
cacheRead: 0.25,
|
||||
cacheWrite: 0,
|
||||
total: 4,
|
||||
totalOrigin: "provider-billed",
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -678,6 +679,7 @@ describe("sanitizeSessionHistory", () => {
|
|||
cacheRead: 0.25,
|
||||
cacheWrite: 0,
|
||||
total: 4,
|
||||
totalOrigin: "provider-billed",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -466,7 +466,10 @@ function normalizeAssistantUsageCost(usage: unknown): AssistantUsageSnapshot["co
|
|||
const cacheRead = cacheReadRaw ?? base.cacheRead;
|
||||
const cacheWrite = cacheWriteRaw ?? base.cacheWrite;
|
||||
const total = totalRaw ?? input + output + cacheRead + cacheWrite;
|
||||
return { input, output, cacheRead, cacheWrite, total };
|
||||
// Keep authoritative provider billing provenance through replay repair. Dropping it
|
||||
// turns a real zero-dollar total back into a local estimate during later accounting.
|
||||
const totalOrigin = cost.totalOrigin === "provider-billed" ? cost.totalOrigin : undefined;
|
||||
return { input, output, cacheRead, cacheWrite, total, ...(totalOrigin ? { totalOrigin } : {}) };
|
||||
}
|
||||
|
||||
function toFiniteCostNumber(value: unknown): number | undefined {
|
||||
|
|
|
|||
|
|
@ -2219,6 +2219,62 @@ describe("openai transport stream", () => {
|
|||
);
|
||||
});
|
||||
|
||||
it("preserves a valid provider-reported usage cost", () => {
|
||||
const model = {
|
||||
id: "openrouter/free",
|
||||
name: "OpenRouter Free",
|
||||
api: "openai-completions",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 8_192,
|
||||
} satisfies Model<"openai-completions">;
|
||||
|
||||
const usage = parseTransportChunkUsage(
|
||||
{
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 5,
|
||||
total_tokens: 15,
|
||||
cost: 0,
|
||||
},
|
||||
model,
|
||||
);
|
||||
|
||||
expect(usage.cost.total).toBe(0);
|
||||
expect(usage.cost.totalOrigin).toBe("provider-billed");
|
||||
});
|
||||
|
||||
it("keeps the catalog estimate for an invalid provider-reported usage cost", () => {
|
||||
const model = {
|
||||
id: "openrouter/free",
|
||||
name: "OpenRouter Free",
|
||||
api: "openai-completions",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 8_192,
|
||||
} satisfies Model<"openai-completions">;
|
||||
|
||||
const usage = parseTransportChunkUsage(
|
||||
{
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 5,
|
||||
total_tokens: 15,
|
||||
cost: -1,
|
||||
},
|
||||
model,
|
||||
);
|
||||
|
||||
expect(usage.cost.total).toBeCloseTo(0.00002);
|
||||
expect(usage.cost.totalOrigin).toBeUndefined();
|
||||
});
|
||||
|
||||
it("clamps uncached prompt usage at zero", () => {
|
||||
const model = {
|
||||
id: "gpt-5",
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ import {
|
|||
type OpenAIToolProjection,
|
||||
} from "@openclaw/ai/internal/openai";
|
||||
import {
|
||||
applyProviderReportedUsageCost,
|
||||
calculateCost,
|
||||
createFirstStreamEventAbortController,
|
||||
createReasoningTagTextPartitioner,
|
||||
|
|
@ -62,7 +63,7 @@ import type {
|
|||
} from "openai/resources/responses/responses.js";
|
||||
import type { ModelCompatConfig } from "../config/types.models.js";
|
||||
import { sha256Hex, sha256HexPrefix } from "../infra/crypto-digest.js";
|
||||
import type { Api, Context, Model } from "../llm/types.js";
|
||||
import type { Api, Context, Model, Usage } from "../llm/types.js";
|
||||
import "../llm/ai-transport-host.js";
|
||||
import { createAssistantMessageEventStream } from "../llm/utils/event-stream.js";
|
||||
import { redactIdentifier } from "../logging/redact-identifier.js";
|
||||
|
|
@ -252,7 +253,7 @@ type MutableAssistantOutput = {
|
|||
cacheWrite: number;
|
||||
reasoningTokens?: number;
|
||||
totalTokens: number;
|
||||
cost: { input: number; output: number; cacheRead: number; cacheWrite: number; total: number };
|
||||
cost: Usage["cost"];
|
||||
};
|
||||
stopReason: string;
|
||||
timestamp: number;
|
||||
|
|
@ -4469,15 +4470,15 @@ export function buildOpenAICompletionsParams(
|
|||
}
|
||||
|
||||
export function parseTransportChunkUsage(
|
||||
rawUsage: NonNullable<ChatCompletionChunk["usage"]>,
|
||||
rawUsage: NonNullable<ChatCompletionChunk["usage"]> & { cost?: unknown },
|
||||
model: Model,
|
||||
) {
|
||||
): MutableAssistantOutput["usage"] {
|
||||
const cachedTokens = rawUsage.prompt_tokens_details?.cached_tokens || 0;
|
||||
const promptTokens = rawUsage.prompt_tokens || 0;
|
||||
const input = Math.max(0, promptTokens - cachedTokens);
|
||||
const outputTokens = rawUsage.completion_tokens || 0;
|
||||
const reasoningTokens = rawUsage.completion_tokens_details?.reasoning_tokens;
|
||||
const usage = {
|
||||
const usage: MutableAssistantOutput["usage"] = {
|
||||
input,
|
||||
output: outputTokens,
|
||||
cacheRead: cachedTokens,
|
||||
|
|
@ -4489,6 +4490,7 @@ export function parseTransportChunkUsage(
|
|||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
calculateCost(model as never, usage as never);
|
||||
applyProviderReportedUsageCost(usage, rawUsage.cost);
|
||||
return usage;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,10 +4,9 @@
|
|||
* output, cache, reasoning, and total token accounting fields.
|
||||
*/
|
||||
import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion";
|
||||
import type { Usage } from "../llm/types.js";
|
||||
|
||||
export type ContextUsage =
|
||||
| { state: "available"; promptTokens: number; totalTokens: number }
|
||||
| { state: "unavailable" };
|
||||
export type ContextUsage = NonNullable<Usage["contextUsage"]>;
|
||||
|
||||
/** Provider/SDK usage payload variants accepted by usage normalization. */
|
||||
export type UsageLike = {
|
||||
|
|
@ -50,6 +49,8 @@ export type UsageLike = {
|
|||
prompt_n?: number;
|
||||
predicted_n?: number;
|
||||
};
|
||||
// Optional cost metadata carried through transcripts for downstream cost accounting.
|
||||
cost?: Partial<Usage["cost"]>;
|
||||
};
|
||||
|
||||
/** Normalized token counts used by runtime accounting. */
|
||||
|
|
@ -73,21 +74,7 @@ export type OpenAiChatCompletionsUsage = {
|
|||
};
|
||||
|
||||
/** Assistant usage snapshot with token counts and computed cost buckets. */
|
||||
export type AssistantUsageSnapshot = {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead: number;
|
||||
cacheWrite: number;
|
||||
contextUsage?: ContextUsage;
|
||||
totalTokens: number;
|
||||
cost: {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead: number;
|
||||
cacheWrite: number;
|
||||
total: number;
|
||||
};
|
||||
};
|
||||
export type AssistantUsageSnapshot = Usage;
|
||||
|
||||
/** Build a zeroed assistant usage snapshot. */
|
||||
export function makeZeroUsageSnapshot(): AssistantUsageSnapshot {
|
||||
|
|
|
|||
|
|
@ -449,6 +449,80 @@ describe("session cost usage", () => {
|
|||
}
|
||||
});
|
||||
|
||||
it("preserves a provider-billed zero total when total provenance is explicit", async () => {
|
||||
const root = await makeSessionCostRoot("cost-provider-billed-zero-origin");
|
||||
const sessionsDir = path.join(root, "agents", "main", "sessions");
|
||||
await fs.mkdir(sessionsDir, { recursive: true });
|
||||
const sessionFile = path.join(sessionsDir, "sess-openrouter-billed-zero.jsonl");
|
||||
const entry = {
|
||||
type: "message",
|
||||
timestamp: "2026-02-05T12:00:00.000Z",
|
||||
message: {
|
||||
role: "assistant",
|
||||
provider: "openrouter",
|
||||
model: "openai/gpt-5.5",
|
||||
content: "provider-billed free route",
|
||||
usage: {
|
||||
input: 10_000,
|
||||
output: 5_000,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 15_000,
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
total: 0,
|
||||
totalOrigin: "provider-billed",
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
await fs.writeFile(sessionFile, transcriptText("sess-openrouter-billed-zero", entry), "utf-8");
|
||||
|
||||
const config = {
|
||||
models: {
|
||||
providers: {
|
||||
openrouter: {
|
||||
models: [
|
||||
{
|
||||
id: "openai/gpt-5.5",
|
||||
cost: { input: 1, output: 2, cacheRead: 0.5, cacheWrite: 0 },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
|
||||
clearGatewayModelPricingCacheState();
|
||||
await withStateDir(root, async () => {
|
||||
const summary = await loadCostUsageSummary({
|
||||
startMs: Date.UTC(2026, 1, 5),
|
||||
endMs: Date.UTC(2026, 1, 5, 23, 59, 59, 999),
|
||||
config,
|
||||
});
|
||||
expect(summary.totals.totalTokens).toBe(15_000);
|
||||
expect(summary.totals.totalCost).toBe(0);
|
||||
expect(summary.totals.missingCostEntries).toBe(0);
|
||||
|
||||
await refreshCostUsageCache({ config, sessionFiles: [sessionFile] });
|
||||
const cached = await loadCostUsageSummaryFromCache({
|
||||
startMs: Date.UTC(2026, 1, 5),
|
||||
endMs: Date.UTC(2026, 1, 5, 23, 59, 59, 999),
|
||||
config,
|
||||
requestRefresh: false,
|
||||
});
|
||||
expect(cached.totals.totalCost).toBe(0);
|
||||
expect(cached.totals.missingCostEntries).toBe(0);
|
||||
|
||||
const logs = await loadSessionLogs({ sessionFile, config });
|
||||
expect(logs?.[0]?.tokens).toBe(15_000);
|
||||
expect(logs?.[0]?.cost).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
it("uses top-level transcript provider and model when recomputing session-log cost", async () => {
|
||||
const root = await makeSessionCostRoot("cost-known-pricing-top-level-metadata");
|
||||
const sessionsDir = path.join(root, "agents", "main", "sessions");
|
||||
|
|
@ -529,7 +603,7 @@ describe("session cost usage", () => {
|
|||
await refreshCostUsageCache({ sessionFiles: [sessionFile] });
|
||||
const cachePath = path.join(sessionsDir, ".usage-cost-cache.json");
|
||||
const cache = JSON.parse(await fs.readFile(cachePath, "utf-8")) as { version: number };
|
||||
cache.version = 5;
|
||||
cache.version = 6;
|
||||
await fs.writeFile(cachePath, `${JSON.stringify(cache)}\n`, "utf-8");
|
||||
|
||||
// The pre-upgrade cache must be treated as stale (not served), forcing a rebuild
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ export type {
|
|||
|
||||
// Bump when the durable cache schema or the meaning of cached totals changes, so
|
||||
// older builds are rebuilt instead of served stale.
|
||||
const USAGE_COST_CACHE_VERSION = 6;
|
||||
const USAGE_COST_CACHE_VERSION = 7;
|
||||
const USAGE_COST_CACHE_FILE = ".usage-cost-cache.json";
|
||||
const USAGE_COST_CACHE_LOCK_WRITE_GRACE_MS = 10_000;
|
||||
const USAGE_COST_CACHE_TEMP_FILE_GRACE_MS = USAGE_COST_CACHE_LOCK_WRITE_GRACE_MS;
|
||||
|
|
@ -835,6 +835,9 @@ function buildSessionCostSummaryFromCacheEntry(params: {
|
|||
};
|
||||
}
|
||||
|
||||
const normalizeUsageCostTotalOrigin = (value: unknown): CostBreakdown["totalOrigin"] =>
|
||||
value === "provider-billed" ? value : undefined;
|
||||
|
||||
const extractCostBreakdown = (usageRaw?: UsageLike | null): CostBreakdown | undefined => {
|
||||
if (!usageRaw || typeof usageRaw !== "object") {
|
||||
return undefined;
|
||||
|
|
@ -856,6 +859,7 @@ const extractCostBreakdown = (usageRaw?: UsageLike | null): CostBreakdown | unde
|
|||
output: asFiniteNumber(cost.output),
|
||||
cacheRead: asFiniteNumber(cost.cacheRead),
|
||||
cacheWrite: asFiniteNumber(cost.cacheWrite),
|
||||
totalOrigin: normalizeUsageCostTotalOrigin(cost.totalOrigin),
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -1141,12 +1145,13 @@ const isModelPricingKnown = (cost: ReturnType<typeof resolveModelCostConfig>): b
|
|||
|
||||
const shouldPreserveRecordedZeroCost = (costBreakdown: CostBreakdown | undefined): boolean =>
|
||||
costBreakdown?.total === 0 &&
|
||||
[
|
||||
costBreakdown.input,
|
||||
costBreakdown.output,
|
||||
costBreakdown.cacheRead,
|
||||
costBreakdown.cacheWrite,
|
||||
].some((value) => value !== undefined && value !== 0);
|
||||
(costBreakdown.totalOrigin === "provider-billed" ||
|
||||
[
|
||||
costBreakdown.input,
|
||||
costBreakdown.output,
|
||||
costBreakdown.cacheRead,
|
||||
costBreakdown.cacheWrite,
|
||||
].some((value) => value !== undefined && value !== 0));
|
||||
|
||||
const shouldRecomputeRecordedZeroCost = (params: {
|
||||
cost: ReturnType<typeof resolveModelCostConfig>;
|
||||
|
|
@ -1303,8 +1308,8 @@ async function scanTranscriptFile(params: {
|
|||
})
|
||||
) {
|
||||
// Fill in missing estimates and override fabricated API-provided zeros
|
||||
// for known-priced models such as DeepSeek V4. Providers that reconcile
|
||||
// only the total keep their authoritative zero when components are nonzero.
|
||||
// for known-priced models such as DeepSeek V4. Providers that mark
|
||||
// the total as provider-billed keep their authoritative zero total.
|
||||
entry.costTotal = estimateUsageCost({ usage: entry.usage, cost });
|
||||
entry.costBreakdown = undefined;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,17 +1,12 @@
|
|||
// Shared session cost and usage accounting type contracts.
|
||||
import type { NormalizedUsage } from "../agents/usage.js";
|
||||
import type { Usage } from "../llm/types.js";
|
||||
import type {
|
||||
SessionUsageTimePoint as SharedSessionUsageTimePoint,
|
||||
SessionUsageTimeSeries as SharedSessionUsageTimeSeries,
|
||||
} from "../shared/session-usage-timeseries-types.js";
|
||||
|
||||
export type CostBreakdown = {
|
||||
total?: number;
|
||||
input?: number;
|
||||
output?: number;
|
||||
cacheRead?: number;
|
||||
cacheWrite?: number;
|
||||
};
|
||||
export type CostBreakdown = Partial<Usage["cost"]>;
|
||||
|
||||
export type ParsedUsageEntry = {
|
||||
usage: NormalizedUsage;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue