diff --git a/test/huggingface-local-app-openclaw.e2e.test.ts b/test/huggingface-local-app-openclaw.e2e.test.ts new file mode 100644 index 00000000000..12c50970c75 --- /dev/null +++ b/test/huggingface-local-app-openclaw.e2e.test.ts @@ -0,0 +1,250 @@ +import fs from "node:fs/promises"; +// Hugging Face local-app CLI contract tests cover OpenClaw's snippet-facing commands. +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { afterEach, describe, expect, it } from "vitest"; +import { + createOpenClawTestInstance, + type OpenClawTestInstance, +} from "./helpers/openclaw-test-instance.js"; + +type CapturedChatRequest = { + path: string; + model?: string; + messages?: Array<{ role?: string; content?: unknown }>; + stream?: boolean; +}; + +type FakeOpenAiServer = { + baseUrl: string; + requests: CapturedChatRequest[]; + close: () => Promise; +}; + +const HF_LOCAL_APP_CASES = [ + { + name: "GGUF llama.cpp", + providerId: "llama-cpp", + modelId: "ggml-org/gemma-4-E4B-it-GGUF:Q4_K_M", + }, + { + name: "MLX LM", + providerId: "mlx-lm", + modelId: "mlx-community/Qwen3-0.6B-4bit", + }, +] as const; + +const instances: OpenClawTestInstance[] = []; +const fakeServers: FakeOpenAiServer[] = []; + +afterEach(async () => { + await Promise.allSettled(instances.splice(0).map((instance) => instance.cleanup())); + await Promise.allSettled(fakeServers.splice(0).map((server) => server.close())); +}); + +describe("Hugging Face OpenClaw local-app CLI contract", () => { + it.each(HF_LOCAL_APP_CASES)( + "runs onboard and local agent commands for $name", + async ({ providerId, modelId }) => { + const fakeServer = await startFakeOpenAiServer({ modelId }); + fakeServers.push(fakeServer); + const instance = await createOpenClawTestInstance({ + name: `hf-local-app-${providerId}`, + env: { + CUSTOM_API_KEY: undefined, + OPENCLAW_TEST_FAST: "1", + }, + }); + instances.push(instance); + + const onboard = await instance.cli( + [ + "onboard", + "--non-interactive", + "--mode", + "local", + "--auth-choice", + "custom-api-key", + "--custom-base-url", + fakeServer.baseUrl, + "--custom-model-id", + modelId, + "--custom-provider-id", + providerId, + "--custom-compatibility", + "openai", + "--custom-text-input", + "--accept-risk", + "--skip-health", + ], + { timeoutMs: 120_000 }, + ); + + expect(onboard.code, onboard.stderr).toBe(0); + expect(onboard.stdout).toContain("Updated config"); + await expectConfiguredLocalProvider(instance.configPath, { + baseUrl: fakeServer.baseUrl, + modelId, + providerId, + }); + + const agent = await instance.cli( + ["agent", "--local", "--agent", "main", "--message", "Hello from Hugging Face"], + { timeoutMs: 120_000 }, + ); + + expect(agent.code, agent.stderr).toBe(0); + expect(agent.stdout).toContain(`hf local app ok: ${providerId}`); + expect(fakeServer.requests).toHaveLength(1); + expect(fakeServer.requests[0]).toMatchObject({ + path: "/v1/chat/completions", + model: modelId, + stream: true, + }); + expect(fakeServer.requests[0]?.messages?.some(messageContainsHello)).toBe(true); + }, + 180_000, + ); +}); + +function messageContainsHello(message: { content?: unknown }): boolean { + const content = message.content; + if (typeof content === "string") { + return content.includes("Hello from Hugging Face"); + } + if (!Array.isArray(content)) { + return false; + } + return content.some((part) => { + if (typeof part === "string") { + return part.includes("Hello from Hugging Face"); + } + if (part && typeof part === "object" && "text" in part) { + return String(part.text).includes("Hello from Hugging Face"); + } + return false; + }); +} + +async function expectConfiguredLocalProvider( + configPath: string, + expected: { baseUrl: string; modelId: string; providerId: string }, +): Promise { + const config = JSON.parse(await fs.readFile(configPath, "utf8")) as { + agents?: { + defaults?: { + model?: { primary?: string }; + }; + }; + models?: { + providers?: Record< + string, + { api?: string; baseUrl?: string; models?: Array<{ id?: string }> } + >; + }; + }; + const provider = config.models?.providers?.[expected.providerId]; + expect(provider).toMatchObject({ + api: "openai-completions", + baseUrl: expected.baseUrl, + }); + expect(provider?.models?.some((model) => model.id === expected.modelId)).toBe(true); + expect(config.agents?.defaults?.model?.primary).toBe( + `${expected.providerId}/${expected.modelId}`, + ); +} + +async function startFakeOpenAiServer(params: { modelId: string }): Promise { + const requests: CapturedChatRequest[] = []; + const server = createServer((req, res) => { + void handleFakeOpenAiRequest(req, res, params.modelId, requests); + }); + + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") { + await new Promise((resolve) => server.close(() => resolve())); + throw new Error("fake OpenAI server did not bind a TCP port"); + } + return { + baseUrl: `http://127.0.0.1:${(address as AddressInfo).port}/v1`, + requests, + close: () => new Promise((resolve) => server.close(() => resolve())), + }; +} + +async function handleFakeOpenAiRequest( + req: IncomingMessage, + res: ServerResponse, + modelId: string, + requests: CapturedChatRequest[], +): Promise { + if (req.method === "GET" && req.url === "/v1/models") { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ object: "list", data: [{ id: modelId, object: "model" }] })); + return; + } + + if (req.method !== "POST" || req.url !== "/v1/chat/completions") { + res.writeHead(404, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: { message: `unexpected ${req.method} ${req.url}` } })); + return; + } + + const rawBody = await readRequestBody(req); + const payload = JSON.parse(rawBody) as { + messages?: Array<{ role?: string; content?: unknown }>; + model?: string; + stream?: boolean; + }; + requests.push({ + path: req.url, + messages: payload.messages, + model: payload.model, + stream: payload.stream, + }); + + res.writeHead(200, { + "cache-control": "no-cache", + connection: "keep-alive", + "content-type": "text/event-stream; charset=utf-8", + }); + const providerId = modelId.startsWith("mlx-community/") ? "mlx-lm" : "llama-cpp"; + const created = Math.floor(Date.now() / 1000); + writeSse(res, { + id: "chatcmpl-hf-openclaw-contract", + object: "chat.completion.chunk", + created, + model: modelId, + choices: [ + { + index: 0, + delta: { role: "assistant", content: `hf local app ok: ${providerId}` }, + finish_reason: null, + }, + ], + }); + writeSse(res, { + id: "chatcmpl-hf-openclaw-contract", + object: "chat.completion.chunk", + created, + model: modelId, + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + res.end("data: [DONE]\n\n"); +} + +async function readRequestBody(req: IncomingMessage): Promise { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk); + } + return Buffer.concat(chunks).toString("utf8"); +} + +function writeSse(res: ServerResponse, event: unknown): void { + res.write(`data: ${JSON.stringify(event)}\n\n`); +}