diff --git a/extensions/microsoft-foundry/index.test.ts b/extensions/microsoft-foundry/index.test.ts index 317b707798b5..a961c0ff1e80 100644 --- a/extensions/microsoft-foundry/index.test.ts +++ b/extensions/microsoft-foundry/index.test.ts @@ -1,4 +1,8 @@ // Microsoft Foundry tests cover index plugin behavior. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; import type { StreamFn } from "openclaw/plugin-sdk/agent-core"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import type { ProviderAuthMethod } from "openclaw/plugin-sdk/core"; @@ -127,6 +131,7 @@ const defaultFoundryModelId = "gpt-5.4"; const defaultFoundryProfileId = "microsoft-foundry:entra"; const defaultFoundryAgentDir = "/tmp/test-agent"; const defaultAzureCliLoginError = "Please run 'az login' to setup account."; +const foundryTokenCacheMaxEntries = 128; let runtimeAuthTestSequence = 0; let runtimeAuthTestTenantId = "tenant-0"; @@ -637,6 +642,265 @@ describe("microsoft-foundry plugin", () => { } }); + it("bounds settled Entra tokens by least-recently-used account tuple", async () => { + const provider = registerProvider(); + const prepareRuntimeAuth = requirePrepareRuntimeAuth(provider); + execFileMock.mockImplementation(async () => ({ + stdout: JSON.stringify({ + accessToken: `token-${execFileMock.mock.calls.length}`, + expiresOn: new Date(Date.now() + 10 * 60_000).toISOString(), + }), + stderr: "", + })); + const prepareForTenant = async (tenantId: string) => { + ensureAuthProfileStoreMock.mockReturnValueOnce(buildEntraProfileStore({ tenantId })); + return await prepareRuntimeAuth(buildFoundryRuntimeAuthContext()); + }; + + for (let index = 0; index < foundryTokenCacheMaxEntries; index += 1) { + await prepareForTenant(`lru-${runtimeAuthTestTenantId}-${index}`); + } + expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries); + + await prepareForTenant(`lru-${runtimeAuthTestTenantId}-0`); + expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries); + + await prepareForTenant(`lru-${runtimeAuthTestTenantId}-${foundryTokenCacheMaxEntries}`); + await prepareForTenant(`lru-${runtimeAuthTestTenantId}-0`); + expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries + 1); + await prepareForTenant(`lru-${runtimeAuthTestTenantId}-1`); + expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries + 2); + }); + + it("reclaims expired tokens before evicting an older live account", async () => { + const prepare = requirePrepareRuntimeAuth(registerProvider()); + const now = Date.now(); + const clock = vi.spyOn(Date, "now").mockReturnValue(now); + execFileMock.mockImplementation(async () => ({ + stdout: JSON.stringify({ + accessToken: "synthetic-expiry-token", + expiresOn: new Date(now + 60 * 60_000).toISOString(), + }), + stderr: "", + })); + const prepareForTenant = async (index: number) => { + ensureAuthProfileStoreMock.mockReturnValueOnce( + buildEntraProfileStore({ tenantId: `expiry-${runtimeAuthTestTenantId}-${index}` }), + ); + return await prepare(buildFoundryRuntimeAuthContext()); + }; + for (let index = 0; index < foundryTokenCacheMaxEntries - 1; index++) { + await prepareForTenant(index); + } + mockAzureCliToken({ accessToken: "synthetic-short-token", expiresInMs: 6 * 60_000 }); + await prepareForTenant(foundryTokenCacheMaxEntries - 1); + clock.mockReturnValue(now + 7 * 60_000); + await prepareForTenant(foundryTokenCacheMaxEntries); + await prepareForTenant(0); + expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries + 1); + await prepareForTenant(foundryTokenCacheMaxEntries - 1); + expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries + 2); + }); + + it("keeps one active refresh while settled account entries churn", async () => { + const prepare = requirePrepareRuntimeAuth(registerProvider()); + const release = createDeferred(); + execFileMock.mockImplementation(async () => ({ + stdout: JSON.stringify({ + accessToken: "synthetic-churn-token", + expiresOn: new Date(Date.now() + 60 * 60_000).toISOString(), + }), + stderr: "", + })); + execFileMock.mockImplementationOnce(async () => { + await release.promise; + return { + stdout: JSON.stringify({ + accessToken: "synthetic-held-token", + expiresOn: new Date(Date.now() + 60 * 60_000).toISOString(), + }), + stderr: "", + }; + }); + const prepareForTenant = async (index: number) => { + ensureAuthProfileStoreMock.mockReturnValueOnce( + buildEntraProfileStore({ tenantId: `churn-${runtimeAuthTestTenantId}-${index}` }), + ); + return await prepare(buildFoundryRuntimeAuthContext()); + }; + const pending = [prepareForTenant(0)]; + try { + for (let index = 1; index <= foundryTokenCacheMaxEntries + 1; index++) { + await prepareForTenant(index); + } + pending.push(prepareForTenant(0)); + expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries + 2); + release.resolve(); + const results = await Promise.all(pending); + expect(results.map((result) => requireRuntimeAuthResult(result).apiKey)).toEqual([ + "synthetic-held-token", + "synthetic-held-token", + ]); + await prepareForTenant(0); + expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries + 2); + } finally { + release.resolve(); + await Promise.allSettled(pending); + } + }); + + it( + "refreshes an evicted account through the real provider and Azure CLI boundary", + { + timeout: 180_000, + }, + async ({ signal }) => { + const { runExec } = await vi.importActual< + typeof import("openclaw/plugin-sdk/process-runtime") + >("openclaw/plugin-sdk/process-runtime"); + const repoRoot = fileURLToPath(new URL("../../", import.meta.url)); + const proofDir = await fs.mkdtemp(path.join(os.tmpdir(), "foundry-cache-proof-")); + const binDir = path.join(proofDir, "bin"); + const homeDir = path.join(proofDir, "home"); + const azureDir = path.join(proofDir, "azure"); + const stateDir = path.join(proofDir, "state"); + try { + await Promise.all([binDir, homeDir, azureDir, stateDir].map((dir) => fs.mkdir(dir))); + const fakeAz = path.join(proofDir, "fake-az.mjs"); + await fs.writeFile( + fakeAz, + String.raw`#!/usr/bin/env node +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +const args = process.argv.slice(2); +assert.deepEqual(args.slice(0, 2), ["account", "get-access-token"]); +const value = (name) => { const index = args.indexOf(name); return index < 0 ? undefined : args[index + 1]; }; +const tenant = value("--tenant"); +assert(tenant && value("--resource") && !value("--subscription")); +fs.appendFileSync(path.join(import.meta.dirname, "calls.jsonl"), JSON.stringify({ tenant }) + "\n"); +process.stdout.write(JSON.stringify({ + accessToken: "synthetic-proof-" + tenant, + expiresOn: new Date(Date.now() + 60 * 60_000).toISOString(), +})); +`, + ); + if (process.platform === "win32") { + await fs.writeFile( + path.join(binDir, "az.cmd"), + `@echo off\r\n"${process.execPath}" "${fakeAz}" %*\r\n`, + ); + } else { + await fs.chmod(fakeAz, 0o700); + await fs.symlink(fakeAz, path.join(binDir, "az")); + } + const script = path.join(proofDir, "proof.mts"); + await fs.writeFile( + script, + String.raw`import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { upsertAuthProfile } from "openclaw/plugin-sdk/provider-auth"; +import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api"; +import type { ProviderPlugin } from "openclaw/plugin-sdk/provider-model-shared"; +const { default: plugin } = await import(process.argv[3]); + +const proofDir = process.argv[2]; +assert(proofDir); +const agentDir = path.join(proofDir, "state", "agents", "main", "agent"); +let provider: ProviderPlugin | undefined; +plugin.register(createTestPluginApi({ registerProvider: (value) => { provider = value; } })); +assert(provider?.prepareRuntimeAuth); +const prepare = provider.prepareRuntimeAuth; +const tenant = (index: number) => "00000000-0000-0000-0000-" + String(index).padStart(12, "0"); +const prepareForTenant = async (index: number) => { + upsertAuthProfile({ + profileId: "microsoft-foundry:proof", + agentDir, + credential: { + type: "api_key", provider: "microsoft-foundry", key: "__entra_id_dynamic__", + metadata: { authMethod: "entra-id", tenantId: tenant(index), api: "openai-responses" }, + }, + }); + const result = await prepare({ + provider: "microsoft-foundry", config: {}, agentDir, env: process.env, + modelId: "synthetic-model", apiKey: "__entra_id_dynamic__", authMode: "api_key", + profileId: "microsoft-foundry:proof", + model: { id: "synthetic-model", name: "synthetic-model", provider: "microsoft-foundry", + api: "openai-responses", baseUrl: "https://example.services.ai.azure.com/openai/v1", + reasoning: false, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1000, maxTokens: 100 }, + }); + assert(result?.apiKey && result.request?.auth?.mode === "authorization-bearer"); + assert(result.apiKey.includes(tenant(index)), "prepared token must belong to the selected synthetic tenant"); +}; +const calls = async () => (await fs.readFile(path.join(proofDir, "calls.jsonl"), "utf8")).trim().split("\n").length; +const started = performance.now(); +for (let index = 0; index < 128; index++) { + await prepareForTenant(index); +} +assert.equal(await calls(), 128); +await prepareForTenant(0); +assert.equal(await calls(), 128); +await prepareForTenant(128); +assert.equal(await calls(), 129); +await prepareForTenant(0); +const afterRetouched = await calls(); +await prepareForTenant(1); +const afterOldest = await calls(); +console.info("[foundry-public-provider-cache-proof]", JSON.stringify({ + seededTuples: 128, afterRetouched, afterOldest, + allTenantBindingsCorrect: true, elapsedMs: performance.now() - started, + rssBytes: process.memoryUsage().rss, +})); +assert.equal(afterRetouched, 129); +assert.equal(afterOldest, 130, "the evicted account must refresh through az"); +`, + ); + // The child has its own real profile store and token cache. Only az is synthetic; + // no ambient credentials or the parent Vitest mocks enter its provider graph. + const { stdout } = await runExec( + process.execPath, + ["--import", "tsx/esm", script, proofDir, new URL("./index.ts", import.meta.url).href], + { + cwd: repoRoot, + signal, + timeoutMs: 150_000, + logOutput: false, + baseEnv: { + PATH: [ + binDir, + path.dirname(process.execPath), + ...(process.platform === "win32" ? [] : ["/usr/bin", "/bin"]), + ].join(path.delimiter), + HOME: homeDir, + USERPROFILE: homeDir, + TMPDIR: proofDir, + TEMP: proofDir, + TMP: proofDir, + SystemRoot: process.env.SystemRoot, + ComSpec: process.env.ComSpec, + OPENCLAW_STATE_DIR: stateDir, + AZURE_CONFIG_DIR: azureDir, + TSX_TSCONFIG_PATH: path.join(repoRoot, "tsconfig.json"), + TSX_DISABLE_CACHE: "1", + NODE_ENV: "test", + CI: "1", + }, + }, + ); + const observed = stdout + .split("\n") + .find((line) => line.startsWith("[foundry-public-provider-cache-proof] ")); + expect(observed).toBeDefined(); + console.info(observed); + } finally { + await fs.rm(proofDir, { recursive: true, force: true }); + } + }, + ); + it("clears failed refresh state so later concurrent retries succeed", async () => { const provider = registerProvider(); const prepareRuntimeAuth = requirePrepareRuntimeAuth(provider); diff --git a/extensions/microsoft-foundry/runtime.ts b/extensions/microsoft-foundry/runtime.ts index fc17210b0d4d..34b2d5086a66 100644 --- a/extensions/microsoft-foundry/runtime.ts +++ b/extensions/microsoft-foundry/runtime.ts @@ -1,4 +1,5 @@ // Microsoft Foundry plugin module implements runtime behavior. +import { pruneMapToMaxSize } from "openclaw/plugin-sdk/collection-runtime"; import type { ProviderPreparedRuntimeAuth, ProviderPrepareRuntimeAuthContext, @@ -27,6 +28,9 @@ import { const cachedTokens = new Map(); const refreshPromises = new Map>(); const FOUNDRY_TOKEN_FALLBACK_LIFETIME_MS = 55 * 60 * 1000; +// Bound settled credential material across profile generations. In-flight +// refresh ownership remains separate so admission never evicts active work. +const FOUNDRY_TOKEN_CACHE_MAX_ENTRIES = 128; async function refreshEntraToken(params?: { scope?: string; @@ -40,10 +44,16 @@ async function refreshEntraToken(params?: { asDateTimestampMs(rawExpiry) ?? resolveExpiresAtMsFromDurationMs(FOUNDRY_TOKEN_FALLBACK_LIFETIME_MS, { nowMs: now }) ?? now; + for (const [cacheKey, cachedToken] of cachedTokens) { + if (cachedToken.expiresAt <= now) { + cachedTokens.delete(cacheKey); + } + } cachedTokens.set(getFoundryTokenCacheKey(params), { token: result.accessToken, expiresAt, }); + pruneMapToMaxSize(cachedTokens, FOUNDRY_TOKEN_CACHE_MAX_ENTRIES); return { apiKey: result.accessToken, expiresAt }; } @@ -107,6 +117,9 @@ export async function prepareFoundryRuntimeAuth( const refreshAfterMs = resolveExpiresAtMsFromDurationMs(TOKEN_REFRESH_MARGIN_MS, { nowMs: now }) ?? now; if (cachedToken && hasValidClock && cachedToken.expiresAt > refreshAfterMs) { + // Map insertion order is the eviction order; touch valid hits to retain active accounts. + cachedTokens.delete(cacheKey); + cachedTokens.set(cacheKey, cachedToken); return { apiKey: cachedToken.token, expiresAt: cachedToken.expiresAt, @@ -116,6 +129,7 @@ export async function prepareFoundryRuntimeAuth( }, }; } + cachedTokens.delete(cacheKey); let refreshPromise = refreshPromises.get(cacheKey); if (!refreshPromise) { refreshPromise = refreshEntraToken({