mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 09:39:25 +00:00
fix(microsoft-foundry): bound Entra token retention across accounts (#127216)
Bound settled Microsoft Foundry Entra credentials to 128 recently used account tuples. Remove unusable and expired entries before capacity eviction while preserving separate single-flight Azure CLI refresh ownership for text and image authentication. Verified recency, expiry-first eviction, active refresh during cache churn, failure cleanup, and the real public-provider/profile-store/az boundary. After 129 distinct tuples, the touched account stays cached and the evicted account causes refresh 130 with correct tenant binding. Exact-head CI #34027488799 passed; no genuine Azure credentials or cross-platform performance ratio claimed. Fixes #127196 Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
Peter Steinberger
GitHub
parent
c2ffbec6fc
commit
811eb132a1
2 changed files with 278 additions and 0 deletions
|
|
@ -1,4 +1,8 @@
|
|||
// Microsoft Foundry tests cover index plugin behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import type { StreamFn } from "openclaw/plugin-sdk/agent-core";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import type { ProviderAuthMethod } from "openclaw/plugin-sdk/core";
|
||||
|
|
@ -127,6 +131,7 @@ const defaultFoundryModelId = "gpt-5.4";
|
|||
const defaultFoundryProfileId = "microsoft-foundry:entra";
|
||||
const defaultFoundryAgentDir = "/tmp/test-agent";
|
||||
const defaultAzureCliLoginError = "Please run 'az login' to setup account.";
|
||||
const foundryTokenCacheMaxEntries = 128;
|
||||
let runtimeAuthTestSequence = 0;
|
||||
let runtimeAuthTestTenantId = "tenant-0";
|
||||
|
||||
|
|
@ -637,6 +642,265 @@ describe("microsoft-foundry plugin", () => {
|
|||
}
|
||||
});
|
||||
|
||||
it("bounds settled Entra tokens by least-recently-used account tuple", async () => {
|
||||
const provider = registerProvider();
|
||||
const prepareRuntimeAuth = requirePrepareRuntimeAuth(provider);
|
||||
execFileMock.mockImplementation(async () => ({
|
||||
stdout: JSON.stringify({
|
||||
accessToken: `token-${execFileMock.mock.calls.length}`,
|
||||
expiresOn: new Date(Date.now() + 10 * 60_000).toISOString(),
|
||||
}),
|
||||
stderr: "",
|
||||
}));
|
||||
const prepareForTenant = async (tenantId: string) => {
|
||||
ensureAuthProfileStoreMock.mockReturnValueOnce(buildEntraProfileStore({ tenantId }));
|
||||
return await prepareRuntimeAuth(buildFoundryRuntimeAuthContext());
|
||||
};
|
||||
|
||||
for (let index = 0; index < foundryTokenCacheMaxEntries; index += 1) {
|
||||
await prepareForTenant(`lru-${runtimeAuthTestTenantId}-${index}`);
|
||||
}
|
||||
expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries);
|
||||
|
||||
await prepareForTenant(`lru-${runtimeAuthTestTenantId}-0`);
|
||||
expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries);
|
||||
|
||||
await prepareForTenant(`lru-${runtimeAuthTestTenantId}-${foundryTokenCacheMaxEntries}`);
|
||||
await prepareForTenant(`lru-${runtimeAuthTestTenantId}-0`);
|
||||
expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries + 1);
|
||||
await prepareForTenant(`lru-${runtimeAuthTestTenantId}-1`);
|
||||
expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries + 2);
|
||||
});
|
||||
|
||||
it("reclaims expired tokens before evicting an older live account", async () => {
|
||||
const prepare = requirePrepareRuntimeAuth(registerProvider());
|
||||
const now = Date.now();
|
||||
const clock = vi.spyOn(Date, "now").mockReturnValue(now);
|
||||
execFileMock.mockImplementation(async () => ({
|
||||
stdout: JSON.stringify({
|
||||
accessToken: "synthetic-expiry-token",
|
||||
expiresOn: new Date(now + 60 * 60_000).toISOString(),
|
||||
}),
|
||||
stderr: "",
|
||||
}));
|
||||
const prepareForTenant = async (index: number) => {
|
||||
ensureAuthProfileStoreMock.mockReturnValueOnce(
|
||||
buildEntraProfileStore({ tenantId: `expiry-${runtimeAuthTestTenantId}-${index}` }),
|
||||
);
|
||||
return await prepare(buildFoundryRuntimeAuthContext());
|
||||
};
|
||||
for (let index = 0; index < foundryTokenCacheMaxEntries - 1; index++) {
|
||||
await prepareForTenant(index);
|
||||
}
|
||||
mockAzureCliToken({ accessToken: "synthetic-short-token", expiresInMs: 6 * 60_000 });
|
||||
await prepareForTenant(foundryTokenCacheMaxEntries - 1);
|
||||
clock.mockReturnValue(now + 7 * 60_000);
|
||||
await prepareForTenant(foundryTokenCacheMaxEntries);
|
||||
await prepareForTenant(0);
|
||||
expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries + 1);
|
||||
await prepareForTenant(foundryTokenCacheMaxEntries - 1);
|
||||
expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries + 2);
|
||||
});
|
||||
|
||||
it("keeps one active refresh while settled account entries churn", async () => {
|
||||
const prepare = requirePrepareRuntimeAuth(registerProvider());
|
||||
const release = createDeferred<void>();
|
||||
execFileMock.mockImplementation(async () => ({
|
||||
stdout: JSON.stringify({
|
||||
accessToken: "synthetic-churn-token",
|
||||
expiresOn: new Date(Date.now() + 60 * 60_000).toISOString(),
|
||||
}),
|
||||
stderr: "",
|
||||
}));
|
||||
execFileMock.mockImplementationOnce(async () => {
|
||||
await release.promise;
|
||||
return {
|
||||
stdout: JSON.stringify({
|
||||
accessToken: "synthetic-held-token",
|
||||
expiresOn: new Date(Date.now() + 60 * 60_000).toISOString(),
|
||||
}),
|
||||
stderr: "",
|
||||
};
|
||||
});
|
||||
const prepareForTenant = async (index: number) => {
|
||||
ensureAuthProfileStoreMock.mockReturnValueOnce(
|
||||
buildEntraProfileStore({ tenantId: `churn-${runtimeAuthTestTenantId}-${index}` }),
|
||||
);
|
||||
return await prepare(buildFoundryRuntimeAuthContext());
|
||||
};
|
||||
const pending = [prepareForTenant(0)];
|
||||
try {
|
||||
for (let index = 1; index <= foundryTokenCacheMaxEntries + 1; index++) {
|
||||
await prepareForTenant(index);
|
||||
}
|
||||
pending.push(prepareForTenant(0));
|
||||
expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries + 2);
|
||||
release.resolve();
|
||||
const results = await Promise.all(pending);
|
||||
expect(results.map((result) => requireRuntimeAuthResult(result).apiKey)).toEqual([
|
||||
"synthetic-held-token",
|
||||
"synthetic-held-token",
|
||||
]);
|
||||
await prepareForTenant(0);
|
||||
expect(execFileMock).toHaveBeenCalledTimes(foundryTokenCacheMaxEntries + 2);
|
||||
} finally {
|
||||
release.resolve();
|
||||
await Promise.allSettled(pending);
|
||||
}
|
||||
});
|
||||
|
||||
it(
|
||||
"refreshes an evicted account through the real provider and Azure CLI boundary",
|
||||
{
|
||||
timeout: 180_000,
|
||||
},
|
||||
async ({ signal }) => {
|
||||
const { runExec } = await vi.importActual<
|
||||
typeof import("openclaw/plugin-sdk/process-runtime")
|
||||
>("openclaw/plugin-sdk/process-runtime");
|
||||
const repoRoot = fileURLToPath(new URL("../../", import.meta.url));
|
||||
const proofDir = await fs.mkdtemp(path.join(os.tmpdir(), "foundry-cache-proof-"));
|
||||
const binDir = path.join(proofDir, "bin");
|
||||
const homeDir = path.join(proofDir, "home");
|
||||
const azureDir = path.join(proofDir, "azure");
|
||||
const stateDir = path.join(proofDir, "state");
|
||||
try {
|
||||
await Promise.all([binDir, homeDir, azureDir, stateDir].map((dir) => fs.mkdir(dir)));
|
||||
const fakeAz = path.join(proofDir, "fake-az.mjs");
|
||||
await fs.writeFile(
|
||||
fakeAz,
|
||||
String.raw`#!/usr/bin/env node
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
const args = process.argv.slice(2);
|
||||
assert.deepEqual(args.slice(0, 2), ["account", "get-access-token"]);
|
||||
const value = (name) => { const index = args.indexOf(name); return index < 0 ? undefined : args[index + 1]; };
|
||||
const tenant = value("--tenant");
|
||||
assert(tenant && value("--resource") && !value("--subscription"));
|
||||
fs.appendFileSync(path.join(import.meta.dirname, "calls.jsonl"), JSON.stringify({ tenant }) + "\n");
|
||||
process.stdout.write(JSON.stringify({
|
||||
accessToken: "synthetic-proof-" + tenant,
|
||||
expiresOn: new Date(Date.now() + 60 * 60_000).toISOString(),
|
||||
}));
|
||||
`,
|
||||
);
|
||||
if (process.platform === "win32") {
|
||||
await fs.writeFile(
|
||||
path.join(binDir, "az.cmd"),
|
||||
`@echo off\r\n"${process.execPath}" "${fakeAz}" %*\r\n`,
|
||||
);
|
||||
} else {
|
||||
await fs.chmod(fakeAz, 0o700);
|
||||
await fs.symlink(fakeAz, path.join(binDir, "az"));
|
||||
}
|
||||
const script = path.join(proofDir, "proof.mts");
|
||||
await fs.writeFile(
|
||||
script,
|
||||
String.raw`import assert from "node:assert/strict";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { upsertAuthProfile } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api";
|
||||
import type { ProviderPlugin } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
const { default: plugin } = await import(process.argv[3]);
|
||||
|
||||
const proofDir = process.argv[2];
|
||||
assert(proofDir);
|
||||
const agentDir = path.join(proofDir, "state", "agents", "main", "agent");
|
||||
let provider: ProviderPlugin | undefined;
|
||||
plugin.register(createTestPluginApi({ registerProvider: (value) => { provider = value; } }));
|
||||
assert(provider?.prepareRuntimeAuth);
|
||||
const prepare = provider.prepareRuntimeAuth;
|
||||
const tenant = (index: number) => "00000000-0000-0000-0000-" + String(index).padStart(12, "0");
|
||||
const prepareForTenant = async (index: number) => {
|
||||
upsertAuthProfile({
|
||||
profileId: "microsoft-foundry:proof",
|
||||
agentDir,
|
||||
credential: {
|
||||
type: "api_key", provider: "microsoft-foundry", key: "__entra_id_dynamic__",
|
||||
metadata: { authMethod: "entra-id", tenantId: tenant(index), api: "openai-responses" },
|
||||
},
|
||||
});
|
||||
const result = await prepare({
|
||||
provider: "microsoft-foundry", config: {}, agentDir, env: process.env,
|
||||
modelId: "synthetic-model", apiKey: "__entra_id_dynamic__", authMode: "api_key",
|
||||
profileId: "microsoft-foundry:proof",
|
||||
model: { id: "synthetic-model", name: "synthetic-model", provider: "microsoft-foundry",
|
||||
api: "openai-responses", baseUrl: "https://example.services.ai.azure.com/openai/v1",
|
||||
reasoning: false, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 1000, maxTokens: 100 },
|
||||
});
|
||||
assert(result?.apiKey && result.request?.auth?.mode === "authorization-bearer");
|
||||
assert(result.apiKey.includes(tenant(index)), "prepared token must belong to the selected synthetic tenant");
|
||||
};
|
||||
const calls = async () => (await fs.readFile(path.join(proofDir, "calls.jsonl"), "utf8")).trim().split("\n").length;
|
||||
const started = performance.now();
|
||||
for (let index = 0; index < 128; index++) {
|
||||
await prepareForTenant(index);
|
||||
}
|
||||
assert.equal(await calls(), 128);
|
||||
await prepareForTenant(0);
|
||||
assert.equal(await calls(), 128);
|
||||
await prepareForTenant(128);
|
||||
assert.equal(await calls(), 129);
|
||||
await prepareForTenant(0);
|
||||
const afterRetouched = await calls();
|
||||
await prepareForTenant(1);
|
||||
const afterOldest = await calls();
|
||||
console.info("[foundry-public-provider-cache-proof]", JSON.stringify({
|
||||
seededTuples: 128, afterRetouched, afterOldest,
|
||||
allTenantBindingsCorrect: true, elapsedMs: performance.now() - started,
|
||||
rssBytes: process.memoryUsage().rss,
|
||||
}));
|
||||
assert.equal(afterRetouched, 129);
|
||||
assert.equal(afterOldest, 130, "the evicted account must refresh through az");
|
||||
`,
|
||||
);
|
||||
// The child has its own real profile store and token cache. Only az is synthetic;
|
||||
// no ambient credentials or the parent Vitest mocks enter its provider graph.
|
||||
const { stdout } = await runExec(
|
||||
process.execPath,
|
||||
["--import", "tsx/esm", script, proofDir, new URL("./index.ts", import.meta.url).href],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
signal,
|
||||
timeoutMs: 150_000,
|
||||
logOutput: false,
|
||||
baseEnv: {
|
||||
PATH: [
|
||||
binDir,
|
||||
path.dirname(process.execPath),
|
||||
...(process.platform === "win32" ? [] : ["/usr/bin", "/bin"]),
|
||||
].join(path.delimiter),
|
||||
HOME: homeDir,
|
||||
USERPROFILE: homeDir,
|
||||
TMPDIR: proofDir,
|
||||
TEMP: proofDir,
|
||||
TMP: proofDir,
|
||||
SystemRoot: process.env.SystemRoot,
|
||||
ComSpec: process.env.ComSpec,
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
AZURE_CONFIG_DIR: azureDir,
|
||||
TSX_TSCONFIG_PATH: path.join(repoRoot, "tsconfig.json"),
|
||||
TSX_DISABLE_CACHE: "1",
|
||||
NODE_ENV: "test",
|
||||
CI: "1",
|
||||
},
|
||||
},
|
||||
);
|
||||
const observed = stdout
|
||||
.split("\n")
|
||||
.find((line) => line.startsWith("[foundry-public-provider-cache-proof] "));
|
||||
expect(observed).toBeDefined();
|
||||
console.info(observed);
|
||||
} finally {
|
||||
await fs.rm(proofDir, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("clears failed refresh state so later concurrent retries succeed", async () => {
|
||||
const provider = registerProvider();
|
||||
const prepareRuntimeAuth = requirePrepareRuntimeAuth(provider);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
// Microsoft Foundry plugin module implements runtime behavior.
|
||||
import { pruneMapToMaxSize } from "openclaw/plugin-sdk/collection-runtime";
|
||||
import type {
|
||||
ProviderPreparedRuntimeAuth,
|
||||
ProviderPrepareRuntimeAuthContext,
|
||||
|
|
@ -27,6 +28,9 @@ import {
|
|||
const cachedTokens = new Map<string, CachedTokenEntry>();
|
||||
const refreshPromises = new Map<string, Promise<{ apiKey: string; expiresAt: number }>>();
|
||||
const FOUNDRY_TOKEN_FALLBACK_LIFETIME_MS = 55 * 60 * 1000;
|
||||
// Bound settled credential material across profile generations. In-flight
|
||||
// refresh ownership remains separate so admission never evicts active work.
|
||||
const FOUNDRY_TOKEN_CACHE_MAX_ENTRIES = 128;
|
||||
|
||||
async function refreshEntraToken(params?: {
|
||||
scope?: string;
|
||||
|
|
@ -40,10 +44,16 @@ async function refreshEntraToken(params?: {
|
|||
asDateTimestampMs(rawExpiry) ??
|
||||
resolveExpiresAtMsFromDurationMs(FOUNDRY_TOKEN_FALLBACK_LIFETIME_MS, { nowMs: now }) ??
|
||||
now;
|
||||
for (const [cacheKey, cachedToken] of cachedTokens) {
|
||||
if (cachedToken.expiresAt <= now) {
|
||||
cachedTokens.delete(cacheKey);
|
||||
}
|
||||
}
|
||||
cachedTokens.set(getFoundryTokenCacheKey(params), {
|
||||
token: result.accessToken,
|
||||
expiresAt,
|
||||
});
|
||||
pruneMapToMaxSize(cachedTokens, FOUNDRY_TOKEN_CACHE_MAX_ENTRIES);
|
||||
return { apiKey: result.accessToken, expiresAt };
|
||||
}
|
||||
|
||||
|
|
@ -107,6 +117,9 @@ export async function prepareFoundryRuntimeAuth(
|
|||
const refreshAfterMs =
|
||||
resolveExpiresAtMsFromDurationMs(TOKEN_REFRESH_MARGIN_MS, { nowMs: now }) ?? now;
|
||||
if (cachedToken && hasValidClock && cachedToken.expiresAt > refreshAfterMs) {
|
||||
// Map insertion order is the eviction order; touch valid hits to retain active accounts.
|
||||
cachedTokens.delete(cacheKey);
|
||||
cachedTokens.set(cacheKey, cachedToken);
|
||||
return {
|
||||
apiKey: cachedToken.token,
|
||||
expiresAt: cachedToken.expiresAt,
|
||||
|
|
@ -116,6 +129,7 @@ export async function prepareFoundryRuntimeAuth(
|
|||
},
|
||||
};
|
||||
}
|
||||
cachedTokens.delete(cacheKey);
|
||||
let refreshPromise = refreshPromises.get(cacheKey);
|
||||
if (!refreshPromise) {
|
||||
refreshPromise = refreshEntraToken({
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue