mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 03:03:12 +00:00
fix(test): recycle unit-fast CI batches (#51884)
* fix(test): recycle unit-fast ci batches * refactor(config): narrow discord timeout import * test(outbound): lighten target plugin stubs * refactor(auth): narrow env api key resolution * docs(auth): restore anthropic vertex sentinel comment
This commit is contained in:
parent
039ea5998e
commit
5069c771e7
6 changed files with 190 additions and 77 deletions
|
|
@ -384,6 +384,32 @@ const singletonBatchLaneCount =
|
|||
);
|
||||
const estimateUnitDurationMs = (file) =>
|
||||
unitTimingManifest.files[file]?.durationMs ?? unitTimingManifest.defaultDurationMs;
|
||||
const splitFilesByDurationBudget = (files, targetDurationMs, estimateDurationMs) => {
|
||||
if (!Number.isFinite(targetDurationMs) || targetDurationMs <= 0 || files.length <= 1) {
|
||||
return [files];
|
||||
}
|
||||
|
||||
const batches = [];
|
||||
let currentBatch = [];
|
||||
let currentDurationMs = 0;
|
||||
|
||||
for (const file of files) {
|
||||
const durationMs = estimateDurationMs(file);
|
||||
if (currentBatch.length > 0 && currentDurationMs + durationMs > targetDurationMs) {
|
||||
batches.push(currentBatch);
|
||||
currentBatch = [];
|
||||
currentDurationMs = 0;
|
||||
}
|
||||
currentBatch.push(file);
|
||||
currentDurationMs += durationMs;
|
||||
}
|
||||
|
||||
if (currentBatch.length > 0) {
|
||||
batches.push(currentBatch);
|
||||
}
|
||||
|
||||
return batches;
|
||||
};
|
||||
const unitSingletonBuckets =
|
||||
singletonBatchLaneCount > 0
|
||||
? packFilesByDuration(unitSingletonBatchFiles, singletonBatchLaneCount, estimateUnitDurationMs)
|
||||
|
|
@ -397,6 +423,11 @@ const unitFastLaneCount = Math.max(
|
|||
1,
|
||||
parseEnvNumber("OPENCLAW_TEST_UNIT_FAST_LANES", defaultUnitFastLaneCount),
|
||||
);
|
||||
const defaultUnitFastBatchTargetMs = isCI && !isWindows ? 45_000 : 0;
|
||||
const unitFastBatchTargetMs = parseEnvNumber(
|
||||
"OPENCLAW_TEST_UNIT_FAST_BATCH_TARGET_MS",
|
||||
defaultUnitFastBatchTargetMs,
|
||||
);
|
||||
// Heap snapshots on current main show long-lived unit-fast workers retaining
|
||||
// transformed Vitest/Vite module graphs rather than app objects. Multiple
|
||||
// bounded unit-fast lanes only help if we also recycle them serially instead
|
||||
|
|
@ -405,26 +436,34 @@ const unitFastBuckets =
|
|||
unitFastLaneCount > 1
|
||||
? packFilesByDuration(unitFastCandidateFiles, unitFastLaneCount, estimateUnitDurationMs)
|
||||
: [unitFastCandidateFiles];
|
||||
const unitFastEntries = unitFastBuckets
|
||||
.filter((files) => files.length > 0)
|
||||
.map((files, index) => ({
|
||||
name: unitFastBuckets.length === 1 ? "unit-fast" : `unit-fast-${String(index + 1)}`,
|
||||
serialPhase: "unit-fast",
|
||||
env: {
|
||||
OPENCLAW_VITEST_INCLUDE_FILE: writeTempJsonArtifact(
|
||||
`vitest-unit-fast-include-${String(index + 1)}`,
|
||||
files,
|
||||
),
|
||||
},
|
||||
args: [
|
||||
"vitest",
|
||||
"run",
|
||||
"--config",
|
||||
"vitest.unit.config.ts",
|
||||
`--pool=${useVmForks ? "vmForks" : "forks"}`,
|
||||
...(disableIsolation ? ["--isolate=false"] : []),
|
||||
],
|
||||
}));
|
||||
const unitFastEntries = unitFastBuckets.flatMap((files, index) => {
|
||||
const laneName = unitFastBuckets.length === 1 ? "unit-fast" : `unit-fast-${String(index + 1)}`;
|
||||
const recycledBatches = splitFilesByDurationBudget(
|
||||
files,
|
||||
unitFastBatchTargetMs,
|
||||
estimateUnitDurationMs,
|
||||
);
|
||||
return recycledBatches
|
||||
.filter((batch) => batch.length > 0)
|
||||
.map((batch, batchIndex) => ({
|
||||
name: recycledBatches.length === 1 ? laneName : `${laneName}-batch-${String(batchIndex + 1)}`,
|
||||
serialPhase: "unit-fast",
|
||||
env: {
|
||||
OPENCLAW_VITEST_INCLUDE_FILE: writeTempJsonArtifact(
|
||||
`vitest-unit-fast-include-${String(index + 1)}-${String(batchIndex + 1)}`,
|
||||
batch,
|
||||
),
|
||||
},
|
||||
args: [
|
||||
"vitest",
|
||||
"run",
|
||||
"--config",
|
||||
"vitest.unit.config.ts",
|
||||
`--pool=${useVmForks ? "vmForks" : "forks"}`,
|
||||
...(disableIsolation ? ["--isolate=false"] : []),
|
||||
],
|
||||
}));
|
||||
});
|
||||
const heavyUnitBuckets = packFilesByDuration(
|
||||
timedHeavyUnitFiles,
|
||||
heavyUnitLaneCount,
|
||||
|
|
|
|||
57
src/agents/model-auth-env.ts
Normal file
57
src/agents/model-auth-env.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import { getEnvApiKey } from "@mariozechner/pi-ai";
|
||||
import { getShellEnvAppliedKeys } from "../infra/shell-env.js";
|
||||
import { normalizeOptionalSecretInput } from "../utils/normalize-secret-input.js";
|
||||
import { hasAnthropicVertexAvailableAuth } from "./anthropic-vertex-provider.js";
|
||||
import { PROVIDER_ENV_API_KEY_CANDIDATES } from "./model-auth-env-vars.js";
|
||||
import { GCP_VERTEX_CREDENTIALS_MARKER } from "./model-auth-markers.js";
|
||||
import { normalizeProviderIdForAuth } from "./provider-id.js";
|
||||
|
||||
export type EnvApiKeyResult = {
|
||||
apiKey: string;
|
||||
source: string;
|
||||
};
|
||||
|
||||
export function resolveEnvApiKey(
|
||||
provider: string,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): EnvApiKeyResult | null {
|
||||
const normalized = normalizeProviderIdForAuth(provider);
|
||||
const applied = new Set(getShellEnvAppliedKeys());
|
||||
const pick = (envVar: string): EnvApiKeyResult | null => {
|
||||
const value = normalizeOptionalSecretInput(env[envVar]);
|
||||
if (!value) {
|
||||
return null;
|
||||
}
|
||||
const source = applied.has(envVar) ? `shell env: ${envVar}` : `env: ${envVar}`;
|
||||
return { apiKey: value, source };
|
||||
};
|
||||
|
||||
const candidates = PROVIDER_ENV_API_KEY_CANDIDATES[normalized];
|
||||
if (candidates) {
|
||||
for (const envVar of candidates) {
|
||||
const resolved = pick(envVar);
|
||||
if (resolved) {
|
||||
return resolved;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (normalized === "google-vertex") {
|
||||
const envKey = getEnvApiKey(normalized);
|
||||
if (!envKey) {
|
||||
return null;
|
||||
}
|
||||
return { apiKey: envKey, source: "gcloud adc" };
|
||||
}
|
||||
|
||||
if (normalized === "anthropic-vertex") {
|
||||
// Vertex AI uses GCP credentials (SA JSON or ADC), not API keys.
|
||||
// Return a sentinel so the model resolver still treats this provider as available.
|
||||
if (hasAnthropicVertexAvailableAuth(env)) {
|
||||
return { apiKey: GCP_VERTEX_CREDENTIALS_MARKER, source: "gcloud adc" };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import path from "node:path";
|
||||
import { type Api, getEnvApiKey, type Model } from "@mariozechner/pi-ai";
|
||||
import { type Api, type Model } from "@mariozechner/pi-ai";
|
||||
import { formatCliCommand } from "../cli/command-format.js";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import type { ModelProviderAuthMode, ModelProviderConfig } from "../config/types.js";
|
||||
|
|
@ -10,7 +10,6 @@ import {
|
|||
normalizeOptionalSecretInput,
|
||||
normalizeSecretInput,
|
||||
} from "../utils/normalize-secret-input.js";
|
||||
import { hasAnthropicVertexAvailableAuth } from "./anthropic-vertex-provider.js";
|
||||
import {
|
||||
type AuthProfileStore,
|
||||
ensureAuthProfileStore,
|
||||
|
|
@ -19,15 +18,14 @@ import {
|
|||
resolveAuthProfileOrder,
|
||||
resolveAuthStorePathForDisplay,
|
||||
} from "./auth-profiles.js";
|
||||
import { PROVIDER_ENV_API_KEY_CANDIDATES } from "./model-auth-env-vars.js";
|
||||
import { resolveEnvApiKey, type EnvApiKeyResult } from "./model-auth-env.js";
|
||||
import {
|
||||
CUSTOM_LOCAL_AUTH_MARKER,
|
||||
GCP_VERTEX_CREDENTIALS_MARKER,
|
||||
isKnownEnvApiKeyMarker,
|
||||
isNonSecretApiKeyMarker,
|
||||
OLLAMA_LOCAL_AUTH_MARKER,
|
||||
} from "./model-auth-markers.js";
|
||||
import { normalizeProviderId, normalizeProviderIdForAuth } from "./model-selection.js";
|
||||
import { normalizeProviderId } from "./model-selection.js";
|
||||
|
||||
export { ensureAuthProfileStore, resolveAuthProfileOrder } from "./auth-profiles.js";
|
||||
|
||||
|
|
@ -395,53 +393,10 @@ export async function resolveApiKeyForProvider(params: {
|
|||
);
|
||||
}
|
||||
|
||||
export type EnvApiKeyResult = { apiKey: string; source: string };
|
||||
export type ModelAuthMode = "api-key" | "oauth" | "token" | "mixed" | "aws-sdk" | "unknown";
|
||||
|
||||
export function resolveEnvApiKey(
|
||||
provider: string,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): EnvApiKeyResult | null {
|
||||
const normalized = normalizeProviderIdForAuth(provider);
|
||||
const applied = new Set(getShellEnvAppliedKeys());
|
||||
const pick = (envVar: string): EnvApiKeyResult | null => {
|
||||
const value = normalizeOptionalSecretInput(env[envVar]);
|
||||
if (!value) {
|
||||
return null;
|
||||
}
|
||||
const source = applied.has(envVar) ? `shell env: ${envVar}` : `env: ${envVar}`;
|
||||
return { apiKey: value, source };
|
||||
};
|
||||
|
||||
const candidates = PROVIDER_ENV_API_KEY_CANDIDATES[normalized];
|
||||
if (candidates) {
|
||||
for (const envVar of candidates) {
|
||||
const resolved = pick(envVar);
|
||||
if (resolved) {
|
||||
return resolved;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (normalized === "google-vertex") {
|
||||
const envKey = getEnvApiKey(normalized);
|
||||
if (!envKey) {
|
||||
return null;
|
||||
}
|
||||
return { apiKey: envKey, source: "gcloud adc" };
|
||||
}
|
||||
|
||||
if (normalized === "anthropic-vertex") {
|
||||
// Vertex AI uses GCP credentials (SA JSON or ADC), not API keys.
|
||||
// Return a sentinel so the model resolver considers this provider available.
|
||||
if (hasAnthropicVertexAvailableAuth(env)) {
|
||||
return { apiKey: GCP_VERTEX_CREDENTIALS_MARKER, source: "gcloud adc" };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
export { resolveEnvApiKey } from "./model-auth-env.js";
|
||||
export type { EnvApiKeyResult } from "./model-auth-env.js";
|
||||
|
||||
export function resolveModelAuthMode(
|
||||
provider?: string,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import {
|
||||
DISCORD_DEFAULT_INBOUND_WORKER_TIMEOUT_MS,
|
||||
DISCORD_DEFAULT_LISTENER_TIMEOUT_MS,
|
||||
} from "../../extensions/discord/runtime-api.js";
|
||||
} from "../../extensions/discord/src/monitor/timeouts.js";
|
||||
import { MEDIA_AUDIO_FIELD_HELP } from "./media-audio-field-metadata.js";
|
||||
import { IRC_FIELD_HELP } from "./schema.irc.js";
|
||||
import { describeTalkSilenceTimeoutDefaults } from "./talk-defaults.js";
|
||||
|
|
|
|||
|
|
@ -1,16 +1,78 @@
|
|||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { telegramPlugin } from "../../../extensions/telegram/index.js";
|
||||
import { whatsappPlugin } from "../../../extensions/whatsapp/index.js";
|
||||
import { parseTelegramTarget } from "../../../extensions/telegram/src/targets.js";
|
||||
import { telegramOutbound, whatsappOutbound } from "../../../test/channel-outbounds.js";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import { setActivePluginRegistry } from "../../plugins/runtime.js";
|
||||
import { createTestRegistry } from "../../test-utils/channel-plugins.js";
|
||||
import { createOutboundTestPlugin, createTestRegistry } from "../../test-utils/channel-plugins.js";
|
||||
import { isWhatsAppGroupJid, normalizeWhatsAppTarget } from "../../whatsapp/normalize.js";
|
||||
import { resolveOutboundTarget } from "./targets.js";
|
||||
|
||||
const telegramMessaging = {
|
||||
parseExplicitTarget: ({ raw }: { raw: string }) => {
|
||||
const target = parseTelegramTarget(raw);
|
||||
return {
|
||||
to: target.chatId,
|
||||
threadId: target.messageThreadId,
|
||||
chatType: target.chatType === "unknown" ? undefined : target.chatType,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
const whatsappMessaging = {
|
||||
inferTargetChatType: ({ to }: { to: string }) => {
|
||||
const normalized = normalizeWhatsAppTarget(to);
|
||||
if (!normalized) {
|
||||
return undefined;
|
||||
}
|
||||
return isWhatsAppGroupJid(normalized) ? ("group" as const) : ("direct" as const);
|
||||
},
|
||||
targetResolver: {
|
||||
hint: "<E.164|group JID>",
|
||||
},
|
||||
};
|
||||
|
||||
export function installResolveOutboundTargetPluginRegistryHooks(): void {
|
||||
beforeEach(() => {
|
||||
setActivePluginRegistry(
|
||||
createTestRegistry([
|
||||
{ pluginId: "whatsapp", plugin: whatsappPlugin, source: "test" },
|
||||
{ pluginId: "telegram", plugin: telegramPlugin, source: "test" },
|
||||
{
|
||||
pluginId: "whatsapp",
|
||||
plugin: {
|
||||
...createOutboundTestPlugin({
|
||||
id: "whatsapp",
|
||||
label: "WhatsApp",
|
||||
outbound: whatsappOutbound,
|
||||
messaging: whatsappMessaging,
|
||||
}),
|
||||
config: {
|
||||
listAccountIds: () => [],
|
||||
resolveDefaultTo: ({ cfg }: { cfg: OpenClawConfig }) =>
|
||||
typeof cfg.channels?.whatsapp?.defaultTo === "string"
|
||||
? cfg.channels.whatsapp.defaultTo
|
||||
: undefined,
|
||||
},
|
||||
},
|
||||
source: "test",
|
||||
},
|
||||
{
|
||||
pluginId: "telegram",
|
||||
plugin: {
|
||||
...createOutboundTestPlugin({
|
||||
id: "telegram",
|
||||
label: "Telegram",
|
||||
outbound: telegramOutbound,
|
||||
messaging: telegramMessaging,
|
||||
}),
|
||||
config: {
|
||||
listAccountIds: () => [],
|
||||
resolveDefaultTo: ({ cfg }: { cfg: OpenClawConfig }) =>
|
||||
typeof cfg.channels?.telegram?.defaultTo === "string"
|
||||
? cfg.channels.telegram.defaultTo
|
||||
: undefined,
|
||||
},
|
||||
},
|
||||
source: "test",
|
||||
},
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { resolveEnvApiKey } from "../agents/model-auth.js";
|
||||
import { resolveEnvApiKey } from "../agents/model-auth-env.js";
|
||||
import type { OpenClawConfig } from "../config/types.js";
|
||||
import type { SecretInput } from "../config/types.secrets.js";
|
||||
import type { WizardPrompter } from "../wizard/prompts.js";
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue