mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-10-19 13:30:37 +00:00
fix(logs): start the source-checkout call-log worker without tsx's disk cache (#17133)
Some checks are pending
Publish to Docker Hub / Resolve Docker release metadata (push) Waiting to run
Publish to Docker Hub / Preflight — amd64 runner availability (push) Blocked by required conditions
Publish to Docker Hub / Build Docker (linux/amd64) (push) Blocked by required conditions
Publish to Docker Hub / Build Docker (linux/arm64) (push) Blocked by required conditions
Publish to Docker Hub / Publish multi-arch manifests (push) Blocked by required conditions
Publish to Docker Hub / omni-build runner offline alert (push) Waiting to run
Lock released branch / lock-branch (push) Waiting to run
Lock released branch / guard-no-push-after-release (push) Waiting to run
Release-Green (continuous) / Validate active release branch (push) Waiting to run
Release-Green (continuous) / Validate main branch (push) Waiting to run
Release-Green (continuous) / Bank ratchet shrinks (push) Waiting to run
Release-Green (continuous) / Baseline headroom (push) Waiting to run
opencode-plugin CI / Test (Node 22) (push) Waiting to run
opencode-plugin CI / Test (Node 24) (push) Waiting to run
opencode-plugin CI / Test v2 (Node 22) (push) Waiting to run
opencode-plugin CI / Test v2 (Node 24) (push) Waiting to run
opencode-plugin CI / Build (push) Blocked by required conditions
Radar Export / publish-export (push) Waiting to run
Release acceptance / Release acceptance (push) Waiting to run
OpenSSF Scorecard / Scorecard analysis (push) Waiting to run
Some checks are pending
Publish to Docker Hub / Resolve Docker release metadata (push) Waiting to run
Publish to Docker Hub / Preflight — amd64 runner availability (push) Blocked by required conditions
Publish to Docker Hub / Build Docker (linux/amd64) (push) Blocked by required conditions
Publish to Docker Hub / Build Docker (linux/arm64) (push) Blocked by required conditions
Publish to Docker Hub / Publish multi-arch manifests (push) Blocked by required conditions
Publish to Docker Hub / omni-build runner offline alert (push) Waiting to run
Lock released branch / lock-branch (push) Waiting to run
Lock released branch / guard-no-push-after-release (push) Waiting to run
Release-Green (continuous) / Validate active release branch (push) Waiting to run
Release-Green (continuous) / Validate main branch (push) Waiting to run
Release-Green (continuous) / Bank ratchet shrinks (push) Waiting to run
Release-Green (continuous) / Baseline headroom (push) Waiting to run
opencode-plugin CI / Test (Node 22) (push) Waiting to run
opencode-plugin CI / Test (Node 24) (push) Waiting to run
opencode-plugin CI / Test v2 (Node 22) (push) Waiting to run
opencode-plugin CI / Test v2 (Node 24) (push) Waiting to run
opencode-plugin CI / Build (push) Blocked by required conditions
Radar Export / publish-export (push) Waiting to run
Release acceptance / Release acceptance (push) Waiting to run
OpenSSF Scorecard / Scorecard analysis (push) Waiting to run
In a source checkout the call-log artifact worker runs through tsx, and tsx indexes its whole disk cache (os.tmpdir()/tsx-<uid>) with one readdirSync the first time a thread transforms a file. On a shared dev box that directory reached ~585k entries and the worker's first write waited 5-7 s on the listing (0.4 s without it). callLogWorkerOptions() now starts a tsx-launched worker with TSX_DISABLE_CACHE=1 on top of the parent's environment; compiled workers (npm, standalone, Docker) keep the default options. Refs #17091, #17099
This commit is contained in:
parent
04b052b8c8
commit
14fe8e9cd6
3 changed files with 114 additions and 1 deletions
|
|
@ -0,0 +1 @@
|
|||
- **fix(logs):** in a source checkout (dev server, tests) the call-log artifact worker starts with `TSX_DISABLE_CACHE=1`: tsx lists its whole disk cache on a thread's first transform, and on a long-lived dev box (585k entries) the worker's first write waited 5-7 s on that listing instead of ~0.4 s. Compiled workers (npm, standalone, Docker) are unchanged.
|
||||
|
|
@ -85,6 +85,21 @@ export function resolveCallLogArtifactWorker(context: WorkerResolutionContext =
|
|||
return { workerFile: entryJs ?? cwdJs, execArgv: [] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Worker options for a resolved worker file. A source checkout runs the worker through tsx, which
|
||||
* lists its whole disk cache (os.tmpdir()/tsx-<uid>) with one readdirSync the first time a thread
|
||||
* transforms a file. A shared dev box grows that directory to hundreds of thousands of entries, and
|
||||
* the worker's first write then waited 5-7 s on the listing (0.4 s without it). The worker only
|
||||
* transforms a handful of small modules, so it runs without the disk cache.
|
||||
*/
|
||||
export function callLogWorkerOptions(execArgv: string[]): {
|
||||
execArgv: string[];
|
||||
env?: NodeJS.ProcessEnv;
|
||||
} {
|
||||
if (!execArgv.includes("tsx/esm")) return { execArgv };
|
||||
return { execArgv, env: { ...process.env, TSX_DISABLE_CACHE: "1" } };
|
||||
}
|
||||
|
||||
function clearIdleTimer(): void {
|
||||
if (!idleTimer) return;
|
||||
clearTimeout(idleTimer);
|
||||
|
|
@ -156,7 +171,10 @@ function ensureWorker(): Worker {
|
|||
const { workerFile, execArgv } = workerFileOverride ?? resolveCallLogArtifactWorker();
|
||||
// Reflect.construct keeps Next/Turbopack from interpreting the runtime-selected
|
||||
// worker path as a build-time glob and tracing tens of thousands of unrelated files.
|
||||
const created = Reflect.construct(Worker, [pathToFileURL(workerFile), { execArgv }]) as Worker;
|
||||
const created = Reflect.construct(Worker, [
|
||||
pathToFileURL(workerFile),
|
||||
callLogWorkerOptions(execArgv),
|
||||
]) as Worker;
|
||||
worker = created;
|
||||
created.on("message", (reply: WorkerReply) => {
|
||||
if (!active || reply.id !== active.id) return;
|
||||
|
|
|
|||
94
tests/unit/call-log-artifact-worker-tsx-cache.test.ts
Normal file
94
tests/unit/call-log-artifact-worker-tsx-cache.test.ts
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
// A source checkout runs the call-log artifact worker through tsx (`--import tsx/esm`). tsx lists
|
||||
// its whole disk cache (os.tmpdir()/tsx-<uid>) the first time a thread transforms a file; on the
|
||||
// shared devbox that cache held 585k entries and the worker's first reply took 5.3-7.5 s against
|
||||
// 0.4 s with TSX_DISABLE_CACHE=1. The probe worker below speaks the writer's protocol and reports
|
||||
// the TSX_DISABLE_CACHE value it was started with in `relPath`.
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-call-log-worker-tsx-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
|
||||
const {
|
||||
writeCallArtifactAsync,
|
||||
closeCallLogArtifactWriter,
|
||||
callLogWorkerOptions,
|
||||
__setCallLogWorkerOverrideForTests,
|
||||
} = await import("../../src/lib/usage/callLogArtifactWriter.ts");
|
||||
|
||||
const probeWorkerFile = path.join(TEST_DATA_DIR, "env-probe-worker.mjs");
|
||||
fs.writeFileSync(
|
||||
probeWorkerFile,
|
||||
[
|
||||
'import { parentPort } from "node:worker_threads";',
|
||||
'parentPort.on("message", ({ id }) => {',
|
||||
' const relPath = process.env.TSX_DISABLE_CACHE ?? "unset";',
|
||||
' parentPort.postMessage({ id, result: { relPath, sizeBytes: 0, sha256: "" } });',
|
||||
"});",
|
||||
"",
|
||||
].join("\n")
|
||||
);
|
||||
|
||||
test.after(async () => {
|
||||
__setCallLogWorkerOverrideForTests(null);
|
||||
await closeCallLogArtifactWriter(0);
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
||||
});
|
||||
|
||||
test("a worker started through tsx skips tsx's disk cache", async () => {
|
||||
__setCallLogWorkerOverrideForTests({
|
||||
workerFile: probeWorkerFile,
|
||||
execArgv: ["--import", "tsx/esm"],
|
||||
});
|
||||
const result = await writeCallArtifactAsync({
|
||||
schemaVersion: 5,
|
||||
summary: {
|
||||
id: "tsx-cache-probe",
|
||||
timestamp: "2026-10-19T00:00:00.000Z",
|
||||
method: "POST",
|
||||
path: "/v1/chat/completions",
|
||||
status: 200,
|
||||
model: "test-model",
|
||||
requestedModel: null,
|
||||
provider: "test-provider",
|
||||
account: "test-account",
|
||||
connectionId: null,
|
||||
duration: 1,
|
||||
tokens: {
|
||||
in: 1,
|
||||
out: 1,
|
||||
cacheRead: null,
|
||||
cacheWrite: null,
|
||||
reasoning: null,
|
||||
compressed: null,
|
||||
},
|
||||
requestType: "chat",
|
||||
sourceFormat: "openai",
|
||||
targetFormat: "openai",
|
||||
apiKeyId: null,
|
||||
apiKeyName: null,
|
||||
comboName: null,
|
||||
comboStepId: null,
|
||||
comboExecutionKey: null,
|
||||
},
|
||||
requestBody: {},
|
||||
responseBody: {},
|
||||
error: null,
|
||||
});
|
||||
assert.equal(result?.relPath, "1");
|
||||
});
|
||||
|
||||
test("a compiled worker keeps the default worker environment", () => {
|
||||
assert.deepEqual(callLogWorkerOptions([]), { execArgv: [] });
|
||||
});
|
||||
|
||||
test("a tsx worker gets the parent environment plus TSX_DISABLE_CACHE", () => {
|
||||
const options = callLogWorkerOptions(["--import", "tsx/esm"]);
|
||||
assert.deepEqual(options.execArgv, ["--import", "tsx/esm"]);
|
||||
assert.equal(options.env?.TSX_DISABLE_CACHE, "1");
|
||||
assert.equal(options.env?.DATA_DIR, process.env.DATA_DIR);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue