fix(call-logs): hold the artifact ceiling when files are already gone (#16354)

Thanks @maxmad64bis — call-log rotation holds the artifact ceiling when the files are already gone, in `src/lib/usage/callLogRotation.ts`, covered by `tests/unit/call-log-file-rotation.test.ts`.

Boarded on release/v3.8.52 at c188c8e682. Focused tests: 145 passed, 0 failed. Changelog integrity passed. File-size kept the inherited executeTargetAttempt.ts overage (1288 > 1284). Three siblings were ejected from this wave because they grow frozen files: #16388 (auth.ts 3588 > 3587), #16356 (imageGeneration.ts 3423 > 3390 and its test 2347 > 2216), #16387 (roundRobinCombo.ts 1294 > 1288). This pull request does not touch those files.

⚠️ base-red inherited: #15306

changelog.d/fixes/16354-call-log-overflow-clear-missing-references.md

@@ -0,0 +1 @@
+- **fix(call-logs):** Hold the artifact ceiling even when files are already gone ([#16354](https://github.com/diegosouzapw/OmniRoute/pull/16354)) — thanks @maxmad64bis

src/lib/usage/callLogRotation.ts

@@ -14,11 +14,7 @@ import {
   selectCallLogIdsBefore,
   selectOverflowArtifactPaths,
 } from "./callLogsBoundedQueries";
-import {
-  CALL_LOGS_DIR,
-  deleteCallArtifact,
-  type CallLogDetailState,
-} from "./callLogArtifacts";
+import { CALL_LOGS_DIR, deleteCallArtifact, type CallLogDetailState } from "./callLogArtifacts";
 import { getCallLogMaxEntries, getCallLogRetentionDays, getCallLogsTableMaxRows } from "../logEnv";
 import { isSqlitePagerCorruptError, notePagerCorruption } from "../db/healthCheck";
 
@@ -54,6 +50,18 @@ export function clearArtifactReference(relativePath: string, nextState: CallLogD
 // under the limit so each DELETE/SELECT stays valid.
 const DELETE_ID_CHUNK_SIZE = 500;
 
+function takeOverflowArtifactSlot(relativePath: string, baseDir: string): boolean {
+  if (deleteCallArtifact(relativePath, baseDir)) {
+    clearArtifactReference(relativePath, "missing");
+    return true;
+  }
+  if (relativePath && fs.existsSync(path.join(baseDir, relativePath))) return false;
+  // Reference without a file: free it like a detail read does,
+  // so later passes select new rows instead of stalling.
+  clearArtifactReference(relativePath, "missing");
+  return true;
+}
+
 function deleteCallLogRowsByIds(ids: string[]): DeleteResult {
   if (ids.length === 0) {
     return { deletedRows: 0, deletedArtifacts: 0 };
@@ -242,12 +250,9 @@ export function cleanupOverflowCallLogFiles(
       if (paths.length === 0) break;
       let progress = 0;
       for (const relativePath of paths) {
-        if (deleteCallArtifact(relativePath, baseDir)) {
-          clearArtifactReference(relativePath, "missing");
-          deleted++;
-          progress++;
-        }
+        if (takeOverflowArtifactSlot(relativePath, baseDir)) progress++;
       }
+      deleted += progress;
       if (progress === 0) break;
     }
     return deleted;

tests/unit/call-log-file-rotation.test.ts

@@ -357,6 +357,228 @@ test("overflow cleanup removes at most 100 artifacts and continues across calls"
   );
 });
 
+test("overflow cleanup frees references whose files are already gone", () => {
+  assert.ok(CALL_LOGS_DIR);
+  fs.rmSync(CALL_LOGS_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+  fs.mkdirSync(CALL_LOGS_DIR, { recursive: true });
+
+  // Nine rows, ceiling 3: rows 8, 7, 6 stay under the ceiling (all with a
+  // file), rows 5..0 are overflow. Files exist for ranks {0, 3, 6, 7, 8} so
+  // every kept rank has its file; ranks {1, 2, 4, 5} are missing.
+  const now = Date.now();
+  const present = new Set([0, 3, 6, 7, 8]);
+  const relPaths: string[] = [];
+  for (let i = 0; i < 9; i++) {
+    const relPath = `2026-04-06/${String(i).padStart(4, "0")}.json`;
+    relPaths.push(relPath);
+    if (present.has(i)) {
+      const absPath = path.join(CALL_LOGS_DIR, relPath);
+      fs.mkdirSync(path.dirname(absPath), { recursive: true });
+      fs.writeFileSync(absPath, "{}");
+    }
+    insertCallLog({
+      id: `gone-${i}`,
+      timestamp: new Date(now + i).toISOString(),
+      artifact_relpath: relPath,
+    });
+  }
+
+  // Overflow window (OFFSET 3) is ranks 5, 4, 3, 2, 1, 0: 2 deletions
+  // (ranks 3, 0) + 4 freed missing references (ranks 5, 4, 2, 1).
+  assert.equal(cleanupOverflowCallLogFiles(CALL_LOGS_DIR, 3), 6);
+
+  const db = core.getDbInstance();
+  assert.equal(
+    (
+      db
+        .prepare("SELECT COUNT(*) AS cnt FROM call_logs WHERE artifact_relpath IS NOT NULL")
+        .get() as {
+        cnt: number;
+      }
+    ).cnt,
+    3
+  );
+  for (const i of [1, 2, 4, 5]) {
+    const row = db
+      .prepare("SELECT detail_state, artifact_relpath FROM call_logs WHERE id = ?")
+      .get(`gone-${i}`) as { detail_state: string; artifact_relpath: string | null };
+    assert.equal(row.detail_state, "missing");
+    assert.equal(row.artifact_relpath, null);
+  }
+  for (const i of [6, 7, 8]) {
+    const row = db
+      .prepare("SELECT detail_state, artifact_relpath FROM call_logs WHERE id = ?")
+      .get(`gone-${i}`) as { detail_state: string; artifact_relpath: string | null };
+    assert.equal(row.detail_state, "ready");
+    assert.notEqual(row.artifact_relpath, null);
+    assert.equal(fs.existsSync(path.join(CALL_LOGS_DIR, relPaths[i])), true);
+  }
+  for (const i of [0, 3]) {
+    assert.equal(fs.existsSync(path.join(CALL_LOGS_DIR, relPaths[i])), false);
+  }
+});
+
+test("overflow cleanup frees a full window of missing files and lets the next pass finish", () => {
+  assert.ok(CALL_LOGS_DIR);
+  fs.rmSync(CALL_LOGS_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+  fs.mkdirSync(CALL_LOGS_DIR, { recursive: true });
+
+  // Six rows without any file, ceiling 2: the 4 oldest are overflow, the 2
+  // newest stay under the ceiling and keep their references (no sweep below
+  // the ceiling).
+  const now = Date.now();
+  for (let i = 0; i < 6; i++) {
+    insertCallLog({
+      id: `empty-${i}`,
+      timestamp: new Date(now + i).toISOString(),
+      artifact_relpath: `2026-04-07/${String(i).padStart(4, "0")}.json`,
+    });
+  }
+
+  assert.equal(cleanupOverflowCallLogFiles(CALL_LOGS_DIR, 2), 4);
+
+  const db = core.getDbInstance();
+  for (const i of [0, 1, 2, 3]) {
+    const row = db
+      .prepare("SELECT detail_state, artifact_relpath FROM call_logs WHERE id = ?")
+      .get(`empty-${i}`) as { detail_state: string; artifact_relpath: string | null };
+    assert.equal(row.detail_state, "missing");
+    assert.equal(row.artifact_relpath, null);
+  }
+  for (const i of [4, 5]) {
+    const row = db
+      .prepare("SELECT detail_state, artifact_relpath FROM call_logs WHERE id = ?")
+      .get(`empty-${i}`) as { detail_state: string; artifact_relpath: string | null };
+    assert.equal(row.detail_state, "ready");
+    assert.notEqual(row.artifact_relpath, null);
+  }
+  assert.equal(cleanupOverflowCallLogFiles(CALL_LOGS_DIR, 2), 0);
+});
+
+test("overflow cleanup keeps references when deletion is refused", () => {
+  assert.ok(CALL_LOGS_DIR);
+  fs.rmSync(CALL_LOGS_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+  fs.mkdirSync(CALL_LOGS_DIR, { recursive: true });
+
+  const dayDir = path.join(CALL_LOGS_DIR, "2026-04-08");
+  fs.mkdirSync(dayDir, { recursive: true });
+  const newerRelPath = "2026-04-08/110000_keep.json";
+  const olderRelPath = "2026-04-08/100000_keep.json";
+  const newerFile = path.join(CALL_LOGS_DIR, newerRelPath);
+  const olderFile = path.join(CALL_LOGS_DIR, olderRelPath);
+  fs.writeFileSync(newerFile, "{}");
+  fs.writeFileSync(olderFile, "{}");
+  insertCallLog({
+    id: "refused-keep",
+    timestamp: "2026-04-08T11:00:00.000Z",
+    artifact_relpath: newerRelPath,
+  });
+  insertCallLog({
+    id: "refused-old",
+    timestamp: "2026-04-08T10:00:00.000Z",
+    artifact_relpath: olderRelPath,
+  });
+
+  const originalRmSync = fs.rmSync;
+  fs.rmSync = (targetPath, options) => {
+    if (targetPath === olderFile) {
+      throw new Error("simulated rm failure");
+    }
+    return originalRmSync.call(fs, targetPath, options);
+  };
+  let freed: number | undefined;
+  try {
+    freed = cleanupOverflowCallLogFiles(CALL_LOGS_DIR, 1);
+  } finally {
+    fs.rmSync = originalRmSync;
+  }
+  assert.equal(freed, 0);
+
+  const db = core.getDbInstance();
+  for (const id of ["refused-keep", "refused-old"]) {
+    const row = db
+      .prepare("SELECT detail_state, artifact_relpath FROM call_logs WHERE id = ?")
+      .get(id) as { detail_state: string; artifact_relpath: string | null };
+    assert.equal(row.detail_state, "ready");
+    assert.notEqual(row.artifact_relpath, null);
+  }
+  assert.equal(fs.existsSync(newerFile), true);
+  assert.equal(fs.existsSync(olderFile), true);
+});
+
+test("overflow cleanup frees an empty artifact reference instead of stalling", () => {
+  assert.ok(CALL_LOGS_DIR);
+  fs.rmSync(CALL_LOGS_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+  fs.mkdirSync(CALL_LOGS_DIR, { recursive: true });
+
+  // Ceiling 2: the two newest rows stay kept (both with a file); the empty
+  // reference and the witness row are both overflow. The harness maps '' to
+  // NULL, so the empty reference is inserted with raw SQL.
+  const now = Date.now();
+  const keptA = "2026-04-09/110000_keep_a.json";
+  const keptB = "2026-04-09/105000_keep_b.json";
+  const witness = "2026-04-09/100000_witness.json";
+  for (const relPath of [keptA, keptB, witness]) {
+    const absPath = path.join(CALL_LOGS_DIR, relPath);
+    fs.mkdirSync(path.dirname(absPath), { recursive: true });
+    fs.writeFileSync(absPath, "{}");
+  }
+  insertCallLog({
+    id: "empty-kept-a",
+    timestamp: new Date(now + 3).toISOString(),
+    artifact_relpath: keptA,
+  });
+  insertCallLog({
+    id: "empty-kept-b",
+    timestamp: new Date(now + 2).toISOString(),
+    artifact_relpath: keptB,
+  });
+  core
+    .getDbInstance()
+    .prepare(
+      `INSERT INTO call_logs (
+        id, timestamp, method, path, status, model, provider, account, detail_state, artifact_relpath,
+        has_request_body
+      )
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+    )
+    .run(
+      "empty-ref",
+      new Date(now + 1).toISOString(),
+      "POST",
+      "/v1/chat/completions",
+      200,
+      "openai/gpt-4.1",
+      "openai",
+      "acct",
+      "ready",
+      "",
+      1
+    );
+  insertCallLog({
+    id: "empty-witness",
+    timestamp: new Date(now).toISOString(),
+    artifact_relpath: witness,
+  });
+
+  assert.equal(cleanupOverflowCallLogFiles(CALL_LOGS_DIR, 2), 2);
+
+  const db = core.getDbInstance();
+  const emptyRow = db
+    .prepare("SELECT detail_state, artifact_relpath FROM call_logs WHERE id = ?")
+    .get("empty-ref") as { detail_state: string; artifact_relpath: string | null };
+  assert.equal(emptyRow.detail_state, "missing");
+  assert.equal(emptyRow.artifact_relpath, null);
+  const witnessRow = db
+    .prepare("SELECT detail_state, artifact_relpath FROM call_logs WHERE id = ?")
+    .get("empty-witness") as { detail_state: string; artifact_relpath: string | null };
+  assert.equal(witnessRow.detail_state, "missing");
+  assert.equal(witnessRow.artifact_relpath, null);
+  assert.equal(fs.existsSync(path.join(CALL_LOGS_DIR, witness)), false);
+  assert.equal(fs.existsSync(path.join(CALL_LOGS_DIR, keptA)), true);
+  assert.equal(fs.existsSync(path.join(CALL_LOGS_DIR, keptB)), true);
+});
+
 test("orphan cleanup scans at most 100 candidates, resumes, and protects fresh files", () => {
   assert.ok(CALL_LOGS_DIR);
   const dayDir = path.join(CALL_LOGS_DIR, "2026-04-04");