diff --git a/crates/ruvector-sota-bench/harness/src/benchmark.ts b/crates/ruvector-sota-bench/harness/src/benchmark.ts index 03d0652ae..b955c3d77 100644 --- a/crates/ruvector-sota-bench/harness/src/benchmark.ts +++ b/crates/ruvector-sota-bench/harness/src/benchmark.ts @@ -89,6 +89,23 @@ export const POLICY_LEVERS: readonly string[] = Object.freeze([ "ef_search", "m", "ef_construction", "k", "runner_set", ]); +/** + * Ranges `normalizePolicy` enforces for the integer levers. + * + * Exported so a declared manifest can be checked against the values the runner + * will actually accept. Without this a manifest can advertise + * `ef_search: {1, 999999}`, look authoritative, and still have every run above + * 4096 rejected at spawn time — a declaration that cannot be effected. + * `runner_set` is absent because it is an enum, validated above. + */ +export const POLICY_LEVER_RANGES: Readonly> = + Object.freeze({ + ef_search: Object.freeze({ minimum: 1, maximum: 4096 }), + m: Object.freeze({ minimum: 2, maximum: 128 }), + ef_construction: Object.freeze({ minimum: 4, maximum: 4096 }), + k: Object.freeze({ minimum: 1, maximum: 100 }), + }); + export function normalizePolicy(policy: Record): BenchmarkPolicy { const runnerSet = policy.runner_set ?? "core"; if (runnerSet !== "core" && runnerSet !== "core-lsm" && runnerSet !== "all") { diff --git a/crates/ruvector-sota-bench/harness/src/flywheel.ts b/crates/ruvector-sota-bench/harness/src/flywheel.ts index 0c4f835c4..034ca077c 100644 --- a/crates/ruvector-sota-bench/harness/src/flywheel.ts +++ b/crates/ruvector-sota-bench/harness/src/flywheel.ts @@ -59,19 +59,20 @@ interface RuvectorScore extends Score { peakRssBytes: number; vetoReasons: string[]; observations: BenchmarkObservation[]; - /** - * Frontier the candidate must not be dominated by. Defaults to the incumbent - * baseline when a run has no wider frontier to hand. - */ - frontier?: ParetoPoint[]; } -/** Objective vector for the in-repo Pareto gate (ADR-335). */ +/** + * Objective vector for the in-repo Pareto gate (ADR-335). + * + * The frontier is the incumbent baseline, and deliberately nothing the + * candidate can influence. An earlier revision let a score carry its own + * `frontier` array, which let the judged object choose its judges: an empty + * array admitted everything, silently and without a reason string. Widening + * this beyond the incumbent must arrive through the caller's options, never + * through the evaluator's score object or a deserialized replay bundle. + */ function paretoPoint(id: string, score: RuvectorScore): ParetoPoint { - return { - id, - values: { primary: score.primary, costPerWin: score.costPerWin, p99Us: score.p99Us }, - }; + return { id, values: { primary: score.primary, costPerWin: score.costPerWin } }; } export interface RuvectorFlywheelOptions { @@ -108,15 +109,26 @@ export function ruvectorPromotionRule(evidence: { if (evidence.anchor && evidence.anchor.candidate < evidence.anchor.baseline) reasons.push("anchor_regressed"); // In-repo Pareto gate (ADR-335): a candidate dominated on the objective // vector cannot be promoted even when the paired test passes, because a - // scalar improvement bought with a strictly worse cost and latency profile - // is a trade rather than a win. Non-finite objectives throw here rather than - // comparing, since NaN defeats `<` in both directions. - const admission = admitToFrontier( - paretoPoint("candidate", candidate), - candidate.frontier ?? [paretoPoint("baseline", baseline)], - ); - if (!admission.admitted) { - reasons.push(`pareto_dominated_by_${admission.dominatedBy.join("_")}`); + // scalar improvement bought with a strictly worse resource profile is a + // trade rather than a win. + // + // The pure API throws on a non-finite or missing objective, which is right + // for a library but wrong here: `@metaharness/flywheel` calls this rule + // without a try/catch from both its generation loop and `verifyReplayBundle`, + // so an escaping throw would abort the run in one case and, in the other, + // destroy the structured `checks` output that the replay assertion below + // depends on. Unusable evidence is a reason to REFUSE the candidate, not to + // kill the harness, so it is converted into a blocking reason here. + try { + const admission = admitToFrontier( + paretoPoint("candidate", candidate), + [paretoPoint("baseline", baseline)], + ); + if (!admission.admitted) { + reasons.push(`pareto_dominated_by_${admission.dominatedBy.join("_")}`); + } + } catch { + reasons.push("non_finite_objective"); } return { promote: reasons.length === 0, reasons }; } diff --git a/crates/ruvector-sota-bench/harness/src/optimizationManifest.ts b/crates/ruvector-sota-bench/harness/src/optimizationManifest.ts index 0e0766f2a..cbb449acc 100644 --- a/crates/ruvector-sota-bench/harness/src/optimizationManifest.ts +++ b/crates/ruvector-sota-bench/harness/src/optimizationManifest.ts @@ -14,6 +14,15 @@ * research manifest from it. The manifest is the standing policy; the research * manifest is one run under that policy. * + * STATUS: this module is currently INERT. Nothing in `runRuvectorFlywheel` + * loads a manifest, and `declaredObjectives()` has no caller — the promotion + * rule uses `DEFAULT_OBJECTIVES` directly. So `promotion.pareto` is a + * *declaration* that is checked for self-consistency, not policy that is + * enforced at runtime. Registration in `schema_validate.py` is likewise by + * directory glob, so naming the schema there is documentation rather than + * wiring. Treat a passing manifest as "this repository has stated its intent + * coherently", not as "this repository is being optimized under it". + * * Scope note, deliberately narrow: NVIDIA's skillpack also carries * domain-specific optimization *knowledge* (what to try, in what order) and an * adversarial-review step performed by a separate agent. This manifest encodes @@ -23,8 +32,8 @@ */ import { readFile } from "node:fs/promises"; -import { POLICY_LEVERS } from "./benchmark.js"; -import type { ObjectiveSpec } from "./pareto.js"; +import { POLICY_LEVER_RANGES, POLICY_LEVERS } from "./benchmark.js"; +import { DEFAULT_OBJECTIVES, type ObjectiveSpec } from "./pareto.js"; export interface ManifestObjective { name: string; @@ -145,7 +154,26 @@ export function assertOptimizationManifest(value: unknown): asserts value is Opt } if (seen.has(name)) throw new Error(`optimization manifest repeats lever ${name}`); seen.add(name); - if (lever.values !== undefined) requireStringArray(lever.values, `lever ${name} values`); + // A declared range or value set must be one the runner can actually + // effect. Otherwise a manifest can advertise a lever setting that + // `normalizePolicy` rejects at spawn time -- an authorization that looks + // real and does nothing. + const runnerRange = POLICY_LEVER_RANGES[name]; + if (lever.values !== undefined) { + const values = requireStringArray(lever.values, `lever ${name} values`); + if (runnerRange) { + for (const raw of values) { + const parsed = Number(raw); + if (!Number.isSafeInteger(parsed) + || parsed < runnerRange.minimum || parsed > runnerRange.maximum) { + throw new Error( + `optimization manifest lever ${name} declares ${raw}, outside the runner's ` + + `[${runnerRange.minimum}, ${runnerRange.maximum}]`, + ); + } + } + } + } if (lever.bounds !== undefined) { const bounds = requireObject(lever.bounds, `lever ${name} bounds`); const minimum = requireFinite(bounds.minimum, `lever ${name} bounds.minimum`); @@ -153,6 +181,15 @@ export function assertOptimizationManifest(value: unknown): asserts value is Opt if (minimum > maximum) { throw new Error(`optimization manifest lever ${name} has minimum above maximum`); } + if (!runnerRange) { + throw new Error(`optimization manifest lever ${name} takes values, not bounds`); + } + if (minimum < runnerRange.minimum || maximum > runnerRange.maximum) { + throw new Error( + `optimization manifest lever ${name} declares [${minimum}, ${maximum}], outside the ` + + `runner's [${runnerRange.minimum}, ${runnerRange.maximum}]`, + ); + } } } @@ -182,14 +219,32 @@ export function assertOptimizationManifest(value: unknown): asserts value is Opt if (!Array.isArray(pareto.objectives) || pareto.objectives.length === 0) { throw new Error("optimization manifest promotion.pareto requires objectives"); } + // Cross-check against the vector the promotion rule actually uses. A + // manifest declaring `{primary, minimize}` is harmless only while this + // module is inert; the day it is wired in, an inverted direction is a + // promotion gate that rewards regressions. Refuse it now, while the cost of + // being wrong is a failing test rather than a bad promotion. + const gateDirections = new Map(DEFAULT_OBJECTIVES.map((o) => [o.name, o.direction])); for (const raw of pareto.objectives) { const objectiveSpec = requireObject(raw, "pareto objective"); - if (typeof objectiveSpec.name !== "string" || !objectiveSpec.name) { + const name = objectiveSpec.name; + if (typeof name !== "string" || !name) { throw new Error("optimization manifest pareto objective requires a name"); } if (objectiveSpec.direction !== "maximize" && objectiveSpec.direction !== "minimize") { + throw new Error(`optimization manifest pareto objective ${name} needs a direction`); + } + const expected = gateDirections.get(name); + if (expected === undefined) { throw new Error( - `optimization manifest pareto objective ${objectiveSpec.name} needs a direction`, + `optimization manifest declares pareto objective ${name}, which the promotion gate ` + + `does not evaluate (it uses ${[...gateDirections.keys()].join(", ")})`, + ); + } + if (expected !== objectiveSpec.direction) { + throw new Error( + `optimization manifest declares ${name} as ${objectiveSpec.direction}, but the promotion ` + + `gate treats it as ${expected}`, ); } } diff --git a/crates/ruvector-sota-bench/harness/src/pareto.ts b/crates/ruvector-sota-bench/harness/src/pareto.ts index 49aa13b85..55cf90bf5 100644 --- a/crates/ruvector-sota-bench/harness/src/pareto.ts +++ b/crates/ruvector-sota-bench/harness/src/pareto.ts @@ -31,16 +31,25 @@ export interface ParetoPoint { } /** - * The promotion objective vector. + * The promotion objective vector: quality against the resource bought with it. * - * `primary` is the aggregate benchmark score (higher is better); `costPerWin` - * and `p99Us` are the resource and tail-latency axes the promotion rule - * already reasons about in `ruvectorPromotionRule`. + * Deliberately TWO axes, not three. Dominance requires "no worse on every + * axis", so each axis added is one more chance for the condition to fail: + * more axes can only ever turn *dominated* into *not dominated*, never the + * reverse. A Pareto gate therefore gets monotonically weaker as axes are + * added, and this gate exists to block trades dressed as wins. + * + * `p99Us` was on this vector and has been removed, because it is not + * independent evidence — it is already a factor of `costPerWin` + * (`memoryMb * p99Us / qps`) and a term inside `primary`'s `darwinScore`. + * Counting it a third time bought no information and cost real blocking + * power: with it present, a candidate burning 10x the memory to halve tail + * latency is *not* dominated and clears the gate; with two axes the incumbent + * dominates it and the gate fires. */ export const DEFAULT_OBJECTIVES: readonly ObjectiveSpec[] = Object.freeze([ Object.freeze({ name: "primary", direction: "maximize" as const }), Object.freeze({ name: "costPerWin", direction: "minimize" as const }), - Object.freeze({ name: "p99Us", direction: "minimize" as const }), ]); function assertObjectives(objectives: readonly ObjectiveSpec[]): void { @@ -75,6 +84,25 @@ function readObjective(point: ParetoPoint, objective: ObjectiveSpec): number { return value; } +/** + * Read every objective of every point before any of them are compared. + * + * This must happen as a separate pass. The comparison loop below short-circuits + * on the first axis where `a` is worse, so validating inside that loop would + * only ever check the axes the loop happens to reach — leaving a non-finite + * value on a later axis completely unexamined. That is not a hypothetical: it + * is precisely how a NaN reaches a comparison and reads as "not worse" on + * every test, which is the failure this module exists to prevent. + */ +function assertComparable( + points: readonly ParetoPoint[], + objectives: readonly ObjectiveSpec[], +): void { + for (const point of points) { + for (const objective of objectives) readObjective(point, objective); + } +} + /** True when `a` is at least as good on every objective and strictly better on one. */ export function dominates( a: ParetoPoint, @@ -82,6 +110,7 @@ export function dominates( objectives: readonly ObjectiveSpec[] = DEFAULT_OBJECTIVES, ): boolean { assertObjectives(objectives); + assertComparable([a, b], objectives); let strictlyBetterSomewhere = false; for (const objective of objectives) { const left = readObjective(a, objective); @@ -107,11 +136,7 @@ export function frontier( objectives: readonly ObjectiveSpec[] = DEFAULT_OBJECTIVES, ): ParetoPoint[] { assertObjectives(objectives); - // Read every objective up front so a malformed point fails the whole call - // rather than only when comparison order happens to reach it. - for (const point of points) { - for (const objective of objectives) readObjective(point, objective); - } + assertComparable(points, objectives); return points.filter( (candidate) => !points.some((other) => other !== candidate && dominates(other, candidate, objectives)), ); @@ -138,6 +163,7 @@ export function admitToFrontier( objectives: readonly ObjectiveSpec[] = DEFAULT_OBJECTIVES, ): FrontierAdmission { assertObjectives(objectives); + assertComparable([candidate, ...current], objectives); const dominatedBy = current .filter((member) => dominates(member, candidate, objectives)) .map((member) => member.id) diff --git a/crates/ruvector-sota-bench/harness/test/flywheel.test.ts b/crates/ruvector-sota-bench/harness/test/flywheel.test.ts index 1a628c3f2..c9413176b 100644 --- a/crates/ruvector-sota-bench/harness/test/flywheel.test.ts +++ b/crates/ruvector-sota-bench/harness/test/flywheel.test.ts @@ -71,7 +71,7 @@ interface ScoreShape { observations: unknown[]; } -function score(primary: number, costPerWin: number, p99Us: number, samples: number[]): ScoreShape { +function score(primary: number, costPerWin: number, samples: number[]): ScoreShape { return { primary, primarySamples: samples, @@ -79,7 +79,7 @@ function score(primary: number, costPerWin: number, p99Us: number, samples: numb recallAt10: 0.96, qps: 2_000, memoryMb: 40, - p99Us, + p99Us: 500, peakRssBytes: 0, costPerWin, regressed: false, @@ -97,14 +97,14 @@ const reasonsFor = (baseline: ScoreShape, candidate: ScoreShape): string[] => test("a candidate dominated by the incumbent is refused on the Pareto gate", () => { // Equal accuracy bought with strictly worse cost and latency: the baseline // dominates it on the objective vector, so the frontier gate must fire. - const baseline = score(0.9, 10, 100, [0.9, 0.9, 0.9, 0.9, 0.9]); - const dominated = score(0.9, 20, 200, [0.9, 0.9, 0.9, 0.9, 0.9]); + const baseline = score(0.9, 10, [0.9, 0.9, 0.9, 0.9, 0.9]); + const dominated = score(0.9, 20, [0.9, 0.9, 0.9, 0.9, 0.9]); assert.ok(reasonsFor(baseline, dominated).includes("pareto_dominated_by_baseline")); }); test("a candidate better on every objective clears the Pareto gate", () => { - const baseline = score(0.90, 20, 200, [0.90, 0.90, 0.90, 0.90, 0.90]); - const better = score(0.95, 10, 100, [0.95, 0.95, 0.95, 0.95, 0.95]); + const baseline = score(0.90, 20, [0.90, 0.90, 0.90, 0.90, 0.90]); + const better = score(0.95, 10, [0.95, 0.95, 0.95, 0.95, 0.95]); const reasons = reasonsFor(baseline, better); assert.ok(!reasons.some((reason) => reason.startsWith("pareto_dominated"))); assert.deepEqual(reasons, []); @@ -114,15 +114,50 @@ test("a trade is not domination: better accuracy at worse cost stays on the fron // Neither point dominates, so the Pareto gate stays silent and the existing // cost rule is what decides. The frontier must not become a second, hidden // cost gate. - const baseline = score(0.90, 10, 100, [0.90, 0.90, 0.90, 0.90, 0.90]); - const traded = score(0.95, 40, 400, [0.95, 0.95, 0.95, 0.95, 0.95]); + const baseline = score(0.90, 10, [0.90, 0.90, 0.90, 0.90, 0.90]); + const traded = score(0.95, 40, [0.95, 0.95, 0.95, 0.95, 0.95]); const reasons = reasonsFor(baseline, traded); assert.ok(!reasons.some((reason) => reason.startsWith("pareto_dominated"))); assert.ok(reasons.includes("resource_cost_worsened")); }); -test("a non-finite objective throws rather than silently promoting", () => { - const baseline = score(0.9, 10, 100, [0.9, 0.9, 0.9, 0.9, 0.9]); - const broken = score(Number.NaN, 10, 100, [0.95, 0.95, 0.95, 0.95, 0.95]); - assert.throws(() => reasonsFor(baseline, broken), /non-finite/); +test("unusable objectives block the candidate instead of killing the run", () => { + // The pure pareto API throws, but @metaharness/flywheel calls this rule with + // no try/catch from both its generation loop and verifyReplayBundle, so an + // escaping throw would abort the run and destroy the replay `checks` output. + // Unusable evidence must REFUSE the candidate, not take down the harness. + // + // Every case below is worse on primary (axis 1) and carries the bad value on + // costPerWin (axis 2) or omits a field entirely — the shapes the comparison + // loop would short-circuit past. + const baseline = score(0.9, 10, [0.9, 0.9, 0.9, 0.9, 0.9]); + const passing = [0.95, 0.95, 0.95, 0.95, 0.95]; + const broken: ScoreShape[] = [ + score(Number.NaN, 10, passing), + score(0.1, Number.NaN, passing), + score(0.1, Number.POSITIVE_INFINITY, passing), + score(0.1, Number.NEGATIVE_INFINITY, passing), + ]; + for (const candidate of broken) { + const reasons = reasonsFor(baseline, candidate); + assert.ok(reasons.includes("non_finite_objective"), JSON.stringify(reasons)); + assert.equal( + ruvectorPromotionRule({ baseline: baseline as never, candidate: candidate as never }).promote, + false, + ); + } + // A sealed score from before this vector existed: costPerWin absent. + const legacy = score(0.1, 0, passing) as Partial; + delete legacy.costPerWin; + const legacyReasons = reasonsFor(baseline, legacy as ScoreShape); + assert.ok(legacyReasons.includes("non_finite_objective"), JSON.stringify(legacyReasons)); +}); + +test("a candidate cannot supply its own frontier to switch the gate off", () => { + // An earlier revision read `candidate.frontier`, so an empty array admitted + // everything silently. The frontier is now the incumbent and nothing on the + // judged object can influence it. + const baseline = score(0.9, 10, [0.9, 0.9, 0.9, 0.9, 0.9]); + const dominated = { ...score(0.5, 1e9, [0.5, 0.5, 0.5, 0.5, 0.5]), frontier: [] }; + assert.ok(reasonsFor(baseline, dominated as ScoreShape).includes("pareto_dominated_by_baseline")); }); diff --git a/crates/ruvector-sota-bench/harness/test/optimizationManifest.test.ts b/crates/ruvector-sota-bench/harness/test/optimizationManifest.test.ts index a41e25d6d..4f3bde335 100644 --- a/crates/ruvector-sota-bench/harness/test/optimizationManifest.test.ts +++ b/crates/ruvector-sota-bench/harness/test/optimizationManifest.test.ts @@ -11,6 +11,7 @@ import { type OptimizationManifest, } from "../src/optimizationManifest.js"; import { POLICY_LEVERS } from "../src/benchmark.js"; +import { DEFAULT_OBJECTIVES } from "../src/pareto.js"; const repoRoot = resolve(import.meta.dirname, "../../../../.."); const FIXTURE = resolve(repoRoot, "schemas/fixtures/optimization-manifest-v1.json"); @@ -29,7 +30,7 @@ test("the shipped fixture manifest is accepted", async () => { assert.equal(manifest.repository, "ruvnet/ruvector"); assert.deepEqual(declaredLevers(manifest), ["ef_search", "m", "ef_construction", "runner_set"]); assert.deepEqual(declaredObjectives(manifest).map((entry) => entry.name), - ["primary", "costPerWin", "p99Us"]); + ["primary", "costPerWin"]); }); test("a lever the native runner would reject is refused", async () => { @@ -164,3 +165,52 @@ test("a manifest that is not valid JSON reports the file", async () => { loadOptimizationManifest(resolve(repoRoot, "README.md")), ); }); + +test("a pareto direction inverted from the real gate is refused", async () => { + // Harmless while the module is inert; an inverted promotion gate the day it + // is wired in. Refuse it now, while being wrong costs a failing test. + const manifest = await fixture(); + manifest.promotion.pareto.objectives = [ + { name: "primary", direction: "minimize" }, + { name: "costPerWin", direction: "minimize" }, + ]; + assert.throws(() => assertOptimizationManifest(manifest), + /declares primary as minimize, but the promotion gate treats it as maximize/); +}); + +test("a pareto objective the gate does not evaluate is refused", async () => { + const manifest = await fixture(); + manifest.promotion.pareto.objectives = [ + { name: "primary", direction: "maximize" }, + { name: "p99Us", direction: "minimize" }, + ]; + assert.throws(() => assertOptimizationManifest(manifest), + /declares pareto objective p99Us, which the promotion gate does not evaluate/); +}); + +test("declared objectives match the vector the promotion rule actually uses", async () => { + const manifest = await fixture(); + assert.deepEqual(declaredObjectives(manifest), [...DEFAULT_OBJECTIVES]); +}); + +test("a lever range the runner would reject is refused", async () => { + // normalizePolicy bounds ef_search to [1, 4096]. A manifest advertising more + // would look authoritative and be rejected at spawn time. + const manifest = await fixture(); + manifest.levers[0] = { name: "ef_search", bounds: { minimum: 1, maximum: 999_999 } }; + assert.throws(() => assertOptimizationManifest(manifest), + /declares \[1, 999999\], outside the runner's \[1, 4096\]/); +}); + +test("a lever value the runner would reject is refused", async () => { + const manifest = await fixture(); + manifest.levers[0] = { name: "ef_search", values: ["64", "999999"] }; + assert.throws(() => assertOptimizationManifest(manifest), + /declares 999999, outside the runner's \[1, 4096\]/); +}); + +test("bounds on an enum lever are refused", async () => { + const manifest = await fixture(); + manifest.levers[3] = { name: "runner_set", bounds: { minimum: 0, maximum: 2 } }; + assert.throws(() => assertOptimizationManifest(manifest), /takes values, not bounds/); +}); diff --git a/crates/ruvector-sota-bench/harness/test/pareto.test.ts b/crates/ruvector-sota-bench/harness/test/pareto.test.ts index f21a3fd2f..2ef97a71b 100644 --- a/crates/ruvector-sota-bench/harness/test/pareto.test.ts +++ b/crates/ruvector-sota-bench/harness/test/pareto.test.ts @@ -8,29 +8,29 @@ import { type ParetoPoint, } from "../src/pareto.js"; -const point = (id: string, primary: number, costPerWin: number, p99Us: number): ParetoPoint => ({ - id, values: { primary, costPerWin, p99Us }, +const point = (id: string, primary: number, costPerWin: number): ParetoPoint => ({ + id, values: { primary, costPerWin }, }); test("dominance requires no worse on every objective and better on one", () => { - const strong = point("strong", 0.9, 10, 100); - const weak = point("weak", 0.8, 20, 200); + const strong = point("strong", 0.9, 10); + const weak = point("weak", 0.8, 20); assert.equal(dominates(strong, weak), true); assert.equal(dominates(weak, strong), false); }); -test("a scalar win bought with worse cost and latency does not dominate", () => { - // The case the gate exists for: higher primary, but strictly worse on both - // resource axes. That is a trade, not a win, so neither point dominates. - const traded = point("traded", 0.95, 40, 400); - const incumbent = point("incumbent", 0.90, 10, 100); +test("a scalar win bought with worse cost does not dominate", () => { + // Higher primary, strictly worse cost. That is a trade, not a win, so + // neither point dominates. + const traded = point("traded", 0.95, 40); + const incumbent = point("incumbent", 0.90, 10); assert.equal(dominates(traded, incumbent), false); assert.equal(dominates(incumbent, traded), false); }); test("identical points do not dominate each other and both stay on the frontier", () => { - const a = point("a", 0.9, 10, 100); - const b = point("b", 0.9, 10, 100); + const a = point("a", 0.9, 10); + const b = point("b", 0.9, 10); assert.equal(dominates(a, b), false); assert.equal(dominates(b, a), false); assert.deepEqual(frontier([a, b]).map((entry) => entry.id), ["a", "b"]); @@ -38,49 +38,85 @@ test("identical points do not dominate each other and both stay on the frontier" test("frontier keeps the non-dominated set in input order", () => { const points = [ - point("dominated", 0.5, 50, 500), - point("fast", 0.8, 30, 50), - point("accurate", 0.95, 60, 300), - point("cheap", 0.7, 5, 400), + point("dominated", 0.5, 50), + point("fast", 0.8, 30), + point("accurate", 0.95, 60), + point("cheap", 0.7, 5), ]; assert.deepEqual(frontier(points).map((entry) => entry.id), ["fast", "accurate", "cheap"]); }); -test("non-finite objectives are refused rather than compared", () => { - // NaN defeats both `<` and `>`, so a NaN candidate would read as - // non-dominated against everything. Refuse it at the choke point instead. - const sane = point("sane", 0.9, 10, 100); +test("non-finite objectives are refused on EVERY axis, not just the first", () => { + // The comparison loop short-circuits on the first axis where a point is + // worse, so validating inside it would only ever check axes the loop + // reaches. A bad value on a LATER axis is the case that silently promotes: + // NaN defeats both `<` and `>`, so it reads as "not worse" everywhere. + // Every point here is deliberately worse on `primary` (axis 1) so the loop + // would return before ever seeing axis 2. + const sane = point("sane", 0.9, 10); for (const bad of [Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]) { - const broken = point("broken", bad, 10, 100); - assert.throws(() => dominates(broken, sane), /non-finite/); - assert.throws(() => dominates(sane, broken), /non-finite/); - assert.throws(() => frontier([sane, broken]), /non-finite/); - assert.throws(() => admitToFrontier(broken, [sane]), /non-finite/); + for (const broken of [ + { id: "axis1", values: { primary: bad, costPerWin: 10 } }, + { id: "axis2", values: { primary: 0.1, costPerWin: bad } }, + ] satisfies ParetoPoint[]) { + assert.throws(() => dominates(broken, sane), /non-finite/, `${broken.id} a-vs-b ${bad}`); + assert.throws(() => dominates(sane, broken), /non-finite/, `${broken.id} b-vs-a ${bad}`); + assert.throws(() => frontier([sane, broken]), /non-finite/, `${broken.id} frontier ${bad}`); + assert.throws(() => admitToFrontier(broken, [sane]), /non-finite/, `${broken.id} admit ${bad}`); + assert.throws(() => admitToFrontier(sane, [broken]), /non-finite/, `${broken.id} member ${bad}`); + } } }); -test("a missing objective is refused rather than treated as zero", () => { - const partial: ParetoPoint = { id: "partial", values: { primary: 0.9 } }; - assert.throws(() => dominates(partial, point("other", 0.8, 10, 100)), /missing objective/); +test("a missing objective is refused on any axis, including a later one", () => { + // The shape an older sealed score carries: a field the current vector + // expects simply is not there. + const sane = point("sane", 0.9, 10); + const missingFirst: ParetoPoint = { id: "missingFirst", values: { costPerWin: 10 } }; + const missingLater: ParetoPoint = { id: "missingLater", values: { primary: 0.1 } }; + for (const partial of [missingFirst, missingLater]) { + assert.throws(() => dominates(partial, sane), /missing objective/, partial.id); + assert.throws(() => dominates(sane, partial), /missing objective/, partial.id); + assert.throws(() => admitToFrontier(partial, [sane]), /missing objective/, partial.id); + assert.throws(() => admitToFrontier(sane, [partial]), /missing objective/, partial.id); + } +}); + +test("adding an axis can only weaken the gate, which is why the vector is two", () => { + // Records the MEDIUM-3 reasoning as an executable fact. Same two points: + // dominated on the shipped vector, NOT dominated once a third axis on which + // the candidate happens to be better is added. + const incumbent: ParetoPoint = { id: "incumbent", values: { primary: 0.9, costPerWin: 10, p99Us: 100 } }; + const heavy: ParetoPoint = { id: "heavy", values: { primary: 0.9, costPerWin: 50, p99Us: 50 } }; + assert.equal(dominates(incumbent, heavy), true); + assert.equal( + dominates(incumbent, heavy, [ + { name: "primary", direction: "maximize" }, + { name: "costPerWin", direction: "minimize" }, + { name: "p99Us", direction: "minimize" }, + ]), + false, + ); + assert.deepEqual(DEFAULT_OBJECTIVES.map((o) => o.name), ["primary", "costPerWin"]); }); test("admission blocks a dominated candidate and names the dominators", () => { - const current = [point("incumbent", 0.9, 10, 100), point("other", 0.85, 8, 90)]; - const worse = point("worse", 0.5, 50, 500); + const current = [point("incumbent", 0.9, 10), point("other", 0.85, 8)]; + const worse = point("worse", 0.5, 50); const admission = admitToFrontier(worse, current); assert.equal(admission.admitted, false); assert.deepEqual(admission.dominatedBy, ["incumbent", "other"]); }); test("an empty frontier admits, because absence of evidence is not domination", () => { - assert.deepEqual(admitToFrontier(point("first", 0.1, 999, 999), []), { + assert.deepEqual(admitToFrontier(point("first", 0.1, 999), []), { admitted: true, dominatedBy: [], }); }); test("objective specs are validated before any comparison", () => { - const a = point("a", 0.9, 10, 100); - const b = point("b", 0.8, 20, 200); + const a = point("a", 0.9, 10); + const b = point("b", 0.8, 20); assert.throws(() => dominates(a, b, []), /at least one objective/); assert.throws( () => dominates(a, b, [{ name: "primary", direction: "sideways" as never }]), diff --git a/schemas/fixtures/optimization-manifest-v1.json b/schemas/fixtures/optimization-manifest-v1.json index fc4ab4395..9f2fafd71 100644 --- a/schemas/fixtures/optimization-manifest-v1.json +++ b/schemas/fixtures/optimization-manifest-v1.json @@ -10,14 +10,43 @@ }, "benchmark": { "binary": "target/release/sota-all", - "runner_sets": ["core", "core-lsm", "all"], + "runner_sets": [ + "core", + "core-lsm", + "all" + ], "isolated": true }, "levers": [ - { "name": "ef_search", "bounds": { "minimum": 1, "maximum": 4096 } }, - { "name": "m", "bounds": { "minimum": 2, "maximum": 128 } }, - { "name": "ef_construction", "bounds": { "minimum": 4, "maximum": 4096 } }, - { "name": "runner_set", "values": ["core", "core-lsm", "all"] } + { + "name": "ef_search", + "bounds": { + "minimum": 1, + "maximum": 4096 + } + }, + { + "name": "m", + "bounds": { + "minimum": 2, + "maximum": 128 + } + }, + { + "name": "ef_construction", + "bounds": { + "minimum": 4, + "maximum": 4096 + } + }, + { + "name": "runner_set", + "values": [ + "core", + "core-lsm", + "all" + ] + } ], "protected_invariants": { "required_veto_providers": [ @@ -48,9 +77,14 @@ }, "pareto": { "objectives": [ - { "name": "primary", "direction": "maximize" }, - { "name": "costPerWin", "direction": "minimize" }, - { "name": "p99Us", "direction": "minimize" } + { + "name": "primary", + "direction": "maximize" + }, + { + "name": "costPerWin", + "direction": "minimize" + } ], "require_non_dominated": true } diff --git a/scripts/research-gate/schema_validate.py b/scripts/research-gate/schema_validate.py index 893126a74..75605f4f9 100644 --- a/scripts/research-gate/schema_validate.py +++ b/scripts/research-gate/schema_validate.py @@ -28,6 +28,9 @@ OVERRIDE_SCHEMA = "research-override-v1.json" BASE_GATE_SCHEMA = "research-base-gate-v1.json" ATTESTATION_SUBJECT_SCHEMA = "research-attestation-subject-v1.json" EMBEDDING_IDENTITY_SCHEMA = "embedding-space-identity-v1.json" +# ADR-335. Registration is by directory glob below, so naming a schema here +# does not wire it into anything -- it is the argument callers pass to +# ``validate_document``. No gate stage consumes an optimization manifest yet. OPTIMIZATION_MANIFEST_SCHEMA = "optimization-manifest-v1.json" diff --git a/scripts/research-gate/tests/test_research_gate.py b/scripts/research-gate/tests/test_research_gate.py index c62071937..5091580b8 100644 --- a/scripts/research-gate/tests/test_research_gate.py +++ b/scripts/research-gate/tests/test_research_gate.py @@ -465,6 +465,26 @@ class SchemaEnforcementTests(unittest.TestCase): # against the offline registry; a network-only $ref would raise here. schema_validate._validator(name) + def test_optimization_manifest_fixture_matches_its_schema(self): + # ADR-335. The manifest is not consumed by any gate stage yet, so + # without this the schema and the fixture could drift apart unnoticed + # and the named constant would be decorative. + fixture = ( + Path(__file__).resolve().parents[3] + / "schemas" / "fixtures" / "optimization-manifest-v1.json" + ) + document = json.loads(fixture.read_text(encoding="utf-8")) + schema_validate.validate_document( + document, schema_validate.OPTIMIZATION_MANIFEST_SCHEMA + ) + # The frontier requirement is a schema constant: a manifest must not be + # able to switch the gate off. + document["promotion"]["pareto"]["require_non_dominated"] = False + with self.assertRaises(schema_validate.SchemaValidationError): + schema_validate.validate_document( + document, schema_validate.OPTIMIZATION_MANIFEST_SCHEMA + ) + def test_manifest_violating_schema_only_rule_is_rejected(self): # selection_rule is required by the schema and by no hand-rolled check, # so this fails only if schema validation actually runs.