mirror of
https://github.com/ruvnet/RuVector.git
synced 2026-08-25 16:42:30 +00:00
fix(sota-bench): validate every Pareto axis, and drop to two objectives
Audit of the new Pareto gate found its headline safety property was
implemented for only one of its three axes, and that when the guard did
fire it took the run down with it.
`dominates` returned out of its comparison loop on the first axis where
the candidate was worse, before `readObjective` ran on the remaining ones,
so the non-finite guard only ever inspected an axis the loop reached.
Through the real promotion rule, a candidate carrying NaN in `costPerWin`,
NaN in `p99Us`, -Infinity in `costPerWin`, or simply MISSING the `p99Us`
field -- which is the shape an older replay bundle carries -- each produced
`{promote: true, reasons: []}`. Nothing else caught it: the cost rule
compares with `>`, and `NaN > x` is false. Both existing tests put the bad
value in `primary`, the one axis where the guard worked, so they appeared
to protect the property and did not. `assertComparable` now pre-reads every
objective of every point before any comparison, in `dominates`, `frontier`
and `admitToFrontier`, and the latter validates frontier members too.
When the guard fired it threw out of the promotion rule. `@metaharness/
flywheel` calls the rule without a try/catch in both `run.js` and inside
`verifyReplayBundle`, so a bad objective aborted the generation loop in-run
and, in replay, prevented the structured `checks` output from ever being
produced. The same class of data therefore yielded either a silent
promotion or a dead run, and never "reject this candidate." The gate call
site now catches and pushes `non_finite_objective`; the pure API still
throws, which is right for a library and wrong for a gate.
THE THIRD AXIS IS REMOVED, reversing this PR's own framing. Dominance
requires "no worse on every axis", so each added axis is one more chance
for that condition to fail: a Pareto gate weakens monotonically as axes
grow, and this one was described as tightening. `p99Us` was also not
independent evidence -- it is already a factor of `costPerWin`
(`memoryMb * p99Us / qps`) and a term inside `primary`'s `darwinScore`, so
it bought no information while costing real blocking power. Concretely, a
candidate burning ten times the memory to halve tail latency is dominated
and blocked under two axes and sailed through under three. A test pins both
halves so the frontier cannot quietly become a second cost gate.
The candidate can no longer choose its own judges: the `frontier` field is
removed from the score outright rather than special-casing the empty array,
because it rode on the object the evaluator produces and a deserialized
bundle carrying `frontier: []` would have switched the gate off silently.
The manifest's declared directions are now cross-checked against the gate's
actual objectives, so a manifest can no longer declare an inverted
direction that would invert promotion the day it is wired in, and declared
bounds must be ones `normalizePolicy` can actually effect. The module now
opens by stating plainly that it is inert: `declaredObjectives()` has no
caller and schema registration happens by glob, so the schema constant was
decorative -- a research-gate test now validates the fixture through it.
All six negative checks bit before their fix. 157 of 158 harness tests pass
(one pre-existing skip), 26 research-gate tests pass.
Co-Authored-By: claude-flow <ruv@ruv.net>
Claude-Session: https://claude.ai/code/session_016QSCkKnxDjqU49NVVpWMK5
This commit is contained in:
ruv
parent
0224fec1b4
commit
db047e0bf2
10 changed files with 375 additions and 87 deletions
|
|
@ -89,6 +89,23 @@ export const POLICY_LEVERS: readonly string[] = Object.freeze([
|
|||
"ef_search", "m", "ef_construction", "k", "runner_set",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Ranges `normalizePolicy` enforces for the integer levers.
|
||||
*
|
||||
* Exported so a declared manifest can be checked against the values the runner
|
||||
* will actually accept. Without this a manifest can advertise
|
||||
* `ef_search: {1, 999999}`, look authoritative, and still have every run above
|
||||
* 4096 rejected at spawn time — a declaration that cannot be effected.
|
||||
* `runner_set` is absent because it is an enum, validated above.
|
||||
*/
|
||||
export const POLICY_LEVER_RANGES: Readonly<Record<string, { minimum: number; maximum: number }>> =
|
||||
Object.freeze({
|
||||
ef_search: Object.freeze({ minimum: 1, maximum: 4096 }),
|
||||
m: Object.freeze({ minimum: 2, maximum: 128 }),
|
||||
ef_construction: Object.freeze({ minimum: 4, maximum: 4096 }),
|
||||
k: Object.freeze({ minimum: 1, maximum: 100 }),
|
||||
});
|
||||
|
||||
export function normalizePolicy(policy: Record<string, string>): BenchmarkPolicy {
|
||||
const runnerSet = policy.runner_set ?? "core";
|
||||
if (runnerSet !== "core" && runnerSet !== "core-lsm" && runnerSet !== "all") {
|
||||
|
|
|
|||
|
|
@ -59,19 +59,20 @@ interface RuvectorScore extends Score {
|
|||
peakRssBytes: number;
|
||||
vetoReasons: string[];
|
||||
observations: BenchmarkObservation[];
|
||||
/**
|
||||
* Frontier the candidate must not be dominated by. Defaults to the incumbent
|
||||
* baseline when a run has no wider frontier to hand.
|
||||
*/
|
||||
frontier?: ParetoPoint[];
|
||||
}
|
||||
|
||||
/** Objective vector for the in-repo Pareto gate (ADR-335). */
|
||||
/**
|
||||
* Objective vector for the in-repo Pareto gate (ADR-335).
|
||||
*
|
||||
* The frontier is the incumbent baseline, and deliberately nothing the
|
||||
* candidate can influence. An earlier revision let a score carry its own
|
||||
* `frontier` array, which let the judged object choose its judges: an empty
|
||||
* array admitted everything, silently and without a reason string. Widening
|
||||
* this beyond the incumbent must arrive through the caller's options, never
|
||||
* through the evaluator's score object or a deserialized replay bundle.
|
||||
*/
|
||||
function paretoPoint(id: string, score: RuvectorScore): ParetoPoint {
|
||||
return {
|
||||
id,
|
||||
values: { primary: score.primary, costPerWin: score.costPerWin, p99Us: score.p99Us },
|
||||
};
|
||||
return { id, values: { primary: score.primary, costPerWin: score.costPerWin } };
|
||||
}
|
||||
|
||||
export interface RuvectorFlywheelOptions {
|
||||
|
|
@ -108,15 +109,26 @@ export function ruvectorPromotionRule(evidence: {
|
|||
if (evidence.anchor && evidence.anchor.candidate < evidence.anchor.baseline) reasons.push("anchor_regressed");
|
||||
// In-repo Pareto gate (ADR-335): a candidate dominated on the objective
|
||||
// vector cannot be promoted even when the paired test passes, because a
|
||||
// scalar improvement bought with a strictly worse cost and latency profile
|
||||
// is a trade rather than a win. Non-finite objectives throw here rather than
|
||||
// comparing, since NaN defeats `<` in both directions.
|
||||
const admission = admitToFrontier(
|
||||
paretoPoint("candidate", candidate),
|
||||
candidate.frontier ?? [paretoPoint("baseline", baseline)],
|
||||
);
|
||||
if (!admission.admitted) {
|
||||
reasons.push(`pareto_dominated_by_${admission.dominatedBy.join("_")}`);
|
||||
// scalar improvement bought with a strictly worse resource profile is a
|
||||
// trade rather than a win.
|
||||
//
|
||||
// The pure API throws on a non-finite or missing objective, which is right
|
||||
// for a library but wrong here: `@metaharness/flywheel` calls this rule
|
||||
// without a try/catch from both its generation loop and `verifyReplayBundle`,
|
||||
// so an escaping throw would abort the run in one case and, in the other,
|
||||
// destroy the structured `checks` output that the replay assertion below
|
||||
// depends on. Unusable evidence is a reason to REFUSE the candidate, not to
|
||||
// kill the harness, so it is converted into a blocking reason here.
|
||||
try {
|
||||
const admission = admitToFrontier(
|
||||
paretoPoint("candidate", candidate),
|
||||
[paretoPoint("baseline", baseline)],
|
||||
);
|
||||
if (!admission.admitted) {
|
||||
reasons.push(`pareto_dominated_by_${admission.dominatedBy.join("_")}`);
|
||||
}
|
||||
} catch {
|
||||
reasons.push("non_finite_objective");
|
||||
}
|
||||
return { promote: reasons.length === 0, reasons };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,15 @@
|
|||
* research manifest from it. The manifest is the standing policy; the research
|
||||
* manifest is one run under that policy.
|
||||
*
|
||||
* STATUS: this module is currently INERT. Nothing in `runRuvectorFlywheel`
|
||||
* loads a manifest, and `declaredObjectives()` has no caller — the promotion
|
||||
* rule uses `DEFAULT_OBJECTIVES` directly. So `promotion.pareto` is a
|
||||
* *declaration* that is checked for self-consistency, not policy that is
|
||||
* enforced at runtime. Registration in `schema_validate.py` is likewise by
|
||||
* directory glob, so naming the schema there is documentation rather than
|
||||
* wiring. Treat a passing manifest as "this repository has stated its intent
|
||||
* coherently", not as "this repository is being optimized under it".
|
||||
*
|
||||
* Scope note, deliberately narrow: NVIDIA's skillpack also carries
|
||||
* domain-specific optimization *knowledge* (what to try, in what order) and an
|
||||
* adversarial-review step performed by a separate agent. This manifest encodes
|
||||
|
|
@ -23,8 +32,8 @@
|
|||
*/
|
||||
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { POLICY_LEVERS } from "./benchmark.js";
|
||||
import type { ObjectiveSpec } from "./pareto.js";
|
||||
import { POLICY_LEVER_RANGES, POLICY_LEVERS } from "./benchmark.js";
|
||||
import { DEFAULT_OBJECTIVES, type ObjectiveSpec } from "./pareto.js";
|
||||
|
||||
export interface ManifestObjective {
|
||||
name: string;
|
||||
|
|
@ -145,7 +154,26 @@ export function assertOptimizationManifest(value: unknown): asserts value is Opt
|
|||
}
|
||||
if (seen.has(name)) throw new Error(`optimization manifest repeats lever ${name}`);
|
||||
seen.add(name);
|
||||
if (lever.values !== undefined) requireStringArray(lever.values, `lever ${name} values`);
|
||||
// A declared range or value set must be one the runner can actually
|
||||
// effect. Otherwise a manifest can advertise a lever setting that
|
||||
// `normalizePolicy` rejects at spawn time -- an authorization that looks
|
||||
// real and does nothing.
|
||||
const runnerRange = POLICY_LEVER_RANGES[name];
|
||||
if (lever.values !== undefined) {
|
||||
const values = requireStringArray(lever.values, `lever ${name} values`);
|
||||
if (runnerRange) {
|
||||
for (const raw of values) {
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isSafeInteger(parsed)
|
||||
|| parsed < runnerRange.minimum || parsed > runnerRange.maximum) {
|
||||
throw new Error(
|
||||
`optimization manifest lever ${name} declares ${raw}, outside the runner's `
|
||||
+ `[${runnerRange.minimum}, ${runnerRange.maximum}]`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (lever.bounds !== undefined) {
|
||||
const bounds = requireObject(lever.bounds, `lever ${name} bounds`);
|
||||
const minimum = requireFinite(bounds.minimum, `lever ${name} bounds.minimum`);
|
||||
|
|
@ -153,6 +181,15 @@ export function assertOptimizationManifest(value: unknown): asserts value is Opt
|
|||
if (minimum > maximum) {
|
||||
throw new Error(`optimization manifest lever ${name} has minimum above maximum`);
|
||||
}
|
||||
if (!runnerRange) {
|
||||
throw new Error(`optimization manifest lever ${name} takes values, not bounds`);
|
||||
}
|
||||
if (minimum < runnerRange.minimum || maximum > runnerRange.maximum) {
|
||||
throw new Error(
|
||||
`optimization manifest lever ${name} declares [${minimum}, ${maximum}], outside the `
|
||||
+ `runner's [${runnerRange.minimum}, ${runnerRange.maximum}]`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -182,14 +219,32 @@ export function assertOptimizationManifest(value: unknown): asserts value is Opt
|
|||
if (!Array.isArray(pareto.objectives) || pareto.objectives.length === 0) {
|
||||
throw new Error("optimization manifest promotion.pareto requires objectives");
|
||||
}
|
||||
// Cross-check against the vector the promotion rule actually uses. A
|
||||
// manifest declaring `{primary, minimize}` is harmless only while this
|
||||
// module is inert; the day it is wired in, an inverted direction is a
|
||||
// promotion gate that rewards regressions. Refuse it now, while the cost of
|
||||
// being wrong is a failing test rather than a bad promotion.
|
||||
const gateDirections = new Map(DEFAULT_OBJECTIVES.map((o) => [o.name, o.direction]));
|
||||
for (const raw of pareto.objectives) {
|
||||
const objectiveSpec = requireObject(raw, "pareto objective");
|
||||
if (typeof objectiveSpec.name !== "string" || !objectiveSpec.name) {
|
||||
const name = objectiveSpec.name;
|
||||
if (typeof name !== "string" || !name) {
|
||||
throw new Error("optimization manifest pareto objective requires a name");
|
||||
}
|
||||
if (objectiveSpec.direction !== "maximize" && objectiveSpec.direction !== "minimize") {
|
||||
throw new Error(`optimization manifest pareto objective ${name} needs a direction`);
|
||||
}
|
||||
const expected = gateDirections.get(name);
|
||||
if (expected === undefined) {
|
||||
throw new Error(
|
||||
`optimization manifest pareto objective ${objectiveSpec.name} needs a direction`,
|
||||
`optimization manifest declares pareto objective ${name}, which the promotion gate `
|
||||
+ `does not evaluate (it uses ${[...gateDirections.keys()].join(", ")})`,
|
||||
);
|
||||
}
|
||||
if (expected !== objectiveSpec.direction) {
|
||||
throw new Error(
|
||||
`optimization manifest declares ${name} as ${objectiveSpec.direction}, but the promotion `
|
||||
+ `gate treats it as ${expected}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,16 +31,25 @@ export interface ParetoPoint {
|
|||
}
|
||||
|
||||
/**
|
||||
* The promotion objective vector.
|
||||
* The promotion objective vector: quality against the resource bought with it.
|
||||
*
|
||||
* `primary` is the aggregate benchmark score (higher is better); `costPerWin`
|
||||
* and `p99Us` are the resource and tail-latency axes the promotion rule
|
||||
* already reasons about in `ruvectorPromotionRule`.
|
||||
* Deliberately TWO axes, not three. Dominance requires "no worse on every
|
||||
* axis", so each axis added is one more chance for the condition to fail:
|
||||
* more axes can only ever turn *dominated* into *not dominated*, never the
|
||||
* reverse. A Pareto gate therefore gets monotonically weaker as axes are
|
||||
* added, and this gate exists to block trades dressed as wins.
|
||||
*
|
||||
* `p99Us` was on this vector and has been removed, because it is not
|
||||
* independent evidence — it is already a factor of `costPerWin`
|
||||
* (`memoryMb * p99Us / qps`) and a term inside `primary`'s `darwinScore`.
|
||||
* Counting it a third time bought no information and cost real blocking
|
||||
* power: with it present, a candidate burning 10x the memory to halve tail
|
||||
* latency is *not* dominated and clears the gate; with two axes the incumbent
|
||||
* dominates it and the gate fires.
|
||||
*/
|
||||
export const DEFAULT_OBJECTIVES: readonly ObjectiveSpec[] = Object.freeze([
|
||||
Object.freeze({ name: "primary", direction: "maximize" as const }),
|
||||
Object.freeze({ name: "costPerWin", direction: "minimize" as const }),
|
||||
Object.freeze({ name: "p99Us", direction: "minimize" as const }),
|
||||
]);
|
||||
|
||||
function assertObjectives(objectives: readonly ObjectiveSpec[]): void {
|
||||
|
|
@ -75,6 +84,25 @@ function readObjective(point: ParetoPoint, objective: ObjectiveSpec): number {
|
|||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read every objective of every point before any of them are compared.
|
||||
*
|
||||
* This must happen as a separate pass. The comparison loop below short-circuits
|
||||
* on the first axis where `a` is worse, so validating inside that loop would
|
||||
* only ever check the axes the loop happens to reach — leaving a non-finite
|
||||
* value on a later axis completely unexamined. That is not a hypothetical: it
|
||||
* is precisely how a NaN reaches a comparison and reads as "not worse" on
|
||||
* every test, which is the failure this module exists to prevent.
|
||||
*/
|
||||
function assertComparable(
|
||||
points: readonly ParetoPoint[],
|
||||
objectives: readonly ObjectiveSpec[],
|
||||
): void {
|
||||
for (const point of points) {
|
||||
for (const objective of objectives) readObjective(point, objective);
|
||||
}
|
||||
}
|
||||
|
||||
/** True when `a` is at least as good on every objective and strictly better on one. */
|
||||
export function dominates(
|
||||
a: ParetoPoint,
|
||||
|
|
@ -82,6 +110,7 @@ export function dominates(
|
|||
objectives: readonly ObjectiveSpec[] = DEFAULT_OBJECTIVES,
|
||||
): boolean {
|
||||
assertObjectives(objectives);
|
||||
assertComparable([a, b], objectives);
|
||||
let strictlyBetterSomewhere = false;
|
||||
for (const objective of objectives) {
|
||||
const left = readObjective(a, objective);
|
||||
|
|
@ -107,11 +136,7 @@ export function frontier(
|
|||
objectives: readonly ObjectiveSpec[] = DEFAULT_OBJECTIVES,
|
||||
): ParetoPoint[] {
|
||||
assertObjectives(objectives);
|
||||
// Read every objective up front so a malformed point fails the whole call
|
||||
// rather than only when comparison order happens to reach it.
|
||||
for (const point of points) {
|
||||
for (const objective of objectives) readObjective(point, objective);
|
||||
}
|
||||
assertComparable(points, objectives);
|
||||
return points.filter(
|
||||
(candidate) => !points.some((other) => other !== candidate && dominates(other, candidate, objectives)),
|
||||
);
|
||||
|
|
@ -138,6 +163,7 @@ export function admitToFrontier(
|
|||
objectives: readonly ObjectiveSpec[] = DEFAULT_OBJECTIVES,
|
||||
): FrontierAdmission {
|
||||
assertObjectives(objectives);
|
||||
assertComparable([candidate, ...current], objectives);
|
||||
const dominatedBy = current
|
||||
.filter((member) => dominates(member, candidate, objectives))
|
||||
.map((member) => member.id)
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ interface ScoreShape {
|
|||
observations: unknown[];
|
||||
}
|
||||
|
||||
function score(primary: number, costPerWin: number, p99Us: number, samples: number[]): ScoreShape {
|
||||
function score(primary: number, costPerWin: number, samples: number[]): ScoreShape {
|
||||
return {
|
||||
primary,
|
||||
primarySamples: samples,
|
||||
|
|
@ -79,7 +79,7 @@ function score(primary: number, costPerWin: number, p99Us: number, samples: numb
|
|||
recallAt10: 0.96,
|
||||
qps: 2_000,
|
||||
memoryMb: 40,
|
||||
p99Us,
|
||||
p99Us: 500,
|
||||
peakRssBytes: 0,
|
||||
costPerWin,
|
||||
regressed: false,
|
||||
|
|
@ -97,14 +97,14 @@ const reasonsFor = (baseline: ScoreShape, candidate: ScoreShape): string[] =>
|
|||
test("a candidate dominated by the incumbent is refused on the Pareto gate", () => {
|
||||
// Equal accuracy bought with strictly worse cost and latency: the baseline
|
||||
// dominates it on the objective vector, so the frontier gate must fire.
|
||||
const baseline = score(0.9, 10, 100, [0.9, 0.9, 0.9, 0.9, 0.9]);
|
||||
const dominated = score(0.9, 20, 200, [0.9, 0.9, 0.9, 0.9, 0.9]);
|
||||
const baseline = score(0.9, 10, [0.9, 0.9, 0.9, 0.9, 0.9]);
|
||||
const dominated = score(0.9, 20, [0.9, 0.9, 0.9, 0.9, 0.9]);
|
||||
assert.ok(reasonsFor(baseline, dominated).includes("pareto_dominated_by_baseline"));
|
||||
});
|
||||
|
||||
test("a candidate better on every objective clears the Pareto gate", () => {
|
||||
const baseline = score(0.90, 20, 200, [0.90, 0.90, 0.90, 0.90, 0.90]);
|
||||
const better = score(0.95, 10, 100, [0.95, 0.95, 0.95, 0.95, 0.95]);
|
||||
const baseline = score(0.90, 20, [0.90, 0.90, 0.90, 0.90, 0.90]);
|
||||
const better = score(0.95, 10, [0.95, 0.95, 0.95, 0.95, 0.95]);
|
||||
const reasons = reasonsFor(baseline, better);
|
||||
assert.ok(!reasons.some((reason) => reason.startsWith("pareto_dominated")));
|
||||
assert.deepEqual(reasons, []);
|
||||
|
|
@ -114,15 +114,50 @@ test("a trade is not domination: better accuracy at worse cost stays on the fron
|
|||
// Neither point dominates, so the Pareto gate stays silent and the existing
|
||||
// cost rule is what decides. The frontier must not become a second, hidden
|
||||
// cost gate.
|
||||
const baseline = score(0.90, 10, 100, [0.90, 0.90, 0.90, 0.90, 0.90]);
|
||||
const traded = score(0.95, 40, 400, [0.95, 0.95, 0.95, 0.95, 0.95]);
|
||||
const baseline = score(0.90, 10, [0.90, 0.90, 0.90, 0.90, 0.90]);
|
||||
const traded = score(0.95, 40, [0.95, 0.95, 0.95, 0.95, 0.95]);
|
||||
const reasons = reasonsFor(baseline, traded);
|
||||
assert.ok(!reasons.some((reason) => reason.startsWith("pareto_dominated")));
|
||||
assert.ok(reasons.includes("resource_cost_worsened"));
|
||||
});
|
||||
|
||||
test("a non-finite objective throws rather than silently promoting", () => {
|
||||
const baseline = score(0.9, 10, 100, [0.9, 0.9, 0.9, 0.9, 0.9]);
|
||||
const broken = score(Number.NaN, 10, 100, [0.95, 0.95, 0.95, 0.95, 0.95]);
|
||||
assert.throws(() => reasonsFor(baseline, broken), /non-finite/);
|
||||
test("unusable objectives block the candidate instead of killing the run", () => {
|
||||
// The pure pareto API throws, but @metaharness/flywheel calls this rule with
|
||||
// no try/catch from both its generation loop and verifyReplayBundle, so an
|
||||
// escaping throw would abort the run and destroy the replay `checks` output.
|
||||
// Unusable evidence must REFUSE the candidate, not take down the harness.
|
||||
//
|
||||
// Every case below is worse on primary (axis 1) and carries the bad value on
|
||||
// costPerWin (axis 2) or omits a field entirely — the shapes the comparison
|
||||
// loop would short-circuit past.
|
||||
const baseline = score(0.9, 10, [0.9, 0.9, 0.9, 0.9, 0.9]);
|
||||
const passing = [0.95, 0.95, 0.95, 0.95, 0.95];
|
||||
const broken: ScoreShape[] = [
|
||||
score(Number.NaN, 10, passing),
|
||||
score(0.1, Number.NaN, passing),
|
||||
score(0.1, Number.POSITIVE_INFINITY, passing),
|
||||
score(0.1, Number.NEGATIVE_INFINITY, passing),
|
||||
];
|
||||
for (const candidate of broken) {
|
||||
const reasons = reasonsFor(baseline, candidate);
|
||||
assert.ok(reasons.includes("non_finite_objective"), JSON.stringify(reasons));
|
||||
assert.equal(
|
||||
ruvectorPromotionRule({ baseline: baseline as never, candidate: candidate as never }).promote,
|
||||
false,
|
||||
);
|
||||
}
|
||||
// A sealed score from before this vector existed: costPerWin absent.
|
||||
const legacy = score(0.1, 0, passing) as Partial<ScoreShape>;
|
||||
delete legacy.costPerWin;
|
||||
const legacyReasons = reasonsFor(baseline, legacy as ScoreShape);
|
||||
assert.ok(legacyReasons.includes("non_finite_objective"), JSON.stringify(legacyReasons));
|
||||
});
|
||||
|
||||
test("a candidate cannot supply its own frontier to switch the gate off", () => {
|
||||
// An earlier revision read `candidate.frontier`, so an empty array admitted
|
||||
// everything silently. The frontier is now the incumbent and nothing on the
|
||||
// judged object can influence it.
|
||||
const baseline = score(0.9, 10, [0.9, 0.9, 0.9, 0.9, 0.9]);
|
||||
const dominated = { ...score(0.5, 1e9, [0.5, 0.5, 0.5, 0.5, 0.5]), frontier: [] };
|
||||
assert.ok(reasonsFor(baseline, dominated as ScoreShape).includes("pareto_dominated_by_baseline"));
|
||||
});
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import {
|
|||
type OptimizationManifest,
|
||||
} from "../src/optimizationManifest.js";
|
||||
import { POLICY_LEVERS } from "../src/benchmark.js";
|
||||
import { DEFAULT_OBJECTIVES } from "../src/pareto.js";
|
||||
|
||||
const repoRoot = resolve(import.meta.dirname, "../../../../..");
|
||||
const FIXTURE = resolve(repoRoot, "schemas/fixtures/optimization-manifest-v1.json");
|
||||
|
|
@ -29,7 +30,7 @@ test("the shipped fixture manifest is accepted", async () => {
|
|||
assert.equal(manifest.repository, "ruvnet/ruvector");
|
||||
assert.deepEqual(declaredLevers(manifest), ["ef_search", "m", "ef_construction", "runner_set"]);
|
||||
assert.deepEqual(declaredObjectives(manifest).map((entry) => entry.name),
|
||||
["primary", "costPerWin", "p99Us"]);
|
||||
["primary", "costPerWin"]);
|
||||
});
|
||||
|
||||
test("a lever the native runner would reject is refused", async () => {
|
||||
|
|
@ -164,3 +165,52 @@ test("a manifest that is not valid JSON reports the file", async () => {
|
|||
loadOptimizationManifest(resolve(repoRoot, "README.md")),
|
||||
);
|
||||
});
|
||||
|
||||
test("a pareto direction inverted from the real gate is refused", async () => {
|
||||
// Harmless while the module is inert; an inverted promotion gate the day it
|
||||
// is wired in. Refuse it now, while being wrong costs a failing test.
|
||||
const manifest = await fixture();
|
||||
manifest.promotion.pareto.objectives = [
|
||||
{ name: "primary", direction: "minimize" },
|
||||
{ name: "costPerWin", direction: "minimize" },
|
||||
];
|
||||
assert.throws(() => assertOptimizationManifest(manifest),
|
||||
/declares primary as minimize, but the promotion gate treats it as maximize/);
|
||||
});
|
||||
|
||||
test("a pareto objective the gate does not evaluate is refused", async () => {
|
||||
const manifest = await fixture();
|
||||
manifest.promotion.pareto.objectives = [
|
||||
{ name: "primary", direction: "maximize" },
|
||||
{ name: "p99Us", direction: "minimize" },
|
||||
];
|
||||
assert.throws(() => assertOptimizationManifest(manifest),
|
||||
/declares pareto objective p99Us, which the promotion gate does not evaluate/);
|
||||
});
|
||||
|
||||
test("declared objectives match the vector the promotion rule actually uses", async () => {
|
||||
const manifest = await fixture();
|
||||
assert.deepEqual(declaredObjectives(manifest), [...DEFAULT_OBJECTIVES]);
|
||||
});
|
||||
|
||||
test("a lever range the runner would reject is refused", async () => {
|
||||
// normalizePolicy bounds ef_search to [1, 4096]. A manifest advertising more
|
||||
// would look authoritative and be rejected at spawn time.
|
||||
const manifest = await fixture();
|
||||
manifest.levers[0] = { name: "ef_search", bounds: { minimum: 1, maximum: 999_999 } };
|
||||
assert.throws(() => assertOptimizationManifest(manifest),
|
||||
/declares \[1, 999999\], outside the runner's \[1, 4096\]/);
|
||||
});
|
||||
|
||||
test("a lever value the runner would reject is refused", async () => {
|
||||
const manifest = await fixture();
|
||||
manifest.levers[0] = { name: "ef_search", values: ["64", "999999"] };
|
||||
assert.throws(() => assertOptimizationManifest(manifest),
|
||||
/declares 999999, outside the runner's \[1, 4096\]/);
|
||||
});
|
||||
|
||||
test("bounds on an enum lever are refused", async () => {
|
||||
const manifest = await fixture();
|
||||
manifest.levers[3] = { name: "runner_set", bounds: { minimum: 0, maximum: 2 } };
|
||||
assert.throws(() => assertOptimizationManifest(manifest), /takes values, not bounds/);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -8,29 +8,29 @@ import {
|
|||
type ParetoPoint,
|
||||
} from "../src/pareto.js";
|
||||
|
||||
const point = (id: string, primary: number, costPerWin: number, p99Us: number): ParetoPoint => ({
|
||||
id, values: { primary, costPerWin, p99Us },
|
||||
const point = (id: string, primary: number, costPerWin: number): ParetoPoint => ({
|
||||
id, values: { primary, costPerWin },
|
||||
});
|
||||
|
||||
test("dominance requires no worse on every objective and better on one", () => {
|
||||
const strong = point("strong", 0.9, 10, 100);
|
||||
const weak = point("weak", 0.8, 20, 200);
|
||||
const strong = point("strong", 0.9, 10);
|
||||
const weak = point("weak", 0.8, 20);
|
||||
assert.equal(dominates(strong, weak), true);
|
||||
assert.equal(dominates(weak, strong), false);
|
||||
});
|
||||
|
||||
test("a scalar win bought with worse cost and latency does not dominate", () => {
|
||||
// The case the gate exists for: higher primary, but strictly worse on both
|
||||
// resource axes. That is a trade, not a win, so neither point dominates.
|
||||
const traded = point("traded", 0.95, 40, 400);
|
||||
const incumbent = point("incumbent", 0.90, 10, 100);
|
||||
test("a scalar win bought with worse cost does not dominate", () => {
|
||||
// Higher primary, strictly worse cost. That is a trade, not a win, so
|
||||
// neither point dominates.
|
||||
const traded = point("traded", 0.95, 40);
|
||||
const incumbent = point("incumbent", 0.90, 10);
|
||||
assert.equal(dominates(traded, incumbent), false);
|
||||
assert.equal(dominates(incumbent, traded), false);
|
||||
});
|
||||
|
||||
test("identical points do not dominate each other and both stay on the frontier", () => {
|
||||
const a = point("a", 0.9, 10, 100);
|
||||
const b = point("b", 0.9, 10, 100);
|
||||
const a = point("a", 0.9, 10);
|
||||
const b = point("b", 0.9, 10);
|
||||
assert.equal(dominates(a, b), false);
|
||||
assert.equal(dominates(b, a), false);
|
||||
assert.deepEqual(frontier([a, b]).map((entry) => entry.id), ["a", "b"]);
|
||||
|
|
@ -38,49 +38,85 @@ test("identical points do not dominate each other and both stay on the frontier"
|
|||
|
||||
test("frontier keeps the non-dominated set in input order", () => {
|
||||
const points = [
|
||||
point("dominated", 0.5, 50, 500),
|
||||
point("fast", 0.8, 30, 50),
|
||||
point("accurate", 0.95, 60, 300),
|
||||
point("cheap", 0.7, 5, 400),
|
||||
point("dominated", 0.5, 50),
|
||||
point("fast", 0.8, 30),
|
||||
point("accurate", 0.95, 60),
|
||||
point("cheap", 0.7, 5),
|
||||
];
|
||||
assert.deepEqual(frontier(points).map((entry) => entry.id), ["fast", "accurate", "cheap"]);
|
||||
});
|
||||
|
||||
test("non-finite objectives are refused rather than compared", () => {
|
||||
// NaN defeats both `<` and `>`, so a NaN candidate would read as
|
||||
// non-dominated against everything. Refuse it at the choke point instead.
|
||||
const sane = point("sane", 0.9, 10, 100);
|
||||
test("non-finite objectives are refused on EVERY axis, not just the first", () => {
|
||||
// The comparison loop short-circuits on the first axis where a point is
|
||||
// worse, so validating inside it would only ever check axes the loop
|
||||
// reaches. A bad value on a LATER axis is the case that silently promotes:
|
||||
// NaN defeats both `<` and `>`, so it reads as "not worse" everywhere.
|
||||
// Every point here is deliberately worse on `primary` (axis 1) so the loop
|
||||
// would return before ever seeing axis 2.
|
||||
const sane = point("sane", 0.9, 10);
|
||||
for (const bad of [Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]) {
|
||||
const broken = point("broken", bad, 10, 100);
|
||||
assert.throws(() => dominates(broken, sane), /non-finite/);
|
||||
assert.throws(() => dominates(sane, broken), /non-finite/);
|
||||
assert.throws(() => frontier([sane, broken]), /non-finite/);
|
||||
assert.throws(() => admitToFrontier(broken, [sane]), /non-finite/);
|
||||
for (const broken of [
|
||||
{ id: "axis1", values: { primary: bad, costPerWin: 10 } },
|
||||
{ id: "axis2", values: { primary: 0.1, costPerWin: bad } },
|
||||
] satisfies ParetoPoint[]) {
|
||||
assert.throws(() => dominates(broken, sane), /non-finite/, `${broken.id} a-vs-b ${bad}`);
|
||||
assert.throws(() => dominates(sane, broken), /non-finite/, `${broken.id} b-vs-a ${bad}`);
|
||||
assert.throws(() => frontier([sane, broken]), /non-finite/, `${broken.id} frontier ${bad}`);
|
||||
assert.throws(() => admitToFrontier(broken, [sane]), /non-finite/, `${broken.id} admit ${bad}`);
|
||||
assert.throws(() => admitToFrontier(sane, [broken]), /non-finite/, `${broken.id} member ${bad}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("a missing objective is refused rather than treated as zero", () => {
|
||||
const partial: ParetoPoint = { id: "partial", values: { primary: 0.9 } };
|
||||
assert.throws(() => dominates(partial, point("other", 0.8, 10, 100)), /missing objective/);
|
||||
test("a missing objective is refused on any axis, including a later one", () => {
|
||||
// The shape an older sealed score carries: a field the current vector
|
||||
// expects simply is not there.
|
||||
const sane = point("sane", 0.9, 10);
|
||||
const missingFirst: ParetoPoint = { id: "missingFirst", values: { costPerWin: 10 } };
|
||||
const missingLater: ParetoPoint = { id: "missingLater", values: { primary: 0.1 } };
|
||||
for (const partial of [missingFirst, missingLater]) {
|
||||
assert.throws(() => dominates(partial, sane), /missing objective/, partial.id);
|
||||
assert.throws(() => dominates(sane, partial), /missing objective/, partial.id);
|
||||
assert.throws(() => admitToFrontier(partial, [sane]), /missing objective/, partial.id);
|
||||
assert.throws(() => admitToFrontier(sane, [partial]), /missing objective/, partial.id);
|
||||
}
|
||||
});
|
||||
|
||||
test("adding an axis can only weaken the gate, which is why the vector is two", () => {
|
||||
// Records the MEDIUM-3 reasoning as an executable fact. Same two points:
|
||||
// dominated on the shipped vector, NOT dominated once a third axis on which
|
||||
// the candidate happens to be better is added.
|
||||
const incumbent: ParetoPoint = { id: "incumbent", values: { primary: 0.9, costPerWin: 10, p99Us: 100 } };
|
||||
const heavy: ParetoPoint = { id: "heavy", values: { primary: 0.9, costPerWin: 50, p99Us: 50 } };
|
||||
assert.equal(dominates(incumbent, heavy), true);
|
||||
assert.equal(
|
||||
dominates(incumbent, heavy, [
|
||||
{ name: "primary", direction: "maximize" },
|
||||
{ name: "costPerWin", direction: "minimize" },
|
||||
{ name: "p99Us", direction: "minimize" },
|
||||
]),
|
||||
false,
|
||||
);
|
||||
assert.deepEqual(DEFAULT_OBJECTIVES.map((o) => o.name), ["primary", "costPerWin"]);
|
||||
});
|
||||
|
||||
test("admission blocks a dominated candidate and names the dominators", () => {
|
||||
const current = [point("incumbent", 0.9, 10, 100), point("other", 0.85, 8, 90)];
|
||||
const worse = point("worse", 0.5, 50, 500);
|
||||
const current = [point("incumbent", 0.9, 10), point("other", 0.85, 8)];
|
||||
const worse = point("worse", 0.5, 50);
|
||||
const admission = admitToFrontier(worse, current);
|
||||
assert.equal(admission.admitted, false);
|
||||
assert.deepEqual(admission.dominatedBy, ["incumbent", "other"]);
|
||||
});
|
||||
|
||||
test("an empty frontier admits, because absence of evidence is not domination", () => {
|
||||
assert.deepEqual(admitToFrontier(point("first", 0.1, 999, 999), []), {
|
||||
assert.deepEqual(admitToFrontier(point("first", 0.1, 999), []), {
|
||||
admitted: true, dominatedBy: [],
|
||||
});
|
||||
});
|
||||
|
||||
test("objective specs are validated before any comparison", () => {
|
||||
const a = point("a", 0.9, 10, 100);
|
||||
const b = point("b", 0.8, 20, 200);
|
||||
const a = point("a", 0.9, 10);
|
||||
const b = point("b", 0.8, 20);
|
||||
assert.throws(() => dominates(a, b, []), /at least one objective/);
|
||||
assert.throws(
|
||||
() => dominates(a, b, [{ name: "primary", direction: "sideways" as never }]),
|
||||
|
|
|
|||
|
|
@ -10,14 +10,43 @@
|
|||
},
|
||||
"benchmark": {
|
||||
"binary": "target/release/sota-all",
|
||||
"runner_sets": ["core", "core-lsm", "all"],
|
||||
"runner_sets": [
|
||||
"core",
|
||||
"core-lsm",
|
||||
"all"
|
||||
],
|
||||
"isolated": true
|
||||
},
|
||||
"levers": [
|
||||
{ "name": "ef_search", "bounds": { "minimum": 1, "maximum": 4096 } },
|
||||
{ "name": "m", "bounds": { "minimum": 2, "maximum": 128 } },
|
||||
{ "name": "ef_construction", "bounds": { "minimum": 4, "maximum": 4096 } },
|
||||
{ "name": "runner_set", "values": ["core", "core-lsm", "all"] }
|
||||
{
|
||||
"name": "ef_search",
|
||||
"bounds": {
|
||||
"minimum": 1,
|
||||
"maximum": 4096
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "m",
|
||||
"bounds": {
|
||||
"minimum": 2,
|
||||
"maximum": 128
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "ef_construction",
|
||||
"bounds": {
|
||||
"minimum": 4,
|
||||
"maximum": 4096
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "runner_set",
|
||||
"values": [
|
||||
"core",
|
||||
"core-lsm",
|
||||
"all"
|
||||
]
|
||||
}
|
||||
],
|
||||
"protected_invariants": {
|
||||
"required_veto_providers": [
|
||||
|
|
@ -48,9 +77,14 @@
|
|||
},
|
||||
"pareto": {
|
||||
"objectives": [
|
||||
{ "name": "primary", "direction": "maximize" },
|
||||
{ "name": "costPerWin", "direction": "minimize" },
|
||||
{ "name": "p99Us", "direction": "minimize" }
|
||||
{
|
||||
"name": "primary",
|
||||
"direction": "maximize"
|
||||
},
|
||||
{
|
||||
"name": "costPerWin",
|
||||
"direction": "minimize"
|
||||
}
|
||||
],
|
||||
"require_non_dominated": true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,9 @@ OVERRIDE_SCHEMA = "research-override-v1.json"
|
|||
BASE_GATE_SCHEMA = "research-base-gate-v1.json"
|
||||
ATTESTATION_SUBJECT_SCHEMA = "research-attestation-subject-v1.json"
|
||||
EMBEDDING_IDENTITY_SCHEMA = "embedding-space-identity-v1.json"
|
||||
# ADR-335. Registration is by directory glob below, so naming a schema here
|
||||
# does not wire it into anything -- it is the argument callers pass to
|
||||
# ``validate_document``. No gate stage consumes an optimization manifest yet.
|
||||
OPTIMIZATION_MANIFEST_SCHEMA = "optimization-manifest-v1.json"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -465,6 +465,26 @@ class SchemaEnforcementTests(unittest.TestCase):
|
|||
# against the offline registry; a network-only $ref would raise here.
|
||||
schema_validate._validator(name)
|
||||
|
||||
def test_optimization_manifest_fixture_matches_its_schema(self):
|
||||
# ADR-335. The manifest is not consumed by any gate stage yet, so
|
||||
# without this the schema and the fixture could drift apart unnoticed
|
||||
# and the named constant would be decorative.
|
||||
fixture = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "schemas" / "fixtures" / "optimization-manifest-v1.json"
|
||||
)
|
||||
document = json.loads(fixture.read_text(encoding="utf-8"))
|
||||
schema_validate.validate_document(
|
||||
document, schema_validate.OPTIMIZATION_MANIFEST_SCHEMA
|
||||
)
|
||||
# The frontier requirement is a schema constant: a manifest must not be
|
||||
# able to switch the gate off.
|
||||
document["promotion"]["pareto"]["require_non_dominated"] = False
|
||||
with self.assertRaises(schema_validate.SchemaValidationError):
|
||||
schema_validate.validate_document(
|
||||
document, schema_validate.OPTIMIZATION_MANIFEST_SCHEMA
|
||||
)
|
||||
|
||||
def test_manifest_violating_schema_only_rule_is_rejected(self):
|
||||
# selection_rule is required by the schema and by no hand-rolled check,
|
||||
# so this fails only if schema validation actually runs.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue