mirror of
https://github.com/signerless/llm-checker.git
synced 2026-10-19 09:54:45 +00:00
fix(#88): unify check/recommend/smart-recommend on one scoring core
The three recommendation surfaces each ran a DIFFERENT scoring engine, so they returned inconsistent recommendations, and only `recommend` had the PR #89 high-capacity right-sizing fix. On high-end hardware (e.g. dual RTX PRO 6000 / ~192GB) `check` and `smart-recommend` still ranked tiny 2B-8B models above large models. Unification approach (DeterministicModelSelector is the canonical core): - New src/models/scoring-core.js exposes `rankModels(models, hardware, options)` backed by DeterministicModelSelector.selectModels (which already carries calculateHighCapacitySizeAdjustment / getHighCapacitySizeTarget). It normalizes any model shape (ExpandedModelsDatabase rows, sql.js variant rows, Ollama scrape models) into the deterministic model shape and keeps a `__source` back-reference so callers can map results to their display shape. - `check` (MultiObjectiveSelector.selectBestModels) now ranks via the core and maps results back into its {compatible, marginal, incompatible} shape with totalScore/components/reasoning preserved. The legacy multi-objective math is retained as `selectBestModelsLegacy`, used only as a defensive fallback if the core throws. - `smart-recommend` (IntelligentSelector.recommend) keeps ScoringEngine for the per-variant score object shape used by display, but re-orders the scored list and overwrites each item's `score.final` using the canonical core (applyUnifiedRanking). ScoringEngine internals are untouched. Each command keeps its own model SOURCE; only the ranking/selection is unified. The high-capacity floor now applies on all three paths, and identical (model, hardware) inputs yield the same ordering everywhere. Tests: full suite 36/36 (was 35/35). New tests/scoring-unification.test.js (registered under Recommendations) drives all three selection entrypoints directly and asserts (a) cross-path consistency of the top pick's size class, (b) high-capacity floor everywhere (no sub-30B #1 pick on 192GB for general/coding/reasoning), and (c) small-hardware sanity (small model on an 8GB CPU-only profile across all three paths). No existing test was modified. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
signerless
parent
35805303ff
commit
55b0c3bf4b
5 changed files with 856 additions and 8 deletions
|
|
@ -10,6 +10,7 @@
|
|||
|
||||
const { MULTI_OBJECTIVE_WEIGHTS } = require('../models/scoring-config');
|
||||
const { normalizePlatform } = require('../utils/platform');
|
||||
const { rankModels } = require('../models/scoring-core');
|
||||
|
||||
class MultiObjectiveSelector {
|
||||
constructor() {
|
||||
|
|
@ -40,23 +41,124 @@ class MultiObjectiveSelector {
|
|||
}
|
||||
|
||||
/**
|
||||
* Select best models using multi-objective ranking
|
||||
* Select best models using the UNIFIED canonical scoring core (issue #88).
|
||||
*
|
||||
* `check` used to rank through this selector's own multi-objective math,
|
||||
* which diverged from `recommend`/`smart-recommend` and never received the
|
||||
* PR #89 high-capacity right-sizing fix. It now routes the ranking through
|
||||
* the shared DeterministicModelSelector core (via scoring-core.rankModels)
|
||||
* so identical (model, hardware) inputs score identically across all three
|
||||
* commands and the high-capacity floor applies here too.
|
||||
*
|
||||
* The output shape is preserved exactly: `{ compatible, marginal,
|
||||
* incompatible }`, each entry being the ORIGINAL model object spread with
|
||||
* `totalScore`, `components { quality, speed, ttfb, context, hardwareMatch }`
|
||||
* and `reasoning`, so downstream `check` rendering and the regression test
|
||||
* (which calls `estimateModelParams` on the returned object) keep working.
|
||||
*/
|
||||
async selectBestModels(hardware, models, category = 'general', topK = 10) {
|
||||
// Step 1: Hard filters - remove incompatible models
|
||||
const inputModels = Array.isArray(models) ? models.filter(Boolean) : [];
|
||||
if (inputModels.length === 0) {
|
||||
return { compatible: [], marginal: [], incompatible: [] };
|
||||
}
|
||||
|
||||
let ranking;
|
||||
try {
|
||||
ranking = await rankModels(inputModels, hardware, { category, topN: inputModels.length });
|
||||
} catch (error) {
|
||||
ranking = null;
|
||||
}
|
||||
|
||||
// Defensive fallback: if the unified core is unavailable for any reason,
|
||||
// fall back to the legacy multi-objective ranking so `check` still works.
|
||||
if (!ranking || !Array.isArray(ranking.candidates)) {
|
||||
return this.selectBestModelsLegacy(hardware, inputModels, category, topK);
|
||||
}
|
||||
|
||||
const scoredModels = [];
|
||||
const rankedSources = new Set();
|
||||
for (const candidate of ranking.candidates) {
|
||||
const source = candidate?.meta?.__source;
|
||||
if (!source) continue;
|
||||
rankedSources.add(source);
|
||||
scoredModels.push(this.mapCoreCandidateToMultiObjective(candidate, source, hardware, category));
|
||||
}
|
||||
|
||||
// Models the canonical core dropped (category filter / budget) are not
|
||||
// viable on this hardware for this use case -> treat as incompatible,
|
||||
// mirroring the previous hard-filter semantics.
|
||||
const incompatibleExtras = inputModels
|
||||
.filter((model) => !rankedSources.has(model))
|
||||
.map((model) => ({
|
||||
...model,
|
||||
totalScore: 0,
|
||||
components: { quality: 0, speed: 0, ttfb: 0, context: 0, hardwareMatch: 0 },
|
||||
reasoning: 'Filtered out by unified scoring core (does not fit hardware/use-case)'
|
||||
}));
|
||||
|
||||
scoredModels.sort((a, b) => b.totalScore - a.totalScore);
|
||||
|
||||
const classified = this.classifyResults(scoredModels, topK);
|
||||
classified.incompatible = [...classified.incompatible, ...incompatibleExtras].slice(0, 5);
|
||||
return classified;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a unified-core candidate back into this selector's multi-objective
|
||||
* output shape. The 0-100 `score` from the deterministic core becomes
|
||||
* `totalScore`; component sub-scores are normalized to 0-1 to match the
|
||||
* historical `components` contract consumed by `check` rendering.
|
||||
*/
|
||||
mapCoreCandidateToMultiObjective(candidate, source, hardware, category) {
|
||||
const components = candidate.components || {};
|
||||
const to01 = (value) => {
|
||||
const num = Number(value);
|
||||
if (!Number.isFinite(num)) return 0;
|
||||
return Math.max(0, Math.min(1, num / 100));
|
||||
};
|
||||
|
||||
const quality = to01(components.Q);
|
||||
const speed = to01(components.S);
|
||||
const context = to01(components.C);
|
||||
// The deterministic core folds hardware fitness into the `F` (fit) plus
|
||||
// `H` (high-capacity right-sizing) components; surface that as the
|
||||
// historical `hardwareMatch` signal so `check` insights stay meaningful.
|
||||
const hardwareMatch = to01((Number(components.F) || 0) + (Number(components.H) || 0));
|
||||
|
||||
return {
|
||||
...source,
|
||||
totalScore: Math.round(candidate.score * 100) / 100,
|
||||
score: Math.round(candidate.score * 100) / 100,
|
||||
components: {
|
||||
quality,
|
||||
speed,
|
||||
ttfb: speed, // ttfb tracks speed; legacy field retained for shape
|
||||
context,
|
||||
hardwareMatch
|
||||
},
|
||||
quant: candidate.quant || source.quant,
|
||||
estimatedRAM: candidate.requiredGB,
|
||||
estimatedTPS: candidate.estTPS,
|
||||
reasoning: candidate.rationale ||
|
||||
this.generateReasoning(source, hardware, quality, hardwareMatch)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy multi-objective ranking, retained ONLY as a defensive fallback if
|
||||
* the unified core throws. Not used on the normal path.
|
||||
*/
|
||||
selectBestModelsLegacy(hardware, models, category = 'general', topK = 10) {
|
||||
const compatibleModels = this.applyHardFilters(hardware, models);
|
||||
|
||||
|
||||
if (compatibleModels.length === 0) {
|
||||
return { compatible: [], marginal: [], incompatible: models };
|
||||
}
|
||||
|
||||
// Step 2: Multi-objective scoring
|
||||
const scoredModels = compatibleModels.map(model =>
|
||||
const scoredModels = compatibleModels.map(model =>
|
||||
this.calculateMultiObjectiveScore(hardware, model, category)
|
||||
).filter(Boolean);
|
||||
|
||||
|
||||
// Step 3: Sort and classify
|
||||
scoredModels.sort((a, b) => b.totalScore - a.totalScore);
|
||||
|
||||
return this.classifyResults(scoredModels, topK);
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ const ScoringEngine = require('./scoring-engine');
|
|||
const UnifiedDetector = require('../hardware/unified-detector');
|
||||
const PolicyManager = require('../policy/policy-manager');
|
||||
const PolicyEngine = require('../policy/policy-engine');
|
||||
const { rankModels } = require('./scoring-core');
|
||||
|
||||
function isPlainObject(value) {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
|
|
@ -66,7 +67,9 @@ class IntelligentSelector {
|
|||
// Apply filters
|
||||
const filtered = this.applyFilters(variants, opts, hardware);
|
||||
|
||||
// Score all filtered variants
|
||||
// Score all filtered variants. ScoringEngine still produces the
|
||||
// per-variant `score` objects (final/components/meta) consumed by the
|
||||
// smart-recommend display, but the RANKING is unified below.
|
||||
const scored = this.scoring.filterAndScore(filtered, hardware, {
|
||||
useCase: opts.useCase,
|
||||
targetContext: opts.targetContext,
|
||||
|
|
@ -75,6 +78,12 @@ class IntelligentSelector {
|
|||
headroom: opts.headroom || 2
|
||||
});
|
||||
|
||||
// Unify ranking with the canonical scoring core (issue #88): re-order
|
||||
// the scored list and rewrite each item's final score using the shared
|
||||
// DeterministicModelSelector so smart-recommend agrees with
|
||||
// `check`/`recommend` and inherits the PR #89 high-capacity floor.
|
||||
await this.applyUnifiedRanking(scored, hardware, opts);
|
||||
|
||||
const policyEngine = this.resolvePolicyEngine(opts);
|
||||
const scoredWithPolicy = policyEngine.evaluateScoredVariants(
|
||||
scored,
|
||||
|
|
@ -114,6 +123,83 @@ class IntelligentSelector {
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-rank the ScoringEngine-scored variants using the canonical scoring
|
||||
* core so smart-recommend's ordering and headline scores match
|
||||
* `check`/`recommend` and inherit the high-capacity right-sizing floor.
|
||||
*
|
||||
* Mutates `scored` in place: it is sorted by the unified score and each
|
||||
* item's `score.final` is overwritten with the canonical 0-100 score.
|
||||
* Component/meta sub-scores are left intact so the existing display (which
|
||||
* shows Q/S/F and estimated TPS) keeps working. If the core cannot rank a
|
||||
* variant (or throws), that item keeps its original ScoringEngine score and
|
||||
* sorts after the unified ones, preserving a sensible fallback ordering.
|
||||
*/
|
||||
async applyUnifiedRanking(scored, hardware, opts = {}) {
|
||||
if (!Array.isArray(scored) || scored.length === 0) return scored;
|
||||
|
||||
let ranking;
|
||||
try {
|
||||
ranking = await rankModels(
|
||||
scored.map((item) => item.variant),
|
||||
hardware,
|
||||
{
|
||||
category: opts.useCase || 'general',
|
||||
optimizeFor: opts.optimizeFor || opts.optimize || 'balanced',
|
||||
runtime: opts.runtime || 'ollama',
|
||||
topN: scored.length
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
return scored; // Defensive: keep original ScoringEngine ordering.
|
||||
}
|
||||
|
||||
if (!ranking || !Array.isArray(ranking.candidates)) return scored;
|
||||
|
||||
// Map each source variant -> its unified score + ordering index.
|
||||
const unifiedByVariant = new Map();
|
||||
ranking.candidates.forEach((candidate, index) => {
|
||||
const source = candidate?.meta?.__source;
|
||||
if (!source) return;
|
||||
unifiedByVariant.set(source, {
|
||||
unifiedScore: Math.round(candidate.score * 10) / 10,
|
||||
rank: index,
|
||||
quant: candidate.quant,
|
||||
estimatedTPS: candidate.estTPS
|
||||
});
|
||||
});
|
||||
|
||||
for (const item of scored) {
|
||||
const unified = unifiedByVariant.get(item.variant);
|
||||
if (!unified) {
|
||||
// Not ranked by the core (e.g. filtered out): sort last and tag
|
||||
// so any downstream tie-breaks are deterministic.
|
||||
item.__unifiedRank = Number.MAX_SAFE_INTEGER;
|
||||
continue;
|
||||
}
|
||||
item.__unifiedRank = unified.rank;
|
||||
if (item.score) {
|
||||
item.score.final = Math.min(100, Math.max(0, Math.round(unified.unifiedScore)));
|
||||
if (item.score.meta) {
|
||||
item.score.meta.unifiedScore = unified.unifiedScore;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
scored.sort((a, b) => {
|
||||
const ra = Number.isFinite(a.__unifiedRank) ? a.__unifiedRank : Number.MAX_SAFE_INTEGER;
|
||||
const rb = Number.isFinite(b.__unifiedRank) ? b.__unifiedRank : Number.MAX_SAFE_INTEGER;
|
||||
if (ra !== rb) return ra - rb;
|
||||
return (b.score?.final || 0) - (a.score?.final || 0);
|
||||
});
|
||||
|
||||
for (const item of scored) {
|
||||
delete item.__unifiedRank;
|
||||
}
|
||||
|
||||
return scored;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve policy engine from explicit options, in-memory policy, or policy file.
|
||||
*/
|
||||
|
|
|
|||
341
src/models/scoring-core.js
Normal file
341
src/models/scoring-core.js
Normal file
|
|
@ -0,0 +1,341 @@
|
|||
/**
|
||||
* Unified Scoring Core (GitHub issue #88)
|
||||
* =======================================
|
||||
*
|
||||
* Historically the three user-facing recommendation surfaces each ran a
|
||||
* DIFFERENT scoring engine, so they could disagree about the best model and
|
||||
* only one of them ("recommend") received the PR #89 high-capacity
|
||||
* right-sizing fix:
|
||||
*
|
||||
* - `check` -> MultiObjectiveSelector (src/ai/multi-objective-selector.js)
|
||||
* - `recommend` -> DeterministicModelSelector (src/models/deterministic-selector.js)
|
||||
* - `smart-recommend` -> ScoringEngine (src/models/scoring-engine.js)
|
||||
*
|
||||
* This module establishes the DeterministicModelSelector as the SINGLE
|
||||
* canonical ranking core (it already carries PR #89's
|
||||
* `calculateHighCapacitySizeAdjustment` / `getHighCapacitySizeTarget`). The
|
||||
* other commands keep their own model SOURCE and DISPLAY shape, but route the
|
||||
* actual ranking/selection through `rankModels()` below so that, for identical
|
||||
* (model, hardware) inputs, every command produces identical scores and the
|
||||
* high-capacity floor applies everywhere.
|
||||
*
|
||||
* The hard part is that each command feeds the engine a different model shape:
|
||||
* - `check` uses ExpandedModelsDatabase rows ({ name: "Llama 3.1 8B", size: "8B", ... })
|
||||
* - `smart-recommend` uses sql.js variant rows ({ model_id, params_b, size_gb, quant, ... })
|
||||
* - `recommend` uses the Ollama scrape/catalog (already deterministic-friendly)
|
||||
*
|
||||
* `normalizeToDeterministic()` converts any of those into the
|
||||
* already-normalized deterministic model shape and keeps a back-reference
|
||||
* (`__source`) so callers can recover their original object after ranking.
|
||||
*/
|
||||
|
||||
const DeterministicModelSelector = require('./deterministic-selector');
|
||||
|
||||
// One shared selector instance: it is stateless w.r.t. the model pool (the
|
||||
// pool is always passed in via options), so sharing it is safe and avoids
|
||||
// re-allocating the large prior tables on every call.
|
||||
const canonicalSelector = new DeterministicModelSelector();
|
||||
|
||||
const PARAM_HINT_REGEX = /(\d+(?:\.\d+)?)\s*([bm])\b/i;
|
||||
|
||||
function toFiniteNumber(value) {
|
||||
if (typeof value === 'number' && Number.isFinite(value)) return value;
|
||||
if (typeof value === 'string' && value.trim() !== '' && Number.isFinite(Number(value))) {
|
||||
return Number(value);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort parameter-count extraction (in billions) from a model's various
|
||||
* shapes. Mirrors the heuristics the individual engines used so the unified
|
||||
* ranking sees the same parameter counts they would have.
|
||||
*/
|
||||
function deriveParamsB(model = {}) {
|
||||
const direct =
|
||||
toFiniteNumber(model.paramsB) ??
|
||||
toFiniteNumber(model.params_b) ??
|
||||
toFiniteNumber(model.parameter_count_b);
|
||||
if (Number.isFinite(direct) && direct > 0) return direct;
|
||||
|
||||
// Parse from explicit "size" / name strings (e.g. "8B", "Qwen 0.5B", "405b").
|
||||
const textCandidates = [model.size, model.parameter_size, model.name, model.model_name, model.model_id, model.model_identifier, model.tag];
|
||||
for (const candidate of textCandidates) {
|
||||
if (typeof candidate !== 'string') continue;
|
||||
const match = candidate.match(PARAM_HINT_REGEX);
|
||||
if (match) {
|
||||
const value = parseFloat(match[1]);
|
||||
if (Number.isFinite(value) && value > 0) {
|
||||
return match[2].toLowerCase() === 'm' ? value / 1000 : value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to inferring from an artifact size in GB (Q4 ~ 0.6GB/B).
|
||||
const sizeGB = deriveSizeGB(model, null);
|
||||
if (Number.isFinite(sizeGB) && sizeGB > 0) {
|
||||
return Math.max(0.5, Math.round((sizeGB / 0.6) * 2) / 2);
|
||||
}
|
||||
|
||||
return 7; // Same neutral fallback the engines used.
|
||||
}
|
||||
|
||||
function deriveSizeGB(model = {}, paramsB = null) {
|
||||
const direct =
|
||||
toFiniteNumber(model.sizeGB) ??
|
||||
toFiniteNumber(model.size_gb) ??
|
||||
toFiniteNumber(model.real_size_gb) ??
|
||||
toFiniteNumber(model.estimated_size_gb);
|
||||
if (Number.isFinite(direct) && direct > 0) return direct;
|
||||
|
||||
// "size" may be a file size like "4.4GB" (vs a parameter count like "8B").
|
||||
if (typeof model.size === 'string') {
|
||||
const gbMatch = model.size.match(/(\d+(?:\.\d+)?)\s*g(?:i)?b\b/i);
|
||||
if (gbMatch) return parseFloat(gbMatch[1]);
|
||||
}
|
||||
if (typeof model.installedSize === 'string') {
|
||||
const gbMatch = model.installedSize.match(/(\d+(?:\.\d+)?)\s*g(?:i)?b\b/i);
|
||||
if (gbMatch) return parseFloat(gbMatch[1]);
|
||||
}
|
||||
|
||||
if (Number.isFinite(paramsB) && paramsB > 0) {
|
||||
return Math.max(0.4, Math.round((paramsB * 0.6 + 0.4) * 10) / 10);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function deriveContext(model = {}) {
|
||||
const candidates = [
|
||||
model.ctxMax,
|
||||
model.context_length,
|
||||
model.contextLength,
|
||||
model.context,
|
||||
model.performance && model.performance.context_length
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate > 0) {
|
||||
return Math.round(candidate);
|
||||
}
|
||||
if (typeof candidate === 'string') {
|
||||
const parsed = canonicalSelector.parseContextLength(candidate);
|
||||
if (Number.isFinite(parsed) && parsed > 0) return parsed;
|
||||
}
|
||||
}
|
||||
return 4096;
|
||||
}
|
||||
|
||||
function deriveQuant(model = {}) {
|
||||
const candidates = [model.quant, model.quantization];
|
||||
for (const candidate of candidates) {
|
||||
if (typeof candidate === 'string' && candidate.trim()) {
|
||||
return canonicalSelector.normalizeQuantization(candidate);
|
||||
}
|
||||
if (Array.isArray(candidate) && candidate.length > 0) {
|
||||
// ExpandedModelsDatabase stores quantization as an array of options;
|
||||
// prefer the highest-quality option that the hierarchy knows about.
|
||||
const normalized = candidate.map((q) => canonicalSelector.normalizeQuantization(q));
|
||||
for (const q of canonicalSelector.quantHierarchy) {
|
||||
if (normalized.includes(q)) return q;
|
||||
}
|
||||
return normalized[0];
|
||||
}
|
||||
}
|
||||
return 'Q4_K_M';
|
||||
}
|
||||
|
||||
function deriveModalities(model = {}) {
|
||||
if (Array.isArray(model.modalities) && model.modalities.length > 0) {
|
||||
return model.modalities.map((m) => String(m).toLowerCase());
|
||||
}
|
||||
const text = [
|
||||
model.name,
|
||||
model.model_name,
|
||||
model.model_id,
|
||||
model.model_identifier,
|
||||
model.specialization,
|
||||
model.category,
|
||||
...(Array.isArray(model.capabilities) ? model.capabilities : [String(model.capabilities || '')]),
|
||||
...(Array.isArray(model.input_types) ? model.input_types : [])
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' ')
|
||||
.toLowerCase();
|
||||
const hasVision = /\b(vision|vl\b|llava|bakllava|moondream|multimodal|image)\b/.test(text);
|
||||
return hasVision ? ['text', 'vision'] : ['text'];
|
||||
}
|
||||
|
||||
function deriveTags(model = {}, modalities = ['text']) {
|
||||
const explicit = Array.isArray(model.tags) ? model.tags.map((t) => String(t).toLowerCase()) : [];
|
||||
const tags = new Set(explicit);
|
||||
|
||||
const text = [
|
||||
model.name,
|
||||
model.model_name,
|
||||
model.model_id,
|
||||
model.model_identifier,
|
||||
model.specialization,
|
||||
model.category,
|
||||
model.tag,
|
||||
...(Array.isArray(model.capabilities)
|
||||
? model.capabilities
|
||||
: [String(model.capabilities || '')]),
|
||||
...(Array.isArray(model.use_cases) ? model.use_cases : [])
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' ')
|
||||
.toLowerCase();
|
||||
|
||||
if (/\b(coder|code|programming|deepseek-coder|starcoder|codellama|codegemma)\b/.test(text)) tags.add('coder');
|
||||
if (/\binstruct\b/.test(text)) tags.add('instruct');
|
||||
if (/\b(chat|assistant|conversation)\b/.test(text)) tags.add('chat');
|
||||
if (/\bembed/.test(text)) tags.add('embedding');
|
||||
if (/\b(reason|math|logic)\b/.test(text)) tags.add('reasoning');
|
||||
if (modalities.includes('vision')) tags.add('vision');
|
||||
|
||||
// Most general/instruct models should compete in the general category even
|
||||
// when no explicit tag exists, matching the permissive engines' behavior.
|
||||
if (tags.size === 0) tags.add('chat');
|
||||
|
||||
return [...tags];
|
||||
}
|
||||
|
||||
function deriveIdentifier(model = {}) {
|
||||
return (
|
||||
model.model_identifier ||
|
||||
model.tag ||
|
||||
model.model_id ||
|
||||
model.model_name ||
|
||||
model.name ||
|
||||
'unknown-model'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert an arbitrary model object (any of the three command shapes) into the
|
||||
* already-normalized deterministic model shape. The original object is kept on
|
||||
* `__source` so callers can map ranking output back to their display shape.
|
||||
*/
|
||||
function normalizeToDeterministic(model = {}) {
|
||||
const paramsB = deriveParamsB(model);
|
||||
const sizeGB = deriveSizeGB(model, paramsB) ?? Math.max(0.4, paramsB * 0.6);
|
||||
const modalities = deriveModalities(model);
|
||||
const tags = deriveTags(model, modalities);
|
||||
const quant = deriveQuant(model);
|
||||
const identifier = deriveIdentifier(model);
|
||||
|
||||
const moeFlag = Boolean(model.isMoE || model.is_moe);
|
||||
const totalParamsB =
|
||||
toFiniteNumber(model.totalParamsB) ?? toFiniteNumber(model.total_params_b) ?? null;
|
||||
const activeParamsB =
|
||||
toFiniteNumber(model.activeParamsB) ?? toFiniteNumber(model.active_params_b) ?? null;
|
||||
|
||||
return {
|
||||
// Canonical deterministic fields (pass `alreadyNormalized` branch).
|
||||
name: identifier,
|
||||
model_identifier: identifier,
|
||||
family: model.family || canonicalSelector.extractFamily(String(identifier)),
|
||||
paramsB,
|
||||
ctxMax: deriveContext(model),
|
||||
quant,
|
||||
sizeGB,
|
||||
modalities,
|
||||
tags,
|
||||
installed: Boolean(model.installed || model.isOllamaInstalled),
|
||||
pulls: toFiniteNumber(model.pulls) ?? toFiniteNumber(model.actual_pulls) ?? 0,
|
||||
availableQuantizations: [quant],
|
||||
sizeByQuant: { [quant]: sizeGB },
|
||||
// MoE passthrough so memory/speed estimation stays accurate.
|
||||
isMoE: moeFlag,
|
||||
is_moe: moeFlag,
|
||||
totalParamsB,
|
||||
activeParamsB,
|
||||
expertCount: toFiniteNumber(model.expertCount) ?? toFiniteNumber(model.expert_count) ?? null,
|
||||
expertsActivePerToken:
|
||||
toFiniteNumber(model.expertsActivePerToken) ??
|
||||
toFiniteNumber(model.experts_active_per_token) ??
|
||||
null,
|
||||
// Freshness defaults: neutral so we never penalize callers that lack
|
||||
// timestamps (the per-command sources already carry these when known).
|
||||
freshnessScore: toFiniteNumber(model.freshnessScore) ?? 55,
|
||||
isStale: Boolean(model.isStale),
|
||||
isDeprecated: Boolean(model.isDeprecated || model.deprecated),
|
||||
source: model.source || 'unified_core',
|
||||
// Back-reference for result mapping.
|
||||
__source: model
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical ranking entrypoint. Returns the deterministic selector's ranked
|
||||
* candidates (each carries `.score`, `.components {Q,S,F,C,H}`, `.meta`, etc.)
|
||||
* for the supplied models/hardware. Each candidate's `meta.__source` points
|
||||
* back to the caller's original model object.
|
||||
*
|
||||
* @param {Array<Object>} models Arbitrary model objects (any command shape).
|
||||
* @param {Object} hardware Hardware profile (any detector shape).
|
||||
* @param {Object} options
|
||||
* @param {string} [options.category='general'] Use-case / category.
|
||||
* @param {string} [options.optimizeFor='balanced']
|
||||
* @param {string} [options.runtime='ollama']
|
||||
* @param {number} [options.topN] Limit (default: all candidates).
|
||||
* @returns {Promise<{category, candidates, total_evaluated, hardware}>}
|
||||
*/
|
||||
async function rankModels(models, hardware, options = {}) {
|
||||
const category = normalizeCategory(options.category || options.useCase || 'general');
|
||||
const pool = (Array.isArray(models) ? models : [])
|
||||
.filter(Boolean)
|
||||
.map(normalizeToDeterministic);
|
||||
|
||||
const result = await canonicalSelector.selectModels(category, {
|
||||
topN: typeof options.topN === 'number' ? options.topN : pool.length || 1,
|
||||
enableProbe: false,
|
||||
silent: true,
|
||||
optimizeFor: options.optimizeFor || options.optimize || options.objective || 'balanced',
|
||||
runtime: options.runtime || 'ollama',
|
||||
hardware: hardware || undefined,
|
||||
installedModels: [],
|
||||
modelPool: pool
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map deterministic-engine categories from the broader vocabulary the commands
|
||||
* use (e.g. `check`/`smart-recommend` accept `chat`, `creative`, `vision`,
|
||||
* `talking`) onto the categories the deterministic core scores.
|
||||
*/
|
||||
function normalizeCategory(category) {
|
||||
const normalized = String(category || 'general').toLowerCase().trim();
|
||||
const map = {
|
||||
general: 'general',
|
||||
chat: 'general',
|
||||
talking: 'general',
|
||||
assistant: 'general',
|
||||
coding: 'coding',
|
||||
code: 'coding',
|
||||
programming: 'coding',
|
||||
reasoning: 'reasoning',
|
||||
math: 'reasoning',
|
||||
multimodal: 'multimodal',
|
||||
vision: 'multimodal',
|
||||
embeddings: 'embeddings',
|
||||
embedding: 'embeddings',
|
||||
summarization: 'summarization',
|
||||
reading: 'reading',
|
||||
creative: 'general',
|
||||
fast: 'general',
|
||||
quality: 'general',
|
||||
longctx: 'reading'
|
||||
};
|
||||
return map[normalized] || 'general';
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
canonicalSelector,
|
||||
rankModels,
|
||||
normalizeToDeterministic,
|
||||
normalizeCategory,
|
||||
deriveParamsB,
|
||||
deriveSizeGB
|
||||
};
|
||||
|
|
@ -18,6 +18,7 @@ const TESTS = [
|
|||
{ name: 'Token speed estimation', file: 'token-speed-estimation.test.js', category: 'Performance' },
|
||||
{ name: 'Deterministic model pool', file: 'deterministic-model-pool-check.js', category: 'Recommendations' },
|
||||
{ name: 'Multi-objective selector regression', file: 'multi-objective-selector-regression.test.js', category: 'Recommendations' },
|
||||
{ name: 'Scoring engine unification', file: 'scoring-unification.test.js', category: 'Recommendations' },
|
||||
{ name: 'Fine-tuning support helper', file: 'fine-tuning-support.test.js', category: 'Recommendations' },
|
||||
{ name: 'Ollama capacity planner', file: 'ollama-capacity-planner.test.js', category: 'Ollama' },
|
||||
{ name: 'Ollama localhost fallback', file: 'ollama-client-localhost-fallback.test.js', category: 'Ollama' },
|
||||
|
|
|
|||
318
tests/scoring-unification.test.js
Normal file
318
tests/scoring-unification.test.js
Normal file
|
|
@ -0,0 +1,318 @@
|
|||
/**
|
||||
* Scoring Unification Integration Test (GitHub issue #88)
|
||||
* ======================================================
|
||||
*
|
||||
* Proves the root-cause fix: the three recommendation surfaces
|
||||
* (`check`, `recommend`, `smart-recommend`) now derive their ranking from ONE
|
||||
* canonical scoring core (DeterministicModelSelector via scoring-core), so:
|
||||
*
|
||||
* a. CONSISTENCY: all three agree on the top pick's size class for the same
|
||||
* logical (model pool, hardware) input.
|
||||
* b. HIGH-CAPACITY FLOOR EVERYWHERE: on a 192GB dual-RTX-PRO-6000 profile none
|
||||
* of the three paths returns a sub-30B model as its #1 pick for
|
||||
* general/coding/reasoning.
|
||||
* c. SMALL-HARDWARE SANITY: on an 8GB CPU-only profile every path still
|
||||
* prefers a small (<30B, and in practice <=8B) model — i.e. the fix did
|
||||
* not just force "always big".
|
||||
*
|
||||
* Each command consumes a DIFFERENT model shape, so we build the SAME logical
|
||||
* models in each path's native shape and call each path's selection entrypoint
|
||||
* directly (no live Ollama / sql.js dependency).
|
||||
*/
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
const MultiObjectiveSelector = require('../src/ai/multi-objective-selector');
|
||||
const DeterministicModelSelector = require('../src/models/deterministic-selector');
|
||||
const IntelligentSelector = require('../src/models/intelligent-selector');
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Hardware profiles
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// The dual RTX PRO 6000 / ~192GB VRAM case from issue #88.
|
||||
function buildHighCapacityHardware() {
|
||||
return {
|
||||
cpu: { brand: 'AMD Threadripper PRO', cores: 32, physicalCores: 32, speed: 4.0, architecture: 'x86_64' },
|
||||
gpu: {
|
||||
model: 'NVIDIA RTX PRO 6000 Blackwell Workstation Edition',
|
||||
vendor: 'NVIDIA',
|
||||
type: 'nvidia',
|
||||
vram: 192,
|
||||
vramPerGPU: 96,
|
||||
gpuCount: 2,
|
||||
isMultiGPU: true,
|
||||
dedicated: true
|
||||
},
|
||||
memory: { total: 192, totalGB: 192 },
|
||||
acceleration: { supports_metal: false, supports_cuda: true, supports_rocm: false },
|
||||
os: { platform: 'linux' },
|
||||
summary: {
|
||||
bestBackend: 'cuda',
|
||||
gpuModel: 'NVIDIA RTX PRO 6000 Blackwell Workstation Edition',
|
||||
effectiveMemory: 180,
|
||||
systemRAM: 192,
|
||||
totalVRAM: 192,
|
||||
isMultiGPU: true,
|
||||
gpuCount: 2,
|
||||
hasDedicatedGPU: true,
|
||||
hasIntegratedGPU: false,
|
||||
hardwareTier: 'very_high',
|
||||
speedCoefficient: 320
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// A small laptop: 8GB RAM, CPU-only.
|
||||
function buildSmallHardware() {
|
||||
return {
|
||||
cpu: { brand: 'Intel Core i5-1135G7', cores: 4, physicalCores: 4, speed: 2.4, architecture: 'x86_64' },
|
||||
gpu: { model: '', vram: 0, dedicated: false },
|
||||
memory: { total: 8, totalGB: 8 },
|
||||
acceleration: { supports_metal: false, supports_cuda: false, supports_rocm: false },
|
||||
os: { platform: 'linux' },
|
||||
summary: {
|
||||
bestBackend: 'cpu',
|
||||
gpuModel: '',
|
||||
effectiveMemory: 8,
|
||||
systemRAM: 8,
|
||||
totalVRAM: 0,
|
||||
hasDedicatedGPU: false,
|
||||
hasIntegratedGPU: false,
|
||||
hardwareTier: 'low',
|
||||
speedCoefficient: 40
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Representative model pool (same logical models, three native shapes).
|
||||
// Each category has a small (<=8B) and a large (>=30B) member so every path is
|
||||
// forced to make a real size choice.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// `check` shape: ExpandedModelsDatabase-style rows.
|
||||
function buildCheckPool() {
|
||||
const mk = (name, sizeB, gb, ctx, cat, quant = ['Q4_K_M']) => ({
|
||||
name,
|
||||
size: `${sizeB}B`,
|
||||
type: 'local',
|
||||
category: cat,
|
||||
requirements: { ram: Math.ceil(gb + 2), vram: 0, cpu_cores: 2, storage: gb },
|
||||
frameworks: ['ollama'],
|
||||
quantization: quant,
|
||||
performance: { context_length: ctx },
|
||||
specialization: cat,
|
||||
installation: { ollama: `ollama pull ${name}` },
|
||||
// size_gb hint so memory math is realistic regardless of param parsing.
|
||||
size_gb: gb
|
||||
});
|
||||
return [
|
||||
mk('uni-chat 8B', 8, 5.0, 65536, 'general'),
|
||||
mk('uni-chat 70B', 70, 42, 65536, 'general'),
|
||||
mk('uni-coder 8B', 8, 5.0, 32768, 'coding'),
|
||||
mk('uni-coder 34B', 34, 20.5, 32768, 'coding'),
|
||||
mk('uni-reason 7B', 7, 4.8, 65536, 'reasoning'),
|
||||
mk('uni-reason 70B', 70, 43, 65536, 'reasoning')
|
||||
];
|
||||
}
|
||||
|
||||
// `recommend` shape: Ollama scrape-style models with variants.
|
||||
function buildRecommendPool() {
|
||||
return [
|
||||
{
|
||||
model_identifier: 'uni-chat',
|
||||
model_name: 'uni-chat',
|
||||
primary_category: 'chat',
|
||||
context_length: '64K',
|
||||
variants: [
|
||||
{ tag: 'uni-chat:8b', size: '8b', quantization: 'Q4_0', real_size_gb: 5.0, categories: ['chat'] },
|
||||
{ tag: 'uni-chat:70b', size: '70b', quantization: 'Q4_0', real_size_gb: 42, categories: ['chat'] }
|
||||
],
|
||||
tags: ['uni-chat:8b', 'uni-chat:70b'],
|
||||
use_cases: ['chat']
|
||||
},
|
||||
{
|
||||
model_identifier: 'uni-coder',
|
||||
model_name: 'uni-coder',
|
||||
primary_category: 'coding',
|
||||
context_length: '32K',
|
||||
variants: [
|
||||
{ tag: 'uni-coder:8b', size: '8b', quantization: 'Q4_0', real_size_gb: 5.0, categories: ['coding'] },
|
||||
{ tag: 'uni-coder:34b', size: '34b', quantization: 'Q4_0', real_size_gb: 20.5, categories: ['coding'] }
|
||||
],
|
||||
tags: ['uni-coder:8b', 'uni-coder:34b'],
|
||||
use_cases: ['coding', 'programming']
|
||||
},
|
||||
{
|
||||
model_identifier: 'uni-reason',
|
||||
model_name: 'uni-reason',
|
||||
primary_category: 'reasoning',
|
||||
context_length: '64K',
|
||||
variants: [
|
||||
{ tag: 'uni-reason:7b', size: '7b', quantization: 'Q4_0', real_size_gb: 4.8, categories: ['reasoning'] },
|
||||
{ tag: 'uni-reason:70b', size: '70b', quantization: 'Q4_0', real_size_gb: 43, categories: ['reasoning'] }
|
||||
],
|
||||
tags: ['uni-reason:7b', 'uni-reason:70b'],
|
||||
use_cases: ['reasoning', 'math']
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
// `smart-recommend` shape: sql.js variant rows.
|
||||
function buildSmartVariants() {
|
||||
return [
|
||||
{ model_id: 'uni-chat', tag: 'uni-chat:8b', params_b: 8, size_gb: 5.0, quant: 'Q4_K_M', context_length: 65536, capabilities: 'chat', family: 'llama3.2', pulls: 5000000 },
|
||||
{ model_id: 'uni-chat', tag: 'uni-chat:70b', params_b: 70, size_gb: 42, quant: 'Q4_K_M', context_length: 65536, capabilities: 'chat', family: 'llama3.3', pulls: 5000000 },
|
||||
{ model_id: 'uni-coder', tag: 'uni-coder:8b', params_b: 8, size_gb: 5.0, quant: 'Q4_K_M', context_length: 32768, capabilities: 'coding', family: 'qwen2.5-coder', pulls: 5000000 },
|
||||
{ model_id: 'uni-coder', tag: 'uni-coder:34b', params_b: 34, size_gb: 20.5, quant: 'Q4_K_M', context_length: 32768, capabilities: 'coding', family: 'qwen2.5-coder', pulls: 5000000 },
|
||||
{ model_id: 'uni-reason', tag: 'uni-reason:7b', params_b: 7, size_gb: 4.8, quant: 'Q4_K_M', context_length: 65536, capabilities: 'reasoning', family: 'deepseek-r1', pulls: 5000000 },
|
||||
{ model_id: 'uni-reason', tag: 'uni-reason:70b', params_b: 70, size_gb: 43, quant: 'Q4_K_M', context_length: 65536, capabilities: 'reasoning', family: 'deepseek-r1', pulls: 5000000 }
|
||||
];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Per-path top-pick params extraction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function parseParamsB(text) {
|
||||
const match = String(text || '').match(/(\d+(?:\.\d+)?)\s*b\b/i);
|
||||
return match ? parseFloat(match[1]) : null;
|
||||
}
|
||||
|
||||
async function checkTopParamsB(hardware, category) {
|
||||
const selector = new MultiObjectiveSelector();
|
||||
const result = await selector.selectBestModels(hardware, buildCheckPool(), category, 20);
|
||||
const top = result.compatible[0] || result.marginal[0];
|
||||
assert.ok(top, `check should return a ranked model for category ${category}`);
|
||||
// ExpandedModelsDatabase rows keep their name/size; derive params from them.
|
||||
const params = selector.estimateModelParams(top) || parseParamsB(top.size) || parseParamsB(top.name);
|
||||
return { params, label: top.name };
|
||||
}
|
||||
|
||||
async function recommendTopParamsB(hardware, category) {
|
||||
const selector = new DeterministicModelSelector();
|
||||
selector.getInstalledModels = async () => [];
|
||||
const recommendations = await selector.getBestModelsForHardware(hardware, buildRecommendPool(), { runtime: 'ollama' });
|
||||
const top = recommendations[category] && recommendations[category].bestModels[0];
|
||||
assert.ok(top, `recommend should return a ranked model for category ${category}`);
|
||||
const params = Number(top.size) || parseParamsB(top.model_identifier);
|
||||
return { params, label: top.model_identifier };
|
||||
}
|
||||
|
||||
function makeStubDetector(hardware) {
|
||||
return {
|
||||
detect: async () => hardware,
|
||||
getHardwareDescription: () => 'unit-test hardware',
|
||||
getHardwareTier: () => (hardware.summary && hardware.summary.hardwareTier) || 'medium',
|
||||
getMaxModelSize: () => (hardware.summary && hardware.summary.effectiveMemory) || 8
|
||||
};
|
||||
}
|
||||
|
||||
async function smartTopParamsB(hardware, useCase) {
|
||||
const selector = new IntelligentSelector({ detector: makeStubDetector(hardware) });
|
||||
const recommendations = await selector.recommend(buildSmartVariants(), {
|
||||
useCase,
|
||||
limit: 10,
|
||||
policyFile: null
|
||||
});
|
||||
const top = recommendations.topPicks.best;
|
||||
assert.ok(top && top.variant, `smart-recommend should return a top pick for use case ${useCase}`);
|
||||
const params = top.variant.params_b || parseParamsB(top.variant.tag);
|
||||
return { params, label: top.variant.tag };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const HIGH_CAPACITY_CATEGORIES = ['general', 'coding', 'reasoning'];
|
||||
|
||||
async function testHighCapacityFloorEverywhere() {
|
||||
const hardware = buildHighCapacityHardware();
|
||||
|
||||
for (const category of HIGH_CAPACITY_CATEGORIES) {
|
||||
const check = await checkTopParamsB(hardware, category);
|
||||
const recommend = await recommendTopParamsB(hardware, category);
|
||||
const smart = await smartTopParamsB(hardware, category);
|
||||
|
||||
assert.ok(
|
||||
check.params >= 30,
|
||||
`[check] ${category}: top pick should be >=30B on 192GB hardware, got ${check.label} (${check.params}B)`
|
||||
);
|
||||
assert.ok(
|
||||
recommend.params >= 30,
|
||||
`[recommend] ${category}: top pick should be >=30B on 192GB hardware, got ${recommend.label} (${recommend.params}B)`
|
||||
);
|
||||
assert.ok(
|
||||
smart.params >= 30,
|
||||
`[smart-recommend] ${category}: top pick should be >=30B on 192GB hardware, got ${smart.label} (${smart.params}B)`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function testConsistencyAcrossPaths() {
|
||||
const hardware = buildHighCapacityHardware();
|
||||
|
||||
for (const category of HIGH_CAPACITY_CATEGORIES) {
|
||||
const check = await checkTopParamsB(hardware, category);
|
||||
const recommend = await recommendTopParamsB(hardware, category);
|
||||
const smart = await smartTopParamsB(hardware, category);
|
||||
|
||||
const sizeClass = (p) => (p >= 30 ? 'large' : 'small');
|
||||
const checkClass = sizeClass(check.params);
|
||||
const recommendClass = sizeClass(recommend.params);
|
||||
const smartClass = sizeClass(smart.params);
|
||||
|
||||
assert.strictEqual(
|
||||
checkClass,
|
||||
recommendClass,
|
||||
`[${category}] check (${check.label} ${check.params}B) and recommend (${recommend.label} ${recommend.params}B) should agree on size class`
|
||||
);
|
||||
assert.strictEqual(
|
||||
recommendClass,
|
||||
smartClass,
|
||||
`[${category}] recommend (${recommend.label} ${recommend.params}B) and smart-recommend (${smart.label} ${smart.params}B) should agree on size class`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function testSmallHardwarePrefersSmallEverywhere() {
|
||||
const hardware = buildSmallHardware();
|
||||
|
||||
for (const category of HIGH_CAPACITY_CATEGORIES) {
|
||||
const check = await checkTopParamsB(hardware, category);
|
||||
const recommend = await recommendTopParamsB(hardware, category);
|
||||
const smart = await smartTopParamsB(hardware, category);
|
||||
|
||||
assert.ok(
|
||||
check.params < 30,
|
||||
`[check] ${category}: small hardware should prefer a small model, got ${check.label} (${check.params}B)`
|
||||
);
|
||||
assert.ok(
|
||||
recommend.params < 30,
|
||||
`[recommend] ${category}: small hardware should prefer a small model, got ${recommend.label} (${recommend.params}B)`
|
||||
);
|
||||
assert.ok(
|
||||
smart.params < 30,
|
||||
`[smart-recommend] ${category}: small hardware should prefer a small model, got ${smart.label} (${smart.params}B)`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function run() {
|
||||
await testHighCapacityFloorEverywhere();
|
||||
await testConsistencyAcrossPaths();
|
||||
await testSmallHardwarePrefersSmallEverywhere();
|
||||
console.log('scoring-unification.test.js: OK');
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
run().catch((error) => {
|
||||
console.error('scoring-unification.test.js: FAILED');
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { run };
|
||||
Loading…
Add table
Add a link
Reference in a new issue