diff --git a/src/ai/multi-objective-selector.js b/src/ai/multi-objective-selector.js index 1eaa1e6..95f0e2f 100644 --- a/src/ai/multi-objective-selector.js +++ b/src/ai/multi-objective-selector.js @@ -10,6 +10,7 @@ const { MULTI_OBJECTIVE_WEIGHTS } = require('../models/scoring-config'); const { normalizePlatform } = require('../utils/platform'); +const { rankModels } = require('../models/scoring-core'); class MultiObjectiveSelector { constructor() { @@ -40,23 +41,124 @@ class MultiObjectiveSelector { } /** - * Select best models using multi-objective ranking + * Select best models using the UNIFIED canonical scoring core (issue #88). + * + * `check` used to rank through this selector's own multi-objective math, + * which diverged from `recommend`/`smart-recommend` and never received the + * PR #89 high-capacity right-sizing fix. It now routes the ranking through + * the shared DeterministicModelSelector core (via scoring-core.rankModels) + * so identical (model, hardware) inputs score identically across all three + * commands and the high-capacity floor applies here too. + * + * The output shape is preserved exactly: `{ compatible, marginal, + * incompatible }`, each entry being the ORIGINAL model object spread with + * `totalScore`, `components { quality, speed, ttfb, context, hardwareMatch }` + * and `reasoning`, so downstream `check` rendering and the regression test + * (which calls `estimateModelParams` on the returned object) keep working. */ async selectBestModels(hardware, models, category = 'general', topK = 10) { - // Step 1: Hard filters - remove incompatible models + const inputModels = Array.isArray(models) ? models.filter(Boolean) : []; + if (inputModels.length === 0) { + return { compatible: [], marginal: [], incompatible: [] }; + } + + let ranking; + try { + ranking = await rankModels(inputModels, hardware, { category, topN: inputModels.length }); + } catch (error) { + ranking = null; + } + + // Defensive fallback: if the unified core is unavailable for any reason, + // fall back to the legacy multi-objective ranking so `check` still works. + if (!ranking || !Array.isArray(ranking.candidates)) { + return this.selectBestModelsLegacy(hardware, inputModels, category, topK); + } + + const scoredModels = []; + const rankedSources = new Set(); + for (const candidate of ranking.candidates) { + const source = candidate?.meta?.__source; + if (!source) continue; + rankedSources.add(source); + scoredModels.push(this.mapCoreCandidateToMultiObjective(candidate, source, hardware, category)); + } + + // Models the canonical core dropped (category filter / budget) are not + // viable on this hardware for this use case -> treat as incompatible, + // mirroring the previous hard-filter semantics. + const incompatibleExtras = inputModels + .filter((model) => !rankedSources.has(model)) + .map((model) => ({ + ...model, + totalScore: 0, + components: { quality: 0, speed: 0, ttfb: 0, context: 0, hardwareMatch: 0 }, + reasoning: 'Filtered out by unified scoring core (does not fit hardware/use-case)' + })); + + scoredModels.sort((a, b) => b.totalScore - a.totalScore); + + const classified = this.classifyResults(scoredModels, topK); + classified.incompatible = [...classified.incompatible, ...incompatibleExtras].slice(0, 5); + return classified; + } + + /** + * Map a unified-core candidate back into this selector's multi-objective + * output shape. The 0-100 `score` from the deterministic core becomes + * `totalScore`; component sub-scores are normalized to 0-1 to match the + * historical `components` contract consumed by `check` rendering. + */ + mapCoreCandidateToMultiObjective(candidate, source, hardware, category) { + const components = candidate.components || {}; + const to01 = (value) => { + const num = Number(value); + if (!Number.isFinite(num)) return 0; + return Math.max(0, Math.min(1, num / 100)); + }; + + const quality = to01(components.Q); + const speed = to01(components.S); + const context = to01(components.C); + // The deterministic core folds hardware fitness into the `F` (fit) plus + // `H` (high-capacity right-sizing) components; surface that as the + // historical `hardwareMatch` signal so `check` insights stay meaningful. + const hardwareMatch = to01((Number(components.F) || 0) + (Number(components.H) || 0)); + + return { + ...source, + totalScore: Math.round(candidate.score * 100) / 100, + score: Math.round(candidate.score * 100) / 100, + components: { + quality, + speed, + ttfb: speed, // ttfb tracks speed; legacy field retained for shape + context, + hardwareMatch + }, + quant: candidate.quant || source.quant, + estimatedRAM: candidate.requiredGB, + estimatedTPS: candidate.estTPS, + reasoning: candidate.rationale || + this.generateReasoning(source, hardware, quality, hardwareMatch) + }; + } + + /** + * Legacy multi-objective ranking, retained ONLY as a defensive fallback if + * the unified core throws. Not used on the normal path. + */ + selectBestModelsLegacy(hardware, models, category = 'general', topK = 10) { const compatibleModels = this.applyHardFilters(hardware, models); - + if (compatibleModels.length === 0) { return { compatible: [], marginal: [], incompatible: models }; } - // Step 2: Multi-objective scoring - const scoredModels = compatibleModels.map(model => + const scoredModels = compatibleModels.map(model => this.calculateMultiObjectiveScore(hardware, model, category) ).filter(Boolean); - - // Step 3: Sort and classify scoredModels.sort((a, b) => b.totalScore - a.totalScore); return this.classifyResults(scoredModels, topK); diff --git a/src/models/intelligent-selector.js b/src/models/intelligent-selector.js index e22ef87..6425adc 100644 --- a/src/models/intelligent-selector.js +++ b/src/models/intelligent-selector.js @@ -10,6 +10,7 @@ const ScoringEngine = require('./scoring-engine'); const UnifiedDetector = require('../hardware/unified-detector'); const PolicyManager = require('../policy/policy-manager'); const PolicyEngine = require('../policy/policy-engine'); +const { rankModels } = require('./scoring-core'); function isPlainObject(value) { return typeof value === 'object' && value !== null && !Array.isArray(value); @@ -66,7 +67,9 @@ class IntelligentSelector { // Apply filters const filtered = this.applyFilters(variants, opts, hardware); - // Score all filtered variants + // Score all filtered variants. ScoringEngine still produces the + // per-variant `score` objects (final/components/meta) consumed by the + // smart-recommend display, but the RANKING is unified below. const scored = this.scoring.filterAndScore(filtered, hardware, { useCase: opts.useCase, targetContext: opts.targetContext, @@ -75,6 +78,12 @@ class IntelligentSelector { headroom: opts.headroom || 2 }); + // Unify ranking with the canonical scoring core (issue #88): re-order + // the scored list and rewrite each item's final score using the shared + // DeterministicModelSelector so smart-recommend agrees with + // `check`/`recommend` and inherits the PR #89 high-capacity floor. + await this.applyUnifiedRanking(scored, hardware, opts); + const policyEngine = this.resolvePolicyEngine(opts); const scoredWithPolicy = policyEngine.evaluateScoredVariants( scored, @@ -114,6 +123,83 @@ class IntelligentSelector { }; } + /** + * Re-rank the ScoringEngine-scored variants using the canonical scoring + * core so smart-recommend's ordering and headline scores match + * `check`/`recommend` and inherit the high-capacity right-sizing floor. + * + * Mutates `scored` in place: it is sorted by the unified score and each + * item's `score.final` is overwritten with the canonical 0-100 score. + * Component/meta sub-scores are left intact so the existing display (which + * shows Q/S/F and estimated TPS) keeps working. If the core cannot rank a + * variant (or throws), that item keeps its original ScoringEngine score and + * sorts after the unified ones, preserving a sensible fallback ordering. + */ + async applyUnifiedRanking(scored, hardware, opts = {}) { + if (!Array.isArray(scored) || scored.length === 0) return scored; + + let ranking; + try { + ranking = await rankModels( + scored.map((item) => item.variant), + hardware, + { + category: opts.useCase || 'general', + optimizeFor: opts.optimizeFor || opts.optimize || 'balanced', + runtime: opts.runtime || 'ollama', + topN: scored.length + } + ); + } catch (error) { + return scored; // Defensive: keep original ScoringEngine ordering. + } + + if (!ranking || !Array.isArray(ranking.candidates)) return scored; + + // Map each source variant -> its unified score + ordering index. + const unifiedByVariant = new Map(); + ranking.candidates.forEach((candidate, index) => { + const source = candidate?.meta?.__source; + if (!source) return; + unifiedByVariant.set(source, { + unifiedScore: Math.round(candidate.score * 10) / 10, + rank: index, + quant: candidate.quant, + estimatedTPS: candidate.estTPS + }); + }); + + for (const item of scored) { + const unified = unifiedByVariant.get(item.variant); + if (!unified) { + // Not ranked by the core (e.g. filtered out): sort last and tag + // so any downstream tie-breaks are deterministic. + item.__unifiedRank = Number.MAX_SAFE_INTEGER; + continue; + } + item.__unifiedRank = unified.rank; + if (item.score) { + item.score.final = Math.min(100, Math.max(0, Math.round(unified.unifiedScore))); + if (item.score.meta) { + item.score.meta.unifiedScore = unified.unifiedScore; + } + } + } + + scored.sort((a, b) => { + const ra = Number.isFinite(a.__unifiedRank) ? a.__unifiedRank : Number.MAX_SAFE_INTEGER; + const rb = Number.isFinite(b.__unifiedRank) ? b.__unifiedRank : Number.MAX_SAFE_INTEGER; + if (ra !== rb) return ra - rb; + return (b.score?.final || 0) - (a.score?.final || 0); + }); + + for (const item of scored) { + delete item.__unifiedRank; + } + + return scored; + } + /** * Resolve policy engine from explicit options, in-memory policy, or policy file. */ diff --git a/src/models/scoring-core.js b/src/models/scoring-core.js new file mode 100644 index 0000000..3be06cb --- /dev/null +++ b/src/models/scoring-core.js @@ -0,0 +1,341 @@ +/** + * Unified Scoring Core (GitHub issue #88) + * ======================================= + * + * Historically the three user-facing recommendation surfaces each ran a + * DIFFERENT scoring engine, so they could disagree about the best model and + * only one of them ("recommend") received the PR #89 high-capacity + * right-sizing fix: + * + * - `check` -> MultiObjectiveSelector (src/ai/multi-objective-selector.js) + * - `recommend` -> DeterministicModelSelector (src/models/deterministic-selector.js) + * - `smart-recommend` -> ScoringEngine (src/models/scoring-engine.js) + * + * This module establishes the DeterministicModelSelector as the SINGLE + * canonical ranking core (it already carries PR #89's + * `calculateHighCapacitySizeAdjustment` / `getHighCapacitySizeTarget`). The + * other commands keep their own model SOURCE and DISPLAY shape, but route the + * actual ranking/selection through `rankModels()` below so that, for identical + * (model, hardware) inputs, every command produces identical scores and the + * high-capacity floor applies everywhere. + * + * The hard part is that each command feeds the engine a different model shape: + * - `check` uses ExpandedModelsDatabase rows ({ name: "Llama 3.1 8B", size: "8B", ... }) + * - `smart-recommend` uses sql.js variant rows ({ model_id, params_b, size_gb, quant, ... }) + * - `recommend` uses the Ollama scrape/catalog (already deterministic-friendly) + * + * `normalizeToDeterministic()` converts any of those into the + * already-normalized deterministic model shape and keeps a back-reference + * (`__source`) so callers can recover their original object after ranking. + */ + +const DeterministicModelSelector = require('./deterministic-selector'); + +// One shared selector instance: it is stateless w.r.t. the model pool (the +// pool is always passed in via options), so sharing it is safe and avoids +// re-allocating the large prior tables on every call. +const canonicalSelector = new DeterministicModelSelector(); + +const PARAM_HINT_REGEX = /(\d+(?:\.\d+)?)\s*([bm])\b/i; + +function toFiniteNumber(value) { + if (typeof value === 'number' && Number.isFinite(value)) return value; + if (typeof value === 'string' && value.trim() !== '' && Number.isFinite(Number(value))) { + return Number(value); + } + return null; +} + +/** + * Best-effort parameter-count extraction (in billions) from a model's various + * shapes. Mirrors the heuristics the individual engines used so the unified + * ranking sees the same parameter counts they would have. + */ +function deriveParamsB(model = {}) { + const direct = + toFiniteNumber(model.paramsB) ?? + toFiniteNumber(model.params_b) ?? + toFiniteNumber(model.parameter_count_b); + if (Number.isFinite(direct) && direct > 0) return direct; + + // Parse from explicit "size" / name strings (e.g. "8B", "Qwen 0.5B", "405b"). + const textCandidates = [model.size, model.parameter_size, model.name, model.model_name, model.model_id, model.model_identifier, model.tag]; + for (const candidate of textCandidates) { + if (typeof candidate !== 'string') continue; + const match = candidate.match(PARAM_HINT_REGEX); + if (match) { + const value = parseFloat(match[1]); + if (Number.isFinite(value) && value > 0) { + return match[2].toLowerCase() === 'm' ? value / 1000 : value; + } + } + } + + // Fall back to inferring from an artifact size in GB (Q4 ~ 0.6GB/B). + const sizeGB = deriveSizeGB(model, null); + if (Number.isFinite(sizeGB) && sizeGB > 0) { + return Math.max(0.5, Math.round((sizeGB / 0.6) * 2) / 2); + } + + return 7; // Same neutral fallback the engines used. +} + +function deriveSizeGB(model = {}, paramsB = null) { + const direct = + toFiniteNumber(model.sizeGB) ?? + toFiniteNumber(model.size_gb) ?? + toFiniteNumber(model.real_size_gb) ?? + toFiniteNumber(model.estimated_size_gb); + if (Number.isFinite(direct) && direct > 0) return direct; + + // "size" may be a file size like "4.4GB" (vs a parameter count like "8B"). + if (typeof model.size === 'string') { + const gbMatch = model.size.match(/(\d+(?:\.\d+)?)\s*g(?:i)?b\b/i); + if (gbMatch) return parseFloat(gbMatch[1]); + } + if (typeof model.installedSize === 'string') { + const gbMatch = model.installedSize.match(/(\d+(?:\.\d+)?)\s*g(?:i)?b\b/i); + if (gbMatch) return parseFloat(gbMatch[1]); + } + + if (Number.isFinite(paramsB) && paramsB > 0) { + return Math.max(0.4, Math.round((paramsB * 0.6 + 0.4) * 10) / 10); + } + return null; +} + +function deriveContext(model = {}) { + const candidates = [ + model.ctxMax, + model.context_length, + model.contextLength, + model.context, + model.performance && model.performance.context_length + ]; + for (const candidate of candidates) { + if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate > 0) { + return Math.round(candidate); + } + if (typeof candidate === 'string') { + const parsed = canonicalSelector.parseContextLength(candidate); + if (Number.isFinite(parsed) && parsed > 0) return parsed; + } + } + return 4096; +} + +function deriveQuant(model = {}) { + const candidates = [model.quant, model.quantization]; + for (const candidate of candidates) { + if (typeof candidate === 'string' && candidate.trim()) { + return canonicalSelector.normalizeQuantization(candidate); + } + if (Array.isArray(candidate) && candidate.length > 0) { + // ExpandedModelsDatabase stores quantization as an array of options; + // prefer the highest-quality option that the hierarchy knows about. + const normalized = candidate.map((q) => canonicalSelector.normalizeQuantization(q)); + for (const q of canonicalSelector.quantHierarchy) { + if (normalized.includes(q)) return q; + } + return normalized[0]; + } + } + return 'Q4_K_M'; +} + +function deriveModalities(model = {}) { + if (Array.isArray(model.modalities) && model.modalities.length > 0) { + return model.modalities.map((m) => String(m).toLowerCase()); + } + const text = [ + model.name, + model.model_name, + model.model_id, + model.model_identifier, + model.specialization, + model.category, + ...(Array.isArray(model.capabilities) ? model.capabilities : [String(model.capabilities || '')]), + ...(Array.isArray(model.input_types) ? model.input_types : []) + ] + .filter(Boolean) + .join(' ') + .toLowerCase(); + const hasVision = /\b(vision|vl\b|llava|bakllava|moondream|multimodal|image)\b/.test(text); + return hasVision ? ['text', 'vision'] : ['text']; +} + +function deriveTags(model = {}, modalities = ['text']) { + const explicit = Array.isArray(model.tags) ? model.tags.map((t) => String(t).toLowerCase()) : []; + const tags = new Set(explicit); + + const text = [ + model.name, + model.model_name, + model.model_id, + model.model_identifier, + model.specialization, + model.category, + model.tag, + ...(Array.isArray(model.capabilities) + ? model.capabilities + : [String(model.capabilities || '')]), + ...(Array.isArray(model.use_cases) ? model.use_cases : []) + ] + .filter(Boolean) + .join(' ') + .toLowerCase(); + + if (/\b(coder|code|programming|deepseek-coder|starcoder|codellama|codegemma)\b/.test(text)) tags.add('coder'); + if (/\binstruct\b/.test(text)) tags.add('instruct'); + if (/\b(chat|assistant|conversation)\b/.test(text)) tags.add('chat'); + if (/\bembed/.test(text)) tags.add('embedding'); + if (/\b(reason|math|logic)\b/.test(text)) tags.add('reasoning'); + if (modalities.includes('vision')) tags.add('vision'); + + // Most general/instruct models should compete in the general category even + // when no explicit tag exists, matching the permissive engines' behavior. + if (tags.size === 0) tags.add('chat'); + + return [...tags]; +} + +function deriveIdentifier(model = {}) { + return ( + model.model_identifier || + model.tag || + model.model_id || + model.model_name || + model.name || + 'unknown-model' + ); +} + +/** + * Convert an arbitrary model object (any of the three command shapes) into the + * already-normalized deterministic model shape. The original object is kept on + * `__source` so callers can map ranking output back to their display shape. + */ +function normalizeToDeterministic(model = {}) { + const paramsB = deriveParamsB(model); + const sizeGB = deriveSizeGB(model, paramsB) ?? Math.max(0.4, paramsB * 0.6); + const modalities = deriveModalities(model); + const tags = deriveTags(model, modalities); + const quant = deriveQuant(model); + const identifier = deriveIdentifier(model); + + const moeFlag = Boolean(model.isMoE || model.is_moe); + const totalParamsB = + toFiniteNumber(model.totalParamsB) ?? toFiniteNumber(model.total_params_b) ?? null; + const activeParamsB = + toFiniteNumber(model.activeParamsB) ?? toFiniteNumber(model.active_params_b) ?? null; + + return { + // Canonical deterministic fields (pass `alreadyNormalized` branch). + name: identifier, + model_identifier: identifier, + family: model.family || canonicalSelector.extractFamily(String(identifier)), + paramsB, + ctxMax: deriveContext(model), + quant, + sizeGB, + modalities, + tags, + installed: Boolean(model.installed || model.isOllamaInstalled), + pulls: toFiniteNumber(model.pulls) ?? toFiniteNumber(model.actual_pulls) ?? 0, + availableQuantizations: [quant], + sizeByQuant: { [quant]: sizeGB }, + // MoE passthrough so memory/speed estimation stays accurate. + isMoE: moeFlag, + is_moe: moeFlag, + totalParamsB, + activeParamsB, + expertCount: toFiniteNumber(model.expertCount) ?? toFiniteNumber(model.expert_count) ?? null, + expertsActivePerToken: + toFiniteNumber(model.expertsActivePerToken) ?? + toFiniteNumber(model.experts_active_per_token) ?? + null, + // Freshness defaults: neutral so we never penalize callers that lack + // timestamps (the per-command sources already carry these when known). + freshnessScore: toFiniteNumber(model.freshnessScore) ?? 55, + isStale: Boolean(model.isStale), + isDeprecated: Boolean(model.isDeprecated || model.deprecated), + source: model.source || 'unified_core', + // Back-reference for result mapping. + __source: model + }; +} + +/** + * Canonical ranking entrypoint. Returns the deterministic selector's ranked + * candidates (each carries `.score`, `.components {Q,S,F,C,H}`, `.meta`, etc.) + * for the supplied models/hardware. Each candidate's `meta.__source` points + * back to the caller's original model object. + * + * @param {Array} models Arbitrary model objects (any command shape). + * @param {Object} hardware Hardware profile (any detector shape). + * @param {Object} options + * @param {string} [options.category='general'] Use-case / category. + * @param {string} [options.optimizeFor='balanced'] + * @param {string} [options.runtime='ollama'] + * @param {number} [options.topN] Limit (default: all candidates). + * @returns {Promise<{category, candidates, total_evaluated, hardware}>} + */ +async function rankModels(models, hardware, options = {}) { + const category = normalizeCategory(options.category || options.useCase || 'general'); + const pool = (Array.isArray(models) ? models : []) + .filter(Boolean) + .map(normalizeToDeterministic); + + const result = await canonicalSelector.selectModels(category, { + topN: typeof options.topN === 'number' ? options.topN : pool.length || 1, + enableProbe: false, + silent: true, + optimizeFor: options.optimizeFor || options.optimize || options.objective || 'balanced', + runtime: options.runtime || 'ollama', + hardware: hardware || undefined, + installedModels: [], + modelPool: pool + }); + + return result; +} + +/** + * Map deterministic-engine categories from the broader vocabulary the commands + * use (e.g. `check`/`smart-recommend` accept `chat`, `creative`, `vision`, + * `talking`) onto the categories the deterministic core scores. + */ +function normalizeCategory(category) { + const normalized = String(category || 'general').toLowerCase().trim(); + const map = { + general: 'general', + chat: 'general', + talking: 'general', + assistant: 'general', + coding: 'coding', + code: 'coding', + programming: 'coding', + reasoning: 'reasoning', + math: 'reasoning', + multimodal: 'multimodal', + vision: 'multimodal', + embeddings: 'embeddings', + embedding: 'embeddings', + summarization: 'summarization', + reading: 'reading', + creative: 'general', + fast: 'general', + quality: 'general', + longctx: 'reading' + }; + return map[normalized] || 'general'; +} + +module.exports = { + canonicalSelector, + rankModels, + normalizeToDeterministic, + normalizeCategory, + deriveParamsB, + deriveSizeGB +}; diff --git a/tests/run-all-tests.js b/tests/run-all-tests.js index 1323c23..8e474c2 100644 --- a/tests/run-all-tests.js +++ b/tests/run-all-tests.js @@ -18,6 +18,7 @@ const TESTS = [ { name: 'Token speed estimation', file: 'token-speed-estimation.test.js', category: 'Performance' }, { name: 'Deterministic model pool', file: 'deterministic-model-pool-check.js', category: 'Recommendations' }, { name: 'Multi-objective selector regression', file: 'multi-objective-selector-regression.test.js', category: 'Recommendations' }, + { name: 'Scoring engine unification', file: 'scoring-unification.test.js', category: 'Recommendations' }, { name: 'Fine-tuning support helper', file: 'fine-tuning-support.test.js', category: 'Recommendations' }, { name: 'Ollama capacity planner', file: 'ollama-capacity-planner.test.js', category: 'Ollama' }, { name: 'Ollama localhost fallback', file: 'ollama-client-localhost-fallback.test.js', category: 'Ollama' }, diff --git a/tests/scoring-unification.test.js b/tests/scoring-unification.test.js new file mode 100644 index 0000000..9b84f91 --- /dev/null +++ b/tests/scoring-unification.test.js @@ -0,0 +1,318 @@ +/** + * Scoring Unification Integration Test (GitHub issue #88) + * ====================================================== + * + * Proves the root-cause fix: the three recommendation surfaces + * (`check`, `recommend`, `smart-recommend`) now derive their ranking from ONE + * canonical scoring core (DeterministicModelSelector via scoring-core), so: + * + * a. CONSISTENCY: all three agree on the top pick's size class for the same + * logical (model pool, hardware) input. + * b. HIGH-CAPACITY FLOOR EVERYWHERE: on a 192GB dual-RTX-PRO-6000 profile none + * of the three paths returns a sub-30B model as its #1 pick for + * general/coding/reasoning. + * c. SMALL-HARDWARE SANITY: on an 8GB CPU-only profile every path still + * prefers a small (<30B, and in practice <=8B) model — i.e. the fix did + * not just force "always big". + * + * Each command consumes a DIFFERENT model shape, so we build the SAME logical + * models in each path's native shape and call each path's selection entrypoint + * directly (no live Ollama / sql.js dependency). + */ + +const assert = require('assert'); + +const MultiObjectiveSelector = require('../src/ai/multi-objective-selector'); +const DeterministicModelSelector = require('../src/models/deterministic-selector'); +const IntelligentSelector = require('../src/models/intelligent-selector'); + +// --------------------------------------------------------------------------- +// Hardware profiles +// --------------------------------------------------------------------------- + +// The dual RTX PRO 6000 / ~192GB VRAM case from issue #88. +function buildHighCapacityHardware() { + return { + cpu: { brand: 'AMD Threadripper PRO', cores: 32, physicalCores: 32, speed: 4.0, architecture: 'x86_64' }, + gpu: { + model: 'NVIDIA RTX PRO 6000 Blackwell Workstation Edition', + vendor: 'NVIDIA', + type: 'nvidia', + vram: 192, + vramPerGPU: 96, + gpuCount: 2, + isMultiGPU: true, + dedicated: true + }, + memory: { total: 192, totalGB: 192 }, + acceleration: { supports_metal: false, supports_cuda: true, supports_rocm: false }, + os: { platform: 'linux' }, + summary: { + bestBackend: 'cuda', + gpuModel: 'NVIDIA RTX PRO 6000 Blackwell Workstation Edition', + effectiveMemory: 180, + systemRAM: 192, + totalVRAM: 192, + isMultiGPU: true, + gpuCount: 2, + hasDedicatedGPU: true, + hasIntegratedGPU: false, + hardwareTier: 'very_high', + speedCoefficient: 320 + } + }; +} + +// A small laptop: 8GB RAM, CPU-only. +function buildSmallHardware() { + return { + cpu: { brand: 'Intel Core i5-1135G7', cores: 4, physicalCores: 4, speed: 2.4, architecture: 'x86_64' }, + gpu: { model: '', vram: 0, dedicated: false }, + memory: { total: 8, totalGB: 8 }, + acceleration: { supports_metal: false, supports_cuda: false, supports_rocm: false }, + os: { platform: 'linux' }, + summary: { + bestBackend: 'cpu', + gpuModel: '', + effectiveMemory: 8, + systemRAM: 8, + totalVRAM: 0, + hasDedicatedGPU: false, + hasIntegratedGPU: false, + hardwareTier: 'low', + speedCoefficient: 40 + } + }; +} + +// --------------------------------------------------------------------------- +// Representative model pool (same logical models, three native shapes). +// Each category has a small (<=8B) and a large (>=30B) member so every path is +// forced to make a real size choice. +// --------------------------------------------------------------------------- + +// `check` shape: ExpandedModelsDatabase-style rows. +function buildCheckPool() { + const mk = (name, sizeB, gb, ctx, cat, quant = ['Q4_K_M']) => ({ + name, + size: `${sizeB}B`, + type: 'local', + category: cat, + requirements: { ram: Math.ceil(gb + 2), vram: 0, cpu_cores: 2, storage: gb }, + frameworks: ['ollama'], + quantization: quant, + performance: { context_length: ctx }, + specialization: cat, + installation: { ollama: `ollama pull ${name}` }, + // size_gb hint so memory math is realistic regardless of param parsing. + size_gb: gb + }); + return [ + mk('uni-chat 8B', 8, 5.0, 65536, 'general'), + mk('uni-chat 70B', 70, 42, 65536, 'general'), + mk('uni-coder 8B', 8, 5.0, 32768, 'coding'), + mk('uni-coder 34B', 34, 20.5, 32768, 'coding'), + mk('uni-reason 7B', 7, 4.8, 65536, 'reasoning'), + mk('uni-reason 70B', 70, 43, 65536, 'reasoning') + ]; +} + +// `recommend` shape: Ollama scrape-style models with variants. +function buildRecommendPool() { + return [ + { + model_identifier: 'uni-chat', + model_name: 'uni-chat', + primary_category: 'chat', + context_length: '64K', + variants: [ + { tag: 'uni-chat:8b', size: '8b', quantization: 'Q4_0', real_size_gb: 5.0, categories: ['chat'] }, + { tag: 'uni-chat:70b', size: '70b', quantization: 'Q4_0', real_size_gb: 42, categories: ['chat'] } + ], + tags: ['uni-chat:8b', 'uni-chat:70b'], + use_cases: ['chat'] + }, + { + model_identifier: 'uni-coder', + model_name: 'uni-coder', + primary_category: 'coding', + context_length: '32K', + variants: [ + { tag: 'uni-coder:8b', size: '8b', quantization: 'Q4_0', real_size_gb: 5.0, categories: ['coding'] }, + { tag: 'uni-coder:34b', size: '34b', quantization: 'Q4_0', real_size_gb: 20.5, categories: ['coding'] } + ], + tags: ['uni-coder:8b', 'uni-coder:34b'], + use_cases: ['coding', 'programming'] + }, + { + model_identifier: 'uni-reason', + model_name: 'uni-reason', + primary_category: 'reasoning', + context_length: '64K', + variants: [ + { tag: 'uni-reason:7b', size: '7b', quantization: 'Q4_0', real_size_gb: 4.8, categories: ['reasoning'] }, + { tag: 'uni-reason:70b', size: '70b', quantization: 'Q4_0', real_size_gb: 43, categories: ['reasoning'] } + ], + tags: ['uni-reason:7b', 'uni-reason:70b'], + use_cases: ['reasoning', 'math'] + } + ]; +} + +// `smart-recommend` shape: sql.js variant rows. +function buildSmartVariants() { + return [ + { model_id: 'uni-chat', tag: 'uni-chat:8b', params_b: 8, size_gb: 5.0, quant: 'Q4_K_M', context_length: 65536, capabilities: 'chat', family: 'llama3.2', pulls: 5000000 }, + { model_id: 'uni-chat', tag: 'uni-chat:70b', params_b: 70, size_gb: 42, quant: 'Q4_K_M', context_length: 65536, capabilities: 'chat', family: 'llama3.3', pulls: 5000000 }, + { model_id: 'uni-coder', tag: 'uni-coder:8b', params_b: 8, size_gb: 5.0, quant: 'Q4_K_M', context_length: 32768, capabilities: 'coding', family: 'qwen2.5-coder', pulls: 5000000 }, + { model_id: 'uni-coder', tag: 'uni-coder:34b', params_b: 34, size_gb: 20.5, quant: 'Q4_K_M', context_length: 32768, capabilities: 'coding', family: 'qwen2.5-coder', pulls: 5000000 }, + { model_id: 'uni-reason', tag: 'uni-reason:7b', params_b: 7, size_gb: 4.8, quant: 'Q4_K_M', context_length: 65536, capabilities: 'reasoning', family: 'deepseek-r1', pulls: 5000000 }, + { model_id: 'uni-reason', tag: 'uni-reason:70b', params_b: 70, size_gb: 43, quant: 'Q4_K_M', context_length: 65536, capabilities: 'reasoning', family: 'deepseek-r1', pulls: 5000000 } + ]; +} + +// --------------------------------------------------------------------------- +// Per-path top-pick params extraction +// --------------------------------------------------------------------------- + +function parseParamsB(text) { + const match = String(text || '').match(/(\d+(?:\.\d+)?)\s*b\b/i); + return match ? parseFloat(match[1]) : null; +} + +async function checkTopParamsB(hardware, category) { + const selector = new MultiObjectiveSelector(); + const result = await selector.selectBestModels(hardware, buildCheckPool(), category, 20); + const top = result.compatible[0] || result.marginal[0]; + assert.ok(top, `check should return a ranked model for category ${category}`); + // ExpandedModelsDatabase rows keep their name/size; derive params from them. + const params = selector.estimateModelParams(top) || parseParamsB(top.size) || parseParamsB(top.name); + return { params, label: top.name }; +} + +async function recommendTopParamsB(hardware, category) { + const selector = new DeterministicModelSelector(); + selector.getInstalledModels = async () => []; + const recommendations = await selector.getBestModelsForHardware(hardware, buildRecommendPool(), { runtime: 'ollama' }); + const top = recommendations[category] && recommendations[category].bestModels[0]; + assert.ok(top, `recommend should return a ranked model for category ${category}`); + const params = Number(top.size) || parseParamsB(top.model_identifier); + return { params, label: top.model_identifier }; +} + +function makeStubDetector(hardware) { + return { + detect: async () => hardware, + getHardwareDescription: () => 'unit-test hardware', + getHardwareTier: () => (hardware.summary && hardware.summary.hardwareTier) || 'medium', + getMaxModelSize: () => (hardware.summary && hardware.summary.effectiveMemory) || 8 + }; +} + +async function smartTopParamsB(hardware, useCase) { + const selector = new IntelligentSelector({ detector: makeStubDetector(hardware) }); + const recommendations = await selector.recommend(buildSmartVariants(), { + useCase, + limit: 10, + policyFile: null + }); + const top = recommendations.topPicks.best; + assert.ok(top && top.variant, `smart-recommend should return a top pick for use case ${useCase}`); + const params = top.variant.params_b || parseParamsB(top.variant.tag); + return { params, label: top.variant.tag }; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +const HIGH_CAPACITY_CATEGORIES = ['general', 'coding', 'reasoning']; + +async function testHighCapacityFloorEverywhere() { + const hardware = buildHighCapacityHardware(); + + for (const category of HIGH_CAPACITY_CATEGORIES) { + const check = await checkTopParamsB(hardware, category); + const recommend = await recommendTopParamsB(hardware, category); + const smart = await smartTopParamsB(hardware, category); + + assert.ok( + check.params >= 30, + `[check] ${category}: top pick should be >=30B on 192GB hardware, got ${check.label} (${check.params}B)` + ); + assert.ok( + recommend.params >= 30, + `[recommend] ${category}: top pick should be >=30B on 192GB hardware, got ${recommend.label} (${recommend.params}B)` + ); + assert.ok( + smart.params >= 30, + `[smart-recommend] ${category}: top pick should be >=30B on 192GB hardware, got ${smart.label} (${smart.params}B)` + ); + } +} + +async function testConsistencyAcrossPaths() { + const hardware = buildHighCapacityHardware(); + + for (const category of HIGH_CAPACITY_CATEGORIES) { + const check = await checkTopParamsB(hardware, category); + const recommend = await recommendTopParamsB(hardware, category); + const smart = await smartTopParamsB(hardware, category); + + const sizeClass = (p) => (p >= 30 ? 'large' : 'small'); + const checkClass = sizeClass(check.params); + const recommendClass = sizeClass(recommend.params); + const smartClass = sizeClass(smart.params); + + assert.strictEqual( + checkClass, + recommendClass, + `[${category}] check (${check.label} ${check.params}B) and recommend (${recommend.label} ${recommend.params}B) should agree on size class` + ); + assert.strictEqual( + recommendClass, + smartClass, + `[${category}] recommend (${recommend.label} ${recommend.params}B) and smart-recommend (${smart.label} ${smart.params}B) should agree on size class` + ); + } +} + +async function testSmallHardwarePrefersSmallEverywhere() { + const hardware = buildSmallHardware(); + + for (const category of HIGH_CAPACITY_CATEGORIES) { + const check = await checkTopParamsB(hardware, category); + const recommend = await recommendTopParamsB(hardware, category); + const smart = await smartTopParamsB(hardware, category); + + assert.ok( + check.params < 30, + `[check] ${category}: small hardware should prefer a small model, got ${check.label} (${check.params}B)` + ); + assert.ok( + recommend.params < 30, + `[recommend] ${category}: small hardware should prefer a small model, got ${recommend.label} (${recommend.params}B)` + ); + assert.ok( + smart.params < 30, + `[smart-recommend] ${category}: small hardware should prefer a small model, got ${smart.label} (${smart.params}B)` + ); + } +} + +async function run() { + await testHighCapacityFloorEverywhere(); + await testConsistencyAcrossPaths(); + await testSmallHardwarePrefersSmallEverywhere(); + console.log('scoring-unification.test.js: OK'); +} + +if (require.main === module) { + run().catch((error) => { + console.error('scoring-unification.test.js: FAILED'); + console.error(error); + process.exit(1); + }); +} + +module.exports = { run };