From 42e6e87484ad5b7572eedcb0bd3d97e12bd8aa5d Mon Sep 17 00:00:00 2001 From: signerless Date: Sun, 6 Sep 2026 15:40:12 +0200 Subject: [PATCH] fix: remove fabricated precision and size fallbacks from Ollama metadata --- docs/ranking-contract-validation.md | 2 +- src/data/registry-ingestors.js | 1 + src/data/registry-recommender.js | 5 ++-- src/models/deterministic-selector.js | 36 +++++++++++++++------------- src/ollama/native-scraper.js | 6 ++--- tests/ranking-contract.test.js | 19 +++++++++++++++ 6 files changed, 47 insertions(+), 22 deletions(-) diff --git a/docs/ranking-contract-validation.md b/docs/ranking-contract-validation.md index c854f5e..f64e2fd 100644 --- a/docs/ranking-contract-validation.md +++ b/docs/ranking-contract-validation.md @@ -6,7 +6,7 @@ registry recommendations, and the desktop source now tracked in this repository. | Case | Corrected behavior | Regression coverage | | --- | --- | --- | -| Artifact precision and size | FP16/BF16/FP32 retain their precision. A fixed Q8 file cannot become Q5 to fit a budget. Complete shard sets contribute their combined size; incomplete sets are excluded. Unknown precision without a known size is ineligible. | `ranking-contract.test.js`, `selector-memory-sizing.test.js`, `deterministic-model-pool-check.js` | +| Artifact precision and size | FP16/BF16/FP32 retain their precision. A fixed Q8 file cannot become Q5 to fit a budget. Complete shard sets contribute their combined size; incomplete sets are excluded. Unknown precision without a known size is ineligible. Missing Ollama metadata no longer invents a 4 GB Q4 artifact, and full-precision resident weights cannot shrink to a stale download-size estimate. | `ranking-contract.test.js`, `selector-memory-sizing.test.js`, `deterministic-model-pool-check.js` | | Model capabilities | Embeddings and rerankers are excluded from generation categories; missing capability data does not imply generation. | `ranking-contract.test.js` | | Benchmark identity | Base, Instruct, size, revision, thinking, and context-extension variants remain distinct. Unsized family measurements cannot score a smaller checkpoint. Quantized repos require an explicit quantization alias to inherit upstream measurements. | `quality-evals.test.js`, `ranking-contract.test.js` | | Accelerator scoring | ROCm, Vulkan and SYCL have explicit estimate profiles. CPU-only mode suppresses accelerator scoring. | `ranking-contract.test.js`, `cpu-only-mode.test.js` | diff --git a/src/data/registry-ingestors.js b/src/data/registry-ingestors.js index f7ecc2b..4ff882a 100644 --- a/src/data/registry-ingestors.js +++ b/src/data/registry-ingestors.js @@ -779,6 +779,7 @@ module.exports = { normalizeOllamaRows, inferFormat, inferQuantization, + inferPrecision, inferRuntimeSupport, isModelArtifactFile, parseParamsB, diff --git a/src/data/registry-recommender.js b/src/data/registry-recommender.js index 94ab2d5..f9ad282 100644 --- a/src/data/registry-recommender.js +++ b/src/data/registry-recommender.js @@ -1,5 +1,5 @@ const ModelDatabase = require('./model-database'); -const { isSupportedHuggingFaceModel } = require('./registry-ingestors'); +const { isSupportedHuggingFaceModel, inferQuantization, inferPrecision } = require('./registry-ingestors'); const DeterministicModelSelector = require('../models/deterministic-selector'); const { normalizePrecision, memoryBudgetGB } = require('../models/ranking-contract'); const { applyCpuOnlyOverride } = require('../hardware/cpu-only'); @@ -93,7 +93,8 @@ function inferFamily(identifier = '') { } function normalizeQuantization(row = {}) { - const raw = row.quantization || row.precision || ''; + const filename = row.filename || row.artifact_name || ''; + const raw = inferQuantization(filename) || inferPrecision(filename) || row.quantization || row.precision || ''; return normalizePrecision(raw); } diff --git a/src/models/deterministic-selector.js b/src/models/deterministic-selector.js index a823a16..692bf8b 100644 --- a/src/models/deterministic-selector.js +++ b/src/models/deterministic-selector.js @@ -1135,7 +1135,7 @@ class DeterministicModelSelector { return this.normalizeQuantization( variant.quantization || variant.quant || - 'Q4_K_M' + 'UNKNOWN' ); } @@ -1210,10 +1210,9 @@ class DeterministicModelSelector { } extractVariantSizeGB(variant, paramsB) { - const candidate = Number(variant.real_size_gb ?? variant.estimated_size_gb ?? variant.size_gb ?? NaN); + const candidate = Number(variant.real_size_gb ?? variant.size_gb ?? NaN); if (Number.isFinite(candidate) && candidate > 0) return candidate; - if (!Number.isFinite(paramsB) || paramsB <= 0) return 0.5; - return Math.max(0.5, Math.round((paramsB * 0.58 + 0.5) * 10) / 10); + return null; // No observed artifact size; estimate from the actual precision later. } inferModalities(model, variantTag = '') { @@ -1303,27 +1302,27 @@ class DeterministicModelSelector { extractParams(details) { // Look for parameter info in ollama show output - const match = details.match(/parameters\s+(\d+\.?\d*)[BM]/i); + const match = details.match(/parameters\s+(\d+\.?\d*)\s*[BM]/i); if (match) { const num = parseFloat(match[1]); return match[0].toUpperCase().includes('B') ? num : num / 1000; } - return 7; // Default fallback + return null; } extractContextLength(details) { - const match = details.match(/context_length\s+(\d+)/i); - return match ? parseInt(match[1]) : 4096; + const match = details.match(/context[ _]length\s+(\d+)/i); + return match ? parseInt(match[1]) : null; } extractQuantization(details) { - const match = details.match(/quantization\s+(Q\d+_[A-Z0-9_]+)/i); - return match ? match[1] : 'Q4_K_M'; + const match = details.match(/quantization\s+(\S+)/i); + return normalizePrecision(match?.[1]); } extractSizeGB(details) { - const match = details.match(/size\s+(\d+\.?\d*)\s*GB/i); - return match ? parseFloat(match[1]) : 4.0; + const match = details.match(/size\s+(\d+\.?\d*)\s*(GB|MB)/i); + return match ? parseFloat(match[1]) / (match[2].toUpperCase() === 'MB' ? 1024 : 1) : null; } extractModalities(details) { @@ -1584,6 +1583,7 @@ class DeterministicModelSelector { } evaluateModel(model, hardware, category, targetCtx, budget, optimizeFor = 'balanced', runtime = 'ollama', options = {}) { + if (!(this.parseBillionsValue(model.totalParamsB || model.paramsB) > 0)) return null; const nativeContext = Number(model.ctxMax); const requiredContext = options.contextPolicy !== 'preferred'; if (requiredContext && (!(nativeContext > 0) || nativeContext < targetCtx)) return null; @@ -1674,7 +1674,9 @@ class DeterministicModelSelector { // A model/tag/file is an immutable artifact. Alternative files are separate rows. if (model?.quant) return [this.normalizeQuantization(model.quant)]; const available = model?.availableQuantizations || Object.keys(model?.sizeByQuant || {}); - if (!available.length && Number(model?.sizeGB) > 0) return ['UNKNOWN']; + if (available.length !== 1 && (model?.model_identifier || !available.length)) { + return Number(model?.sizeGB) > 0 ? ['UNKNOWN'] : []; + } return [...new Set(available.map(normalizePrecision))] .sort((a, b) => (precisionProfile(b).bytes || 0) - (precisionProfile(a).bytes || 0)); } @@ -1716,9 +1718,11 @@ class DeterministicModelSelector { ? parameterProfile.totalParamsB : parameterProfile.effectiveParamsB; const modeledWeightGB = bpp == null ? Infinity : weightParamsB * bpp; - // A real observed artifact size always wins for weight memory — never let - // an MoE "sparse inference" assumption discard a measured on-disk size. - const useObservedArtifactSize = Number.isFinite(observedWeightGB) && observedWeightGB > 0; + // Full-precision resident weights cannot shrink to a compressed download + // or an old Q4 size estimate. Quantized artifacts retain their observed size. + const fullPrecision = ['FP16', 'BF16', 'FP32'].includes(normalizedQuant); + const useObservedArtifactSize = Number.isFinite(observedWeightGB) && observedWeightGB > 0 && + (!fullPrecision || observedWeightGB >= modeledWeightGB); const modelMemGB = useObservedArtifactSize ? observedWeightGB : modeledWeightGB; const effectiveCtx = Number.isFinite(Number(ctx)) && Number(ctx) > 0 ? Number(ctx) : 4096; diff --git a/src/ollama/native-scraper.js b/src/ollama/native-scraper.js index 4430e33..01c8d7d 100644 --- a/src/ollama/native-scraper.js +++ b/src/ollama/native-scraper.js @@ -569,7 +569,7 @@ class OllamaNativeScraper { quantization: quantization, command: `ollama pull ${tag}`, estimated_size_gb: this.estimateModelSizeGB(tag), - real_size_gb: realSizeGB || this.estimateModelSizeGB(tag) + real_size_gb: realSizeGB || null }; }); @@ -586,8 +586,8 @@ class OllamaNativeScraper { } extractQuantizationFromTag(tag) { - const quantMatch = tag.match(/\b(q\d+_[km]?_?[ms]?|fp16|fp32|int8|int4)\b/i); - return quantMatch ? quantMatch[0].toUpperCase() : 'Q4_0'; // Default assumption + const quantMatch = tag.match(/\b(i?q\d+(?:_[a-z0-9]+)*|bf16|f16|f32|fp16|fp32|int8|int4)\b/i); + return quantMatch ? quantMatch[0].toUpperCase() : 'UNKNOWN'; } extractRealSizeFromHTML(html, tag) { diff --git a/tests/ranking-contract.test.js b/tests/ranking-contract.test.js index 8fe8874..084594f 100644 --- a/tests/ranking-contract.test.js +++ b/tests/ranking-contract.test.js @@ -19,6 +19,22 @@ for (const quant of ['FP16', 'BF16', 'FP32']) { assert.strictEqual(selector.evaluateModel({ ...base, quant }, hardware, 'general', 4096, 8), null); } assert.strictEqual(precisionProfile('IQ4_XS').bytes, 0.58); +assert.strictEqual(selector.extractQuantization('quantization F16'), 'FP16'); +assert.ok(selector.estimateRequiredGB({ ...base, sizeGB: 4 }, 'FP16', 4096) > 14, 'a stale Q4 size must not shrink FP16 resident weights'); +assert.strictEqual(selector.extractQuantization('quantization BF16'), 'BF16'); +assert.strictEqual(selector.extractQuantization(''), 'UNKNOWN'); +assert.strictEqual(selector.extractSizeGB(''), null); +assert.strictEqual(selector.extractParams('parameters 7 B'), 7); +assert.strictEqual(selector.evaluateModel({ ...base, paramsB: null, sizeGB: 4 }, hardware, 'general', 4096, 8), null); +assert.strictEqual(selector.extractContextLength(''), null); +assert.strictEqual(selector.extractVariantSizeGB({ estimated_size_gb: 4 }, 7), null); +const fpVariant = selector.convertOllamaModelToDeterministicModels({ model_identifier: 'test', primary_category: 'general', + variants: [{ tag: 'test:7b-fp16', quantization: 'FP16', size: '7b' }] })[0]; +assert.strictEqual(fpVariant.quant, 'FP16'); +assert.ok(selector.estimateRequiredGB(fpVariant, 'FP16', 4096) > 14); +assert.strictEqual(selector.selectBestQuantization(fpVariant, 8, 4096), null); +assert.deepStrictEqual(selector.getQuantizationCandidates({ model_identifier: 'unknown:7b', availableQuantizations: ['Q8_0', 'Q4_K_M'] }), []); + assert.strictEqual(selector.estimateRequiredGB({ ...base, quant: 'unknown' }, 'unknown', 4096), Infinity); const fixed = { ...base, quant: 'Q8_0', model_identifier: 'qwen:7b-q8_0', availableQuantizations: ['Q8_0', 'Q4_K_M'], sizeByQuant: { Q8_0: 7.6, Q4_K_M: 4.5 } }; @@ -32,6 +48,9 @@ const shard = { source_id: 'huggingface', repo_id: 'test/model-7B-GGUF', canonical_model_id: 'test/model-7B-GGUF', repo_tasks: ['text-generation'], format: 'gguf', quantization: 'Q8_0', size_gb: 4, parameter_count_b: 7, context_length: 4096, runtime_support: ['ollama', 'llama.cpp'] }; +const stalePrecision = artifactToSelectorModel({ ...shard, filename: 'model-7b-f16.gguf', quantization: 'Q4_K_M' }); +assert.strictEqual(stalePrecision.quant, 'FP16', 'the concrete file precision overrides stale metadata'); +assert.ok(selector.estimateRequiredGB(stalePrecision, 'FP16', 4096) > 14); const shards = [1, 2].map(i => ({ ...shard, filename: `model-Q8_0-0000${i}-of-00002.gguf` })); assert.strictEqual(groupWeightShards(shards.slice(1)).length, 0, 'incomplete shard sets are not installable'); const grouped = groupWeightShards(shards);