fix: remove fabricated precision and size fallbacks from Ollama metadata

This commit is contained in:
signerless 2026-09-06 15:40:12 +02:00
parent d0c5a414d0
commit 42e6e87484
6 changed files with 47 additions and 22 deletions

View file

@ -6,7 +6,7 @@ registry recommendations, and the desktop source now tracked in this repository.
| Case | Corrected behavior | Regression coverage |
| --- | --- | --- |
| Artifact precision and size | FP16/BF16/FP32 retain their precision. A fixed Q8 file cannot become Q5 to fit a budget. Complete shard sets contribute their combined size; incomplete sets are excluded. Unknown precision without a known size is ineligible. | `ranking-contract.test.js`, `selector-memory-sizing.test.js`, `deterministic-model-pool-check.js` |
| Artifact precision and size | FP16/BF16/FP32 retain their precision. A fixed Q8 file cannot become Q5 to fit a budget. Complete shard sets contribute their combined size; incomplete sets are excluded. Unknown precision without a known size is ineligible. Missing Ollama metadata no longer invents a 4 GB Q4 artifact, and full-precision resident weights cannot shrink to a stale download-size estimate. | `ranking-contract.test.js`, `selector-memory-sizing.test.js`, `deterministic-model-pool-check.js` |
| Model capabilities | Embeddings and rerankers are excluded from generation categories; missing capability data does not imply generation. | `ranking-contract.test.js` |
| Benchmark identity | Base, Instruct, size, revision, thinking, and context-extension variants remain distinct. Unsized family measurements cannot score a smaller checkpoint. Quantized repos require an explicit quantization alias to inherit upstream measurements. | `quality-evals.test.js`, `ranking-contract.test.js` |
| Accelerator scoring | ROCm, Vulkan and SYCL have explicit estimate profiles. CPU-only mode suppresses accelerator scoring. | `ranking-contract.test.js`, `cpu-only-mode.test.js` |

View file

@ -779,6 +779,7 @@ module.exports = {
normalizeOllamaRows,
inferFormat,
inferQuantization,
inferPrecision,
inferRuntimeSupport,
isModelArtifactFile,
parseParamsB,

View file

@ -1,5 +1,5 @@
const ModelDatabase = require('./model-database');
const { isSupportedHuggingFaceModel } = require('./registry-ingestors');
const { isSupportedHuggingFaceModel, inferQuantization, inferPrecision } = require('./registry-ingestors');
const DeterministicModelSelector = require('../models/deterministic-selector');
const { normalizePrecision, memoryBudgetGB } = require('../models/ranking-contract');
const { applyCpuOnlyOverride } = require('../hardware/cpu-only');
@ -93,7 +93,8 @@ function inferFamily(identifier = '') {
}
function normalizeQuantization(row = {}) {
const raw = row.quantization || row.precision || '';
const filename = row.filename || row.artifact_name || '';
const raw = inferQuantization(filename) || inferPrecision(filename) || row.quantization || row.precision || '';
return normalizePrecision(raw);
}

View file

@ -1135,7 +1135,7 @@ class DeterministicModelSelector {
return this.normalizeQuantization(
variant.quantization ||
variant.quant ||
'Q4_K_M'
'UNKNOWN'
);
}
@ -1210,10 +1210,9 @@ class DeterministicModelSelector {
}
extractVariantSizeGB(variant, paramsB) {
const candidate = Number(variant.real_size_gb ?? variant.estimated_size_gb ?? variant.size_gb ?? NaN);
const candidate = Number(variant.real_size_gb ?? variant.size_gb ?? NaN);
if (Number.isFinite(candidate) && candidate > 0) return candidate;
if (!Number.isFinite(paramsB) || paramsB <= 0) return 0.5;
return Math.max(0.5, Math.round((paramsB * 0.58 + 0.5) * 10) / 10);
return null; // No observed artifact size; estimate from the actual precision later.
}
inferModalities(model, variantTag = '') {
@ -1303,27 +1302,27 @@ class DeterministicModelSelector {
extractParams(details) {
// Look for parameter info in ollama show output
const match = details.match(/parameters\s+(\d+\.?\d*)[BM]/i);
const match = details.match(/parameters\s+(\d+\.?\d*)\s*[BM]/i);
if (match) {
const num = parseFloat(match[1]);
return match[0].toUpperCase().includes('B') ? num : num / 1000;
}
return 7; // Default fallback
return null;
}
extractContextLength(details) {
const match = details.match(/context_length\s+(\d+)/i);
return match ? parseInt(match[1]) : 4096;
const match = details.match(/context[ _]length\s+(\d+)/i);
return match ? parseInt(match[1]) : null;
}
extractQuantization(details) {
const match = details.match(/quantization\s+(Q\d+_[A-Z0-9_]+)/i);
return match ? match[1] : 'Q4_K_M';
const match = details.match(/quantization\s+(\S+)/i);
return normalizePrecision(match?.[1]);
}
extractSizeGB(details) {
const match = details.match(/size\s+(\d+\.?\d*)\s*GB/i);
return match ? parseFloat(match[1]) : 4.0;
const match = details.match(/size\s+(\d+\.?\d*)\s*(GB|MB)/i);
return match ? parseFloat(match[1]) / (match[2].toUpperCase() === 'MB' ? 1024 : 1) : null;
}
extractModalities(details) {
@ -1584,6 +1583,7 @@ class DeterministicModelSelector {
}
evaluateModel(model, hardware, category, targetCtx, budget, optimizeFor = 'balanced', runtime = 'ollama', options = {}) {
if (!(this.parseBillionsValue(model.totalParamsB || model.paramsB) > 0)) return null;
const nativeContext = Number(model.ctxMax);
const requiredContext = options.contextPolicy !== 'preferred';
if (requiredContext && (!(nativeContext > 0) || nativeContext < targetCtx)) return null;
@ -1674,7 +1674,9 @@ class DeterministicModelSelector {
// A model/tag/file is an immutable artifact. Alternative files are separate rows.
if (model?.quant) return [this.normalizeQuantization(model.quant)];
const available = model?.availableQuantizations || Object.keys(model?.sizeByQuant || {});
if (!available.length && Number(model?.sizeGB) > 0) return ['UNKNOWN'];
if (available.length !== 1 && (model?.model_identifier || !available.length)) {
return Number(model?.sizeGB) > 0 ? ['UNKNOWN'] : [];
}
return [...new Set(available.map(normalizePrecision))]
.sort((a, b) => (precisionProfile(b).bytes || 0) - (precisionProfile(a).bytes || 0));
}
@ -1716,9 +1718,11 @@ class DeterministicModelSelector {
? parameterProfile.totalParamsB
: parameterProfile.effectiveParamsB;
const modeledWeightGB = bpp == null ? Infinity : weightParamsB * bpp;
// A real observed artifact size always wins for weight memory — never let
// an MoE "sparse inference" assumption discard a measured on-disk size.
const useObservedArtifactSize = Number.isFinite(observedWeightGB) && observedWeightGB > 0;
// Full-precision resident weights cannot shrink to a compressed download
// or an old Q4 size estimate. Quantized artifacts retain their observed size.
const fullPrecision = ['FP16', 'BF16', 'FP32'].includes(normalizedQuant);
const useObservedArtifactSize = Number.isFinite(observedWeightGB) && observedWeightGB > 0 &&
(!fullPrecision || observedWeightGB >= modeledWeightGB);
const modelMemGB = useObservedArtifactSize ? observedWeightGB : modeledWeightGB;
const effectiveCtx = Number.isFinite(Number(ctx)) && Number(ctx) > 0 ? Number(ctx) : 4096;

View file

@ -569,7 +569,7 @@ class OllamaNativeScraper {
quantization: quantization,
command: `ollama pull ${tag}`,
estimated_size_gb: this.estimateModelSizeGB(tag),
real_size_gb: realSizeGB || this.estimateModelSizeGB(tag)
real_size_gb: realSizeGB || null
};
});
@ -586,8 +586,8 @@ class OllamaNativeScraper {
}
extractQuantizationFromTag(tag) {
const quantMatch = tag.match(/\b(q\d+_[km]?_?[ms]?|fp16|fp32|int8|int4)\b/i);
return quantMatch ? quantMatch[0].toUpperCase() : 'Q4_0'; // Default assumption
const quantMatch = tag.match(/\b(i?q\d+(?:_[a-z0-9]+)*|bf16|f16|f32|fp16|fp32|int8|int4)\b/i);
return quantMatch ? quantMatch[0].toUpperCase() : 'UNKNOWN';
}
extractRealSizeFromHTML(html, tag) {

View file

@ -19,6 +19,22 @@ for (const quant of ['FP16', 'BF16', 'FP32']) {
assert.strictEqual(selector.evaluateModel({ ...base, quant }, hardware, 'general', 4096, 8), null);
}
assert.strictEqual(precisionProfile('IQ4_XS').bytes, 0.58);
assert.strictEqual(selector.extractQuantization('quantization F16'), 'FP16');
assert.ok(selector.estimateRequiredGB({ ...base, sizeGB: 4 }, 'FP16', 4096) > 14, 'a stale Q4 size must not shrink FP16 resident weights');
assert.strictEqual(selector.extractQuantization('quantization BF16'), 'BF16');
assert.strictEqual(selector.extractQuantization(''), 'UNKNOWN');
assert.strictEqual(selector.extractSizeGB(''), null);
assert.strictEqual(selector.extractParams('parameters 7 B'), 7);
assert.strictEqual(selector.evaluateModel({ ...base, paramsB: null, sizeGB: 4 }, hardware, 'general', 4096, 8), null);
assert.strictEqual(selector.extractContextLength(''), null);
assert.strictEqual(selector.extractVariantSizeGB({ estimated_size_gb: 4 }, 7), null);
const fpVariant = selector.convertOllamaModelToDeterministicModels({ model_identifier: 'test', primary_category: 'general',
variants: [{ tag: 'test:7b-fp16', quantization: 'FP16', size: '7b' }] })[0];
assert.strictEqual(fpVariant.quant, 'FP16');
assert.ok(selector.estimateRequiredGB(fpVariant, 'FP16', 4096) > 14);
assert.strictEqual(selector.selectBestQuantization(fpVariant, 8, 4096), null);
assert.deepStrictEqual(selector.getQuantizationCandidates({ model_identifier: 'unknown:7b', availableQuantizations: ['Q8_0', 'Q4_K_M'] }), []);
assert.strictEqual(selector.estimateRequiredGB({ ...base, quant: 'unknown' }, 'unknown', 4096), Infinity);
const fixed = { ...base, quant: 'Q8_0', model_identifier: 'qwen:7b-q8_0',
availableQuantizations: ['Q8_0', 'Q4_K_M'], sizeByQuant: { Q8_0: 7.6, Q4_K_M: 4.5 } };
@ -32,6 +48,9 @@ const shard = { source_id: 'huggingface', repo_id: 'test/model-7B-GGUF',
canonical_model_id: 'test/model-7B-GGUF', repo_tasks: ['text-generation'], format: 'gguf',
quantization: 'Q8_0', size_gb: 4, parameter_count_b: 7, context_length: 4096,
runtime_support: ['ollama', 'llama.cpp'] };
const stalePrecision = artifactToSelectorModel({ ...shard, filename: 'model-7b-f16.gguf', quantization: 'Q4_K_M' });
assert.strictEqual(stalePrecision.quant, 'FP16', 'the concrete file precision overrides stale metadata');
assert.ok(selector.estimateRequiredGB(stalePrecision, 'FP16', 4096) > 14);
const shards = [1, 2].map(i => ({ ...shard, filename: `model-Q8_0-0000${i}-of-00002.gguf` }));
assert.strictEqual(groupWeightShards(shards.slice(1)).length, 0, 'incomplete shard sets are not installable');
const grouped = groupWeightShards(shards);