mirror of
https://github.com/signerless/llm-checker.git
synced 2026-10-19 09:54:45 +00:00
fix: remove fabricated precision and size fallbacks from Ollama metadata
This commit is contained in:
parent
d0c5a414d0
commit
42e6e87484
6 changed files with 47 additions and 22 deletions
|
|
@ -6,7 +6,7 @@ registry recommendations, and the desktop source now tracked in this repository.
|
|||
|
||||
| Case | Corrected behavior | Regression coverage |
|
||||
| --- | --- | --- |
|
||||
| Artifact precision and size | FP16/BF16/FP32 retain their precision. A fixed Q8 file cannot become Q5 to fit a budget. Complete shard sets contribute their combined size; incomplete sets are excluded. Unknown precision without a known size is ineligible. | `ranking-contract.test.js`, `selector-memory-sizing.test.js`, `deterministic-model-pool-check.js` |
|
||||
| Artifact precision and size | FP16/BF16/FP32 retain their precision. A fixed Q8 file cannot become Q5 to fit a budget. Complete shard sets contribute their combined size; incomplete sets are excluded. Unknown precision without a known size is ineligible. Missing Ollama metadata no longer invents a 4 GB Q4 artifact, and full-precision resident weights cannot shrink to a stale download-size estimate. | `ranking-contract.test.js`, `selector-memory-sizing.test.js`, `deterministic-model-pool-check.js` |
|
||||
| Model capabilities | Embeddings and rerankers are excluded from generation categories; missing capability data does not imply generation. | `ranking-contract.test.js` |
|
||||
| Benchmark identity | Base, Instruct, size, revision, thinking, and context-extension variants remain distinct. Unsized family measurements cannot score a smaller checkpoint. Quantized repos require an explicit quantization alias to inherit upstream measurements. | `quality-evals.test.js`, `ranking-contract.test.js` |
|
||||
| Accelerator scoring | ROCm, Vulkan and SYCL have explicit estimate profiles. CPU-only mode suppresses accelerator scoring. | `ranking-contract.test.js`, `cpu-only-mode.test.js` |
|
||||
|
|
|
|||
|
|
@ -779,6 +779,7 @@ module.exports = {
|
|||
normalizeOllamaRows,
|
||||
inferFormat,
|
||||
inferQuantization,
|
||||
inferPrecision,
|
||||
inferRuntimeSupport,
|
||||
isModelArtifactFile,
|
||||
parseParamsB,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
const ModelDatabase = require('./model-database');
|
||||
const { isSupportedHuggingFaceModel } = require('./registry-ingestors');
|
||||
const { isSupportedHuggingFaceModel, inferQuantization, inferPrecision } = require('./registry-ingestors');
|
||||
const DeterministicModelSelector = require('../models/deterministic-selector');
|
||||
const { normalizePrecision, memoryBudgetGB } = require('../models/ranking-contract');
|
||||
const { applyCpuOnlyOverride } = require('../hardware/cpu-only');
|
||||
|
|
@ -93,7 +93,8 @@ function inferFamily(identifier = '') {
|
|||
}
|
||||
|
||||
function normalizeQuantization(row = {}) {
|
||||
const raw = row.quantization || row.precision || '';
|
||||
const filename = row.filename || row.artifact_name || '';
|
||||
const raw = inferQuantization(filename) || inferPrecision(filename) || row.quantization || row.precision || '';
|
||||
return normalizePrecision(raw);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1135,7 +1135,7 @@ class DeterministicModelSelector {
|
|||
return this.normalizeQuantization(
|
||||
variant.quantization ||
|
||||
variant.quant ||
|
||||
'Q4_K_M'
|
||||
'UNKNOWN'
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -1210,10 +1210,9 @@ class DeterministicModelSelector {
|
|||
}
|
||||
|
||||
extractVariantSizeGB(variant, paramsB) {
|
||||
const candidate = Number(variant.real_size_gb ?? variant.estimated_size_gb ?? variant.size_gb ?? NaN);
|
||||
const candidate = Number(variant.real_size_gb ?? variant.size_gb ?? NaN);
|
||||
if (Number.isFinite(candidate) && candidate > 0) return candidate;
|
||||
if (!Number.isFinite(paramsB) || paramsB <= 0) return 0.5;
|
||||
return Math.max(0.5, Math.round((paramsB * 0.58 + 0.5) * 10) / 10);
|
||||
return null; // No observed artifact size; estimate from the actual precision later.
|
||||
}
|
||||
|
||||
inferModalities(model, variantTag = '') {
|
||||
|
|
@ -1303,27 +1302,27 @@ class DeterministicModelSelector {
|
|||
|
||||
extractParams(details) {
|
||||
// Look for parameter info in ollama show output
|
||||
const match = details.match(/parameters\s+(\d+\.?\d*)[BM]/i);
|
||||
const match = details.match(/parameters\s+(\d+\.?\d*)\s*[BM]/i);
|
||||
if (match) {
|
||||
const num = parseFloat(match[1]);
|
||||
return match[0].toUpperCase().includes('B') ? num : num / 1000;
|
||||
}
|
||||
return 7; // Default fallback
|
||||
return null;
|
||||
}
|
||||
|
||||
extractContextLength(details) {
|
||||
const match = details.match(/context_length\s+(\d+)/i);
|
||||
return match ? parseInt(match[1]) : 4096;
|
||||
const match = details.match(/context[ _]length\s+(\d+)/i);
|
||||
return match ? parseInt(match[1]) : null;
|
||||
}
|
||||
|
||||
extractQuantization(details) {
|
||||
const match = details.match(/quantization\s+(Q\d+_[A-Z0-9_]+)/i);
|
||||
return match ? match[1] : 'Q4_K_M';
|
||||
const match = details.match(/quantization\s+(\S+)/i);
|
||||
return normalizePrecision(match?.[1]);
|
||||
}
|
||||
|
||||
extractSizeGB(details) {
|
||||
const match = details.match(/size\s+(\d+\.?\d*)\s*GB/i);
|
||||
return match ? parseFloat(match[1]) : 4.0;
|
||||
const match = details.match(/size\s+(\d+\.?\d*)\s*(GB|MB)/i);
|
||||
return match ? parseFloat(match[1]) / (match[2].toUpperCase() === 'MB' ? 1024 : 1) : null;
|
||||
}
|
||||
|
||||
extractModalities(details) {
|
||||
|
|
@ -1584,6 +1583,7 @@ class DeterministicModelSelector {
|
|||
}
|
||||
|
||||
evaluateModel(model, hardware, category, targetCtx, budget, optimizeFor = 'balanced', runtime = 'ollama', options = {}) {
|
||||
if (!(this.parseBillionsValue(model.totalParamsB || model.paramsB) > 0)) return null;
|
||||
const nativeContext = Number(model.ctxMax);
|
||||
const requiredContext = options.contextPolicy !== 'preferred';
|
||||
if (requiredContext && (!(nativeContext > 0) || nativeContext < targetCtx)) return null;
|
||||
|
|
@ -1674,7 +1674,9 @@ class DeterministicModelSelector {
|
|||
// A model/tag/file is an immutable artifact. Alternative files are separate rows.
|
||||
if (model?.quant) return [this.normalizeQuantization(model.quant)];
|
||||
const available = model?.availableQuantizations || Object.keys(model?.sizeByQuant || {});
|
||||
if (!available.length && Number(model?.sizeGB) > 0) return ['UNKNOWN'];
|
||||
if (available.length !== 1 && (model?.model_identifier || !available.length)) {
|
||||
return Number(model?.sizeGB) > 0 ? ['UNKNOWN'] : [];
|
||||
}
|
||||
return [...new Set(available.map(normalizePrecision))]
|
||||
.sort((a, b) => (precisionProfile(b).bytes || 0) - (precisionProfile(a).bytes || 0));
|
||||
}
|
||||
|
|
@ -1716,9 +1718,11 @@ class DeterministicModelSelector {
|
|||
? parameterProfile.totalParamsB
|
||||
: parameterProfile.effectiveParamsB;
|
||||
const modeledWeightGB = bpp == null ? Infinity : weightParamsB * bpp;
|
||||
// A real observed artifact size always wins for weight memory — never let
|
||||
// an MoE "sparse inference" assumption discard a measured on-disk size.
|
||||
const useObservedArtifactSize = Number.isFinite(observedWeightGB) && observedWeightGB > 0;
|
||||
// Full-precision resident weights cannot shrink to a compressed download
|
||||
// or an old Q4 size estimate. Quantized artifacts retain their observed size.
|
||||
const fullPrecision = ['FP16', 'BF16', 'FP32'].includes(normalizedQuant);
|
||||
const useObservedArtifactSize = Number.isFinite(observedWeightGB) && observedWeightGB > 0 &&
|
||||
(!fullPrecision || observedWeightGB >= modeledWeightGB);
|
||||
const modelMemGB = useObservedArtifactSize ? observedWeightGB : modeledWeightGB;
|
||||
const effectiveCtx = Number.isFinite(Number(ctx)) && Number(ctx) > 0 ? Number(ctx) : 4096;
|
||||
|
||||
|
|
|
|||
|
|
@ -569,7 +569,7 @@ class OllamaNativeScraper {
|
|||
quantization: quantization,
|
||||
command: `ollama pull ${tag}`,
|
||||
estimated_size_gb: this.estimateModelSizeGB(tag),
|
||||
real_size_gb: realSizeGB || this.estimateModelSizeGB(tag)
|
||||
real_size_gb: realSizeGB || null
|
||||
};
|
||||
});
|
||||
|
||||
|
|
@ -586,8 +586,8 @@ class OllamaNativeScraper {
|
|||
}
|
||||
|
||||
extractQuantizationFromTag(tag) {
|
||||
const quantMatch = tag.match(/\b(q\d+_[km]?_?[ms]?|fp16|fp32|int8|int4)\b/i);
|
||||
return quantMatch ? quantMatch[0].toUpperCase() : 'Q4_0'; // Default assumption
|
||||
const quantMatch = tag.match(/\b(i?q\d+(?:_[a-z0-9]+)*|bf16|f16|f32|fp16|fp32|int8|int4)\b/i);
|
||||
return quantMatch ? quantMatch[0].toUpperCase() : 'UNKNOWN';
|
||||
}
|
||||
|
||||
extractRealSizeFromHTML(html, tag) {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,22 @@ for (const quant of ['FP16', 'BF16', 'FP32']) {
|
|||
assert.strictEqual(selector.evaluateModel({ ...base, quant }, hardware, 'general', 4096, 8), null);
|
||||
}
|
||||
assert.strictEqual(precisionProfile('IQ4_XS').bytes, 0.58);
|
||||
assert.strictEqual(selector.extractQuantization('quantization F16'), 'FP16');
|
||||
assert.ok(selector.estimateRequiredGB({ ...base, sizeGB: 4 }, 'FP16', 4096) > 14, 'a stale Q4 size must not shrink FP16 resident weights');
|
||||
assert.strictEqual(selector.extractQuantization('quantization BF16'), 'BF16');
|
||||
assert.strictEqual(selector.extractQuantization(''), 'UNKNOWN');
|
||||
assert.strictEqual(selector.extractSizeGB(''), null);
|
||||
assert.strictEqual(selector.extractParams('parameters 7 B'), 7);
|
||||
assert.strictEqual(selector.evaluateModel({ ...base, paramsB: null, sizeGB: 4 }, hardware, 'general', 4096, 8), null);
|
||||
assert.strictEqual(selector.extractContextLength(''), null);
|
||||
assert.strictEqual(selector.extractVariantSizeGB({ estimated_size_gb: 4 }, 7), null);
|
||||
const fpVariant = selector.convertOllamaModelToDeterministicModels({ model_identifier: 'test', primary_category: 'general',
|
||||
variants: [{ tag: 'test:7b-fp16', quantization: 'FP16', size: '7b' }] })[0];
|
||||
assert.strictEqual(fpVariant.quant, 'FP16');
|
||||
assert.ok(selector.estimateRequiredGB(fpVariant, 'FP16', 4096) > 14);
|
||||
assert.strictEqual(selector.selectBestQuantization(fpVariant, 8, 4096), null);
|
||||
assert.deepStrictEqual(selector.getQuantizationCandidates({ model_identifier: 'unknown:7b', availableQuantizations: ['Q8_0', 'Q4_K_M'] }), []);
|
||||
|
||||
assert.strictEqual(selector.estimateRequiredGB({ ...base, quant: 'unknown' }, 'unknown', 4096), Infinity);
|
||||
const fixed = { ...base, quant: 'Q8_0', model_identifier: 'qwen:7b-q8_0',
|
||||
availableQuantizations: ['Q8_0', 'Q4_K_M'], sizeByQuant: { Q8_0: 7.6, Q4_K_M: 4.5 } };
|
||||
|
|
@ -32,6 +48,9 @@ const shard = { source_id: 'huggingface', repo_id: 'test/model-7B-GGUF',
|
|||
canonical_model_id: 'test/model-7B-GGUF', repo_tasks: ['text-generation'], format: 'gguf',
|
||||
quantization: 'Q8_0', size_gb: 4, parameter_count_b: 7, context_length: 4096,
|
||||
runtime_support: ['ollama', 'llama.cpp'] };
|
||||
const stalePrecision = artifactToSelectorModel({ ...shard, filename: 'model-7b-f16.gguf', quantization: 'Q4_K_M' });
|
||||
assert.strictEqual(stalePrecision.quant, 'FP16', 'the concrete file precision overrides stale metadata');
|
||||
assert.ok(selector.estimateRequiredGB(stalePrecision, 'FP16', 4096) > 14);
|
||||
const shards = [1, 2].map(i => ({ ...shard, filename: `model-Q8_0-0000${i}-of-00002.gguf` }));
|
||||
assert.strictEqual(groupWeightShards(shards.slice(1)).length, 0, 'incomplete shard sets are not installable');
|
||||
const grouped = groupWeightShards(shards);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue