diff --git a/README.md b/README.md index 82063ba..e802c06 100644 --- a/README.md +++ b/README.md @@ -508,7 +508,7 @@ rules: | Command | Description | |---------|-------------| | `sync` | Refresh the local SQLite model catalog from Ollama | -| `quality-sync` | Refresh public quality benchmarks from HF Open LLM, LMArena, BigCodeBench, EvalPlus, LiveBench and MMMU; supports `--sources` and `--json` | +| `quality-sync` | Refresh public quality benchmarks from HF Open LLM, LMArena, BigCodeBench, EvalPlus, LiveBench, MMMU and Hugging Face model-card eval results; supports `--sources` and `--json` | | `search ` | Search the synced Ollama catalog; add `--registry`/`--source` to search the multi-source registry (HF + Ollama + GPT4All) with `--max-params`/`--runtime`/`--format` filters | | `smart-recommend` | Advanced recommendations using the full scoring engine | @@ -533,7 +533,9 @@ llm-checker registry-recommend --category coding --runtime mlx llm-checker registry-search qwen --source huggingface --runtime vllm --max-params 24 ``` -Refresh the quality scores used by `check`, `recommend` and `registry-recommend`: +The packaged snapshot ships with these scores, so a fresh install ranks by +measurements where they exist. Refresh the quality scores used by `check`, +`recommend` and `registry-recommend`: ```bash llm-checker quality-sync @@ -544,6 +546,11 @@ HF Open LLM contributes MMLU-PRO, BBH, GPQA, MuSR, MATH Level 5 and IFEval from official, available, original checkpoints. It supplies no coding score. LMArena contributes overall human preference Elo to general and conversation recommendations, with attribution to its [CC-BY-4.0 dataset](https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset). +LiveBench releases newer than the curated list are discovered from its site. +`hf_eval_results` reads the evaluation results publishers attach to their own +Hugging Face repos (GPQA, MMLU-Pro, HLE, SWE-bench, Terminal-Bench, AIME, ...). +They are self-reported, labeled as not independent, and only used when no +independent board measured the model. The sources are downloaded from their official Parquet snapshots without an account. Failed refreshes retain cached scores. Recommendations expose `qualitySource` as `measured` or `estimated`, including the measured source, diff --git a/bin/enhanced_cli.js b/bin/enhanced_cli.js index 84dd1ba..6041967 100755 --- a/bin/enhanced_cli.js +++ b/bin/enhanced_cli.js @@ -5429,7 +5429,7 @@ program program .command('quality-sync') .description('Refresh public benchmark scores used for model quality recommendations') - .option('-s, --sources ', 'Comma-separated sources: hf_open_llm,lmarena,bigcodebench,evalplus,livebench,mmmu') + .option('-s, --sources ', 'Comma-separated sources: hf_open_llm,lmarena,bigcodebench,evalplus,livebench,mmmu,hf_eval_results') .option('--db ', 'Model database path (defaults to the local catalog)') .option('-j, --json', 'Output the sync and coverage report as JSON') .action(async (options) => { diff --git a/docs/reference/changelog.md b/docs/reference/changelog.md index 9804c1a..667c38f 100644 --- a/docs/reference/changelog.md +++ b/docs/reference/changelog.md @@ -21,6 +21,13 @@ Unreleased — exact catalog metadata - Existing databases adopt a newer packaged snapshot; `sync` keeps the registry's Ollama rows instead of deleting them. - MCP `recommend` and `project_recommend` honor the requested category. +- The packaged snapshot ships benchmark scores; existing databases adopt + them unless `quality-sync` refreshed a source more recently. New + `hf_eval_results` source (self-reported model-card results), and LiveBench + releases newer than the curated list are discovered automatically. +- Ollama tags record their license from the manifest (`sync --exact`). +- Base/pretrained checkpoints rank below instruct builds; Ollama aliases and + quantizations of one base checkpoint collapse into one recommendation. 3.8.1 — ModelVet attribution (2026-08-04) ------------------------------------------- diff --git a/package.json b/package.json index 749f88e..faf5b2c 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "list-models": "node bin/enhanced_cli.js list-models", "ai-check": "node bin/enhanced_cli.js ai-check", "ai-run": "node bin/enhanced_cli.js ai-run", - "sync:seed": "node bin/enhanced_cli.js sync --force --exact --quiet && node scripts/update-seed-db.js && node scripts/update-registry-seed.js", + "sync:seed": "node bin/enhanced_cli.js sync --force --exact --quiet && node scripts/update-seed-db.js && node scripts/update-registry-seed.js && node scripts/update-quality-seed.js", "sync:registry-seed": "node scripts/update-registry-seed.js", "benchmark": "cd ml-model && python python/benchmark_collector.py", "train-ai": "cd ml-model && python python/train_model.py", diff --git a/scripts/update-quality-seed.js b/scripts/update-quality-seed.js new file mode 100644 index 0000000..6489d63 --- /dev/null +++ b/scripts/update-quality-seed.js @@ -0,0 +1,51 @@ +#!/usr/bin/env node +'use strict'; + +/** + * Refresh the benchmark tables inside the packaged snapshot so a fresh install + * ranks by measured quality. A source that cannot be fetched keeps the scores + * the snapshot already holds; the build fails only when no source has data. + */ + +const path = require('path'); + +const ModelDatabase = require('../src/data/model-database'); +const { QualityEvals, SOURCES } = require('../src/data/quality-evals'); + +const rootDir = path.resolve(__dirname, '..'); +const seedDbPath = path.join(rootDir, 'src', 'data', 'seed', 'models.db'); + +async function main() { + const database = new ModelDatabase({ + dbPath: seedDbPath, + seedDbPath: path.join(rootDir, 'missing-seed.db'), + disableRegistrySeedImport: true + }); + await database.initialize(); + try { + const quality = new QualityEvals(database); + for (const source of Object.keys(SOURCES)) { + try { + const report = await quality.ingest(source); + console.log(`[quality-seed] ${source}: ${report.rows} scores`); + } catch (error) { + console.log(`[quality-seed] ${source}: kept previous scores (${error.message})`); + } + } + const catalog = database.all('SELECT name FROM models'); + quality.refreshCatalogCohort(catalog); + const stats = quality.stats(); + const coverage = quality.coverage(catalog); + console.log(`[quality-seed] ${stats.evals} scores from ${stats.sources.length} sources; ` + + `${coverage.measured}/${coverage.total} catalog families measured`); + if (stats.evals === 0) throw new Error('No benchmark source returned data'); + database.vacuum(); + } finally { + database.close(); + } +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/src/data/model-database.js b/src/data/model-database.js index d120ead..44a3708 100644 --- a/src/data/model-database.js +++ b/src/data/model-database.js @@ -932,6 +932,8 @@ class ModelDatabase { imported = true; } + if (this.importQualityFromSnapshot(seed)) imported = true; + const hasOllamaArtifacts = this.get(`SELECT 1 AS present FROM model_artifacts WHERE source_id = 'ollama' LIMIT 1`); if ((catalogIsNewer || !hasOllamaArtifacts) && this.getModelCount() > 0) { this.rebuildOllamaRegistry(); @@ -970,6 +972,48 @@ class ModelDatabase { this.setLastSync(seed.getLastSync()); } + /** + * Benchmark scores ship in the snapshot so a fresh install ranks by + * measurements, not only by parameter count. A source the user refreshed + * later with `quality-sync` is kept. + */ + importQualityFromSnapshot(seed) { + const seedHasQuality = seed.get(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'quality_sources'`); + if (!seedHasQuality) return false; + const { QualityEvals } = require('./quality-evals'); + const quality = new QualityEvals(this); + const local = new Map(this.all(`SELECT id, fetched_at FROM quality_sources`).map((row) => [row.id, row.fetched_at])); + let imported = false; + for (const source of seed.all(`SELECT * FROM quality_sources`)) { + const localFetchedAt = local.get(source.id); + if (localFetchedAt && String(localFetchedAt) >= String(source.fetched_at || '')) continue; + this.run(`DELETE FROM quality_evals WHERE source_id = ?`, [source.id]); + this.run(` + INSERT INTO quality_sources (id, display_name, data_url, homepage_url, independent, fetched_at, row_count, payload_sha256) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + display_name = excluded.display_name, data_url = excluded.data_url, + homepage_url = excluded.homepage_url, independent = excluded.independent, + fetched_at = excluded.fetched_at, row_count = excluded.row_count, + payload_sha256 = excluded.payload_sha256 + `, [source.id, source.display_name, source.data_url, source.homepage_url, source.independent, + source.fetched_at, source.row_count, source.payload_sha256]); + for (const row of seed.all(`SELECT * FROM quality_evals WHERE source_id = ?`, [source.id])) { + this.run(` + INSERT OR REPLACE INTO quality_evals + (source_id, bench_model_name, bench_model_url, family_key, params_b, active_params_b, is_moe, + variant_role, metric, category, raw_score, raw_scale_max, eval_precision, fetched_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, [row.source_id, row.bench_model_name, row.bench_model_url, row.family_key, row.params_b, + row.active_params_b, row.is_moe, row.variant_role, row.metric, row.category, row.raw_score, + row.raw_scale_max, row.eval_precision, row.fetched_at]); + } + imported = true; + } + if (imported) quality.refreshCatalogCohort(this.all(`SELECT name FROM models`)); + return imported; + } + importRegistrySource(seed, source) { this.clearRegistrySource(source.id); this.upsertRegistrySource({ ...source, metadata: this.parseJson(source.metadata, {}) }); diff --git a/src/data/quality-evals.js b/src/data/quality-evals.js index 6c897e9..8295bcc 100644 --- a/src/data/quality-evals.js +++ b/src/data/quality-evals.js @@ -229,6 +229,19 @@ const SOURCES = { // of models; the union covers far more of our catalog than the latest // alone, and newer releases win on conflict. releases: ['2025_04_25', '2025_05_30', '2025_11_25', '2025_12_23', '2026_01_08', '2026_06_25'], + // The site bundle lists every release. Newer ones are added + // automatically; older ones used a different question set and stay out. + async discoverReleases(request) { + const page = await request(this.homepage); + if (!page.ok) return []; + const script = (await page.text()).match(/src="\.?\/?(static\/js\/main\.[^"]+\.js)"/); + if (!script) return []; + const bundle = await request(`${this.homepage}/${script[1]}`); + if (!bundle.ok) return []; + const lists = (await bundle.text()).match(/\[("20\d{2}-\d{2}-\d{2}",?){4,}\]/g) || []; + const dates = lists.map((list) => JSON.parse(list)).sort((a, b) => b.length - a.length)[0] || []; + return dates.map((date) => date.replace(/-/g, '_')); + }, // Authoritative task -> category map, from categories_.json. categories: { Reasoning: ['theory_of_mind', 'zebra_puzzle', 'spatial', 'logic_with_navigation'], @@ -286,6 +299,84 @@ const SOURCES = { }, }, + hf_eval_results: { + displayName: 'Hugging Face model-card evaluation results (self-reported)', + homepage: 'https://huggingface.co/docs/hub/eval-results', + url: 'https://huggingface.co/api/models?expand[]=evalResults', + // Results publishers attach to their own repos. Almost none are + // verified, so they rank below every independent board. + independent: false, + datasets: { + 'TIGER-Lab/MMLU-Pro': ['hf_eval_mmlu_pro', 'general'], + 'Idavidrein/gpqa': ['hf_eval_gpqa_diamond', 'reasoning', /diamond/i], + 'cais/hle': ['hf_eval_hle', 'reasoning'], + 'openai/gsm8k': ['hf_eval_gsm8k', 'reasoning'], + 'MathArena/aime_2026': ['hf_eval_aime_2026', 'reasoning'], + 'MathArena/hmmt_feb_2026': ['hf_eval_hmmt_feb_2026', 'reasoning'], + 'SWE-bench/SWE-bench_Verified': ['hf_eval_swe_bench_verified', 'coding'], + 'ScaleAI/SWE-bench_Pro': ['hf_eval_swe_bench_pro', 'coding'], + 'harborframework/terminal-bench-2.0': ['hf_eval_terminal_bench_2_0', 'coding'], + 'harborframework/terminal-bench-2.1': ['hf_eval_terminal_bench_2_1', 'coding'], + 'google/IFEval': ['hf_eval_ifeval', 'talking'], + 'MMMU/MMMU': ['hf_eval_mmmu', 'multimodal'], + 'lmms-lab/MMMU': ['hf_eval_mmmu', 'multimodal'] + }, + pipelines: ['text-generation', 'image-text-to-text'], + perPipeline: 3000, + async collect(request) { + const models = []; + for (const pipeline of this.pipelines) { + let url = `https://huggingface.co/api/models?pipeline_tag=${pipeline}&sort=downloads&direction=-1&limit=1000&expand[]=evalResults`; + let fetched = 0; + while (url && fetched < this.perPipeline) { + const response = await request(url); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const page = await response.json(); + if (!Array.isArray(page) || page.length === 0) break; + fetched += page.length; + models.push(...page.filter((model) => Array.isArray(model.evalResults) && model.evalResults.length)); + const link = response.headers?.get?.('link') || ''; + url = (link.match(/<([^>]+)>;\s*rel="next"/i) || [])[1] || null; + } + } + return { rows: this.parse(models), payload: JSON.stringify(models) }; + }, + parse(models) { + const latest = new Map(); + for (const model of models) { + if (typeof model?.id !== 'string' || !/^[\w.-]+\/[\w.-]+$/.test(model.id)) continue; + for (const entry of model.evalResults || []) { + const data = entry?.data || {}; + const spec = this.datasets[data.dataset?.id]; + if (!spec) continue; + const [metric, category, taskPattern] = spec; + if (taskPattern && !taskPattern.test(String(data.dataset?.task_id || ''))) continue; + const value = finiteNumber(data.value); + if (value == null || value < 0 || value > 100) continue; + const key = `${model.id}::${metric}`; + const previous = latest.get(key); + if (previous && String(previous.date) >= String(data.date || '')) continue; + latest.set(key, { + date: data.date || '', + row: { + benchModelName: model.id, + benchModelUrl: data.source?.url || `https://huggingface.co/${model.id}`, + paramsB: null, + isMoe: 0, + variantRole: 'instruct', + metric, + category, + rawScore: value, + rawScaleMax: 100, + evalPrecision: entry.verified ? 'verified' : 'self-reported', + } + }); + } + } + return [...latest.values()].map((entry) => entry.row); + }, + }, + mmmu: { displayName: 'MMMU (multimodal understanding)', homepage: 'https://mmmu-benchmark.github.io', @@ -449,11 +540,19 @@ class QualityEvals { let payload = ''; const request = (url) => fetchImpl(url, { redirect: 'follow', signal: AbortSignal.timeout(timeoutMs) }); - if (Array.isArray(spec.releases)) { + if (typeof spec.collect === 'function') { + ({ rows, payload } = await spec.collect(request)); + } else if (Array.isArray(spec.releases)) { // Merge releases oldest-first so a newer re-run of the same model // overwrites the older score rather than duplicating it. const seen = new Map(); - for (const release of spec.releases) { + let discovered = []; + try { + discovered = typeof spec.discoverReleases === 'function' ? await spec.discoverReleases(request) : []; + } catch { discovered = []; } + const oldest = [...spec.releases].sort()[0]; + const releases = [...new Set([...spec.releases, ...discovered.filter((release) => release >= oldest)])].sort(); + for (const release of releases) { const url = `https://livebench.ai/table_${release}.csv`; let body; try { diff --git a/src/models/deterministic-selector.js b/src/models/deterministic-selector.js index 9e64997..09ceea8 100644 --- a/src/models/deterministic-selector.js +++ b/src/models/deterministic-selector.js @@ -1887,16 +1887,21 @@ class DeterministicModelSelector { // first. LiveBench closes the gap for reasoning, creative and chat — // the three categories that previously had no task signal at all and // therefore ranked purely by model size. + // Self-reported model-card results (hf_eval_*) come last: they only + // speak for models no independent board has measured. const BENCH_FOR_CATEGORY = { coding: [ 'bcb_hard_instruct', 'bcb_instruct', 'livebench_coding', 'livebench_agentic_coding', 'humaneval_plus', 'mbpp_plus', 'bcb_complete', + 'hf_eval_swe_bench_verified', 'hf_eval_terminal_bench_2_1', 'hf_eval_terminal_bench_2_0', + 'hf_eval_swe_bench_pro', ], - reasoning: ['livebench_reasoning', 'hf_bbh', 'hf_gpqa', 'livebench_mathematics', 'hf_math_lvl5', 'hf_musr'], + reasoning: ['livebench_reasoning', 'hf_bbh', 'hf_gpqa', 'livebench_mathematics', 'hf_math_lvl5', 'hf_musr', + 'hf_eval_gpqa_diamond', 'hf_eval_hle', 'hf_eval_aime_2026', 'hf_eval_hmmt_feb_2026', 'hf_eval_gsm8k'], creative: ['livebench_language'], - talking: ['lmarena_chat', 'hf_ifeval', 'livebench_if'], - general: ['lmarena_general', 'hf_mmlu_pro', 'livebench_data_analysis'], - multimodal: ['mmmu_val'], + talking: ['lmarena_chat', 'hf_ifeval', 'livebench_if', 'hf_eval_ifeval'], + general: ['lmarena_general', 'hf_mmlu_pro', 'livebench_data_analysis', 'hf_eval_mmlu_pro'], + multimodal: ['mmmu_val', 'hf_eval_mmmu'], }; const wanted = BENCH_FOR_CATEGORY[category]; if (!wanted) return null; // no benchmark speaks to this category yet diff --git a/tests/packaged-snapshot-refresh.test.js b/tests/packaged-snapshot-refresh.test.js index 020c4e4..81df921 100644 --- a/tests/packaged-snapshot-refresh.test.js +++ b/tests/packaged-snapshot-refresh.test.js @@ -41,10 +41,18 @@ async function run() { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'llm-checker-snapshot-refresh-')); try { const seedPath = path.join(tempDir, 'seed.db'); - (await buildDatabase(seedPath, { + const seedDatabase = await buildDatabase(seedPath, { lastSync: '2026-09-01T00:00:00.000Z', hfIngestedAt: '2026-09-01T00:00:00.000Z', modelId: 'qwen3', hfRepo: 'org/new-model-GGUF' - })).close(); + }); + const { QualityEvals } = require('../src/data/quality-evals'); + const seedQuality = new QualityEvals(seedDatabase); + seedQuality.db.prepare(`INSERT INTO quality_sources (id, display_name, data_url, independent, fetched_at, row_count) + VALUES ('livebench', 'LiveBench', 'https://livebench.ai', 1, '2026-09-01T00:00:00.000Z', 1)`).run(); + seedQuality.db.prepare(`INSERT INTO quality_evals (source_id, bench_model_name, family_key, params_b, variant_role, + metric, category, raw_score, raw_scale_max) VALUES ('livebench', 'qwen3-8b', 'qwen3', 8, 'instruct', + 'livebench_coding', 'coding', 61.5, 100)`).run(); + seedDatabase.close(); const userPath = path.join(tempDir, 'user.db'); const user = await buildDatabase(userPath, { @@ -66,6 +74,10 @@ async function run() { .map((row) => row.repo_id), ['org/new-model-GGUF'], 'newer registry source adopted'); assert.deepStrictEqual(upgraded.all(`SELECT artifact_name FROM model_artifacts WHERE source_id = 'ollama'`) .map((row) => row.artifact_name), ['qwen3:8b'], 'Ollama registry rows follow the catalog'); + assert.deepStrictEqual(upgraded.all('SELECT bench_model_name, raw_score FROM quality_evals').map((row) => ({ ...row })), + [{ bench_model_name: 'qwen3-8b', raw_score: 61.5 }], 'benchmark scores ship with the snapshot'); + assert.deepStrictEqual(upgraded.all('SELECT family_key FROM catalog_families').map((row) => row.family_key), ['qwen3'], + 'the cohort follows the imported catalog'); const benchmark = upgraded.get('SELECT model_id, tag, tokens_per_second FROM benchmarks'); assert.deepStrictEqual({ ...benchmark }, { model_id: 'llama3.1', tag: 'llama3.1:8b', tokens_per_second: 42 }, 'speed measurements survive'); diff --git a/tests/quality-evals.test.js b/tests/quality-evals.test.js index f770d0c..4d9aa49 100644 --- a/tests/quality-evals.test.js +++ b/tests/quality-evals.test.js @@ -438,6 +438,50 @@ test('refresh invalidates percentiles and catalog clear/reopen preserves quality assert.deepStrictEqual(new QualityEvals(q.modelDatabase).stats(), before); }); +test('model-card eval results keep mapped percent metrics, the newest result, and are self-reported', () => { + const spec = SOURCES.hf_eval_results; + assert.strictEqual(spec.independent, false); + const result = (dataset, value, extra = {}) => ({ + verified: false, + data: { dataset: { id: dataset, task_id: extra.task || 'main' }, value, date: extra.date || '2026-06-01', source: { url: 'https://example.invalid' } } + }); + const rows = spec.parse([ + { id: 'Qwen/Qwen3-8B', evalResults: [ + result('TIGER-Lab/MMLU-Pro', 70, { date: '2026-01-01' }), + result('TIGER-Lab/MMLU-Pro', 72.5, { date: '2026-06-01' }), + result('Idavidrein/gpqa', 60, { task: 'gpqa_diamond' }), + result('Idavidrein/gpqa', 75, { task: 'main' }), + result('internlm/WildClawBench', 605), + result('SWE-bench/SWE-bench_Verified', 140) + ] }, + { id: 'not a repo', evalResults: [result('TIGER-Lab/MMLU-Pro', 50)] } + ]); + assert.deepStrictEqual(rows.map((row) => [row.benchModelName, row.metric, row.category, row.rawScore]).sort(), [ + ['Qwen/Qwen3-8B', 'hf_eval_gpqa_diamond', 'reasoning', 60], + ['Qwen/Qwen3-8B', 'hf_eval_mmlu_pro', 'general', 72.5] + ], 'unmapped datasets, other GPQA splits and out-of-range values are dropped'); + assert.ok(rows.every((row) => row.evalPrecision === 'self-reported')); +}); + +test('LiveBench adds newer releases listed by the site and ignores older question sets', async () => { + const { q } = await seed(); + const requested = []; + const csv = 'model,code_generation,code_completion\nqwen2.5-coder-32b-instruct,70,60\n'; + const fetchImpl = async (url) => { + requested.push(url); + if (url === SOURCES.livebench.homepage) return new Response(''); + if (url.endsWith('/static/js/main.abc.js')) { + return new Response('x=["2024-06-24","2025-04-25","2026-06-25","2026-12-01"],y=1'); + } + if (/table_\d{4}_\d{2}_\d{2}\.csv$/.test(url)) return new Response(csv); + return new Response('missing', { status: 404 }); + }; + await q.ingest('livebench', { fetchImpl }); + const tables = requested.filter((url) => url.endsWith('.csv')).map((url) => url.match(/table_(.+)\.csv/)[1]); + assert.ok(tables.includes('2026_12_01'), 'a release newer than the curated list is fetched'); + assert.ok(!tables.includes('2024_06_24'), 'releases older than the curated list are not mixed in'); +}); + async function run() { testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'llm-checker-quality-')); const backends = ['wasm'];