mirror of
https://github.com/signerless/llm-checker.git
synced 2026-10-19 18:04:26 +00:00
2bfd8fdc4c
- `sync:seed` runs scripts/update-quality-seed.js, which refreshes every quality source inside the snapshot (a failed source keeps its previous scores). A fresh install had no benchmark data and ranked every model by parameter count until the user ran `quality-sync`. - Existing databases import a snapshot's quality source when it is newer than their own copy, and rebuild the catalog cohort. - New `hf_eval_results` source: GPQA, MMLU-Pro, HLE, SWE-bench, Terminal-Bench, GSM8K, AIME/HMMT, IFEval and MMMU results attached to Hub repos. Self-reported, marked not independent, last in each category's metric preference. - LiveBench discovers releases from the site bundle and adds those newer than the curated list; older question sets stay out.
51 lines
1.8 KiB
JavaScript
51 lines
1.8 KiB
JavaScript
#!/usr/bin/env node
|
|
'use strict';
|
|
|
|
/**
|
|
* Refresh the benchmark tables inside the packaged snapshot so a fresh install
|
|
* ranks by measured quality. A source that cannot be fetched keeps the scores
|
|
* the snapshot already holds; the build fails only when no source has data.
|
|
*/
|
|
|
|
const path = require('path');
|
|
|
|
const ModelDatabase = require('../src/data/model-database');
|
|
const { QualityEvals, SOURCES } = require('../src/data/quality-evals');
|
|
|
|
const rootDir = path.resolve(__dirname, '..');
|
|
const seedDbPath = path.join(rootDir, 'src', 'data', 'seed', 'models.db');
|
|
|
|
async function main() {
|
|
const database = new ModelDatabase({
|
|
dbPath: seedDbPath,
|
|
seedDbPath: path.join(rootDir, 'missing-seed.db'),
|
|
disableRegistrySeedImport: true
|
|
});
|
|
await database.initialize();
|
|
try {
|
|
const quality = new QualityEvals(database);
|
|
for (const source of Object.keys(SOURCES)) {
|
|
try {
|
|
const report = await quality.ingest(source);
|
|
console.log(`[quality-seed] ${source}: ${report.rows} scores`);
|
|
} catch (error) {
|
|
console.log(`[quality-seed] ${source}: kept previous scores (${error.message})`);
|
|
}
|
|
}
|
|
const catalog = database.all('SELECT name FROM models');
|
|
quality.refreshCatalogCohort(catalog);
|
|
const stats = quality.stats();
|
|
const coverage = quality.coverage(catalog);
|
|
console.log(`[quality-seed] ${stats.evals} scores from ${stats.sources.length} sources; ` +
|
|
`${coverage.measured}/${coverage.total} catalog families measured`);
|
|
if (stats.evals === 0) throw new Error('No benchmark source returned data');
|
|
database.vacuum();
|
|
} finally {
|
|
database.close();
|
|
}
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(error);
|
|
process.exit(1);
|
|
});
|