mirror of
https://github.com/AlexsJones/llmfit.git
synced 2026-08-30 05:31:40 +00:00
7b9cc973ae
Adds the Community Benchmarks workflow: any PR touching llmfit-core/data/community/ runs scripts/validate_community_benchmarks.py, which checks JSON schema conformance (schema.json, draft-07), path/naming conventions (community/<slug>/<timestamp>-<hash>.json), file size caps, and cross-field sanity the schema can't express (tps ordering, plausible hardware bounds, submission timestamps within the feature's lifetime). Whole-directory validation, so repo integrity is re-checked on every touch; also runs on push to main. Seeds the directory with the first genuine submission (Intel Arc 140V / Core Ultra 7 258V, gemma-3 Q8 via llama-server, 3.8 tok/s) — previously attempted in #713/#716 — which also exercises the workflow on this PR. Closes the CI-validation follow-up named in #712. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
40 lines
1.1 KiB
YAML
40 lines
1.1 KiB
YAML
name: Community Benchmarks
|
|
|
|
# Validates benchmark submissions contributed via `llmfit bench --share`
|
|
# (PRs adding files under llmfit-core/data/community/): JSON schema
|
|
# conformance, path/naming conventions, and cross-field sanity checks.
|
|
|
|
'on':
|
|
pull_request:
|
|
paths:
|
|
- 'llmfit-core/data/community/**'
|
|
push:
|
|
branches: [main]
|
|
paths:
|
|
- 'llmfit-core/data/community/**'
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
validate:
|
|
name: Validate submissions
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
|
with:
|
|
python-version: '3.12'
|
|
|
|
- name: Install jsonschema
|
|
run: pip install jsonschema
|
|
|
|
# Validates the whole directory, not just changed files: repo-wide
|
|
# integrity is the invariant, and the directory stays small.
|
|
- name: Validate community submissions
|
|
run: python3 scripts/validate_community_benchmarks.py
|