mirror of
https://github.com/signerless/llm-checker.git
synced 2026-10-19 09:54:45 +00:00
chore: commit all local changes
This commit is contained in:
parent
72d3761103
commit
bfa9755aec
55 changed files with 352302 additions and 190 deletions
1
.tmp-feature-check/ai_check.exit
Normal file
1
.tmp-feature-check/ai_check.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=0
|
||||
50
.tmp-feature-check/ai_check.out
Normal file
50
.tmp-feature-check/ai_check.out
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
|
||||
llm-checker | AI Check
|
||||
--------------------------------
|
||||
- Checking Ollama availability...
|
||||
- AI-Check Mode: Meta-evaluation in progress...
|
||||
│ Found 213 models in Ollama database
|
||||
│ 213 models match general category
|
||||
|
||||
AI-CHECK MODE
|
||||
╭─────────────────────────────────────────────────────────────────
|
||||
│ Category: GENERAL
|
||||
│ AI Weight: 30% + Deterministic: 70%
|
||||
│ Candidates Found: 1
|
||||
│ Hardware: 12 cores, 24GB RAM, apple_silicon
|
||||
╰
|
||||
|
||||
AI EVALUATOR STATUS
|
||||
╭──────────────────────────────────────────────────
|
||||
│ Model: qwen2.5-coder:7b
|
||||
│ 🔬 Evaluating: 20 models (showing top 1)
|
||||
│ 🔬 Status: Running AI evaluation...
|
||||
╰
|
||||
|
||||
AI-CHECK RESULTS
|
||||
╭─────────────────────────────────────────────────────────────────
|
||||
│ Evaluator: qwen2.5-coder:7b
|
||||
│ Category: GENERAL
|
||||
│ Models Evaluated: 1
|
||||
│ 📝 Note: AI-evaluated using qwen2.5-coder:7b
|
||||
╰
|
||||
╔═══════════════╤════════╤═════════════╤════════════╤═════════╤══════════╤═════════╤══════════════╗
|
||||
║ Model │ Size │ Det Score │ AI Score │ Final │ RAM │ Speed │ Status ║
|
||||
╟───────────────┼────────┼─────────────┼────────────┼─────────┼──────────┼─────────┼──────────────╢
|
||||
║ qwen2.5-coder │ 7B │ 74/100 │ 90/100 │ 79/100 │ 7.9/19GB │ 24t/s │ 🌐 Available ║
|
||||
╚═══════════════╧════════╧═════════════╧════════════╧═════════╧══════════╧═════════╧══════════════╝
|
||||
|
||||
|
||||
AI-POWERED RECOMMENDATION
|
||||
╭──────────────────────────────────────────────────
|
||||
│ Best Model: qwen2.5-coder
|
||||
│ Final Score: 79/100
|
||||
│ ⚖️ Det: 74 + AI: 90
|
||||
│
|
||||
│ 📥 Install command:
|
||||
│ ollama pull qwen2.5-coder
|
||||
│
|
||||
│ Why this model?
|
||||
│ • fits in 7.907344/19.200000000000003GB, Q8_0, coder-tuned, 7B is sweet spot, Metal backend
|
||||
│ • AI: High performance and efficiency.
|
||||
╰
|
||||
1
.tmp-feature-check/ai_run.exit
Normal file
1
.tmp-feature-check/ai_run.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
0
|
||||
18
.tmp-feature-check/ai_run.out
Normal file
18
.tmp-feature-check/ai_run.out
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
|
||||
llm-checker | AI Run
|
||||
------------------------------
|
||||
- Checking Ollama availability...
|
||||
- Selecting best model and launching...
|
||||
🔍 Using comprehensive model database for selection...
|
||||
Evaluating against 213 models from database
|
||||
Best model llama2 not installed locally
|
||||
🔄 Using best local alternative: codellama:latest
|
||||
✔ Selected codellama:latest (intelligent_database_comprehensive, 87% confidence)
|
||||
|
||||
Launching codellama:latest...
|
||||
Tip: Type /bye to exit the chat when finished
|
||||
|
||||
[?2026h[?25l[1G⠙ [K[?25h[?2026l[?2026h[?25l[1G⠹ [K[?25h[?2026l[?2026h[?25l[1G⠸ [K[?25h[?2026l[?2026h[?25l[1G⠼ [K[?25h[?2026l[?2026h[?25l[1G⠴ [K[?25h[?2026l[?2026h[?25l[1G⠦ [K[?25h[?2026l[?2026h[?25l[1G⠧ [K[?25h[?2026l[?2026h[?25l[1G⠇ [K[?25h[?2026l[?2026h[?25l[1G⠏ [K[?25h[?2026l[?2026h[?25l[1G⠏ [K[?25h[?2026l[?2026h[?25l[1G⠙ [K[?25h[?2026l[?2026h[?25l[1G⠹ [K[?25h[?2026l[?2026h[?25l[1G⠸ [K[?25h[?2026l[?2026h[?25l[1G⠸ [K[?25h[?2026l[?2026h[?25l[1G⠴ [K[?25h[?2026l[?2026h[?25l[1G⠴ [K[?25h[?2026l[?2026h[?25l[1G⠧ [K[?25h[?2026l[?2026h[?25l[1G⠇ [K[?25h[?2026l[?2026h[?25l[1G⠏ [K[?25h[?2026l[?2026h[?25l[1G⠏ [K[?25h[?2026l[?2026h[?25l[1G⠙ [K[?25h[?2026l[?2026h[?25l[1G⠹ [K[?25h[?2026l[?25l[?2026h[?25l[1G[K[?25h[?2026l[2K[1G[?25h
|
||||
[?25l[?25hOK[?25l[?25h.[?25l[?25h
|
||||
|
||||
[?25l[?25h
|
||||
1
.tmp-feature-check/audit_export.exit
Normal file
1
.tmp-feature-check/audit_export.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=3
|
||||
17
.tmp-feature-check/audit_export.out
Normal file
17
.tmp-feature-check/audit_export.out
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
|
||||
POLICY SUMMARY (AUDIT CHECK)
|
||||
╭─────────────────────────────────────────────────────────────────
|
||||
│ File: ~/llm-checker/.tmp-feature-check/policy.yaml
|
||||
│ Mode: enforce | Action: error
|
||||
│ Total checked: 19
|
||||
│ Pass: 0 | Fail: 19
|
||||
│ Suppressed: 0 | Exceptions: 0
|
||||
│ Top violations:
|
||||
│ - BACKEND_NOT_ALLOWED: 19
|
||||
│ - INSUFFICIENT_RAM: 19
|
||||
│ - LICENSE_MISSING: 19
|
||||
│ Enforcement result: blocking violations detected (exit 3)
|
||||
╰─────────────────────────────────────────────────────────────────
|
||||
|
||||
AUDIT EXPORT
|
||||
JSON: ~/llm-checker/.tmp-feature-check/audit.json
|
||||
1
.tmp-feature-check/audit_export_audit_mode.exit
Normal file
1
.tmp-feature-check/audit_export_audit_mode.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
0
|
||||
17
.tmp-feature-check/audit_export_audit_mode.out
Normal file
17
.tmp-feature-check/audit_export_audit_mode.out
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
|
||||
POLICY SUMMARY (AUDIT CHECK)
|
||||
╭─────────────────────────────────────────────────────────────────
|
||||
│ File: ~/llm-checker/.tmp-feature-check/policy.yaml
|
||||
│ Mode: audit | Action: warn
|
||||
│ Total checked: 19
|
||||
│ Pass: 0 | Fail: 19
|
||||
│ Suppressed: 0 | Exceptions: 0
|
||||
│ Top violations:
|
||||
│ - BACKEND_NOT_ALLOWED: 19
|
||||
│ - INSUFFICIENT_RAM: 19
|
||||
│ - LICENSE_MISSING: 19
|
||||
│ Audit mode: violations reported, command exits with code 0
|
||||
╰─────────────────────────────────────────────────────────────────
|
||||
|
||||
AUDIT EXPORT
|
||||
JSON: ~/llm-checker/.tmp-feature-check/audit-auditmode.json
|
||||
1
.tmp-feature-check/calibrate.exit
Normal file
1
.tmp-feature-check/calibrate.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=0
|
||||
9
.tmp-feature-check/calibrate.out
Normal file
9
.tmp-feature-check/calibrate.out
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
|
||||
CALIBRATION ARTIFACTS GENERATED
|
||||
╭────────────────────────────────────────────────────────────────────────
|
||||
│ Suite: ~/llm-checker/docs/fixtures/calibration/sample-suite.jsonl
|
||||
│ Runtime: ollama | Objective: balanced
|
||||
│ Models: 2
|
||||
│ Execution mode: dry-run
|
||||
│ Result: ~/llm-checker/.tmp-feature-check/calibration.json
|
||||
╰────────────────────────────────────────────────────────────────────────
|
||||
1
.tmp-feature-check/check.exit
Normal file
1
.tmp-feature-check/check.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=0
|
||||
31
.tmp-feature-check/check.out
Normal file
31
.tmp-feature-check/check.out
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
|
||||
llm-checker | Compatibility Check
|
||||
-------------------------------------------
|
||||
Analyzing your system... done
|
||||
|
||||
SYSTEM SUMMARY
|
||||
──────────────────────────────────────────────────
|
||||
CPU: M4 Pro (12 cores)
|
||||
Memory: 24GB RAM
|
||||
GPU: Apple M4 Pro
|
||||
Architecture: Apple Silicon
|
||||
Hardware Tier: HIGH
|
||||
|
||||
RECOMMENDED MODEL
|
||||
──────────────────────────────────────────────────
|
||||
Model: Llama 3.1 8B
|
||||
Size: 4.9GB
|
||||
Compatibility Score: 72.3/100
|
||||
Reason: Best general-purpose model for your hardware
|
||||
Estimated Speed: 41 tokens/sec
|
||||
Runtime: Ollama
|
||||
Status: Available for installation
|
||||
|
||||
Pull: ollama pull llama3.1:8b
|
||||
|
||||
QUICK START
|
||||
──────────────────────────────────────────────────
|
||||
1. Install the recommended model:
|
||||
ollama pull llama3.1:8b
|
||||
2. Start using it:
|
||||
ollama run llama3.1:8b
|
||||
1
.tmp-feature-check/demo.exit
Normal file
1
.tmp-feature-check/demo.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=0
|
||||
51
.tmp-feature-check/demo.out
Normal file
51
.tmp-feature-check/demo.out
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
|
||||
llm-checker | Demo
|
||||
----------------------------
|
||||
|
||||
LLM Checker - Enhanced Progress Demo
|
||||
This demonstrates the new step-by-step progress display with visual indicators
|
||||
────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
████░░░░░░░░░░░░░░░░ 20% [1/5] System Detection
|
||||
└─ Scanning hardware specifications...
|
||||
├─ CPU detected: Apple M4 Pro (12 cores) (1.0s)
|
||||
└─ Memory detected: 24GB unified memory (1.5s)
|
||||
Apple M4 Pro, 24GB RAM, Apple Silicon GPU
|
||||
└─ 1.5s
|
||||
|
||||
████████░░░░░░░░░░░░ 40% [2/5] Database Sync
|
||||
└─ Updating model database...
|
||||
3,247 models in database
|
||||
Database synchronized
|
||||
└─ 801ms
|
||||
|
||||
████████████░░░░░░░░ 60% [3/5] Compatibility Analysis
|
||||
└─ Running mathematical heuristics...
|
||||
├─ Analyzing hardware requirements... (1.2s)
|
||||
└─ Calculating performance scores... (1.8s)
|
||||
127 compatible models found
|
||||
Compatibility analysis complete
|
||||
└─ 1.8s
|
||||
|
||||
████████████████░░░░ 80% [4/5] AI Evaluation
|
||||
└─ Running intelligent model selection...
|
||||
├─ Mathematical heuristics applied (914ms)
|
||||
Top 15 models selected by AI
|
||||
AI evaluation complete
|
||||
└─ 919ms
|
||||
|
||||
████████████████████ 100% [5/5] Smart Recommendations
|
||||
└─ Generating personalized suggestions...
|
||||
├─ Analyzing use case: general (603ms)
|
||||
└─ Generating Ollama commands... (1.0s)
|
||||
23 recommendations generated
|
||||
└─ 1.0s
|
||||
────────────────────────────────────────────────────────────
|
||||
LLM Model Analysis & Compatibility Demo complete!
|
||||
Analysis complete! Found optimal models for your hardware
|
||||
Total time: 6.0s
|
||||
|
||||
Demo completed successfully!
|
||||
\nNow try running: llm-checker check
|
||||
For silent mode: llm-checker check --no-verbose
|
||||
1
.tmp-feature-check/hw_detect.exit
Normal file
1
.tmp-feature-check/hw_detect.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=0
|
||||
180
.tmp-feature-check/hw_detect.out
Normal file
180
.tmp-feature-check/hw_detect.out
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
{
|
||||
"backends": {
|
||||
"cpu": {
|
||||
"available": true,
|
||||
"info": {
|
||||
"brand": "Apple M4 Pro",
|
||||
"vendor": "Apple",
|
||||
"cores": {
|
||||
"physical": 12,
|
||||
"logical": 12,
|
||||
"performance": 6,
|
||||
"efficiency": 6
|
||||
},
|
||||
"frequency": {
|
||||
"base": 2400,
|
||||
"max": 2400
|
||||
},
|
||||
"cache": {
|
||||
"l1d": 64,
|
||||
"l1i": 128,
|
||||
"l2": 4,
|
||||
"l3": 0
|
||||
},
|
||||
"capabilities": {
|
||||
"sse": false,
|
||||
"sse2": false,
|
||||
"sse3": false,
|
||||
"ssse3": false,
|
||||
"sse4_1": false,
|
||||
"sse4_2": false,
|
||||
"avx": false,
|
||||
"avx2": false,
|
||||
"avx512": false,
|
||||
"avx512_vnni": false,
|
||||
"amx": false,
|
||||
"fma": false,
|
||||
"f16c": true,
|
||||
"neon": true,
|
||||
"sve": false,
|
||||
"dotprod": true,
|
||||
"bestSimd": "NEON"
|
||||
},
|
||||
"architecture": "arm64",
|
||||
"backend": "cpu",
|
||||
"speedCoefficient": 70
|
||||
}
|
||||
},
|
||||
"metal": {
|
||||
"available": true,
|
||||
"info": {
|
||||
"chip": "Apple M4 Pro",
|
||||
"variant": "pro",
|
||||
"generation": 4,
|
||||
"cores": {
|
||||
"performance": 8,
|
||||
"efficiency": 4,
|
||||
"total": 12
|
||||
},
|
||||
"gpu": {
|
||||
"cores": 16,
|
||||
"model": "Apple M4 Pro"
|
||||
},
|
||||
"neuralEngine": {
|
||||
"cores": 0
|
||||
},
|
||||
"memory": {
|
||||
"unified": 24,
|
||||
"bandwidth": 273
|
||||
},
|
||||
"capabilities": {
|
||||
"metal": true,
|
||||
"metalVersion": "4",
|
||||
"fp16": true,
|
||||
"int8": true,
|
||||
"amx": true
|
||||
},
|
||||
"backend": "metal",
|
||||
"speedCoefficient": 270
|
||||
}
|
||||
}
|
||||
},
|
||||
"primary": {
|
||||
"type": "metal",
|
||||
"name": "Apple Metal",
|
||||
"info": {
|
||||
"chip": "Apple M4 Pro",
|
||||
"variant": "pro",
|
||||
"generation": 4,
|
||||
"cores": {
|
||||
"performance": 8,
|
||||
"efficiency": 4,
|
||||
"total": 12
|
||||
},
|
||||
"gpu": {
|
||||
"cores": 16,
|
||||
"model": "Apple M4 Pro"
|
||||
},
|
||||
"neuralEngine": {
|
||||
"cores": 0
|
||||
},
|
||||
"memory": {
|
||||
"unified": 24,
|
||||
"bandwidth": 273
|
||||
},
|
||||
"capabilities": {
|
||||
"metal": true,
|
||||
"metalVersion": "4",
|
||||
"fp16": true,
|
||||
"int8": true,
|
||||
"amx": true
|
||||
},
|
||||
"backend": "metal",
|
||||
"speedCoefficient": 270
|
||||
}
|
||||
},
|
||||
"cpu": {
|
||||
"brand": "Apple M4 Pro",
|
||||
"vendor": "Apple",
|
||||
"cores": {
|
||||
"physical": 12,
|
||||
"logical": 12,
|
||||
"performance": 6,
|
||||
"efficiency": 6
|
||||
},
|
||||
"frequency": {
|
||||
"base": 2400,
|
||||
"max": 2400
|
||||
},
|
||||
"cache": {
|
||||
"l1d": 64,
|
||||
"l1i": 128,
|
||||
"l2": 4,
|
||||
"l3": 0
|
||||
},
|
||||
"capabilities": {
|
||||
"sse": false,
|
||||
"sse2": false,
|
||||
"sse3": false,
|
||||
"ssse3": false,
|
||||
"sse4_1": false,
|
||||
"sse4_2": false,
|
||||
"avx": false,
|
||||
"avx2": false,
|
||||
"avx512": false,
|
||||
"avx512_vnni": false,
|
||||
"amx": false,
|
||||
"fma": false,
|
||||
"f16c": true,
|
||||
"neon": true,
|
||||
"sve": false,
|
||||
"dotprod": true,
|
||||
"bestSimd": "NEON"
|
||||
},
|
||||
"architecture": "arm64",
|
||||
"backend": "cpu",
|
||||
"speedCoefficient": 70
|
||||
},
|
||||
"summary": {
|
||||
"bestBackend": "metal",
|
||||
"backendName": "Apple Metal",
|
||||
"totalVRAM": 24,
|
||||
"effectiveMemory": 17,
|
||||
"speedCoefficient": 270,
|
||||
"isMultiGPU": false,
|
||||
"gpuCount": 1,
|
||||
"gpuModel": "Apple M4 Pro",
|
||||
"gpuInventory": "Apple M4 Pro",
|
||||
"gpuModels": [
|
||||
{
|
||||
"name": "Apple M4 Pro",
|
||||
"count": 1
|
||||
}
|
||||
],
|
||||
"hasHeterogeneousGPU": false,
|
||||
"cpuModel": "Apple M4 Pro",
|
||||
"systemRAM": 24
|
||||
},
|
||||
"fingerprint": "apple-apple-m4-pro-24gb",
|
||||
"timestamp": 1771419210820
|
||||
}
|
||||
1
.tmp-feature-check/installed.exit
Normal file
1
.tmp-feature-check/installed.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=0
|
||||
94
.tmp-feature-check/installed.out
Normal file
94
.tmp-feature-check/installed.out
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
- Analyzing installed models...
|
||||
✔ Found 9 installed models
|
||||
[
|
||||
{
|
||||
"name": "codellama:latest",
|
||||
"displayName": "codellama latest",
|
||||
"size": "7B",
|
||||
"fileSizeGB": 3.6,
|
||||
"quantization": "Q4_0",
|
||||
"useCase": "coding",
|
||||
"score": 79,
|
||||
"command": "ollama run codellama:latest"
|
||||
},
|
||||
{
|
||||
"name": "llava:7b",
|
||||
"displayName": "llava 7b",
|
||||
"size": "7B",
|
||||
"fileSizeGB": 4.4,
|
||||
"quantization": "Q4_0",
|
||||
"useCase": "multimodal",
|
||||
"score": 79,
|
||||
"command": "ollama run llava:7b"
|
||||
},
|
||||
{
|
||||
"name": "codellama:7b",
|
||||
"displayName": "codellama 7b",
|
||||
"size": "7B",
|
||||
"fileSizeGB": 3.6,
|
||||
"quantization": "Q4_0",
|
||||
"useCase": "coding",
|
||||
"score": 79,
|
||||
"command": "ollama run codellama:7b"
|
||||
},
|
||||
{
|
||||
"name": "llama3.2:3b",
|
||||
"displayName": "llama3.2 3b",
|
||||
"size": "3.2B",
|
||||
"fileSizeGB": 1.9,
|
||||
"quantization": "Q4_K_M",
|
||||
"useCase": "general",
|
||||
"score": 79,
|
||||
"command": "ollama run llama3.2:3b"
|
||||
},
|
||||
{
|
||||
"name": "llama3.2:1b",
|
||||
"displayName": "llama3.2 1b",
|
||||
"size": "1.2B",
|
||||
"fileSizeGB": 1.2,
|
||||
"quantization": "Q8_0",
|
||||
"useCase": "general",
|
||||
"score": 79,
|
||||
"command": "ollama run llama3.2:1b"
|
||||
},
|
||||
{
|
||||
"name": "qwen2.5-coder:7b",
|
||||
"displayName": "qwen2.5-coder 7b",
|
||||
"size": "7.6B",
|
||||
"fileSizeGB": 4.4,
|
||||
"quantization": "Q4_K_M",
|
||||
"useCase": "coding",
|
||||
"score": 78,
|
||||
"command": "ollama run qwen2.5-coder:7b"
|
||||
},
|
||||
{
|
||||
"name": "qwen2.5-coder:1.5b",
|
||||
"displayName": "qwen2.5-coder 1.5b",
|
||||
"size": "1.5B",
|
||||
"fileSizeGB": 0.9,
|
||||
"quantization": "Q4_K_M",
|
||||
"useCase": "coding",
|
||||
"score": 78,
|
||||
"command": "ollama run qwen2.5-coder:1.5b"
|
||||
},
|
||||
{
|
||||
"name": "deepseek-v3.1:671b-cloud",
|
||||
"displayName": "deepseek-v3.1 671b-cloud",
|
||||
"size": "671.0B",
|
||||
"fileSizeGB": 0,
|
||||
"quantization": "FP8_E4M3",
|
||||
"useCase": "general",
|
||||
"score": 75,
|
||||
"command": "ollama run deepseek-v3.1:671b-cloud"
|
||||
},
|
||||
{
|
||||
"name": "gpt-oss:20b",
|
||||
"displayName": "gpt-oss 20b",
|
||||
"size": "20.9B",
|
||||
"fileSizeGB": 12.8,
|
||||
"quantization": "MXFP4",
|
||||
"useCase": "general",
|
||||
"score": 70,
|
||||
"command": "ollama run gpt-oss:20b"
|
||||
}
|
||||
]
|
||||
1
.tmp-feature-check/list_models.exit
Normal file
1
.tmp-feature-check/list_models.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=0
|
||||
337
.tmp-feature-check/list_models.out
Normal file
337
.tmp-feature-check/list_models.out
Normal file
|
|
@ -0,0 +1,337 @@
|
|||
- 📋 Loading models database...
|
||||
✔ ✅ Found 213 models (showing 3)
|
||||
[
|
||||
{
|
||||
"model_identifier": "llama3.1",
|
||||
"model_name": "llama3.1",
|
||||
"description": "",
|
||||
"labels": [],
|
||||
"pulls": 0,
|
||||
"tags": [
|
||||
"llama3.1:latest",
|
||||
"llama3.1:8b",
|
||||
"llama3.1:70b",
|
||||
"llama3.1:405b"
|
||||
],
|
||||
"last_updated": "Unknown",
|
||||
"url": "https://ollama.com/library/llama3.1",
|
||||
"namespace": null,
|
||||
"model_type": "official",
|
||||
"variants": [
|
||||
{
|
||||
"tag": "llama3.1:latest",
|
||||
"size": "unknown",
|
||||
"quantization": "Q4_0",
|
||||
"command": "ollama pull llama3.1:latest",
|
||||
"estimated_size_gb": 1,
|
||||
"real_size_gb": 4.9,
|
||||
"categories": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tag": "llama3.1:8b",
|
||||
"size": "8b",
|
||||
"quantization": "Q4_0",
|
||||
"command": "ollama pull llama3.1:8b",
|
||||
"estimated_size_gb": 8,
|
||||
"real_size_gb": 4.9,
|
||||
"categories": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tag": "llama3.1:70b",
|
||||
"size": "70b",
|
||||
"quantization": "Q4_0",
|
||||
"command": "ollama pull llama3.1:70b",
|
||||
"estimated_size_gb": 70,
|
||||
"real_size_gb": 43,
|
||||
"categories": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tag": "llama3.1:405b",
|
||||
"size": "405b",
|
||||
"quantization": "Q4_0",
|
||||
"command": "ollama pull llama3.1:405b",
|
||||
"estimated_size_gb": 405,
|
||||
"real_size_gb": 243,
|
||||
"categories": [
|
||||
"chat"
|
||||
]
|
||||
}
|
||||
],
|
||||
"detailed_description": "Llama 3.1 is a new state-of-the-art model from Meta available in 8B, 70B and 405B parameter sizes.",
|
||||
"parameters": {},
|
||||
"quantizations": [],
|
||||
"model_sizes": [
|
||||
"8b",
|
||||
"70b",
|
||||
"405b",
|
||||
"4.9gb",
|
||||
"43gb",
|
||||
"243gb",
|
||||
"952b"
|
||||
],
|
||||
"category": "talking",
|
||||
"use_cases": [
|
||||
"chat",
|
||||
"conversation",
|
||||
"assistant"
|
||||
],
|
||||
"main_size": "8b",
|
||||
"actual_pulls": 0,
|
||||
"context_length": "128K",
|
||||
"input_types": [
|
||||
"text",
|
||||
"image",
|
||||
"code"
|
||||
],
|
||||
"detailed_scraped_at": "2026-02-18T08:17:27.647Z",
|
||||
"categories": [
|
||||
"chat"
|
||||
],
|
||||
"primary_category": "chat"
|
||||
},
|
||||
{
|
||||
"model_identifier": "deepseek-r1",
|
||||
"model_name": "deepseek-r1",
|
||||
"description": "",
|
||||
"labels": [],
|
||||
"pulls": 0,
|
||||
"tags": [
|
||||
"deepseek-r1:671b",
|
||||
"deepseek-r1:8b",
|
||||
"deepseek-r1:1.5b",
|
||||
"deepseek-r1:7b",
|
||||
"deepseek-r1:14b",
|
||||
"deepseek-r1:32b",
|
||||
"deepseek-r1:70b",
|
||||
"deepseek-r1:latest"
|
||||
],
|
||||
"last_updated": "Unknown",
|
||||
"url": "https://ollama.com/library/deepseek-r1",
|
||||
"namespace": null,
|
||||
"model_type": "official",
|
||||
"variants": [
|
||||
{
|
||||
"tag": "deepseek-r1:671b",
|
||||
"size": "671b",
|
||||
"quantization": "Q4_0",
|
||||
"command": "ollama pull deepseek-r1:671b",
|
||||
"estimated_size_gb": 671,
|
||||
"real_size_gb": 404,
|
||||
"categories": [
|
||||
"chat",
|
||||
"reasoning"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tag": "deepseek-r1:8b",
|
||||
"size": "8b",
|
||||
"quantization": "Q4_0",
|
||||
"command": "ollama pull deepseek-r1:8b",
|
||||
"estimated_size_gb": 8,
|
||||
"real_size_gb": 5.2,
|
||||
"categories": [
|
||||
"chat",
|
||||
"reasoning"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tag": "deepseek-r1:1.5b",
|
||||
"size": "1.5b",
|
||||
"quantization": "Q4_0",
|
||||
"command": "ollama pull deepseek-r1:1.5b",
|
||||
"estimated_size_gb": 1.5,
|
||||
"real_size_gb": 1.1,
|
||||
"categories": [
|
||||
"chat",
|
||||
"reasoning"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tag": "deepseek-r1:7b",
|
||||
"size": "7b",
|
||||
"quantization": "Q4_0",
|
||||
"command": "ollama pull deepseek-r1:7b",
|
||||
"estimated_size_gb": 7,
|
||||
"real_size_gb": 4.7,
|
||||
"categories": [
|
||||
"chat",
|
||||
"reasoning"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tag": "deepseek-r1:14b",
|
||||
"size": "14b",
|
||||
"quantization": "Q4_0",
|
||||
"command": "ollama pull deepseek-r1:14b",
|
||||
"estimated_size_gb": 14,
|
||||
"real_size_gb": 9,
|
||||
"categories": [
|
||||
"chat",
|
||||
"reasoning"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tag": "deepseek-r1:32b",
|
||||
"size": "32b",
|
||||
"quantization": "Q4_0",
|
||||
"command": "ollama pull deepseek-r1:32b",
|
||||
"estimated_size_gb": 32,
|
||||
"real_size_gb": 20,
|
||||
"categories": [
|
||||
"chat",
|
||||
"reasoning"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tag": "deepseek-r1:70b",
|
||||
"size": "70b",
|
||||
"quantization": "Q4_0",
|
||||
"command": "ollama pull deepseek-r1:70b",
|
||||
"estimated_size_gb": 70,
|
||||
"real_size_gb": 43,
|
||||
"categories": [
|
||||
"chat",
|
||||
"reasoning"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tag": "deepseek-r1:latest",
|
||||
"size": "unknown",
|
||||
"quantization": "Q4_0",
|
||||
"command": "ollama pull deepseek-r1:latest",
|
||||
"estimated_size_gb": 1,
|
||||
"real_size_gb": 5.2,
|
||||
"categories": [
|
||||
"chat",
|
||||
"reasoning"
|
||||
]
|
||||
}
|
||||
],
|
||||
"detailed_description": "DeepSeek-R1 is a family of open reasoning models with performance approaching that of leading models, such as O3 and Gemini 2.5 Pro.",
|
||||
"parameters": {},
|
||||
"quantizations": [],
|
||||
"model_sizes": [
|
||||
"1.5b",
|
||||
"7b",
|
||||
"8b",
|
||||
"14b",
|
||||
"32b",
|
||||
"70b",
|
||||
"671b",
|
||||
"5.2gb",
|
||||
"1.1gb",
|
||||
"4.7gb",
|
||||
"9.0gb",
|
||||
"20gb",
|
||||
"43gb",
|
||||
"404gb"
|
||||
],
|
||||
"category": "reasoning",
|
||||
"use_cases": [
|
||||
"reasoning",
|
||||
"mathematics",
|
||||
"logic"
|
||||
],
|
||||
"main_size": "1.5b",
|
||||
"actual_pulls": 0,
|
||||
"context_length": "160K",
|
||||
"input_types": [
|
||||
"text",
|
||||
"image",
|
||||
"code"
|
||||
],
|
||||
"detailed_scraped_at": "2026-02-18T08:17:27.855Z",
|
||||
"categories": [
|
||||
"chat",
|
||||
"reasoning"
|
||||
],
|
||||
"primary_category": "reasoning"
|
||||
},
|
||||
{
|
||||
"model_identifier": "llama3.2",
|
||||
"model_name": "llama3.2",
|
||||
"description": "",
|
||||
"labels": [],
|
||||
"pulls": 0,
|
||||
"tags": [
|
||||
"llama3.2:1b",
|
||||
"llama3.2:latest",
|
||||
"llama3.2:3b"
|
||||
],
|
||||
"last_updated": "Unknown",
|
||||
"url": "https://ollama.com/library/llama3.2",
|
||||
"namespace": null,
|
||||
"model_type": "official",
|
||||
"variants": [
|
||||
{
|
||||
"tag": "llama3.2:1b",
|
||||
"size": "1b",
|
||||
"quantization": "Q4_0",
|
||||
"command": "ollama pull llama3.2:1b",
|
||||
"estimated_size_gb": 1,
|
||||
"real_size_gb": 1.3,
|
||||
"categories": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tag": "llama3.2:latest",
|
||||
"size": "unknown",
|
||||
"quantization": "Q4_0",
|
||||
"command": "ollama pull llama3.2:latest",
|
||||
"estimated_size_gb": 1,
|
||||
"real_size_gb": 2,
|
||||
"categories": [
|
||||
"chat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tag": "llama3.2:3b",
|
||||
"size": "3b",
|
||||
"quantization": "Q4_0",
|
||||
"command": "ollama pull llama3.2:3b",
|
||||
"estimated_size_gb": 3,
|
||||
"real_size_gb": 2,
|
||||
"categories": [
|
||||
"chat"
|
||||
]
|
||||
}
|
||||
],
|
||||
"detailed_description": "Meta's Llama 3.2 goes small with 1B and 3B models.",
|
||||
"parameters": {},
|
||||
"quantizations": [],
|
||||
"model_sizes": [
|
||||
"1b",
|
||||
"3b",
|
||||
"2.0gb",
|
||||
"1.3gb",
|
||||
"929b",
|
||||
"2.6b"
|
||||
],
|
||||
"category": "talking",
|
||||
"use_cases": [
|
||||
"chat",
|
||||
"conversation",
|
||||
"assistant"
|
||||
],
|
||||
"main_size": "1b",
|
||||
"actual_pulls": 0,
|
||||
"context_length": "128K",
|
||||
"input_types": [
|
||||
"text",
|
||||
"image",
|
||||
"code"
|
||||
],
|
||||
"detailed_scraped_at": "2026-02-18T08:17:27.748Z",
|
||||
"categories": [
|
||||
"chat"
|
||||
],
|
||||
"primary_category": "chat"
|
||||
}
|
||||
]
|
||||
1
.tmp-feature-check/mcp_setup.exit
Normal file
1
.tmp-feature-check/mcp_setup.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=0
|
||||
38
.tmp-feature-check/mcp_setup.out
Normal file
38
.tmp-feature-check/mcp_setup.out
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
{
|
||||
"recommended": {
|
||||
"command": "claude",
|
||||
"args": [
|
||||
"mcp",
|
||||
"add",
|
||||
"llm-checker",
|
||||
"--",
|
||||
"llm-checker-mcp"
|
||||
],
|
||||
"commandLine": "claude mcp add llm-checker -- llm-checker-mcp"
|
||||
},
|
||||
"alternatives": [
|
||||
{
|
||||
"command": "claude",
|
||||
"args": [
|
||||
"mcp",
|
||||
"add",
|
||||
"llm-checker",
|
||||
"--",
|
||||
"npx",
|
||||
"llm-checker-mcp"
|
||||
],
|
||||
"commandLine": "claude mcp add llm-checker -- npx llm-checker-mcp"
|
||||
}
|
||||
],
|
||||
"claudeDesktop": {
|
||||
"configPath": "~/Library/Application Support/Claude/claude_desktop_config.json",
|
||||
"snippet": {
|
||||
"mcpServers": {
|
||||
"llm-checker": {
|
||||
"command": "llm-checker-mcp",
|
||||
"args": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
1
.tmp-feature-check/ollama.exit
Normal file
1
.tmp-feature-check/ollama.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=0
|
||||
62
.tmp-feature-check/ollama.out
Normal file
62
.tmp-feature-check/ollama.out
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
|
||||
llm-checker | Ollama Integration
|
||||
------------------------------------------
|
||||
- Checking Ollama integration...
|
||||
|
||||
|
||||
██░░░░░░░░░░░░░░░░░░ 13% [1/8] System Detection
|
||||
└─ Scanning hardware specifications...
|
||||
├─ CPU detected: M4 Pro (12 cores) (525ms)
|
||||
└─ Memory detected: 24GB unified memory (726ms)
|
||||
M4 Pro, 24GB RAM, Apple M4 Pro
|
||||
└─ 729ms
|
||||
├─ CPU detected: M4 Pro (12 cores) (729ms)
|
||||
└─ Memory detected: 24GB unified memory (930ms)
|
||||
M4 Pro, 24GB RAM, Apple M4 Pro
|
||||
└─ 930ms
|
||||
|
||||
█████░░░░░░░░░░░░░░░ 25% [2/8] Database Sync
|
||||
└─ Loading model database...
|
||||
177 models in database
|
||||
Database synchronized
|
||||
└─ 0ms
|
||||
|
||||
███████░░░░░░░░░░░░░ 38% [3/8] Model Analysis
|
||||
└─ Loading base model definitions...
|
||||
Loaded 26 base models
|
||||
Base models loaded
|
||||
└─ 0ms
|
||||
|
||||
██████████░░░░░░░░░░ 50% [4/8] Ollama Integration
|
||||
└─ Connecting to Ollama and checking installed models...
|
||||
Ollama connected with 9 locally installed models
|
||||
Ollama integration complete
|
||||
└─ 56ms
|
||||
|
||||
████████████░░░░░░░░ 63% [5/8] Model Filtering
|
||||
└─ Applying user-specified filters...
|
||||
└─ Cloud models excluded (0ms)
|
||||
26/26 models selected
|
||||
└─ 0ms
|
||||
|
||||
███████████████░░░░░ 75% [6/8] Compatibility Analysis
|
||||
└─ Running mathematical heuristics and hardware matching...
|
||||
11 compatible, 8 marginal
|
||||
└─ 2ms
|
||||
|
||||
█████████████████░░░ 88% [7/8] Performance Analysis
|
||||
└─ Estimating model performance and speeds...
|
||||
Performance data for 19 models
|
||||
└─ 1ms
|
||||
|
||||
████████████████████ 100% [8/8] Smart Recommendations
|
||||
└─ Generating personalized model suggestions...
|
||||
└─ Generating apple_silicon recommendations... (561ms)
|
||||
6 recommendations generated
|
||||
└─ 561ms
|
||||
────────────────────────────────────────────────────────────
|
||||
LLM Model Analysis & Compatibility Check complete!
|
||||
Found 11 compatible (8 marginal) models for your hardware
|
||||
Total time: 1.6s
|
||||
|
||||
✔ Ollama integration active
|
||||
1
.tmp-feature-check/ollama_plan.exit
Normal file
1
.tmp-feature-check/ollama_plan.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=0
|
||||
99
.tmp-feature-check/ollama_plan.out
Normal file
99
.tmp-feature-check/ollama_plan.out
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
{
|
||||
"generated_at": "2026-02-18T12:52:57.994Z",
|
||||
"selection": {
|
||||
"requested": [],
|
||||
"selected": [
|
||||
"codellama:latest",
|
||||
"qwen2.5-coder:7b",
|
||||
"llava:7b",
|
||||
"codellama:7b",
|
||||
"gpt-oss:20b",
|
||||
"llama3.2:3b",
|
||||
"llama3.2:1b",
|
||||
"qwen2.5-coder:1.5b"
|
||||
],
|
||||
"missing": []
|
||||
},
|
||||
"plan": {
|
||||
"objective": "balanced",
|
||||
"inputs": {
|
||||
"targetContext": 8192,
|
||||
"targetConcurrency": 2
|
||||
},
|
||||
"hardware": {
|
||||
"backend": "metal",
|
||||
"backendName": "Apple Metal",
|
||||
"rawCapacityGB": 17,
|
||||
"reserveGB": 2,
|
||||
"memoryBudgetGB": 15
|
||||
},
|
||||
"models": [
|
||||
{
|
||||
"name": "gpt-oss:20b",
|
||||
"size": "20.9B",
|
||||
"fileSizeGB": 12.8,
|
||||
"paramsB": 20.9,
|
||||
"estimatedBaseMemoryGB": 13.5
|
||||
}
|
||||
],
|
||||
"envelope": {
|
||||
"context": {
|
||||
"requested": 8192,
|
||||
"recommended": 3674,
|
||||
"min_safe": 2048,
|
||||
"max_for_parallel_1": 3674,
|
||||
"max_for_recommended_parallel": 3674
|
||||
},
|
||||
"parallel": {
|
||||
"requested": 2,
|
||||
"recommended": 1,
|
||||
"max_at_requested_ctx": 0
|
||||
},
|
||||
"loaded_models": {
|
||||
"requested": 2,
|
||||
"recommended": 1,
|
||||
"max_at_recommended_settings": 1
|
||||
}
|
||||
},
|
||||
"recommendation": {
|
||||
"num_ctx": 3674,
|
||||
"num_parallel": 1,
|
||||
"max_loaded_models": 1,
|
||||
"max_queue": 4,
|
||||
"keep_alive": "15m",
|
||||
"flash_attention": "1"
|
||||
},
|
||||
"memory": {
|
||||
"budgetGB": 15,
|
||||
"requestedEstimatedGB": 20.19,
|
||||
"recommendedEstimatedGB": 15,
|
||||
"utilizationPercent": 100
|
||||
},
|
||||
"risk": {
|
||||
"level": "critical",
|
||||
"score": 100
|
||||
},
|
||||
"fallback": {
|
||||
"num_ctx": 3674,
|
||||
"num_parallel": 1,
|
||||
"max_loaded_models": 1,
|
||||
"estimated_memory_gb": 15
|
||||
},
|
||||
"shell": {
|
||||
"env": {
|
||||
"OLLAMA_NUM_CTX": "3674",
|
||||
"OLLAMA_NUM_PARALLEL": "1",
|
||||
"OLLAMA_MAX_LOADED_MODELS": "1",
|
||||
"OLLAMA_MAX_QUEUE": "4",
|
||||
"OLLAMA_KEEP_ALIVE": "15m",
|
||||
"OLLAMA_FLASH_ATTENTION": "1"
|
||||
}
|
||||
},
|
||||
"notes": [
|
||||
"Requested settings exceed available memory budget; reduced settings are recommended.",
|
||||
"Context reduced from 8192 to 3674 to avoid memory pressure.",
|
||||
"Parallelism reduced from 2 to 1 to keep memory stable.",
|
||||
"Loaded models capped at 1 for this objective and memory budget."
|
||||
]
|
||||
}
|
||||
}
|
||||
41
.tmp-feature-check/policy.yaml
Normal file
41
.tmp-feature-check/policy.yaml
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
version: 1
|
||||
org: your-org
|
||||
mode: audit
|
||||
rules:
|
||||
models:
|
||||
allow:
|
||||
- qwen2.5-coder:*
|
||||
- llama3.1:*
|
||||
deny:
|
||||
- "*uncensored*"
|
||||
max_size_gb: 24
|
||||
max_params_b: 32
|
||||
allowed_quantizations:
|
||||
- Q4_K_M
|
||||
- Q5_K_M
|
||||
- Q8_0
|
||||
runtime:
|
||||
required_backends:
|
||||
- metal
|
||||
- cuda
|
||||
min_ram_gb: 32
|
||||
local_only: true
|
||||
compliance:
|
||||
approved_licenses:
|
||||
- mit
|
||||
- apache-2.0
|
||||
- llama
|
||||
enforcement:
|
||||
on_violation: warn
|
||||
exit_code: 3
|
||||
allow_exceptions: true
|
||||
exceptions:
|
||||
- model: deepseek-r1:32b
|
||||
reason: Approved PoC
|
||||
approver: security@example.com
|
||||
expires_at: 2026-06-30
|
||||
reporting:
|
||||
formats:
|
||||
- json
|
||||
- csv
|
||||
- sarif
|
||||
1
.tmp-feature-check/policy_init.exit
Normal file
1
.tmp-feature-check/policy_init.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=0
|
||||
1
.tmp-feature-check/policy_init.out
Normal file
1
.tmp-feature-check/policy_init.out
Normal file
|
|
@ -0,0 +1 @@
|
|||
Policy file created: ~/llm-checker/.tmp-feature-check/policy.yaml
|
||||
1
.tmp-feature-check/policy_validate.exit
Normal file
1
.tmp-feature-check/policy_validate.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=0
|
||||
1
.tmp-feature-check/policy_validate.out
Normal file
1
.tmp-feature-check/policy_validate.out
Normal file
|
|
@ -0,0 +1 @@
|
|||
Policy is valid (enforce mode): ~/llm-checker/.tmp-feature-check/policy.yaml
|
||||
1
.tmp-feature-check/recommend.exit
Normal file
1
.tmp-feature-check/recommend.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=0
|
||||
68
.tmp-feature-check/recommend.out
Normal file
68
.tmp-feature-check/recommend.out
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
|
||||
llm-checker | Recommendations
|
||||
---------------------------------------
|
||||
Generating recommendations... done
|
||||
|
||||
SYSTEM INFORMATION
|
||||
╭──────────────────────────────────────────────────
|
||||
│ CPU: M4 Pro (12 cores, 2.4GHz)
|
||||
│ Architecture: Apple Silicon
|
||||
│ RAM: 24GB
|
||||
│ GPU: Apple M4 Pro
|
||||
│ VRAM: 24GB (Integrated)
|
||||
│ Hardware Tier: HIGH
|
||||
╰
|
||||
|
||||
INTELLIGENT RECOMMENDATIONS BY CATEGORY
|
||||
╭─────────────────────────────────────────────────────────────────
|
||||
│ Hardware Tier: HIGH | Models Analyzed: 213
|
||||
│ Optimization: BALANCED
|
||||
│
|
||||
│ BEST OVERALL: yi:latest
|
||||
│ Command: ollama pull yi:latest
|
||||
│ Score: 88/100 | Category: general
|
||||
│ Quantization: Q4_K_M
|
||||
│
|
||||
│ Coding (Coding):
|
||||
│ deepseek-coder:6.7b (6.7B)
|
||||
│ Score: 87/100 | Pulls: 0
|
||||
│ Quantization: Q4_K_M
|
||||
│ Command: ollama pull deepseek-coder:6.7b
|
||||
│
|
||||
│ Reasoning (Reasoning):
|
||||
│ deepseek-coder-v2:latest (16B)
|
||||
│ Score: 91/100 | Pulls: 0
|
||||
│ Quantization: Q4_K_M
|
||||
│ Command: ollama pull deepseek-coder-v2:latest
|
||||
│
|
||||
│ Multimodal (Multimodal):
|
||||
│ qwen2.5vl:7b (7B)
|
||||
│ Score: 87/100 | Pulls: 0
|
||||
│ Quantization: Q4_K_M
|
||||
│ Command: ollama pull qwen2.5vl:7b
|
||||
│
|
||||
│ Creative (Creative):
|
||||
│ yi:latest (6B)
|
||||
│ Score: 87/100 | Pulls: 0
|
||||
│ Quantization: Q4_K_M
|
||||
│ Command: ollama pull yi:latest
|
||||
│
|
||||
│ Talking (Chat):
|
||||
│ yi:latest (6B)
|
||||
│ Score: 87/100 | Pulls: 0
|
||||
│ Quantization: Q4_K_M
|
||||
│ Command: ollama pull yi:latest
|
||||
│
|
||||
│ Reading (Reading):
|
||||
│ qwen3:1.7b (1.7B)
|
||||
│ Score: 84/100 | Pulls: 0
|
||||
│ Quantization: Q4_K_M
|
||||
│ Command: ollama pull qwen3:1.7b
|
||||
│
|
||||
│ General (General):
|
||||
│ yi:latest (6B)
|
||||
│ Score: 88/100 | Pulls: 0
|
||||
│ Quantization: Q4_K_M
|
||||
│ Command: ollama pull yi:latest
|
||||
│
|
||||
╰
|
||||
1
.tmp-feature-check/search.exit
Normal file
1
.tmp-feature-check/search.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=0
|
||||
1606
.tmp-feature-check/search.out
Normal file
1606
.tmp-feature-check/search.out
Normal file
File diff suppressed because it is too large
Load diff
1
.tmp-feature-check/smart_recommend.exit
Normal file
1
.tmp-feature-check/smart_recommend.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
exit=0
|
||||
348432
.tmp-feature-check/smart_recommend.out
Normal file
348432
.tmp-feature-check/smart_recommend.out
Normal file
File diff suppressed because it is too large
Load diff
1
.tmp-feature-check/sync.exit
Normal file
1
.tmp-feature-check/sync.exit
Normal file
|
|
@ -0,0 +1 @@
|
|||
0
|
||||
0
.tmp-feature-check/sync.out
Normal file
0
.tmp-feature-check/sync.out
Normal file
|
|
@ -254,6 +254,12 @@ npm install -g llm-checker
|
|||
claude mcp add llm-checker -- llm-checker-mcp
|
||||
```
|
||||
|
||||
Or generate the exact command directly from the CLI:
|
||||
|
||||
```bash
|
||||
llm-checker mcp-setup
|
||||
```
|
||||
|
||||
Or with npx (no global install needed):
|
||||
|
||||
```bash
|
||||
|
|
@ -352,6 +358,7 @@ llm-checker search "qwen coder" --json
|
|||
| `calibrate` | Generate calibration result + routing policy artifacts from a JSONL prompt suite |
|
||||
| `installed` | Rank your installed Ollama models by compatibility |
|
||||
| `ollama-plan` | Compute safe Ollama runtime env vars (`NUM_CTX`, `NUM_PARALLEL`, `MAX_LOADED_MODELS`) for selected local models |
|
||||
| `mcp-setup` | Print/apply Claude MCP setup command and config snippet (`--apply`, `--json`, `--npx`) |
|
||||
|
||||
### Advanced Commands (require `sql.js`)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
const { estimateTokenSpeedFromHardware } = require('../src/utils/token-speed-estimator');
|
||||
|
||||
class PerformanceAnalyzer {
|
||||
constructor() {
|
||||
this.benchmarkCache = new Map();
|
||||
|
|
@ -284,113 +286,57 @@ class PerformanceAnalyzer {
|
|||
}
|
||||
|
||||
calculateRealisticPerformance(model, hardware) {
|
||||
// Parse model size
|
||||
const modelSizeB = this.parseModelSize(model.size);
|
||||
|
||||
// Get hardware specifics
|
||||
const cpuModel = hardware.cpu?.brand || hardware.cpu?.model || '';
|
||||
const gpuModel = hardware.gpu?.model || '';
|
||||
const cores = hardware.cpu?.physicalCores || hardware.cpu?.cores || 1;
|
||||
const baseSpeed = hardware.cpu?.speed || 2.4;
|
||||
const vramGB = hardware.gpu?.vram || 0;
|
||||
const memoryTotal = hardware.memory?.total || 8;
|
||||
|
||||
// Hardware type detection
|
||||
const isAppleSilicon = hardware.cpu?.architecture === 'Apple Silicon' || (
|
||||
process.platform === 'darwin' && (
|
||||
gpuModel.toLowerCase().includes('apple') ||
|
||||
gpuModel.toLowerCase().includes('m1') ||
|
||||
gpuModel.toLowerCase().includes('m2') ||
|
||||
gpuModel.toLowerCase().includes('m3') ||
|
||||
gpuModel.toLowerCase().includes('m4')
|
||||
)
|
||||
);
|
||||
const isIntegratedGPU = /iris.*xe|iris.*graphics|uhd.*graphics|vega.*integrated|radeon.*graphics/i.test(gpuModel);
|
||||
const hasDedicatedGPU = vramGB > 0 && !isIntegratedGPU && !isAppleSilicon;
|
||||
|
||||
let tokensPerSecond;
|
||||
|
||||
if (isAppleSilicon) {
|
||||
// Apple Silicon - realistic but optimistic due to unified memory
|
||||
let baseTPS = 20; // More realistic baseline
|
||||
if (gpuModel.toLowerCase().includes('m4 pro')) baseTPS = 30;
|
||||
else if (gpuModel.toLowerCase().includes('m4')) baseTPS = 25;
|
||||
else if (gpuModel.toLowerCase().includes('m3 pro')) baseTPS = 28;
|
||||
else if (gpuModel.toLowerCase().includes('m3')) baseTPS = 22;
|
||||
else if (gpuModel.toLowerCase().includes('m2 pro')) baseTPS = 25;
|
||||
else if (gpuModel.toLowerCase().includes('m2')) baseTPS = 20;
|
||||
else if (gpuModel.toLowerCase().includes('m1 pro')) baseTPS = 22;
|
||||
else if (gpuModel.toLowerCase().includes('m1')) baseTPS = 18;
|
||||
|
||||
// Memory scaling for Apple Silicon
|
||||
if (memoryTotal >= 64) baseTPS *= 1.2;
|
||||
else if (memoryTotal >= 32) baseTPS *= 1.1;
|
||||
|
||||
tokensPerSecond = Math.max(6, Math.round(baseTPS / Math.max(0.7, modelSizeB)));
|
||||
|
||||
} else if (hasDedicatedGPU) {
|
||||
// Dedicated GPU - much better but still realistic
|
||||
let gpuTPS = 25;
|
||||
if (gpuModel.toLowerCase().includes('rtx 50')) gpuTPS = 60;
|
||||
else if (gpuModel.toLowerCase().includes('rtx 40')) gpuTPS = 45;
|
||||
else if (gpuModel.toLowerCase().includes('rtx 30')) gpuTPS = 35;
|
||||
else if (gpuModel.toLowerCase().includes('rtx 20')) gpuTPS = 28;
|
||||
else if (vramGB >= 16) gpuTPS = 40;
|
||||
else if (vramGB >= 8) gpuTPS = 30;
|
||||
else if (vramGB >= 4) gpuTPS = 25;
|
||||
|
||||
tokensPerSecond = Math.max(8, Math.round(gpuTPS / Math.max(0.4, modelSizeB)));
|
||||
|
||||
} else {
|
||||
// CPU-only or integrated GPU - most conservative and realistic
|
||||
const hasAVX512 = cpuModel.toLowerCase().includes('intel') &&
|
||||
(cpuModel.includes('12th') || cpuModel.includes('13th') || cpuModel.includes('14th'));
|
||||
const hasAVX2 = cpuModel.toLowerCase().includes('intel') || cpuModel.toLowerCase().includes('amd');
|
||||
|
||||
// Base CPU performance - very conservative
|
||||
let cpuK = 1.2; // Much more realistic
|
||||
if (hasAVX512) cpuK = 2.0;
|
||||
else if (hasAVX2) cpuK = 1.6;
|
||||
|
||||
// Threading efficiency (realistic diminishing returns)
|
||||
const effectiveThreads = Math.min(cores, 6); // CPU inference doesn't scale linearly
|
||||
|
||||
// iGPU small boost
|
||||
const iGpuMultiplier = isIntegratedGPU ? 1.2 : 1.0;
|
||||
|
||||
// Memory pressure factor
|
||||
const memoryPressure = Math.min(1.0, Math.max(0.6, memoryTotal / (modelSizeB * 2)));
|
||||
|
||||
const baseTPS = (cpuK * baseSpeed * effectiveThreads * iGpuMultiplier * memoryPressure) / Math.max(2.0, modelSizeB);
|
||||
|
||||
// Realistic CPU caps based on hardware
|
||||
const maxCPUTPS = hasAVX512 ? 18 : (isIntegratedGPU ? 12 : 8);
|
||||
tokensPerSecond = Math.max(1, Math.min(maxCPUTPS, Math.round(baseTPS)));
|
||||
}
|
||||
const speedProfile = estimateTokenSpeedFromHardware(hardware, {
|
||||
modelSizeB,
|
||||
modelName: model.name
|
||||
});
|
||||
const tokensPerSecond = speedProfile.tokensPerSecond;
|
||||
|
||||
return {
|
||||
estimatedTokensPerSecond: Math.round(tokensPerSecond),
|
||||
estimatedTokensPerSecond: tokensPerSecond,
|
||||
confidence: this.calculateConfidence(hardware, model),
|
||||
factors: {
|
||||
cpu: cpuModel,
|
||||
memory: memoryTotal,
|
||||
gpu: hasDedicatedGPU ? 'dedicated' : (isIntegratedGPU ? 'integrated' : 'cpu_only'),
|
||||
cpu: hardware.cpu?.brand || hardware.cpu?.model || 'Unknown CPU',
|
||||
memory: hardware.memory?.total || 0,
|
||||
gpu: speedProfile.backend,
|
||||
modelSize: modelSizeB,
|
||||
architecture: isAppleSilicon ? 'Apple Silicon' : 'x86'
|
||||
architecture: hardware.cpu?.architecture || 'unknown',
|
||||
sizeScale: speedProfile.sizeScale,
|
||||
memoryFactor: speedProfile.memoryFactor
|
||||
},
|
||||
category: this.categorizePerformance(Math.round(tokensPerSecond)),
|
||||
category: this.categorizePerformance(tokensPerSecond),
|
||||
loadTimeEstimate: this.estimateLoadTime(model, hardware)
|
||||
};
|
||||
}
|
||||
|
||||
parseModelSize(sizeString) {
|
||||
const match = sizeString.match(/(\d+\.?\d*)[BM]/i);
|
||||
if (!match) return 1;
|
||||
if (typeof sizeString !== 'string' || !sizeString.trim()) return 1;
|
||||
|
||||
const num = parseFloat(match[1]);
|
||||
const unit = match[0].slice(-1).toUpperCase();
|
||||
const normalized = sizeString.trim().toUpperCase();
|
||||
|
||||
return unit === 'B' ? num : num / 1000; // Convert M to B
|
||||
// Parameter notation (e.g. 8B, 774M)
|
||||
const paramMatch = normalized.match(/(\d+\.?\d*)\s*([BM])\b/);
|
||||
if (paramMatch) {
|
||||
const num = parseFloat(paramMatch[1]);
|
||||
const unit = paramMatch[2];
|
||||
return unit === 'B' ? num : num / 1000;
|
||||
}
|
||||
|
||||
// File-size notation fallback (e.g. 4.9GB) -> rough Q4 param estimate
|
||||
const gbMatch = normalized.match(/(\d+\.?\d*)\s*GB\b/);
|
||||
if (gbMatch) {
|
||||
const sizeGB = parseFloat(gbMatch[1]);
|
||||
return Math.max(0.5, sizeGB / 0.62);
|
||||
}
|
||||
|
||||
const mbMatch = normalized.match(/(\d+\.?\d*)\s*MB\b/);
|
||||
if (mbMatch) {
|
||||
const sizeGB = parseFloat(mbMatch[1]) / 1024;
|
||||
return Math.max(0.5, sizeGB / 0.62);
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
calculateConfidence(hardware, model) {
|
||||
|
|
@ -502,4 +448,4 @@ class PerformanceAnalyzer {
|
|||
}
|
||||
}
|
||||
|
||||
module.exports = PerformanceAnalyzer;
|
||||
module.exports = PerformanceAnalyzer;
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ const COMMAND_HEADER_LABELS = {
|
|||
'smart-recommend': 'Smart Recommend',
|
||||
search: 'Model Search',
|
||||
sync: 'Database Sync',
|
||||
'mcp-setup': 'Claude MCP Setup',
|
||||
check: 'Compatibility Check',
|
||||
installed: 'Installed Models',
|
||||
'ai-check': 'AI Check',
|
||||
|
|
@ -612,6 +613,60 @@ async function checkOllamaAndExit() {
|
|||
}
|
||||
}
|
||||
|
||||
function quoteCliArg(value) {
|
||||
const stringValue = String(value ?? '');
|
||||
if (!stringValue) return '""';
|
||||
if (/^[A-Za-z0-9._:/=-]+$/.test(stringValue)) return stringValue;
|
||||
return `"${stringValue.replace(/(["\\$`])/g, '\\$1')}"`;
|
||||
}
|
||||
|
||||
function getClaudeDesktopConfigPath() {
|
||||
const homeDir = os.homedir();
|
||||
if (process.platform === 'darwin') {
|
||||
return path.join(homeDir, 'Library', 'Application Support', 'Claude', 'claude_desktop_config.json');
|
||||
}
|
||||
if (process.platform === 'win32') {
|
||||
const appData = process.env.APPDATA || path.join(homeDir, 'AppData', 'Roaming');
|
||||
return path.join(appData, 'Claude', 'claude_desktop_config.json');
|
||||
}
|
||||
return path.join(homeDir, '.config', 'Claude', 'claude_desktop_config.json');
|
||||
}
|
||||
|
||||
function buildClaudeMcpSetup(useNpx = false, serverName = 'llm-checker') {
|
||||
const normalizedServerName = String(serverName || 'llm-checker').trim() || 'llm-checker';
|
||||
const runner = useNpx ? ['npx', 'llm-checker-mcp'] : ['llm-checker-mcp'];
|
||||
const claudeArgs = ['mcp', 'add', normalizedServerName, '--', ...runner];
|
||||
const commandLine = ['claude', ...claudeArgs].map(quoteCliArg).join(' ');
|
||||
const desktopServerConfig = useNpx
|
||||
? { command: 'npx', args: ['llm-checker-mcp'] }
|
||||
: { command: 'llm-checker-mcp', args: [] };
|
||||
|
||||
return {
|
||||
serverName: normalizedServerName,
|
||||
useNpx: Boolean(useNpx),
|
||||
claudeArgs,
|
||||
commandLine,
|
||||
desktopConfigPath: getClaudeDesktopConfigPath(),
|
||||
desktopConfig: {
|
||||
mcpServers: {
|
||||
[normalizedServerName]: desktopServerConfig
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function runExternalCommand(command, args) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, {
|
||||
stdio: 'inherit',
|
||||
env: process.env
|
||||
});
|
||||
|
||||
child.on('error', reject);
|
||||
child.on('close', (code) => resolve(code));
|
||||
});
|
||||
}
|
||||
|
||||
function parsePositiveIntegerOption(rawValue, optionName) {
|
||||
const parsed = Number(rawValue);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
|
|
@ -629,13 +684,32 @@ function parseNonNegativeNumberOption(rawValue, optionName) {
|
|||
}
|
||||
|
||||
function selectModelsForPlan(installedModels, requestedModels = []) {
|
||||
const runnableModels = (installedModels || []).filter((model) => {
|
||||
const name = String(model?.name || '').toLowerCase();
|
||||
const source = String(model?.source || '').toLowerCase();
|
||||
const type = String(model?.type || model?.model_type || '').toLowerCase();
|
||||
const fileSizeGB = Number(model?.fileSizeGB) || 0;
|
||||
|
||||
const cloudTagged = (
|
||||
name.includes('-cloud') ||
|
||||
name.endsWith(':cloud') ||
|
||||
source.includes('cloud') ||
|
||||
type === 'cloud' ||
|
||||
type === 'remote' ||
|
||||
type === 'hosted'
|
||||
);
|
||||
|
||||
// Keep only models that are actually present locally for memory planning.
|
||||
return !(cloudTagged && fileSizeGB <= 0);
|
||||
});
|
||||
|
||||
const requested = Array.isArray(requestedModels)
|
||||
? requestedModels.map((model) => String(model || '').trim()).filter(Boolean)
|
||||
: [];
|
||||
|
||||
if (!requested.length) {
|
||||
return {
|
||||
selected: installedModels.slice(),
|
||||
selected: runnableModels.slice(),
|
||||
missing: []
|
||||
};
|
||||
}
|
||||
|
|
@ -647,24 +721,24 @@ function selectModelsForPlan(installedModels, requestedModels = []) {
|
|||
for (const request of requested) {
|
||||
const normalized = request.toLowerCase();
|
||||
|
||||
let match = installedModels.find(
|
||||
let match = runnableModels.find(
|
||||
(model) => String(model.name || '').toLowerCase() === normalized
|
||||
);
|
||||
|
||||
if (!match) {
|
||||
match = installedModels.find((model) =>
|
||||
match = runnableModels.find((model) =>
|
||||
String(model.name || '').toLowerCase().startsWith(`${normalized}:`)
|
||||
);
|
||||
}
|
||||
|
||||
if (!match) {
|
||||
match = installedModels.find(
|
||||
match = runnableModels.find(
|
||||
(model) => String(model.family || '').toLowerCase() === normalized
|
||||
);
|
||||
}
|
||||
|
||||
if (!match) {
|
||||
match = installedModels.find((model) =>
|
||||
match = runnableModels.find((model) =>
|
||||
String(model.name || '').toLowerCase().includes(normalized)
|
||||
);
|
||||
}
|
||||
|
|
@ -2439,6 +2513,78 @@ function displayPolicySummary(commandName, policyConfig, evaluation, enforcement
|
|||
console.log(chalk.magenta('╰' + '─'.repeat(65)));
|
||||
}
|
||||
|
||||
program
|
||||
.command('mcp-setup')
|
||||
.description('Show or apply Claude MCP setup for llm-checker')
|
||||
.option('--name <server-name>', 'MCP server name in Claude', 'llm-checker')
|
||||
.option('--npx', 'Use npx llm-checker-mcp instead of global llm-checker-mcp')
|
||||
.option('--apply', 'Run `claude mcp add ...` automatically')
|
||||
.option('-j, --json', 'Output setup details as JSON')
|
||||
.action(async (options) => {
|
||||
const primarySetup = buildClaudeMcpSetup(Boolean(options.npx), options.name);
|
||||
const alternateSetup = buildClaudeMcpSetup(!Boolean(options.npx), options.name);
|
||||
|
||||
if (options.json) {
|
||||
console.log(JSON.stringify({
|
||||
recommended: {
|
||||
command: 'claude',
|
||||
args: primarySetup.claudeArgs,
|
||||
commandLine: primarySetup.commandLine
|
||||
},
|
||||
alternatives: [
|
||||
{
|
||||
command: 'claude',
|
||||
args: alternateSetup.claudeArgs,
|
||||
commandLine: alternateSetup.commandLine
|
||||
}
|
||||
],
|
||||
claudeDesktop: {
|
||||
configPath: primarySetup.desktopConfigPath,
|
||||
snippet: primarySetup.desktopConfig
|
||||
}
|
||||
}, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
showAsciiArt('mcp-setup');
|
||||
|
||||
console.log(chalk.blue.bold('\nClaude Code MCP Setup'));
|
||||
console.log(chalk.white('\nRecommended command:'));
|
||||
console.log(chalk.cyan(` ${primarySetup.commandLine}`));
|
||||
|
||||
console.log(chalk.white('\nAlternative command:'));
|
||||
console.log(chalk.gray(` ${alternateSetup.commandLine}`));
|
||||
|
||||
console.log(chalk.white('\nClaude Desktop config path (manual):'));
|
||||
console.log(chalk.gray(` ${primarySetup.desktopConfigPath}`));
|
||||
console.log(chalk.white('\nConfig snippet:'));
|
||||
console.log(chalk.gray(JSON.stringify(primarySetup.desktopConfig, null, 2)));
|
||||
|
||||
if (!options.apply) {
|
||||
console.log(chalk.green('\nTip: run with --apply to execute the command automatically.'));
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(chalk.blue('\nApplying MCP setup via Claude CLI...\n'));
|
||||
try {
|
||||
const exitCode = await runExternalCommand('claude', primarySetup.claudeArgs);
|
||||
if (exitCode === 0) {
|
||||
console.log(chalk.green('\nClaude MCP setup applied successfully.'));
|
||||
} else {
|
||||
console.error(chalk.red(`\nClaude command exited with code ${exitCode}.`));
|
||||
process.exit(exitCode || 1);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error && error.code === 'ENOENT') {
|
||||
console.error(chalk.red('Could not find `claude` in PATH.'));
|
||||
console.log(chalk.yellow('Run the printed command manually once Claude CLI is installed.'));
|
||||
} else {
|
||||
console.error(chalk.red(`Failed to apply MCP setup: ${error.message}`));
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
|
||||
const policyCommand = program
|
||||
.command('policy')
|
||||
.description('Manage enterprise policy files (policy.yaml)')
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ const { OllamaNativeScraper } = require('../ollama/native-scraper');
|
|||
const crypto = require('crypto');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const fetch = require('node-fetch');
|
||||
const fetch = require('../utils/fetch');
|
||||
|
||||
class AICheckSelector {
|
||||
constructor() {
|
||||
|
|
@ -803,4 +803,4 @@ Return JSON with this structure:
|
|||
}
|
||||
}
|
||||
|
||||
module.exports = AICheckSelector;
|
||||
module.exports = AICheckSelector;
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ const fs = require('fs');
|
|||
const path = require('path');
|
||||
const os = require('os');
|
||||
const { spawn } = require('child_process');
|
||||
const fetch = require('../utils/fetch');
|
||||
const { DETERMINISTIC_WEIGHTS } = require('./scoring-config');
|
||||
const {
|
||||
parseBillionsValue: parseMoEBillionsValue,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
const { estimateTokenSpeedFromHardware } = require('../utils/token-speed-estimator');
|
||||
|
||||
class ExpandedModelsDatabase {
|
||||
constructor() {
|
||||
this.models = this.initializeExpandedModels();
|
||||
|
|
@ -958,88 +960,12 @@ class ExpandedModelsDatabase {
|
|||
}
|
||||
|
||||
calculateRealisticTokensPerSecond(model, hardware) {
|
||||
// Extract model parameters from name or size
|
||||
const modelParams = this.extractModelParams(model);
|
||||
|
||||
// Get hardware specifics
|
||||
const cpuModel = hardware.cpu?.brand || hardware.cpu?.model || '';
|
||||
const gpuModel = hardware.gpu?.model || '';
|
||||
const cores = hardware.cpu?.physicalCores || hardware.cpu?.cores || 1;
|
||||
const baseSpeed = hardware.cpu?.speed || 2.4;
|
||||
const vramGB = hardware.gpu?.vram || 0;
|
||||
|
||||
// Check hardware type
|
||||
const isAppleSilicon = process.platform === 'darwin' && (
|
||||
gpuModel.toLowerCase().includes('apple') ||
|
||||
gpuModel.toLowerCase().includes('m1') ||
|
||||
gpuModel.toLowerCase().includes('m2') ||
|
||||
gpuModel.toLowerCase().includes('m3') ||
|
||||
gpuModel.toLowerCase().includes('m4')
|
||||
);
|
||||
const isIntegratedGPU = /iris.*xe|iris.*graphics|uhd.*graphics|vega.*integrated|radeon.*graphics/i.test(gpuModel);
|
||||
const hasDedicatedGPU = vramGB > 0 && !isIntegratedGPU && !isAppleSilicon;
|
||||
|
||||
let tokensPerSecond;
|
||||
|
||||
if (isAppleSilicon) {
|
||||
// Apple Silicon unified memory - more optimistic but realistic
|
||||
let baseTPS = 25;
|
||||
if (gpuModel.toLowerCase().includes('m4 pro')) baseTPS = 35;
|
||||
else if (gpuModel.toLowerCase().includes('m4')) baseTPS = 30;
|
||||
else if (gpuModel.toLowerCase().includes('m3 pro')) baseTPS = 30;
|
||||
else if (gpuModel.toLowerCase().includes('m3')) baseTPS = 25;
|
||||
else if (gpuModel.toLowerCase().includes('m2 pro')) baseTPS = 28;
|
||||
else if (gpuModel.toLowerCase().includes('m2')) baseTPS = 22;
|
||||
else if (gpuModel.toLowerCase().includes('m1 pro')) baseTPS = 25;
|
||||
else if (gpuModel.toLowerCase().includes('m1')) baseTPS = 20;
|
||||
|
||||
// Scale by model size (Apple Silicon handles larger models better)
|
||||
tokensPerSecond = Math.max(8, Math.round(baseTPS / Math.max(0.8, modelParams)));
|
||||
|
||||
} else if (hasDedicatedGPU) {
|
||||
// Dedicated GPU - much better performance
|
||||
let gpuTPS = 30;
|
||||
if (gpuModel.toLowerCase().includes('gb10') ||
|
||||
gpuModel.toLowerCase().includes('grace blackwell') ||
|
||||
gpuModel.toLowerCase().includes('dgx spark')) gpuTPS = 90;
|
||||
else if (gpuModel.toLowerCase().includes('h100')) gpuTPS = 120;
|
||||
else if (gpuModel.toLowerCase().includes('a100')) gpuTPS = 95;
|
||||
else if (gpuModel.toLowerCase().includes('rtx 50')) gpuTPS = 65;
|
||||
else if (gpuModel.toLowerCase().includes('rtx 40')) gpuTPS = 50;
|
||||
else if (gpuModel.toLowerCase().includes('rtx 30')) gpuTPS = 40;
|
||||
else if (gpuModel.toLowerCase().includes('rtx 20')) gpuTPS = 30;
|
||||
else if (gpuModel.toLowerCase().includes('p100')) gpuTPS = 32;
|
||||
else if (vramGB >= 16) gpuTPS = 45;
|
||||
else if (vramGB >= 8) gpuTPS = 35;
|
||||
else if (vramGB >= 4) gpuTPS = 25;
|
||||
|
||||
// Scale by model size for GPU
|
||||
tokensPerSecond = Math.max(10, Math.round(gpuTPS / Math.max(0.5, modelParams)));
|
||||
|
||||
} else {
|
||||
// CPU-only or integrated GPU - most realistic and conservative
|
||||
const hasAVX512 = cpuModel.toLowerCase().includes('intel') &&
|
||||
(cpuModel.includes('12th') || cpuModel.includes('13th') || cpuModel.includes('14th'));
|
||||
const hasAVX2 = cpuModel.toLowerCase().includes('intel') || cpuModel.toLowerCase().includes('amd');
|
||||
|
||||
// Base CPU coefficient - much more conservative
|
||||
let cpuK = 1.8; // Conservative baseline
|
||||
if (hasAVX512) cpuK = 2.6;
|
||||
else if (hasAVX2) cpuK = 2.2;
|
||||
|
||||
// iGPU boost (small)
|
||||
const iGpuMultiplier = isIntegratedGPU ? 1.3 : 1.0;
|
||||
|
||||
// Calculate with realistic threading limits
|
||||
const effectiveThreads = Math.min(cores, 8); // Diminishing returns after 8 threads
|
||||
const baseTPS = (cpuK * baseSpeed * effectiveThreads * iGpuMultiplier) / Math.max(1.5, modelParams);
|
||||
|
||||
// Apply realistic CPU limits
|
||||
const maxCPUTPS = hasAVX512 ? 25 : (isIntegratedGPU ? 20 : 15);
|
||||
tokensPerSecond = Math.max(2, Math.min(maxCPUTPS, Math.round(baseTPS)));
|
||||
}
|
||||
|
||||
return tokensPerSecond;
|
||||
const speedProfile = estimateTokenSpeedFromHardware(hardware, {
|
||||
modelSizeB: modelParams,
|
||||
modelName: model.name
|
||||
});
|
||||
return speedProfile.tokensPerSecond;
|
||||
}
|
||||
|
||||
extractModelParams(model) {
|
||||
|
|
@ -1047,11 +973,12 @@ class ExpandedModelsDatabase {
|
|||
const name = model.name.toLowerCase();
|
||||
|
||||
// Look for patterns like "7b", "3.8b", "0.5b", etc.
|
||||
const paramMatch = name.match(/(\d+\.?\d*)[bm](?:\s|$)/);
|
||||
const paramMatch = name.match(/(\d+\.?\d*)\s*([bm])(?:\s|$)/);
|
||||
if (paramMatch) {
|
||||
const value = parseFloat(paramMatch[1]);
|
||||
const unit = paramMatch[2].toLowerCase();
|
||||
// Convert millions to billions if needed
|
||||
return paramMatch[1].includes('m') ? value / 1000 : value;
|
||||
return unit === 'm' ? value / 1000 : value;
|
||||
}
|
||||
|
||||
// Fallback to size-based estimation
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
const fetch = require('node-fetch');
|
||||
const fetch = require('../utils/fetch');
|
||||
|
||||
class OllamaClient {
|
||||
constructor(baseURL = null) {
|
||||
|
|
@ -377,6 +377,29 @@ class OllamaClient {
|
|||
}
|
||||
}
|
||||
|
||||
calculateTokensPerSecond(data, totalTimeMs) {
|
||||
const evalCount = Number(data?.eval_count) || 0;
|
||||
const evalDurationNs = Number(data?.eval_duration) || 0;
|
||||
const totalSeconds = Math.max(0, Number(totalTimeMs) || 0) / 1000;
|
||||
|
||||
const evalTokensPerSecond = evalDurationNs > 0 && evalCount > 0
|
||||
? (evalCount / (evalDurationNs / 1_000_000_000))
|
||||
: 0;
|
||||
|
||||
const endToEndTokensPerSecond = totalSeconds > 0 && evalCount > 0
|
||||
? (evalCount / totalSeconds)
|
||||
: 0;
|
||||
|
||||
// Prefer eval-only throughput when available because it excludes load/setup overhead.
|
||||
const preferred = evalTokensPerSecond > 0 ? evalTokensPerSecond : endToEndTokensPerSecond;
|
||||
|
||||
return {
|
||||
tokensPerSecond: Math.round(preferred * 10) / 10,
|
||||
evalTokensPerSecond: Math.round(evalTokensPerSecond * 10) / 10,
|
||||
endToEndTokensPerSecond: Math.round(endToEndTokensPerSecond * 10) / 10
|
||||
};
|
||||
}
|
||||
|
||||
async testModelPerformance(modelName, testPrompt = "Hello, how are you?") {
|
||||
const availability = await this.checkOllamaAvailability();
|
||||
if (!availability.available) {
|
||||
|
|
@ -413,13 +436,15 @@ class OllamaClient {
|
|||
const endTime = Date.now();
|
||||
|
||||
const totalTime = endTime - startTime;
|
||||
const tokensGenerated = data.eval_count || 50;
|
||||
const tokensPerSecond = Math.round((tokensGenerated / (totalTime / 1000)) * 10) / 10;
|
||||
const tokensGenerated = Number(data.eval_count) || 0;
|
||||
const speed = this.calculateTokensPerSecond(data, totalTime);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
responseTime: totalTime,
|
||||
tokensPerSecond,
|
||||
tokensPerSecond: speed.tokensPerSecond,
|
||||
evalTokensPerSecond: speed.evalTokensPerSecond,
|
||||
endToEndTokensPerSecond: speed.endToEndTokensPerSecond,
|
||||
tokensGenerated,
|
||||
loadTime: data.load_duration ? Math.round(data.load_duration / 1000000) : null,
|
||||
evalTime: data.eval_duration ? Math.round(data.eval_duration / 1000000) : null,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ const { animateBanner, renderPersistentBanner } = require('./cli-theme');
|
|||
const PRIMARY_COMMAND_PRIORITY = [
|
||||
'check',
|
||||
'help',
|
||||
'mcp-setup',
|
||||
'recommend',
|
||||
'ai-run',
|
||||
'ollama-plan',
|
||||
|
|
@ -89,6 +90,90 @@ function tokenizeArgString(rawInput = '') {
|
|||
return tokens;
|
||||
}
|
||||
|
||||
function getRequiredOptionPrompts(commandMeta) {
|
||||
const commandOptions = Array.isArray(commandMeta?.command?.options)
|
||||
? commandMeta.command.options
|
||||
: [];
|
||||
|
||||
const isMandatoryOption = (option) => {
|
||||
if (!option) return false;
|
||||
if (typeof option.mandatory === 'boolean') {
|
||||
return option.mandatory;
|
||||
}
|
||||
return Boolean(option.required);
|
||||
};
|
||||
|
||||
return commandOptions
|
||||
.filter((option) => isMandatoryOption(option))
|
||||
.map((option) => {
|
||||
const flag = option.long || option.short || option.flags.split(/[,\s|]+/).find((token) => token.startsWith('-'));
|
||||
const attributeName =
|
||||
typeof option.attributeName === 'function'
|
||||
? option.attributeName()
|
||||
: String(flag || 'required').replace(/^-+/, '').replace(/[^a-zA-Z0-9]+/g, '_');
|
||||
const optionType = typeof option.isBoolean === 'function' && option.isBoolean() ? 'boolean' : 'value';
|
||||
const promptName = `required_${attributeName}`;
|
||||
const optionFlags = option.flags || flag || attributeName;
|
||||
const message =
|
||||
optionType === 'boolean'
|
||||
? `Required flag ${optionFlags} (enable?):`
|
||||
: `Required option ${optionFlags}:`;
|
||||
|
||||
return {
|
||||
promptName,
|
||||
flag,
|
||||
optionFlags,
|
||||
variadic: Boolean(option.variadic),
|
||||
optionType,
|
||||
message
|
||||
};
|
||||
})
|
||||
.filter((entry) => Boolean(entry.flag));
|
||||
}
|
||||
|
||||
function normalizeVariadicValue(rawValue) {
|
||||
const tokens = tokenizeArgString(rawValue);
|
||||
const values = [];
|
||||
|
||||
tokens.forEach((token) => {
|
||||
String(token)
|
||||
.split(',')
|
||||
.map((piece) => piece.trim())
|
||||
.filter(Boolean)
|
||||
.forEach((piece) => values.push(piece));
|
||||
});
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
function buildRequiredOptionArgs(requiredOptionPrompts, answers) {
|
||||
const args = [];
|
||||
|
||||
requiredOptionPrompts.forEach((entry) => {
|
||||
const answer = answers ? answers[entry.promptName] : undefined;
|
||||
|
||||
if (entry.optionType === 'boolean') {
|
||||
if (answer) args.push(entry.flag);
|
||||
return;
|
||||
}
|
||||
|
||||
const rawValue = String(answer || '').trim();
|
||||
if (!rawValue) return;
|
||||
|
||||
if (entry.variadic) {
|
||||
const values = normalizeVariadicValue(rawValue);
|
||||
if (values.length > 0) {
|
||||
args.push(entry.flag, ...values);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
args.push(entry.flag, rawValue);
|
||||
});
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
function buildCommandCatalog(program) {
|
||||
return (program.commands || [])
|
||||
.map((command) => ({
|
||||
|
|
@ -227,6 +312,7 @@ function renderPanel(state, catalog, primaryCommands, options = {}) {
|
|||
async function collectCommandArgs(commandMeta) {
|
||||
const prompts = [];
|
||||
const requiredPrompts = REQUIRED_ARG_PROMPTS[commandMeta.name] || [];
|
||||
const requiredOptionPrompts = getRequiredOptionPrompts(commandMeta);
|
||||
const promptOptionalArgs = commandMeta.name !== 'help';
|
||||
|
||||
for (const requiredPrompt of requiredPrompts) {
|
||||
|
|
@ -234,15 +320,45 @@ async function collectCommandArgs(commandMeta) {
|
|||
type: 'input',
|
||||
name: requiredPrompt.name,
|
||||
message: requiredPrompt.message,
|
||||
prefix: ' ',
|
||||
validate: requiredPrompt.validate
|
||||
});
|
||||
}
|
||||
|
||||
for (const requiredOptionPrompt of requiredOptionPrompts) {
|
||||
prompts.push({
|
||||
type: requiredOptionPrompt.optionType === 'boolean' ? 'confirm' : 'input',
|
||||
name: requiredOptionPrompt.promptName,
|
||||
message: requiredOptionPrompt.message,
|
||||
prefix: ' ',
|
||||
default: requiredOptionPrompt.optionType === 'boolean' ? true : '',
|
||||
validate:
|
||||
requiredOptionPrompt.optionType === 'boolean'
|
||||
? undefined
|
||||
: (value) => {
|
||||
const trimmed = String(value || '').trim();
|
||||
if (!trimmed) {
|
||||
return `Provide a value for ${requiredOptionPrompt.optionFlags}`;
|
||||
}
|
||||
|
||||
if (requiredOptionPrompt.variadic) {
|
||||
const values = normalizeVariadicValue(trimmed);
|
||||
if (values.length === 0) {
|
||||
return `Provide at least one value for ${requiredOptionPrompt.optionFlags}`;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (promptOptionalArgs) {
|
||||
prompts.push({
|
||||
type: 'input',
|
||||
name: 'extraArgs',
|
||||
message: 'Optional extra params (example: --json --limit 5):',
|
||||
prefix: ' ',
|
||||
default: ''
|
||||
});
|
||||
}
|
||||
|
|
@ -259,6 +375,8 @@ async function collectCommandArgs(commandMeta) {
|
|||
if (value) args.push(value);
|
||||
}
|
||||
|
||||
args.push(...buildRequiredOptionArgs(requiredOptionPrompts, answers));
|
||||
|
||||
if (promptOptionalArgs) {
|
||||
const extraArgs = String(answers.extraArgs || '').trim();
|
||||
if (extraArgs) {
|
||||
|
|
@ -371,7 +489,8 @@ async function launchInteractivePanel(options) {
|
|||
{
|
||||
type: 'input',
|
||||
name: 'continue',
|
||||
message: 'Press Enter to return to the panel'
|
||||
message: 'Press Enter to return to the panel',
|
||||
prefix: ' '
|
||||
}
|
||||
]);
|
||||
} catch (error) {
|
||||
|
|
@ -379,7 +498,8 @@ async function launchInteractivePanel(options) {
|
|||
{
|
||||
type: 'input',
|
||||
name: 'continue',
|
||||
message: `Command failed (${error.message}). Press Enter to continue`
|
||||
message: `Command failed (${error.message}). Press Enter to continue`,
|
||||
prefix: ' '
|
||||
}
|
||||
]);
|
||||
} finally {
|
||||
|
|
@ -468,6 +588,9 @@ module.exports = {
|
|||
launchInteractivePanel,
|
||||
__private: {
|
||||
tokenizeArgString,
|
||||
getRequiredOptionPrompts,
|
||||
buildRequiredOptionArgs,
|
||||
normalizeVariadicValue,
|
||||
buildCommandCatalog,
|
||||
buildPrimaryCommands,
|
||||
getVisibleCommands,
|
||||
|
|
|
|||
17
src/utils/fetch.js
Normal file
17
src/utils/fetch.js
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
let nodeFetchPromise = null;
|
||||
|
||||
function getNodeFetch() {
|
||||
if (!nodeFetchPromise) {
|
||||
nodeFetchPromise = import('node-fetch').then((mod) => mod.default || mod);
|
||||
}
|
||||
return nodeFetchPromise;
|
||||
}
|
||||
|
||||
function fetchWithFallback(...args) {
|
||||
if (typeof globalThis.fetch === 'function') {
|
||||
return globalThis.fetch(...args);
|
||||
}
|
||||
return getNodeFetch().then((fetchImpl) => fetchImpl(...args));
|
||||
}
|
||||
|
||||
module.exports = fetchWithFallback;
|
||||
207
src/utils/token-speed-estimator.js
Normal file
207
src/utils/token-speed-estimator.js
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
function toNumber(value, fallback = 0) {
|
||||
if (typeof value === 'number' && Number.isFinite(value)) return value;
|
||||
if (typeof value === 'string') {
|
||||
const trimmed = value.trim();
|
||||
if (trimmed) {
|
||||
const parsed = Number(trimmed);
|
||||
if (Number.isFinite(parsed)) return parsed;
|
||||
}
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function normalizeModelSizeB(rawModelSizeB) {
|
||||
const value = toNumber(rawModelSizeB, 1);
|
||||
return Math.max(0.7, value);
|
||||
}
|
||||
|
||||
function detectAppleSilicon(architecture, cpuModel, gpuModel) {
|
||||
const signal = `${architecture} ${cpuModel} ${gpuModel}`.toLowerCase();
|
||||
return (
|
||||
signal.includes('apple silicon') ||
|
||||
/(^|\s)m[1-4](\s|$)/.test(signal) ||
|
||||
signal.includes('apple m1') ||
|
||||
signal.includes('apple m2') ||
|
||||
signal.includes('apple m3') ||
|
||||
signal.includes('apple m4')
|
||||
);
|
||||
}
|
||||
|
||||
function isIntegratedGPU(gpuModel = '') {
|
||||
return /iris.*xe|iris.*graphics|uhd.*graphics|vega.*integrated|radeon.*graphics|intel.*integrated|integrated/i.test(gpuModel);
|
||||
}
|
||||
|
||||
function getAppleSiliconBaseline(cpuModel, gpuModel) {
|
||||
const signal = `${cpuModel} ${gpuModel}`.toLowerCase();
|
||||
const profiles = [
|
||||
{ pattern: /m4 ultra/, tps7b: 95 },
|
||||
{ pattern: /m4 max/, tps7b: 65 },
|
||||
{ pattern: /m4 pro/, tps7b: 43 },
|
||||
{ pattern: /m4/, tps7b: 30 },
|
||||
{ pattern: /m3 ultra/, tps7b: 88 },
|
||||
{ pattern: /m3 max/, tps7b: 58 },
|
||||
{ pattern: /m3 pro/, tps7b: 34 },
|
||||
{ pattern: /m3/, tps7b: 27 },
|
||||
{ pattern: /m2 ultra/, tps7b: 80 },
|
||||
{ pattern: /m2 max/, tps7b: 52 },
|
||||
{ pattern: /m2 pro/, tps7b: 32 },
|
||||
{ pattern: /m2/, tps7b: 24 },
|
||||
{ pattern: /m1 ultra/, tps7b: 72 },
|
||||
{ pattern: /m1 max/, tps7b: 48 },
|
||||
{ pattern: /m1 pro/, tps7b: 30 },
|
||||
{ pattern: /m1/, tps7b: 22 }
|
||||
];
|
||||
|
||||
for (const profile of profiles) {
|
||||
if (profile.pattern.test(signal)) {
|
||||
return profile.tps7b;
|
||||
}
|
||||
}
|
||||
return 24;
|
||||
}
|
||||
|
||||
function getDedicatedGPUBaseline(gpuModel, vramGB) {
|
||||
const signal = (gpuModel || '').toLowerCase();
|
||||
const profiles = [
|
||||
{ pattern: /h100/, tps7b: 170 },
|
||||
{ pattern: /a100/, tps7b: 130 },
|
||||
{ pattern: /rtx 50|rtx50|blackwell/, tps7b: 105 },
|
||||
{ pattern: /rtx 4090/, tps7b: 80 },
|
||||
{ pattern: /rtx 4080/, tps7b: 65 },
|
||||
{ pattern: /rtx 3090/, tps7b: 62 },
|
||||
{ pattern: /rtx 3080/, tps7b: 50 },
|
||||
{ pattern: /rtx 3070/, tps7b: 42 },
|
||||
{ pattern: /rtx 3060/, tps7b: 34 },
|
||||
{ pattern: /rtx 20/, tps7b: 26 },
|
||||
{ pattern: /rx 79|7900|7800/, tps7b: 52 },
|
||||
{ pattern: /rx 69|6800/, tps7b: 42 }
|
||||
];
|
||||
|
||||
for (const profile of profiles) {
|
||||
if (profile.pattern.test(signal)) {
|
||||
return profile.tps7b;
|
||||
}
|
||||
}
|
||||
|
||||
if (vramGB >= 24) return 60;
|
||||
if (vramGB >= 16) return 48;
|
||||
if (vramGB >= 12) return 40;
|
||||
if (vramGB >= 8) return 30;
|
||||
if (vramGB >= 4) return 18;
|
||||
return 14;
|
||||
}
|
||||
|
||||
function hasAVX512(cpuModel = '') {
|
||||
const signal = cpuModel.toLowerCase();
|
||||
return signal.includes('avx512') ||
|
||||
(signal.includes('intel') &&
|
||||
(signal.includes('12th') || signal.includes('13th') || signal.includes('14th')));
|
||||
}
|
||||
|
||||
function hasAVX2(cpuModel = '') {
|
||||
const signal = cpuModel.toLowerCase();
|
||||
return signal.includes('avx2') || signal.includes('intel') || signal.includes('amd');
|
||||
}
|
||||
|
||||
function getCPUBaseline(cpuModel, cores, baseSpeedGHz, integratedGpuAssist) {
|
||||
let baseline = 4 + (cores * 0.6) + ((Math.max(1.5, baseSpeedGHz) - 2.0) * 2.5);
|
||||
|
||||
if (hasAVX512(cpuModel)) baseline += 3;
|
||||
else if (hasAVX2(cpuModel)) baseline += 1.5;
|
||||
|
||||
if (integratedGpuAssist) baseline += 1;
|
||||
|
||||
const maxBaseline = integratedGpuAssist ? 22 : 18;
|
||||
return Math.max(3, Math.min(maxBaseline, baseline));
|
||||
}
|
||||
|
||||
function calculateSizeScale(modelSizeB) {
|
||||
const scale = Math.pow(7 / modelSizeB, 0.72);
|
||||
return Math.max(0.18, Math.min(2.2, scale));
|
||||
}
|
||||
|
||||
function calculateMemoryFactor(modelSizeB, availableInferenceMemoryGB) {
|
||||
const estimatedWorkingSetGB = (modelSizeB * 0.75) + 2;
|
||||
const ratio = availableInferenceMemoryGB / Math.max(1, estimatedWorkingSetGB);
|
||||
|
||||
if (ratio >= 1.2) return 1.05;
|
||||
if (ratio >= 1.0) return 1.0;
|
||||
if (ratio >= 0.75) return 0.85;
|
||||
if (ratio >= 0.6) return 0.65;
|
||||
return 0.45;
|
||||
}
|
||||
|
||||
function estimateTokenSpeedFromHardware(hardware = {}, options = {}) {
|
||||
const cpuModel = String(hardware.cpu?.brand || hardware.cpu?.model || '');
|
||||
const gpuModel = String(hardware.gpu?.model || '');
|
||||
const architecture = String(hardware.cpu?.architecture || '');
|
||||
|
||||
const modelSizeB = normalizeModelSizeB(options.modelSizeB);
|
||||
const cores = Math.max(1, toNumber(hardware.cpu?.physicalCores || hardware.cpu?.cores, 1));
|
||||
const baseSpeedGHz = Math.max(1.5, toNumber(hardware.cpu?.speed || hardware.cpu?.speedMax, 2.4));
|
||||
|
||||
const memoryTotalGB = Math.max(
|
||||
2,
|
||||
toNumber(
|
||||
hardware.memory?.total ||
|
||||
hardware.memory?.totalGB ||
|
||||
hardware.memory_gb,
|
||||
8
|
||||
)
|
||||
);
|
||||
|
||||
const vramGB = Math.max(
|
||||
0,
|
||||
toNumber(
|
||||
hardware.gpu?.vram ||
|
||||
hardware.gpu?.vramGB ||
|
||||
hardware.gpu?.totalVRAM ||
|
||||
hardware.gpu?.memory?.total,
|
||||
0
|
||||
)
|
||||
);
|
||||
|
||||
const appleSilicon = detectAppleSilicon(architecture, cpuModel, gpuModel);
|
||||
const integrated = isIntegratedGPU(gpuModel);
|
||||
const dedicatedGPU = vramGB > 0 && !integrated && !appleSilicon;
|
||||
|
||||
let baselineTPS7B;
|
||||
let backend;
|
||||
let availableInferenceMemoryGB;
|
||||
|
||||
if (appleSilicon) {
|
||||
backend = 'metal';
|
||||
baselineTPS7B = getAppleSiliconBaseline(cpuModel, gpuModel);
|
||||
availableInferenceMemoryGB = memoryTotalGB * 0.82;
|
||||
} else if (dedicatedGPU) {
|
||||
backend = 'gpu';
|
||||
baselineTPS7B = getDedicatedGPUBaseline(gpuModel, vramGB);
|
||||
availableInferenceMemoryGB = vramGB + Math.min(memoryTotalGB * 0.15, 8);
|
||||
} else {
|
||||
backend = integrated ? 'integrated' : 'cpu';
|
||||
baselineTPS7B = getCPUBaseline(cpuModel, cores, baseSpeedGHz, integrated);
|
||||
availableInferenceMemoryGB = memoryTotalGB * 0.65;
|
||||
}
|
||||
|
||||
const sizeScale = calculateSizeScale(modelSizeB);
|
||||
const memoryFactor = calculateMemoryFactor(modelSizeB, availableInferenceMemoryGB);
|
||||
|
||||
const maxTPS = backend === 'metal' ? 140 : backend === 'gpu' ? 220 : 35;
|
||||
const minTPS = backend === 'cpu' ? 1 : 2;
|
||||
|
||||
const estimated = baselineTPS7B * sizeScale * memoryFactor;
|
||||
const tokensPerSecond = Math.max(minTPS, Math.min(maxTPS, Math.round(estimated)));
|
||||
|
||||
return {
|
||||
tokensPerSecond,
|
||||
backend,
|
||||
baselineTPS7B: Math.round(baselineTPS7B * 10) / 10,
|
||||
sizeScale: Math.round(sizeScale * 1000) / 1000,
|
||||
memoryFactor: Math.round(memoryFactor * 1000) / 1000,
|
||||
modelSizeB
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
estimateTokenSpeedFromHardware
|
||||
};
|
||||
|
|
@ -1,7 +1,11 @@
|
|||
const assert = require('assert');
|
||||
const { Command } = require('commander');
|
||||
const {
|
||||
__private: {
|
||||
tokenizeArgString,
|
||||
getRequiredOptionPrompts,
|
||||
buildRequiredOptionArgs,
|
||||
normalizeVariadicValue,
|
||||
buildCommandCatalog,
|
||||
buildPrimaryCommands,
|
||||
getVisibleCommands,
|
||||
|
|
@ -30,6 +34,43 @@ function run() {
|
|||
|
||||
assert.strictEqual(truncateText('short', 10), 'short');
|
||||
assert.strictEqual(truncateText('abcdefghij', 6), 'abc...');
|
||||
assert.deepStrictEqual(
|
||||
normalizeVariadicValue('qwen2.5-coder:7b llama3.2:3b,mistral:7b'),
|
||||
['qwen2.5-coder:7b', 'llama3.2:3b', 'mistral:7b']
|
||||
);
|
||||
|
||||
const calibrateCommand = new Command('calibrate');
|
||||
calibrateCommand.requiredOption('--suite <file>', 'Prompt suite path');
|
||||
calibrateCommand.requiredOption('--models <identifiers...>', 'Models');
|
||||
calibrateCommand.requiredOption('--output <file>', 'Output path');
|
||||
calibrateCommand.option('--runtime <runtime>', 'Runtime backend', 'ollama');
|
||||
calibrateCommand.option('--dry-run', 'Dry run mode');
|
||||
|
||||
const requiredOptionPrompts = getRequiredOptionPrompts({ command: calibrateCommand });
|
||||
assert.deepStrictEqual(
|
||||
requiredOptionPrompts.map((entry) => entry.optionFlags),
|
||||
['--suite <file>', '--models <identifiers...>', '--output <file>'],
|
||||
'only mandatory options should be discovered for interactive prompts'
|
||||
);
|
||||
|
||||
const requiredOptionArgs = buildRequiredOptionArgs(requiredOptionPrompts, {
|
||||
required_suite: './suite.jsonl',
|
||||
required_models: 'qwen2.5-coder:7b, llama3.2:3b',
|
||||
required_output: './calibration.json'
|
||||
});
|
||||
assert.deepStrictEqual(
|
||||
requiredOptionArgs,
|
||||
[
|
||||
'--suite',
|
||||
'./suite.jsonl',
|
||||
'--models',
|
||||
'qwen2.5-coder:7b',
|
||||
'llama3.2:3b',
|
||||
'--output',
|
||||
'./calibration.json'
|
||||
],
|
||||
'required prompt answers should translate to runnable CLI args'
|
||||
);
|
||||
|
||||
const mockProgram = {
|
||||
commands: [
|
||||
|
|
|
|||
136
tests/local-features-e2e.test.js
Normal file
136
tests/local-features-e2e.test.js
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
const assert = require('assert');
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const { spawnSync } = require('child_process');
|
||||
|
||||
const BIN_PATH = path.resolve(__dirname, '..', 'bin', 'enhanced_cli.js');
|
||||
const SAMPLE_SUITE = path.resolve(
|
||||
__dirname,
|
||||
'..',
|
||||
'docs',
|
||||
'fixtures',
|
||||
'calibration',
|
||||
'sample-suite.jsonl'
|
||||
);
|
||||
|
||||
function stripAnsi(text = '') {
|
||||
return String(text).replace(/\u001b\[[0-9;]*m/g, '');
|
||||
}
|
||||
|
||||
function runCli(args, options = {}) {
|
||||
const result = spawnSync(process.execPath, [BIN_PATH, ...args], {
|
||||
cwd: options.cwd || process.cwd(),
|
||||
encoding: 'utf8',
|
||||
timeout: options.timeoutMs || 120000,
|
||||
env: {
|
||||
...process.env,
|
||||
NO_COLOR: '1'
|
||||
}
|
||||
});
|
||||
return {
|
||||
...result,
|
||||
stdout: stripAnsi(result.stdout || ''),
|
||||
stderr: stripAnsi(result.stderr || '')
|
||||
};
|
||||
}
|
||||
|
||||
function assertExitOk(result, context) {
|
||||
const combined = `${result.stdout}\n${result.stderr}`.trim();
|
||||
assert.strictEqual(result.status, 0, `${context}\n${combined}`);
|
||||
}
|
||||
|
||||
function run() {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'llm-checker-local-e2e-'));
|
||||
try {
|
||||
const calibrationOutput = path.join(tempDir, 'calibration-result.json');
|
||||
const policyPath = path.join(tempDir, 'policy.yaml');
|
||||
const auditPath = path.join(tempDir, 'audit.json');
|
||||
|
||||
const mcpSetupResult = runCli(['mcp-setup', '--json']);
|
||||
assertExitOk(mcpSetupResult, 'mcp-setup --json failed');
|
||||
const mcpSetup = JSON.parse(mcpSetupResult.stdout);
|
||||
assert.strictEqual(mcpSetup.recommended.command, 'claude');
|
||||
assert.ok(Array.isArray(mcpSetup.recommended.args) && mcpSetup.recommended.args.length >= 4);
|
||||
|
||||
const hwDetectResult = runCli(['hw-detect', '--json']);
|
||||
assertExitOk(hwDetectResult, 'hw-detect --json failed');
|
||||
const hardware = JSON.parse(hwDetectResult.stdout);
|
||||
assert.ok(hardware && hardware.summary, 'hardware summary should exist');
|
||||
|
||||
const checkResult = runCli(['check', '--use-case', 'general', '--limit', '1', '--runtime', 'ollama', '--no-verbose']);
|
||||
assertExitOk(checkResult, 'check command failed');
|
||||
assert.ok(checkResult.stdout.includes('RECOMMENDED MODEL'), 'check output should include recommendation block');
|
||||
|
||||
const recommendResult = runCli(['recommend', '--no-verbose']);
|
||||
assertExitOk(recommendResult, 'recommend command failed');
|
||||
assert.ok(
|
||||
recommendResult.stdout.includes('INTELLIGENT RECOMMENDATIONS BY CATEGORY'),
|
||||
'recommend output should include recommendations section'
|
||||
);
|
||||
|
||||
const calibrateResult = runCli([
|
||||
'calibrate',
|
||||
'--suite',
|
||||
SAMPLE_SUITE,
|
||||
'--models',
|
||||
'qwen2.5-coder:7b,llama3.2:3b',
|
||||
'--output',
|
||||
calibrationOutput,
|
||||
'--dry-run'
|
||||
]);
|
||||
assertExitOk(calibrateResult, 'calibrate command failed');
|
||||
assert.ok(fs.existsSync(calibrationOutput), 'calibration output should exist');
|
||||
const calibrationJson = JSON.parse(fs.readFileSync(calibrationOutput, 'utf8'));
|
||||
assert.strictEqual(calibrationJson.execution_mode, 'dry-run');
|
||||
|
||||
const policyInitResult = runCli(['policy', 'init', '--file', policyPath, '--force']);
|
||||
assertExitOk(policyInitResult, 'policy init failed');
|
||||
assert.ok(fs.existsSync(policyPath), 'policy file should exist');
|
||||
|
||||
let policyContent = fs.readFileSync(policyPath, 'utf8');
|
||||
policyContent = policyContent.replace(/mode:\s*enforce/g, 'mode: audit');
|
||||
fs.writeFileSync(policyPath, policyContent, 'utf8');
|
||||
|
||||
const policyValidateResult = runCli(['policy', 'validate', '--file', policyPath]);
|
||||
assertExitOk(policyValidateResult, 'policy validate failed');
|
||||
|
||||
const auditResult = runCli([
|
||||
'audit',
|
||||
'export',
|
||||
'--policy',
|
||||
policyPath,
|
||||
'--command',
|
||||
'recommend',
|
||||
'--format',
|
||||
'json',
|
||||
'--out',
|
||||
auditPath,
|
||||
'--runtime',
|
||||
'ollama',
|
||||
'--optimize',
|
||||
'balanced',
|
||||
'--no-verbose'
|
||||
]);
|
||||
assertExitOk(auditResult, 'audit export failed');
|
||||
assert.ok(fs.existsSync(auditPath), 'audit output should exist');
|
||||
const auditJson = JSON.parse(fs.readFileSync(auditPath, 'utf8'));
|
||||
assert.strictEqual(auditJson.command, 'recommend');
|
||||
|
||||
console.log('local-features-e2e.test.js: OK');
|
||||
} finally {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
try {
|
||||
run();
|
||||
} catch (error) {
|
||||
console.error('local-features-e2e.test.js: FAILED');
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { run };
|
||||
246
tests/ollama-client-speed-metrics.test.js
Normal file
246
tests/ollama-client-speed-metrics.test.js
Normal file
|
|
@ -0,0 +1,246 @@
|
|||
const assert = require('assert');
|
||||
const path = require('path');
|
||||
const { spawnSync } = require('child_process');
|
||||
const OllamaClient = require('../src/ollama/client');
|
||||
const fetch = require('../src/utils/fetch');
|
||||
const DeterministicModelSelector = require('../src/models/deterministic-selector');
|
||||
const AICheckSelector = require('../src/models/ai-check-selector');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
const CLI = path.resolve(ROOT, 'bin', 'enhanced_cli.js');
|
||||
|
||||
function stripAnsi(text = '') {
|
||||
return String(text).replace(/\u001b\[[0-9;]*m/g, '');
|
||||
}
|
||||
|
||||
function roundToSingleDecimal(value) {
|
||||
return Math.round(Number(value || 0) * 10) / 10;
|
||||
}
|
||||
|
||||
function isCloudPlaceholder(model) {
|
||||
const name = String(model?.name || '').toLowerCase();
|
||||
const source = String(model?.source || '').toLowerCase();
|
||||
const type = String(model?.type || model?.model_type || '').toLowerCase();
|
||||
const fileSizeGB = Number(model?.fileSizeGB) || 0;
|
||||
|
||||
const cloudTagged = (
|
||||
name.includes('-cloud') ||
|
||||
name.endsWith(':cloud') ||
|
||||
source.includes('cloud') ||
|
||||
type === 'cloud' ||
|
||||
type === 'remote' ||
|
||||
type === 'hosted'
|
||||
);
|
||||
|
||||
return cloudTagged && fileSizeGB <= 0;
|
||||
}
|
||||
|
||||
function parseModelSizeB(sizeLabel = '') {
|
||||
const match = String(sizeLabel).trim().toUpperCase().match(/^(\d+\.?\d*)\s*([BM])$/);
|
||||
if (!match) return 1;
|
||||
const value = parseFloat(match[1]);
|
||||
return match[2] === 'M' ? value / 1000 : value;
|
||||
}
|
||||
|
||||
function runCli(args) {
|
||||
return spawnSync(process.execPath, [CLI, ...args], {
|
||||
cwd: ROOT,
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, NO_COLOR: '1' }
|
||||
});
|
||||
}
|
||||
|
||||
async function run() {
|
||||
const client = new OllamaClient('http://localhost:11434');
|
||||
const availability = await client.checkOllamaAvailability();
|
||||
assert.strictEqual(
|
||||
availability.available,
|
||||
true,
|
||||
`Ollama must be available for real integration test. Details: ${JSON.stringify(availability)}`
|
||||
);
|
||||
|
||||
const versionResponse = await fetch('http://localhost:11434/api/version', {
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
});
|
||||
assert(versionResponse.ok, `Expected /api/version to return HTTP 200, got ${versionResponse.status}`);
|
||||
const versionBody = await versionResponse.json();
|
||||
assert(versionBody.version, 'Expected version payload from Ollama');
|
||||
|
||||
const fallbackFetchScript = `
|
||||
globalThis.fetch = undefined;
|
||||
const fetch = require('./src/utils/fetch');
|
||||
fetch('http://localhost:11434/api/version', { headers: { 'Content-Type': 'application/json' } })
|
||||
.then(async (res) => {
|
||||
if (!res.ok) throw new Error('HTTP ' + res.status);
|
||||
const body = await res.json();
|
||||
if (!body.version) throw new Error('Missing version');
|
||||
process.stdout.write(body.version);
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error(error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
`;
|
||||
const fallbackFetchResult = spawnSync(process.execPath, ['-e', fallbackFetchScript], {
|
||||
cwd: ROOT,
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, NO_COLOR: '1' }
|
||||
});
|
||||
assert.strictEqual(
|
||||
fallbackFetchResult.status,
|
||||
0,
|
||||
`node-fetch fallback path failed: ${fallbackFetchResult.stderr || fallbackFetchResult.stdout}`
|
||||
);
|
||||
|
||||
const localModels = await client.getLocalModels();
|
||||
assert(localModels.length > 0, 'Expected at least one local Ollama model');
|
||||
const runnableModels = localModels.filter((model) => !isCloudPlaceholder(model) && Number(model.fileSizeGB) > 0);
|
||||
assert(runnableModels.length > 0, 'Expected at least one runnable local model (non-cloud)');
|
||||
|
||||
const probeModel = [...runnableModels].sort((a, b) => a.fileSizeGB - b.fileSizeGB)[0];
|
||||
const probeModelSizeB = parseModelSizeB(probeModel.size);
|
||||
|
||||
const generationPrompt = 'Explica en 80 palabras por que las pruebas de integracion son utiles.';
|
||||
const generationRequest = {
|
||||
model: probeModel.name,
|
||||
prompt: generationPrompt,
|
||||
stream: false,
|
||||
options: { num_predict: 120, temperature: 0.2 }
|
||||
};
|
||||
|
||||
const generationStartMs = Date.now();
|
||||
const generationResponse = await fetch('http://localhost:11434/api/generate', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(generationRequest)
|
||||
});
|
||||
assert(generationResponse.ok, `Expected /api/generate to return 200, got ${generationResponse.status}`);
|
||||
const generationData = await generationResponse.json();
|
||||
const totalTimeMs = Date.now() - generationStartMs;
|
||||
|
||||
assert((Number(generationData.eval_count) || 0) > 0, 'Expected eval_count > 0 from real generation');
|
||||
|
||||
const speed = client.calculateTokensPerSecond(generationData, totalTimeMs);
|
||||
const expectedEvalTokensPerSecond = Number(generationData.eval_duration) > 0
|
||||
? roundToSingleDecimal(Number(generationData.eval_count) / (Number(generationData.eval_duration) / 1_000_000_000))
|
||||
: 0;
|
||||
const expectedEndToEndTokensPerSecond = roundToSingleDecimal(
|
||||
Number(generationData.eval_count) / Math.max(totalTimeMs / 1000, 0.001)
|
||||
);
|
||||
const expectedPreferred = expectedEvalTokensPerSecond > 0
|
||||
? expectedEvalTokensPerSecond
|
||||
: expectedEndToEndTokensPerSecond;
|
||||
|
||||
assert.strictEqual(
|
||||
speed.evalTokensPerSecond,
|
||||
expectedEvalTokensPerSecond,
|
||||
`Eval throughput mismatch. Expected ${expectedEvalTokensPerSecond}, got ${speed.evalTokensPerSecond}`
|
||||
);
|
||||
assert.strictEqual(
|
||||
speed.endToEndTokensPerSecond,
|
||||
expectedEndToEndTokensPerSecond,
|
||||
`End-to-end throughput mismatch. Expected ${expectedEndToEndTokensPerSecond}, got ${speed.endToEndTokensPerSecond}`
|
||||
);
|
||||
assert.strictEqual(
|
||||
speed.tokensPerSecond,
|
||||
expectedPreferred,
|
||||
`Preferred throughput mismatch. Expected ${expectedPreferred}, got ${speed.tokensPerSecond}`
|
||||
);
|
||||
|
||||
const performance = await client.testModelPerformance(probeModel.name, 'Responde solo: ok');
|
||||
assert.strictEqual(performance.success, true, 'Expected testModelPerformance success');
|
||||
assert(performance.tokensPerSecond > 0, `Expected positive tokensPerSecond, got ${performance.tokensPerSecond}`);
|
||||
assert(
|
||||
Number.isFinite(performance.evalTokensPerSecond) && Number.isFinite(performance.endToEndTokensPerSecond),
|
||||
'Expected evalTokensPerSecond and endToEndTokensPerSecond in performance result'
|
||||
);
|
||||
if (performance.evalTokensPerSecond > 0) {
|
||||
assert.strictEqual(
|
||||
performance.tokensPerSecond,
|
||||
performance.evalTokensPerSecond,
|
||||
'When eval throughput exists, tokensPerSecond must match eval throughput'
|
||||
);
|
||||
}
|
||||
|
||||
const deterministicSelector = new DeterministicModelSelector();
|
||||
const probeTPS = await deterministicSelector.runSingleProbe(probeModel.name, 'general');
|
||||
assert(Number.isFinite(probeTPS) && probeTPS > 0, `Expected positive real probe TPS, got ${probeTPS}`);
|
||||
|
||||
const aiCheckSelector = new AICheckSelector();
|
||||
const aiResult = await aiCheckSelector.callOllamaEvaluator(probeModel.name, {
|
||||
hardware: { category: 'general' },
|
||||
candidates: [
|
||||
{
|
||||
name: probeModel.name,
|
||||
paramsB: probeModelSizeB,
|
||||
quant: String(probeModel.quantization || 'Q4'),
|
||||
requiredGB: Math.max(1, Number(probeModel.fileSizeGB) || 1),
|
||||
budgetGB: 24,
|
||||
installed: true
|
||||
}
|
||||
]
|
||||
});
|
||||
assert.strictEqual(aiResult.winner, probeModel.name, `Expected evaluator winner to be ${probeModel.name}`);
|
||||
assert(Array.isArray(aiResult.ranking), 'Expected ranking array in evaluator response');
|
||||
assert.strictEqual(aiResult.ranking.length, 1, 'Expected single ranking entry with one candidate');
|
||||
assert.strictEqual(aiResult.ranking[0].name, probeModel.name, 'Expected ranking to include selected probe model');
|
||||
|
||||
const planResult = runCli(['ollama-plan', '--json']);
|
||||
assert.strictEqual(
|
||||
planResult.status,
|
||||
0,
|
||||
`ollama-plan --json failed: ${stripAnsi(planResult.stderr || planResult.stdout)}`
|
||||
);
|
||||
const planPayload = JSON.parse(planResult.stdout);
|
||||
const selectedModels = planPayload.selection?.selected || [];
|
||||
assert(selectedModels.length > 0, 'Expected selected models from ollama-plan');
|
||||
assert(
|
||||
selectedModels.every((name) => !/-cloud|:cloud$/i.test(String(name))),
|
||||
`Cloud placeholder models should be excluded from default plan selection: ${selectedModels.join(', ')}`
|
||||
);
|
||||
|
||||
const cloudPlaceholder = localModels.find((model) => isCloudPlaceholder(model));
|
||||
if (cloudPlaceholder) {
|
||||
assert(
|
||||
!selectedModels.includes(cloudPlaceholder.name),
|
||||
`Cloud placeholder model ${cloudPlaceholder.name} must not be in default selected models`
|
||||
);
|
||||
|
||||
const cloudOnlyResult = runCli(['ollama-plan', '--models', cloudPlaceholder.name, '--json']);
|
||||
assert.notStrictEqual(
|
||||
cloudOnlyResult.status,
|
||||
0,
|
||||
'Expected non-zero exit when planning only a cloud placeholder model'
|
||||
);
|
||||
const cloudOutput = stripAnsi(`${cloudOnlyResult.stdout}\n${cloudOnlyResult.stderr}`);
|
||||
assert(
|
||||
cloudOutput.includes('No matching local models found'),
|
||||
`Expected cloud-only selection to fail with local model message. Output: ${cloudOutput}`
|
||||
);
|
||||
}
|
||||
|
||||
const singleModelPlanResult = runCli(['ollama-plan', '--models', probeModel.name, '--json']);
|
||||
assert.strictEqual(
|
||||
singleModelPlanResult.status,
|
||||
0,
|
||||
`ollama-plan for ${probeModel.name} failed: ${stripAnsi(singleModelPlanResult.stderr || singleModelPlanResult.stdout)}`
|
||||
);
|
||||
const singleModelPlan = JSON.parse(singleModelPlanResult.stdout);
|
||||
assert.deepStrictEqual(singleModelPlan.selection.selected, [probeModel.name]);
|
||||
assert(
|
||||
singleModelPlan.plan?.memory?.recommendedEstimatedGB <= (singleModelPlan.plan?.memory?.budgetGB + 0.2),
|
||||
'Expected recommended memory to fit within budget for single-model plan'
|
||||
);
|
||||
|
||||
console.log('ollama-client-speed-metrics.test.js: OK');
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
run().catch((error) => {
|
||||
console.error('ollama-client-speed-metrics.test.js: FAILED');
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { run };
|
||||
|
|
@ -11,11 +11,14 @@ const TESTS = [
|
|||
{ name: 'CUDA Jetson detection', file: 'cuda-jetson-detection.test.js', category: 'Hardware' },
|
||||
{ name: 'Hardware simulation scoring', file: 'hardware-simulation-tests.js', category: 'Hardware' },
|
||||
{ name: 'Hardware detector regression', file: 'hardware-detector-regression.js', category: 'Hardware' },
|
||||
{ name: 'Token speed estimation', file: 'token-speed-estimation.test.js', category: 'Performance' },
|
||||
{ name: 'Deterministic model pool', file: 'deterministic-model-pool-check.js', category: 'Recommendations' },
|
||||
{ name: 'Ollama capacity planner', file: 'ollama-capacity-planner.test.js', category: 'Ollama' },
|
||||
{ name: 'Ollama speed metrics', file: 'ollama-client-speed-metrics.test.js', category: 'Ollama' },
|
||||
{ name: 'Runtime + speculative decoding', file: 'runtime-specdec-tests.js', category: 'Runtime' },
|
||||
{ name: 'CLI interface smoke', file: 'ui-cli-smoke.test.js', category: 'UI' },
|
||||
{ name: 'CLI interactive panel helpers', file: 'cli-interactive-panel.test.js', category: 'UI' },
|
||||
{ name: 'Local features end-to-end', file: 'local-features-e2e.test.js', category: 'E2E' },
|
||||
{ name: 'Calibration schema', file: 'calibration-schema.test.js', category: 'Calibration' },
|
||||
{ name: 'Calibration command', file: 'calibrate-command.test.js', category: 'Calibration' },
|
||||
{ name: 'Calibration end-to-end', file: 'calibration-e2e-integration.test.js', category: 'Calibration' },
|
||||
|
|
|
|||
98
tests/token-speed-estimation.test.js
Normal file
98
tests/token-speed-estimation.test.js
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
const assert = require('assert');
|
||||
const HardwareDetector = require('../src/hardware/detector');
|
||||
const PerformanceAnalyzer = require('../analyzer/performance');
|
||||
const ExpandedModelsDatabase = require('../src/models/expanded_database');
|
||||
const { estimateTokenSpeedFromHardware } = require('../src/utils/token-speed-estimator');
|
||||
|
||||
function isM4ProMac(hardware) {
|
||||
const cpuBrand = String(hardware?.cpu?.brand || '').toLowerCase();
|
||||
const gpuModel = String(hardware?.gpu?.model || '').toLowerCase();
|
||||
return (
|
||||
String(hardware?.os?.platform || '').toLowerCase() === 'darwin' &&
|
||||
String(hardware?.cpu?.architecture || '').toLowerCase().includes('apple') &&
|
||||
(cpuBrand.includes('m4 pro') || gpuModel.includes('m4 pro'))
|
||||
);
|
||||
}
|
||||
|
||||
async function run() {
|
||||
const detector = new HardwareDetector();
|
||||
const hardware = await detector.getSystemInfo(true);
|
||||
|
||||
assert.strictEqual(
|
||||
String(hardware.os?.platform || '').toLowerCase(),
|
||||
'darwin',
|
||||
`Expected macOS platform, got ${hardware.os?.platform}`
|
||||
);
|
||||
assert(
|
||||
String(hardware.cpu?.architecture || '').toLowerCase().includes('apple'),
|
||||
`Expected Apple Silicon architecture, got ${hardware.cpu?.architecture}`
|
||||
);
|
||||
assert(
|
||||
Number(hardware.memory?.total) >= 16,
|
||||
`Expected at least 16GB RAM on reference Mac, got ${hardware.memory?.total}GB`
|
||||
);
|
||||
|
||||
const speed3B = estimateTokenSpeedFromHardware(hardware, { modelSizeB: 3, modelName: 'llama3.2:3b' });
|
||||
const speed8B = estimateTokenSpeedFromHardware(hardware, { modelSizeB: 8, modelName: 'llama3.1:8b' });
|
||||
const speed13B = estimateTokenSpeedFromHardware(hardware, { modelSizeB: 13, modelName: 'llama3.1:13b' });
|
||||
|
||||
assert.strictEqual(speed3B.backend, 'metal', `Expected metal backend on Apple Silicon, got ${speed3B.backend}`);
|
||||
assert(
|
||||
speed3B.tokensPerSecond > speed8B.tokensPerSecond && speed8B.tokensPerSecond > speed13B.tokensPerSecond,
|
||||
`Expected monotonic scaling 3B > 8B > 13B, got ${speed3B.tokensPerSecond} > ${speed8B.tokensPerSecond} > ${speed13B.tokensPerSecond}`
|
||||
);
|
||||
assert(speed8B.tokensPerSecond > 0, `Expected positive TPS for 8B, got ${speed8B.tokensPerSecond}`);
|
||||
|
||||
if (isM4ProMac(hardware)) {
|
||||
assert(
|
||||
speed8B.tokensPerSecond >= 30 && speed8B.tokensPerSecond <= 70,
|
||||
`M4 Pro 8B estimate out of expected band (30-70 t/s), got ${speed8B.tokensPerSecond}`
|
||||
);
|
||||
}
|
||||
|
||||
const analyzer = new PerformanceAnalyzer();
|
||||
const parsedSizeFromGB = analyzer.parseModelSize('4.9GB');
|
||||
assert(
|
||||
parsedSizeFromGB >= 7 && parsedSizeFromGB <= 9.5,
|
||||
`parseModelSize should infer params from GB notation, got ${parsedSizeFromGB}`
|
||||
);
|
||||
|
||||
const perfEstimate = analyzer.calculateRealisticPerformance(
|
||||
{ name: 'llama3.1:8b', size: '8B', requirements: { ram: 10 } },
|
||||
hardware
|
||||
);
|
||||
assert.strictEqual(
|
||||
perfEstimate.estimatedTokensPerSecond,
|
||||
speed8B.tokensPerSecond,
|
||||
`Performance analyzer should use shared estimator (expected ${speed8B.tokensPerSecond}, got ${perfEstimate.estimatedTokensPerSecond})`
|
||||
);
|
||||
|
||||
const database = new ExpandedModelsDatabase();
|
||||
const dbSpeed = database.calculateRealisticTokensPerSecond(
|
||||
{ name: 'llama3.1:8b', size: '8B' },
|
||||
hardware
|
||||
);
|
||||
assert.strictEqual(
|
||||
dbSpeed,
|
||||
speed8B.tokensPerSecond,
|
||||
`Expanded database estimator should align with shared estimator (expected ${speed8B.tokensPerSecond}, got ${dbSpeed})`
|
||||
);
|
||||
|
||||
const tinyParams = database.extractModelParams({ name: 'all-minilm:22m', size: '22M' });
|
||||
assert(
|
||||
tinyParams > 0 && tinyParams < 0.1,
|
||||
`extractModelParams should convert M suffix to billions, got ${tinyParams}`
|
||||
);
|
||||
|
||||
console.log('token-speed-estimation.test.js: OK');
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
run().catch((error) => {
|
||||
console.error('token-speed-estimation.test.js: FAILED');
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { run };
|
||||
|
|
@ -25,6 +25,7 @@ function run() {
|
|||
assert.ok(stripAnsi(help.stdout).includes('recommend'), 'top-level help should list recommend command');
|
||||
assert.ok(stripAnsi(help.stdout).includes('calibrate'), 'top-level help should list calibrate command');
|
||||
assert.ok(stripAnsi(help.stdout).includes('ollama-plan'), 'top-level help should list ollama-plan command');
|
||||
assert.ok(stripAnsi(help.stdout).includes('mcp-setup'), 'top-level help should list mcp-setup command');
|
||||
|
||||
const commandHelp = runCli(['help']);
|
||||
assert.strictEqual(commandHelp.status, 0, stripAnsi(commandHelp.stderr || commandHelp.stdout));
|
||||
|
|
@ -84,6 +85,23 @@ function run() {
|
|||
'ollama-plan help should expose objective option'
|
||||
);
|
||||
|
||||
const mcpSetupHelp = runCli(['mcp-setup', '--help']);
|
||||
assert.strictEqual(mcpSetupHelp.status, 0, stripAnsi(mcpSetupHelp.stderr || mcpSetupHelp.stdout));
|
||||
assert.ok(
|
||||
stripAnsi(mcpSetupHelp.stdout).includes('Show or apply Claude MCP setup for llm-checker'),
|
||||
'mcp-setup help should describe command purpose'
|
||||
);
|
||||
|
||||
const mcpSetupJson = runCli(['mcp-setup', '--json']);
|
||||
assert.strictEqual(mcpSetupJson.status, 0, stripAnsi(mcpSetupJson.stderr || mcpSetupJson.stdout));
|
||||
const parsedMcpSetup = JSON.parse(stripAnsi(mcpSetupJson.stdout));
|
||||
assert.strictEqual(parsedMcpSetup.recommended.command, 'claude', 'mcp-setup json should target claude CLI');
|
||||
assert.ok(
|
||||
Array.isArray(parsedMcpSetup.recommended.args) &&
|
||||
parsedMcpSetup.recommended.args.slice(0, 2).join(' ') === 'mcp add',
|
||||
'mcp-setup json should include claude mcp add args'
|
||||
);
|
||||
|
||||
console.log('ui-cli-smoke.test.js: OK');
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue