chore: commit all local changes

This commit is contained in:
signerless 2026-02-18 13:57:28 +01:00
parent 72d3761103
commit bfa9755aec
55 changed files with 352302 additions and 190 deletions

View file

@ -0,0 +1 @@
exit=0

View file

@ -0,0 +1,50 @@
llm-checker | AI Check
--------------------------------
- Checking Ollama availability...
- AI-Check Mode: Meta-evaluation in progress...
│ Found 213 models in Ollama database
│ 213 models match general category
AI-CHECK MODE
╭─────────────────────────────────────────────────────────────────
│ Category: GENERAL
│ AI Weight: 30% + Deterministic: 70%
│ Candidates Found: 1
│ Hardware: 12 cores, 24GB RAM, apple_silicon
╰
AI EVALUATOR STATUS
╭──────────────────────────────────────────────────
│ Model: qwen2.5-coder:7b
│ 🔬 Evaluating: 20 models (showing top 1)
│ 🔬 Status: Running AI evaluation...
╰
AI-CHECK RESULTS
╭─────────────────────────────────────────────────────────────────
│ Evaluator: qwen2.5-coder:7b
│ Category: GENERAL
│ Models Evaluated: 1
│ 📝 Note: AI-evaluated using qwen2.5-coder:7b
╰
╔═══════════════╤════════╤═════════════╤════════════╤═════════╤══════════╤═════════╤══════════════╗
║ Model │ Size │ Det Score │ AI Score │ Final │ RAM │ Speed │ Status ║
╟───────────────┼────────┼─────────────┼────────────┼─────────┼──────────┼─────────┼──────────────╢
║ qwen2.5-coder │ 7B │ 74/100 │ 90/100 │ 79/100 │ 7.9/19GB │ 24t/s │ 🌐 Available ║
╚═══════════════╧════════╧═════════════╧════════════╧═════════╧══════════╧═════════╧══════════════╝
AI-POWERED RECOMMENDATION
╭──────────────────────────────────────────────────
│ Best Model: qwen2.5-coder
│ Final Score: 79/100
│ ⚖️ Det: 74 + AI: 90
│
│ 📥 Install command:
│ ollama pull qwen2.5-coder
│
│ Why this model?
│ • fits in 7.907344/19.200000000000003GB, Q8_0, coder-tuned, 7B is sweet spot, Metal backend
│ • AI: High performance and efficiency.
╰

View file

@ -0,0 +1 @@
0

View file

@ -0,0 +1,18 @@
llm-checker | AI Run
------------------------------
- Checking Ollama availability...
- Selecting best model and launching...
🔍 Using comprehensive model database for selection...
Evaluating against 213 models from database
Best model llama2 not installed locally
🔄 Using best local alternative: codellama:latest
✔ Selected codellama:latest (intelligent_database_comprehensive, 87% confidence)
Launching codellama:latest...
Tip: Type /bye to exit the chat when finished
[?2026h[?25l⠙ [?25h[?2026l[?2026h[?25l⠹ [?25h[?2026l[?2026h[?25l⠸ [?25h[?2026l[?2026h[?25l⠼ [?25h[?2026l[?2026h[?25l⠴ [?25h[?2026l[?2026h[?25l⠦ [?25h[?2026l[?2026h[?25l⠧ [?25h[?2026l[?2026h[?25l⠇ [?25h[?2026l[?2026h[?25l⠏ [?25h[?2026l[?2026h[?25l⠏ [?25h[?2026l[?2026h[?25l⠙ [?25h[?2026l[?2026h[?25l⠹ [?25h[?2026l[?2026h[?25l⠸ [?25h[?2026l[?2026h[?25l⠸ [?25h[?2026l[?2026h[?25l⠴ [?25h[?2026l[?2026h[?25l⠴ [?25h[?2026l[?2026h[?25l⠧ [?25h[?2026l[?2026h[?25l⠇ [?25h[?2026l[?2026h[?25l⠏ [?25h[?2026l[?2026h[?25l⠏ [?25h[?2026l[?2026h[?25l⠙ [?25h[?2026l[?2026h[?25l⠹ [?25h[?2026l[?25l[?2026h[?25l[?25h[?2026l[?25h
[?25l[?25hOK[?25l[?25h.[?25l[?25h
[?25l[?25h

View file

@ -0,0 +1 @@
exit=3

View file

@ -0,0 +1,17 @@
POLICY SUMMARY (AUDIT CHECK)
╭─────────────────────────────────────────────────────────────────
│ File: ~/llm-checker/.tmp-feature-check/policy.yaml
│ Mode: enforce | Action: error
│ Total checked: 19
│ Pass: 0 | Fail: 19
│ Suppressed: 0 | Exceptions: 0
│ Top violations:
│ - BACKEND_NOT_ALLOWED: 19
│ - INSUFFICIENT_RAM: 19
│ - LICENSE_MISSING: 19
│ Enforcement result: blocking violations detected (exit 3)
╰─────────────────────────────────────────────────────────────────
AUDIT EXPORT
JSON: ~/llm-checker/.tmp-feature-check/audit.json

View file

@ -0,0 +1 @@
0

View file

@ -0,0 +1,17 @@
POLICY SUMMARY (AUDIT CHECK)
╭─────────────────────────────────────────────────────────────────
│ File: ~/llm-checker/.tmp-feature-check/policy.yaml
│ Mode: audit | Action: warn
│ Total checked: 19
│ Pass: 0 | Fail: 19
│ Suppressed: 0 | Exceptions: 0
│ Top violations:
│ - BACKEND_NOT_ALLOWED: 19
│ - INSUFFICIENT_RAM: 19
│ - LICENSE_MISSING: 19
│ Audit mode: violations reported, command exits with code 0
╰─────────────────────────────────────────────────────────────────
AUDIT EXPORT
JSON: ~/llm-checker/.tmp-feature-check/audit-auditmode.json

View file

@ -0,0 +1 @@
exit=0

View file

@ -0,0 +1,9 @@
CALIBRATION ARTIFACTS GENERATED
╭────────────────────────────────────────────────────────────────────────
│ Suite: ~/llm-checker/docs/fixtures/calibration/sample-suite.jsonl
│ Runtime: ollama | Objective: balanced
│ Models: 2
│ Execution mode: dry-run
│ Result: ~/llm-checker/.tmp-feature-check/calibration.json
╰────────────────────────────────────────────────────────────────────────

View file

@ -0,0 +1 @@
exit=0

View file

@ -0,0 +1,31 @@
llm-checker | Compatibility Check
-------------------------------------------
Analyzing your system... done
SYSTEM SUMMARY
──────────────────────────────────────────────────
CPU: M4 Pro (12 cores)
Memory: 24GB RAM
GPU: Apple M4 Pro
Architecture: Apple Silicon
Hardware Tier: HIGH
RECOMMENDED MODEL
──────────────────────────────────────────────────
Model: Llama 3.1 8B
Size: 4.9GB
Compatibility Score: 72.3/100
Reason: Best general-purpose model for your hardware
Estimated Speed: 41 tokens/sec
Runtime: Ollama
Status: Available for installation
Pull: ollama pull llama3.1:8b
QUICK START
──────────────────────────────────────────────────
1. Install the recommended model:
ollama pull llama3.1:8b
2. Start using it:
ollama run llama3.1:8b

View file

@ -0,0 +1 @@
exit=0

View file

@ -0,0 +1,51 @@
llm-checker | Demo
----------------------------
LLM Checker - Enhanced Progress Demo
This demonstrates the new step-by-step progress display with visual indicators
────────────────────────────────────────────────────────────
████░░░░░░░░░░░░░░░░ 20% [1/5] System Detection
└─ Scanning hardware specifications...
├─ CPU detected: Apple M4 Pro (12 cores) (1.0s)
└─ Memory detected: 24GB unified memory (1.5s)
Apple M4 Pro, 24GB RAM, Apple Silicon GPU
└─ 1.5s
████████░░░░░░░░░░░░ 40% [2/5] Database Sync
└─ Updating model database...
3,247 models in database
Database synchronized
└─ 801ms
████████████░░░░░░░░ 60% [3/5] Compatibility Analysis
└─ Running mathematical heuristics...
├─ Analyzing hardware requirements... (1.2s)
└─ Calculating performance scores... (1.8s)
127 compatible models found
Compatibility analysis complete
└─ 1.8s
████████████████░░░░ 80% [4/5] AI Evaluation
└─ Running intelligent model selection...
├─ Mathematical heuristics applied (914ms)
Top 15 models selected by AI
AI evaluation complete
└─ 919ms
████████████████████ 100% [5/5] Smart Recommendations
└─ Generating personalized suggestions...
├─ Analyzing use case: general (603ms)
└─ Generating Ollama commands... (1.0s)
23 recommendations generated
└─ 1.0s
────────────────────────────────────────────────────────────
LLM Model Analysis & Compatibility Demo complete!
Analysis complete! Found optimal models for your hardware
Total time: 6.0s
Demo completed successfully!
\nNow try running: llm-checker check
For silent mode: llm-checker check --no-verbose

View file

@ -0,0 +1 @@
exit=0

View file

@ -0,0 +1,180 @@
{
"backends": {
"cpu": {
"available": true,
"info": {
"brand": "Apple M4 Pro",
"vendor": "Apple",
"cores": {
"physical": 12,
"logical": 12,
"performance": 6,
"efficiency": 6
},
"frequency": {
"base": 2400,
"max": 2400
},
"cache": {
"l1d": 64,
"l1i": 128,
"l2": 4,
"l3": 0
},
"capabilities": {
"sse": false,
"sse2": false,
"sse3": false,
"ssse3": false,
"sse4_1": false,
"sse4_2": false,
"avx": false,
"avx2": false,
"avx512": false,
"avx512_vnni": false,
"amx": false,
"fma": false,
"f16c": true,
"neon": true,
"sve": false,
"dotprod": true,
"bestSimd": "NEON"
},
"architecture": "arm64",
"backend": "cpu",
"speedCoefficient": 70
}
},
"metal": {
"available": true,
"info": {
"chip": "Apple M4 Pro",
"variant": "pro",
"generation": 4,
"cores": {
"performance": 8,
"efficiency": 4,
"total": 12
},
"gpu": {
"cores": 16,
"model": "Apple M4 Pro"
},
"neuralEngine": {
"cores": 0
},
"memory": {
"unified": 24,
"bandwidth": 273
},
"capabilities": {
"metal": true,
"metalVersion": "4",
"fp16": true,
"int8": true,
"amx": true
},
"backend": "metal",
"speedCoefficient": 270
}
}
},
"primary": {
"type": "metal",
"name": "Apple Metal",
"info": {
"chip": "Apple M4 Pro",
"variant": "pro",
"generation": 4,
"cores": {
"performance": 8,
"efficiency": 4,
"total": 12
},
"gpu": {
"cores": 16,
"model": "Apple M4 Pro"
},
"neuralEngine": {
"cores": 0
},
"memory": {
"unified": 24,
"bandwidth": 273
},
"capabilities": {
"metal": true,
"metalVersion": "4",
"fp16": true,
"int8": true,
"amx": true
},
"backend": "metal",
"speedCoefficient": 270
}
},
"cpu": {
"brand": "Apple M4 Pro",
"vendor": "Apple",
"cores": {
"physical": 12,
"logical": 12,
"performance": 6,
"efficiency": 6
},
"frequency": {
"base": 2400,
"max": 2400
},
"cache": {
"l1d": 64,
"l1i": 128,
"l2": 4,
"l3": 0
},
"capabilities": {
"sse": false,
"sse2": false,
"sse3": false,
"ssse3": false,
"sse4_1": false,
"sse4_2": false,
"avx": false,
"avx2": false,
"avx512": false,
"avx512_vnni": false,
"amx": false,
"fma": false,
"f16c": true,
"neon": true,
"sve": false,
"dotprod": true,
"bestSimd": "NEON"
},
"architecture": "arm64",
"backend": "cpu",
"speedCoefficient": 70
},
"summary": {
"bestBackend": "metal",
"backendName": "Apple Metal",
"totalVRAM": 24,
"effectiveMemory": 17,
"speedCoefficient": 270,
"isMultiGPU": false,
"gpuCount": 1,
"gpuModel": "Apple M4 Pro",
"gpuInventory": "Apple M4 Pro",
"gpuModels": [
{
"name": "Apple M4 Pro",
"count": 1
}
],
"hasHeterogeneousGPU": false,
"cpuModel": "Apple M4 Pro",
"systemRAM": 24
},
"fingerprint": "apple-apple-m4-pro-24gb",
"timestamp": 1771419210820
}

View file

@ -0,0 +1 @@
exit=0

View file

@ -0,0 +1,94 @@
- Analyzing installed models...
✔ Found 9 installed models
[
{
"name": "codellama:latest",
"displayName": "codellama latest",
"size": "7B",
"fileSizeGB": 3.6,
"quantization": "Q4_0",
"useCase": "coding",
"score": 79,
"command": "ollama run codellama:latest"
},
{
"name": "llava:7b",
"displayName": "llava 7b",
"size": "7B",
"fileSizeGB": 4.4,
"quantization": "Q4_0",
"useCase": "multimodal",
"score": 79,
"command": "ollama run llava:7b"
},
{
"name": "codellama:7b",
"displayName": "codellama 7b",
"size": "7B",
"fileSizeGB": 3.6,
"quantization": "Q4_0",
"useCase": "coding",
"score": 79,
"command": "ollama run codellama:7b"
},
{
"name": "llama3.2:3b",
"displayName": "llama3.2 3b",
"size": "3.2B",
"fileSizeGB": 1.9,
"quantization": "Q4_K_M",
"useCase": "general",
"score": 79,
"command": "ollama run llama3.2:3b"
},
{
"name": "llama3.2:1b",
"displayName": "llama3.2 1b",
"size": "1.2B",
"fileSizeGB": 1.2,
"quantization": "Q8_0",
"useCase": "general",
"score": 79,
"command": "ollama run llama3.2:1b"
},
{
"name": "qwen2.5-coder:7b",
"displayName": "qwen2.5-coder 7b",
"size": "7.6B",
"fileSizeGB": 4.4,
"quantization": "Q4_K_M",
"useCase": "coding",
"score": 78,
"command": "ollama run qwen2.5-coder:7b"
},
{
"name": "qwen2.5-coder:1.5b",
"displayName": "qwen2.5-coder 1.5b",
"size": "1.5B",
"fileSizeGB": 0.9,
"quantization": "Q4_K_M",
"useCase": "coding",
"score": 78,
"command": "ollama run qwen2.5-coder:1.5b"
},
{
"name": "deepseek-v3.1:671b-cloud",
"displayName": "deepseek-v3.1 671b-cloud",
"size": "671.0B",
"fileSizeGB": 0,
"quantization": "FP8_E4M3",
"useCase": "general",
"score": 75,
"command": "ollama run deepseek-v3.1:671b-cloud"
},
{
"name": "gpt-oss:20b",
"displayName": "gpt-oss 20b",
"size": "20.9B",
"fileSizeGB": 12.8,
"quantization": "MXFP4",
"useCase": "general",
"score": 70,
"command": "ollama run gpt-oss:20b"
}
]

View file

@ -0,0 +1 @@
exit=0

View file

@ -0,0 +1,337 @@
- 📋 Loading models database...
✔ ✅ Found 213 models (showing 3)
[
{
"model_identifier": "llama3.1",
"model_name": "llama3.1",
"description": "",
"labels": [],
"pulls": 0,
"tags": [
"llama3.1:latest",
"llama3.1:8b",
"llama3.1:70b",
"llama3.1:405b"
],
"last_updated": "Unknown",
"url": "https://ollama.com/library/llama3.1",
"namespace": null,
"model_type": "official",
"variants": [
{
"tag": "llama3.1:latest",
"size": "unknown",
"quantization": "Q4_0",
"command": "ollama pull llama3.1:latest",
"estimated_size_gb": 1,
"real_size_gb": 4.9,
"categories": [
"chat"
]
},
{
"tag": "llama3.1:8b",
"size": "8b",
"quantization": "Q4_0",
"command": "ollama pull llama3.1:8b",
"estimated_size_gb": 8,
"real_size_gb": 4.9,
"categories": [
"chat"
]
},
{
"tag": "llama3.1:70b",
"size": "70b",
"quantization": "Q4_0",
"command": "ollama pull llama3.1:70b",
"estimated_size_gb": 70,
"real_size_gb": 43,
"categories": [
"chat"
]
},
{
"tag": "llama3.1:405b",
"size": "405b",
"quantization": "Q4_0",
"command": "ollama pull llama3.1:405b",
"estimated_size_gb": 405,
"real_size_gb": 243,
"categories": [
"chat"
]
}
],
"detailed_description": "Llama 3.1 is a new state-of-the-art model from Meta available in 8B, 70B and 405B parameter sizes.",
"parameters": {},
"quantizations": [],
"model_sizes": [
"8b",
"70b",
"405b",
"4.9gb",
"43gb",
"243gb",
"952b"
],
"category": "talking",
"use_cases": [
"chat",
"conversation",
"assistant"
],
"main_size": "8b",
"actual_pulls": 0,
"context_length": "128K",
"input_types": [
"text",
"image",
"code"
],
"detailed_scraped_at": "2026-02-18T08:17:27.647Z",
"categories": [
"chat"
],
"primary_category": "chat"
},
{
"model_identifier": "deepseek-r1",
"model_name": "deepseek-r1",
"description": "",
"labels": [],
"pulls": 0,
"tags": [
"deepseek-r1:671b",
"deepseek-r1:8b",
"deepseek-r1:1.5b",
"deepseek-r1:7b",
"deepseek-r1:14b",
"deepseek-r1:32b",
"deepseek-r1:70b",
"deepseek-r1:latest"
],
"last_updated": "Unknown",
"url": "https://ollama.com/library/deepseek-r1",
"namespace": null,
"model_type": "official",
"variants": [
{
"tag": "deepseek-r1:671b",
"size": "671b",
"quantization": "Q4_0",
"command": "ollama pull deepseek-r1:671b",
"estimated_size_gb": 671,
"real_size_gb": 404,
"categories": [
"chat",
"reasoning"
]
},
{
"tag": "deepseek-r1:8b",
"size": "8b",
"quantization": "Q4_0",
"command": "ollama pull deepseek-r1:8b",
"estimated_size_gb": 8,
"real_size_gb": 5.2,
"categories": [
"chat",
"reasoning"
]
},
{
"tag": "deepseek-r1:1.5b",
"size": "1.5b",
"quantization": "Q4_0",
"command": "ollama pull deepseek-r1:1.5b",
"estimated_size_gb": 1.5,
"real_size_gb": 1.1,
"categories": [
"chat",
"reasoning"
]
},
{
"tag": "deepseek-r1:7b",
"size": "7b",
"quantization": "Q4_0",
"command": "ollama pull deepseek-r1:7b",
"estimated_size_gb": 7,
"real_size_gb": 4.7,
"categories": [
"chat",
"reasoning"
]
},
{
"tag": "deepseek-r1:14b",
"size": "14b",
"quantization": "Q4_0",
"command": "ollama pull deepseek-r1:14b",
"estimated_size_gb": 14,
"real_size_gb": 9,
"categories": [
"chat",
"reasoning"
]
},
{
"tag": "deepseek-r1:32b",
"size": "32b",
"quantization": "Q4_0",
"command": "ollama pull deepseek-r1:32b",
"estimated_size_gb": 32,
"real_size_gb": 20,
"categories": [
"chat",
"reasoning"
]
},
{
"tag": "deepseek-r1:70b",
"size": "70b",
"quantization": "Q4_0",
"command": "ollama pull deepseek-r1:70b",
"estimated_size_gb": 70,
"real_size_gb": 43,
"categories": [
"chat",
"reasoning"
]
},
{
"tag": "deepseek-r1:latest",
"size": "unknown",
"quantization": "Q4_0",
"command": "ollama pull deepseek-r1:latest",
"estimated_size_gb": 1,
"real_size_gb": 5.2,
"categories": [
"chat",
"reasoning"
]
}
],
"detailed_description": "DeepSeek-R1 is a family of open reasoning models with performance approaching that of leading models, such as O3 and Gemini 2.5 Pro.",
"parameters": {},
"quantizations": [],
"model_sizes": [
"1.5b",
"7b",
"8b",
"14b",
"32b",
"70b",
"671b",
"5.2gb",
"1.1gb",
"4.7gb",
"9.0gb",
"20gb",
"43gb",
"404gb"
],
"category": "reasoning",
"use_cases": [
"reasoning",
"mathematics",
"logic"
],
"main_size": "1.5b",
"actual_pulls": 0,
"context_length": "160K",
"input_types": [
"text",
"image",
"code"
],
"detailed_scraped_at": "2026-02-18T08:17:27.855Z",
"categories": [
"chat",
"reasoning"
],
"primary_category": "reasoning"
},
{
"model_identifier": "llama3.2",
"model_name": "llama3.2",
"description": "",
"labels": [],
"pulls": 0,
"tags": [
"llama3.2:1b",
"llama3.2:latest",
"llama3.2:3b"
],
"last_updated": "Unknown",
"url": "https://ollama.com/library/llama3.2",
"namespace": null,
"model_type": "official",
"variants": [
{
"tag": "llama3.2:1b",
"size": "1b",
"quantization": "Q4_0",
"command": "ollama pull llama3.2:1b",
"estimated_size_gb": 1,
"real_size_gb": 1.3,
"categories": [
"chat"
]
},
{
"tag": "llama3.2:latest",
"size": "unknown",
"quantization": "Q4_0",
"command": "ollama pull llama3.2:latest",
"estimated_size_gb": 1,
"real_size_gb": 2,
"categories": [
"chat"
]
},
{
"tag": "llama3.2:3b",
"size": "3b",
"quantization": "Q4_0",
"command": "ollama pull llama3.2:3b",
"estimated_size_gb": 3,
"real_size_gb": 2,
"categories": [
"chat"
]
}
],
"detailed_description": "Meta's Llama 3.2 goes small with 1B and 3B models.",
"parameters": {},
"quantizations": [],
"model_sizes": [
"1b",
"3b",
"2.0gb",
"1.3gb",
"929b",
"2.6b"
],
"category": "talking",
"use_cases": [
"chat",
"conversation",
"assistant"
],
"main_size": "1b",
"actual_pulls": 0,
"context_length": "128K",
"input_types": [
"text",
"image",
"code"
],
"detailed_scraped_at": "2026-02-18T08:17:27.748Z",
"categories": [
"chat"
],
"primary_category": "chat"
}
]

View file

@ -0,0 +1 @@
exit=0

View file

@ -0,0 +1,38 @@
{
"recommended": {
"command": "claude",
"args": [
"mcp",
"add",
"llm-checker",
"--",
"llm-checker-mcp"
],
"commandLine": "claude mcp add llm-checker -- llm-checker-mcp"
},
"alternatives": [
{
"command": "claude",
"args": [
"mcp",
"add",
"llm-checker",
"--",
"npx",
"llm-checker-mcp"
],
"commandLine": "claude mcp add llm-checker -- npx llm-checker-mcp"
}
],
"claudeDesktop": {
"configPath": "~/Library/Application Support/Claude/claude_desktop_config.json",
"snippet": {
"mcpServers": {
"llm-checker": {
"command": "llm-checker-mcp",
"args": []
}
}
}
}
}

View file

@ -0,0 +1 @@
exit=0

View file

@ -0,0 +1,62 @@
llm-checker | Ollama Integration
------------------------------------------
- Checking Ollama integration...
██░░░░░░░░░░░░░░░░░░ 13% [1/8] System Detection
└─ Scanning hardware specifications...
├─ CPU detected: M4 Pro (12 cores) (525ms)
└─ Memory detected: 24GB unified memory (726ms)
M4 Pro, 24GB RAM, Apple M4 Pro
└─ 729ms
├─ CPU detected: M4 Pro (12 cores) (729ms)
└─ Memory detected: 24GB unified memory (930ms)
M4 Pro, 24GB RAM, Apple M4 Pro
└─ 930ms
█████░░░░░░░░░░░░░░░ 25% [2/8] Database Sync
└─ Loading model database...
177 models in database
Database synchronized
└─ 0ms
███████░░░░░░░░░░░░░ 38% [3/8] Model Analysis
└─ Loading base model definitions...
Loaded 26 base models
Base models loaded
└─ 0ms
██████████░░░░░░░░░░ 50% [4/8] Ollama Integration
└─ Connecting to Ollama and checking installed models...
Ollama connected with 9 locally installed models
Ollama integration complete
└─ 56ms
████████████░░░░░░░░ 63% [5/8] Model Filtering
└─ Applying user-specified filters...
└─ Cloud models excluded (0ms)
26/26 models selected
└─ 0ms
███████████████░░░░░ 75% [6/8] Compatibility Analysis
└─ Running mathematical heuristics and hardware matching...
11 compatible, 8 marginal
└─ 2ms
█████████████████░░░ 88% [7/8] Performance Analysis
└─ Estimating model performance and speeds...
Performance data for 19 models
└─ 1ms
████████████████████ 100% [8/8] Smart Recommendations
└─ Generating personalized model suggestions...
└─ Generating apple_silicon recommendations... (561ms)
6 recommendations generated
└─ 561ms
────────────────────────────────────────────────────────────
LLM Model Analysis & Compatibility Check complete!
Found 11 compatible (8 marginal) models for your hardware
Total time: 1.6s
✔ Ollama integration active

View file

@ -0,0 +1 @@
exit=0

View file

@ -0,0 +1,99 @@
{
"generated_at": "2026-02-18T12:52:57.994Z",
"selection": {
"requested": [],
"selected": [
"codellama:latest",
"qwen2.5-coder:7b",
"llava:7b",
"codellama:7b",
"gpt-oss:20b",
"llama3.2:3b",
"llama3.2:1b",
"qwen2.5-coder:1.5b"
],
"missing": []
},
"plan": {
"objective": "balanced",
"inputs": {
"targetContext": 8192,
"targetConcurrency": 2
},
"hardware": {
"backend": "metal",
"backendName": "Apple Metal",
"rawCapacityGB": 17,
"reserveGB": 2,
"memoryBudgetGB": 15
},
"models": [
{
"name": "gpt-oss:20b",
"size": "20.9B",
"fileSizeGB": 12.8,
"paramsB": 20.9,
"estimatedBaseMemoryGB": 13.5
}
],
"envelope": {
"context": {
"requested": 8192,
"recommended": 3674,
"min_safe": 2048,
"max_for_parallel_1": 3674,
"max_for_recommended_parallel": 3674
},
"parallel": {
"requested": 2,
"recommended": 1,
"max_at_requested_ctx": 0
},
"loaded_models": {
"requested": 2,
"recommended": 1,
"max_at_recommended_settings": 1
}
},
"recommendation": {
"num_ctx": 3674,
"num_parallel": 1,
"max_loaded_models": 1,
"max_queue": 4,
"keep_alive": "15m",
"flash_attention": "1"
},
"memory": {
"budgetGB": 15,
"requestedEstimatedGB": 20.19,
"recommendedEstimatedGB": 15,
"utilizationPercent": 100
},
"risk": {
"level": "critical",
"score": 100
},
"fallback": {
"num_ctx": 3674,
"num_parallel": 1,
"max_loaded_models": 1,
"estimated_memory_gb": 15
},
"shell": {
"env": {
"OLLAMA_NUM_CTX": "3674",
"OLLAMA_NUM_PARALLEL": "1",
"OLLAMA_MAX_LOADED_MODELS": "1",
"OLLAMA_MAX_QUEUE": "4",
"OLLAMA_KEEP_ALIVE": "15m",
"OLLAMA_FLASH_ATTENTION": "1"
}
},
"notes": [
"Requested settings exceed available memory budget; reduced settings are recommended.",
"Context reduced from 8192 to 3674 to avoid memory pressure.",
"Parallelism reduced from 2 to 1 to keep memory stable.",
"Loaded models capped at 1 for this objective and memory budget."
]
}
}

View file

@ -0,0 +1,41 @@
version: 1
org: your-org
mode: audit
rules:
models:
allow:
- qwen2.5-coder:*
- llama3.1:*
deny:
- "*uncensored*"
max_size_gb: 24
max_params_b: 32
allowed_quantizations:
- Q4_K_M
- Q5_K_M
- Q8_0
runtime:
required_backends:
- metal
- cuda
min_ram_gb: 32
local_only: true
compliance:
approved_licenses:
- mit
- apache-2.0
- llama
enforcement:
on_violation: warn
exit_code: 3
allow_exceptions: true
exceptions:
- model: deepseek-r1:32b
reason: Approved PoC
approver: security@example.com
expires_at: 2026-06-30
reporting:
formats:
- json
- csv
- sarif

View file

@ -0,0 +1 @@
exit=0

View file

@ -0,0 +1 @@
Policy file created: ~/llm-checker/.tmp-feature-check/policy.yaml

View file

@ -0,0 +1 @@
exit=0

View file

@ -0,0 +1 @@
Policy is valid (enforce mode): ~/llm-checker/.tmp-feature-check/policy.yaml

View file

@ -0,0 +1 @@
exit=0

View file

@ -0,0 +1,68 @@
llm-checker | Recommendations
---------------------------------------
Generating recommendations... done
SYSTEM INFORMATION
╭──────────────────────────────────────────────────
│ CPU: M4 Pro (12 cores, 2.4GHz)
│ Architecture: Apple Silicon
│ RAM: 24GB
│ GPU: Apple M4 Pro
│ VRAM: 24GB (Integrated)
│ Hardware Tier: HIGH
╰
INTELLIGENT RECOMMENDATIONS BY CATEGORY
╭─────────────────────────────────────────────────────────────────
│ Hardware Tier: HIGH | Models Analyzed: 213
│ Optimization: BALANCED
│
│ BEST OVERALL: yi:latest
│ Command: ollama pull yi:latest
│ Score: 88/100 | Category: general
│ Quantization: Q4_K_M
│
│ Coding (Coding):
│ deepseek-coder:6.7b (6.7B)
│ Score: 87/100 | Pulls: 0
│ Quantization: Q4_K_M
│ Command: ollama pull deepseek-coder:6.7b
│
│ Reasoning (Reasoning):
│ deepseek-coder-v2:latest (16B)
│ Score: 91/100 | Pulls: 0
│ Quantization: Q4_K_M
│ Command: ollama pull deepseek-coder-v2:latest
│
│ Multimodal (Multimodal):
│ qwen2.5vl:7b (7B)
│ Score: 87/100 | Pulls: 0
│ Quantization: Q4_K_M
│ Command: ollama pull qwen2.5vl:7b
│
│ Creative (Creative):
│ yi:latest (6B)
│ Score: 87/100 | Pulls: 0
│ Quantization: Q4_K_M
│ Command: ollama pull yi:latest
│
│ Talking (Chat):
│ yi:latest (6B)
│ Score: 87/100 | Pulls: 0
│ Quantization: Q4_K_M
│ Command: ollama pull yi:latest
│
│ Reading (Reading):
│ qwen3:1.7b (1.7B)
│ Score: 84/100 | Pulls: 0
│ Quantization: Q4_K_M
│ Command: ollama pull qwen3:1.7b
│
│ General (General):
│ yi:latest (6B)
│ Score: 88/100 | Pulls: 0
│ Quantization: Q4_K_M
│ Command: ollama pull yi:latest
│
╰

View file

@ -0,0 +1 @@
exit=0

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1 @@
exit=0

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1 @@
0

View file

View file

@ -254,6 +254,12 @@ npm install -g llm-checker
claude mcp add llm-checker -- llm-checker-mcp
```
Or generate the exact command directly from the CLI:
```bash
llm-checker mcp-setup
```
Or with npx (no global install needed):
```bash
@ -352,6 +358,7 @@ llm-checker search "qwen coder" --json
| `calibrate` | Generate calibration result + routing policy artifacts from a JSONL prompt suite |
| `installed` | Rank your installed Ollama models by compatibility |
| `ollama-plan` | Compute safe Ollama runtime env vars (`NUM_CTX`, `NUM_PARALLEL`, `MAX_LOADED_MODELS`) for selected local models |
| `mcp-setup` | Print/apply Claude MCP setup command and config snippet (`--apply`, `--json`, `--npx`) |
### Advanced Commands (require `sql.js`)

View file

@ -1,3 +1,5 @@
const { estimateTokenSpeedFromHardware } = require('../src/utils/token-speed-estimator');
class PerformanceAnalyzer {
constructor() {
this.benchmarkCache = new Map();
@ -284,113 +286,57 @@ class PerformanceAnalyzer {
}
calculateRealisticPerformance(model, hardware) {
// Parse model size
const modelSizeB = this.parseModelSize(model.size);
// Get hardware specifics
const cpuModel = hardware.cpu?.brand || hardware.cpu?.model || '';
const gpuModel = hardware.gpu?.model || '';
const cores = hardware.cpu?.physicalCores || hardware.cpu?.cores || 1;
const baseSpeed = hardware.cpu?.speed || 2.4;
const vramGB = hardware.gpu?.vram || 0;
const memoryTotal = hardware.memory?.total || 8;
// Hardware type detection
const isAppleSilicon = hardware.cpu?.architecture === 'Apple Silicon' || (
process.platform === 'darwin' && (
gpuModel.toLowerCase().includes('apple') ||
gpuModel.toLowerCase().includes('m1') ||
gpuModel.toLowerCase().includes('m2') ||
gpuModel.toLowerCase().includes('m3') ||
gpuModel.toLowerCase().includes('m4')
)
);
const isIntegratedGPU = /iris.*xe|iris.*graphics|uhd.*graphics|vega.*integrated|radeon.*graphics/i.test(gpuModel);
const hasDedicatedGPU = vramGB > 0 && !isIntegratedGPU && !isAppleSilicon;
let tokensPerSecond;
if (isAppleSilicon) {
// Apple Silicon - realistic but optimistic due to unified memory
let baseTPS = 20; // More realistic baseline
if (gpuModel.toLowerCase().includes('m4 pro')) baseTPS = 30;
else if (gpuModel.toLowerCase().includes('m4')) baseTPS = 25;
else if (gpuModel.toLowerCase().includes('m3 pro')) baseTPS = 28;
else if (gpuModel.toLowerCase().includes('m3')) baseTPS = 22;
else if (gpuModel.toLowerCase().includes('m2 pro')) baseTPS = 25;
else if (gpuModel.toLowerCase().includes('m2')) baseTPS = 20;
else if (gpuModel.toLowerCase().includes('m1 pro')) baseTPS = 22;
else if (gpuModel.toLowerCase().includes('m1')) baseTPS = 18;
// Memory scaling for Apple Silicon
if (memoryTotal >= 64) baseTPS *= 1.2;
else if (memoryTotal >= 32) baseTPS *= 1.1;
tokensPerSecond = Math.max(6, Math.round(baseTPS / Math.max(0.7, modelSizeB)));
} else if (hasDedicatedGPU) {
// Dedicated GPU - much better but still realistic
let gpuTPS = 25;
if (gpuModel.toLowerCase().includes('rtx 50')) gpuTPS = 60;
else if (gpuModel.toLowerCase().includes('rtx 40')) gpuTPS = 45;
else if (gpuModel.toLowerCase().includes('rtx 30')) gpuTPS = 35;
else if (gpuModel.toLowerCase().includes('rtx 20')) gpuTPS = 28;
else if (vramGB >= 16) gpuTPS = 40;
else if (vramGB >= 8) gpuTPS = 30;
else if (vramGB >= 4) gpuTPS = 25;
tokensPerSecond = Math.max(8, Math.round(gpuTPS / Math.max(0.4, modelSizeB)));
} else {
// CPU-only or integrated GPU - most conservative and realistic
const hasAVX512 = cpuModel.toLowerCase().includes('intel') &&
(cpuModel.includes('12th') || cpuModel.includes('13th') || cpuModel.includes('14th'));
const hasAVX2 = cpuModel.toLowerCase().includes('intel') || cpuModel.toLowerCase().includes('amd');
// Base CPU performance - very conservative
let cpuK = 1.2; // Much more realistic
if (hasAVX512) cpuK = 2.0;
else if (hasAVX2) cpuK = 1.6;
// Threading efficiency (realistic diminishing returns)
const effectiveThreads = Math.min(cores, 6); // CPU inference doesn't scale linearly
// iGPU small boost
const iGpuMultiplier = isIntegratedGPU ? 1.2 : 1.0;
// Memory pressure factor
const memoryPressure = Math.min(1.0, Math.max(0.6, memoryTotal / (modelSizeB * 2)));
const baseTPS = (cpuK * baseSpeed * effectiveThreads * iGpuMultiplier * memoryPressure) / Math.max(2.0, modelSizeB);
// Realistic CPU caps based on hardware
const maxCPUTPS = hasAVX512 ? 18 : (isIntegratedGPU ? 12 : 8);
tokensPerSecond = Math.max(1, Math.min(maxCPUTPS, Math.round(baseTPS)));
}
const speedProfile = estimateTokenSpeedFromHardware(hardware, {
modelSizeB,
modelName: model.name
});
const tokensPerSecond = speedProfile.tokensPerSecond;
return {
estimatedTokensPerSecond: Math.round(tokensPerSecond),
estimatedTokensPerSecond: tokensPerSecond,
confidence: this.calculateConfidence(hardware, model),
factors: {
cpu: cpuModel,
memory: memoryTotal,
gpu: hasDedicatedGPU ? 'dedicated' : (isIntegratedGPU ? 'integrated' : 'cpu_only'),
cpu: hardware.cpu?.brand || hardware.cpu?.model || 'Unknown CPU',
memory: hardware.memory?.total || 0,
gpu: speedProfile.backend,
modelSize: modelSizeB,
architecture: isAppleSilicon ? 'Apple Silicon' : 'x86'
architecture: hardware.cpu?.architecture || 'unknown',
sizeScale: speedProfile.sizeScale,
memoryFactor: speedProfile.memoryFactor
},
category: this.categorizePerformance(Math.round(tokensPerSecond)),
category: this.categorizePerformance(tokensPerSecond),
loadTimeEstimate: this.estimateLoadTime(model, hardware)
};
}
parseModelSize(sizeString) {
const match = sizeString.match(/(\d+\.?\d*)[BM]/i);
if (!match) return 1;
if (typeof sizeString !== 'string' || !sizeString.trim()) return 1;
const num = parseFloat(match[1]);
const unit = match[0].slice(-1).toUpperCase();
const normalized = sizeString.trim().toUpperCase();
return unit === 'B' ? num : num / 1000; // Convert M to B
// Parameter notation (e.g. 8B, 774M)
const paramMatch = normalized.match(/(\d+\.?\d*)\s*([BM])\b/);
if (paramMatch) {
const num = parseFloat(paramMatch[1]);
const unit = paramMatch[2];
return unit === 'B' ? num : num / 1000;
}
// File-size notation fallback (e.g. 4.9GB) -> rough Q4 param estimate
const gbMatch = normalized.match(/(\d+\.?\d*)\s*GB\b/);
if (gbMatch) {
const sizeGB = parseFloat(gbMatch[1]);
return Math.max(0.5, sizeGB / 0.62);
}
const mbMatch = normalized.match(/(\d+\.?\d*)\s*MB\b/);
if (mbMatch) {
const sizeGB = parseFloat(mbMatch[1]) / 1024;
return Math.max(0.5, sizeGB / 0.62);
}
return 1;
}
calculateConfidence(hardware, model) {
@ -502,4 +448,4 @@ class PerformanceAnalyzer {
}
}
module.exports = PerformanceAnalyzer;
module.exports = PerformanceAnalyzer;

View file

@ -57,6 +57,7 @@ const COMMAND_HEADER_LABELS = {
'smart-recommend': 'Smart Recommend',
search: 'Model Search',
sync: 'Database Sync',
'mcp-setup': 'Claude MCP Setup',
check: 'Compatibility Check',
installed: 'Installed Models',
'ai-check': 'AI Check',
@ -612,6 +613,60 @@ async function checkOllamaAndExit() {
}
}
function quoteCliArg(value) {
const stringValue = String(value ?? '');
if (!stringValue) return '""';
if (/^[A-Za-z0-9._:/=-]+$/.test(stringValue)) return stringValue;
return `"${stringValue.replace(/(["\\$`])/g, '\\$1')}"`;
}
function getClaudeDesktopConfigPath() {
const homeDir = os.homedir();
if (process.platform === 'darwin') {
return path.join(homeDir, 'Library', 'Application Support', 'Claude', 'claude_desktop_config.json');
}
if (process.platform === 'win32') {
const appData = process.env.APPDATA || path.join(homeDir, 'AppData', 'Roaming');
return path.join(appData, 'Claude', 'claude_desktop_config.json');
}
return path.join(homeDir, '.config', 'Claude', 'claude_desktop_config.json');
}
function buildClaudeMcpSetup(useNpx = false, serverName = 'llm-checker') {
const normalizedServerName = String(serverName || 'llm-checker').trim() || 'llm-checker';
const runner = useNpx ? ['npx', 'llm-checker-mcp'] : ['llm-checker-mcp'];
const claudeArgs = ['mcp', 'add', normalizedServerName, '--', ...runner];
const commandLine = ['claude', ...claudeArgs].map(quoteCliArg).join(' ');
const desktopServerConfig = useNpx
? { command: 'npx', args: ['llm-checker-mcp'] }
: { command: 'llm-checker-mcp', args: [] };
return {
serverName: normalizedServerName,
useNpx: Boolean(useNpx),
claudeArgs,
commandLine,
desktopConfigPath: getClaudeDesktopConfigPath(),
desktopConfig: {
mcpServers: {
[normalizedServerName]: desktopServerConfig
}
}
};
}
async function runExternalCommand(command, args) {
return new Promise((resolve, reject) => {
const child = spawn(command, args, {
stdio: 'inherit',
env: process.env
});
child.on('error', reject);
child.on('close', (code) => resolve(code));
});
}
function parsePositiveIntegerOption(rawValue, optionName) {
const parsed = Number(rawValue);
if (!Number.isFinite(parsed) || parsed <= 0) {
@ -629,13 +684,32 @@ function parseNonNegativeNumberOption(rawValue, optionName) {
}
function selectModelsForPlan(installedModels, requestedModels = []) {
const runnableModels = (installedModels || []).filter((model) => {
const name = String(model?.name || '').toLowerCase();
const source = String(model?.source || '').toLowerCase();
const type = String(model?.type || model?.model_type || '').toLowerCase();
const fileSizeGB = Number(model?.fileSizeGB) || 0;
const cloudTagged = (
name.includes('-cloud') ||
name.endsWith(':cloud') ||
source.includes('cloud') ||
type === 'cloud' ||
type === 'remote' ||
type === 'hosted'
);
// Keep only models that are actually present locally for memory planning.
return !(cloudTagged && fileSizeGB <= 0);
});
const requested = Array.isArray(requestedModels)
? requestedModels.map((model) => String(model || '').trim()).filter(Boolean)
: [];
if (!requested.length) {
return {
selected: installedModels.slice(),
selected: runnableModels.slice(),
missing: []
};
}
@ -647,24 +721,24 @@ function selectModelsForPlan(installedModels, requestedModels = []) {
for (const request of requested) {
const normalized = request.toLowerCase();
let match = installedModels.find(
let match = runnableModels.find(
(model) => String(model.name || '').toLowerCase() === normalized
);
if (!match) {
match = installedModels.find((model) =>
match = runnableModels.find((model) =>
String(model.name || '').toLowerCase().startsWith(`${normalized}:`)
);
}
if (!match) {
match = installedModels.find(
match = runnableModels.find(
(model) => String(model.family || '').toLowerCase() === normalized
);
}
if (!match) {
match = installedModels.find((model) =>
match = runnableModels.find((model) =>
String(model.name || '').toLowerCase().includes(normalized)
);
}
@ -2439,6 +2513,78 @@ function displayPolicySummary(commandName, policyConfig, evaluation, enforcement
console.log(chalk.magenta('╰' + '─'.repeat(65)));
}
program
.command('mcp-setup')
.description('Show or apply Claude MCP setup for llm-checker')
.option('--name <server-name>', 'MCP server name in Claude', 'llm-checker')
.option('--npx', 'Use npx llm-checker-mcp instead of global llm-checker-mcp')
.option('--apply', 'Run `claude mcp add ...` automatically')
.option('-j, --json', 'Output setup details as JSON')
.action(async (options) => {
const primarySetup = buildClaudeMcpSetup(Boolean(options.npx), options.name);
const alternateSetup = buildClaudeMcpSetup(!Boolean(options.npx), options.name);
if (options.json) {
console.log(JSON.stringify({
recommended: {
command: 'claude',
args: primarySetup.claudeArgs,
commandLine: primarySetup.commandLine
},
alternatives: [
{
command: 'claude',
args: alternateSetup.claudeArgs,
commandLine: alternateSetup.commandLine
}
],
claudeDesktop: {
configPath: primarySetup.desktopConfigPath,
snippet: primarySetup.desktopConfig
}
}, null, 2));
return;
}
showAsciiArt('mcp-setup');
console.log(chalk.blue.bold('\nClaude Code MCP Setup'));
console.log(chalk.white('\nRecommended command:'));
console.log(chalk.cyan(` ${primarySetup.commandLine}`));
console.log(chalk.white('\nAlternative command:'));
console.log(chalk.gray(` ${alternateSetup.commandLine}`));
console.log(chalk.white('\nClaude Desktop config path (manual):'));
console.log(chalk.gray(` ${primarySetup.desktopConfigPath}`));
console.log(chalk.white('\nConfig snippet:'));
console.log(chalk.gray(JSON.stringify(primarySetup.desktopConfig, null, 2)));
if (!options.apply) {
console.log(chalk.green('\nTip: run with --apply to execute the command automatically.'));
return;
}
console.log(chalk.blue('\nApplying MCP setup via Claude CLI...\n'));
try {
const exitCode = await runExternalCommand('claude', primarySetup.claudeArgs);
if (exitCode === 0) {
console.log(chalk.green('\nClaude MCP setup applied successfully.'));
} else {
console.error(chalk.red(`\nClaude command exited with code ${exitCode}.`));
process.exit(exitCode || 1);
}
} catch (error) {
if (error && error.code === 'ENOENT') {
console.error(chalk.red('Could not find `claude` in PATH.'));
console.log(chalk.yellow('Run the printed command manually once Claude CLI is installed.'));
} else {
console.error(chalk.red(`Failed to apply MCP setup: ${error.message}`));
}
process.exit(1);
}
});
const policyCommand = program
.command('policy')
.description('Manage enterprise policy files (policy.yaml)')

View file

@ -10,7 +10,7 @@ const { OllamaNativeScraper } = require('../ollama/native-scraper');
const crypto = require('crypto');
const fs = require('fs');
const path = require('path');
const fetch = require('node-fetch');
const fetch = require('../utils/fetch');
class AICheckSelector {
constructor() {
@ -803,4 +803,4 @@ Return JSON with this structure:
}
}
module.exports = AICheckSelector;
module.exports = AICheckSelector;

View file

@ -9,6 +9,7 @@ const fs = require('fs');
const path = require('path');
const os = require('os');
const { spawn } = require('child_process');
const fetch = require('../utils/fetch');
const { DETERMINISTIC_WEIGHTS } = require('./scoring-config');
const {
parseBillionsValue: parseMoEBillionsValue,

View file

@ -1,3 +1,5 @@
const { estimateTokenSpeedFromHardware } = require('../utils/token-speed-estimator');
class ExpandedModelsDatabase {
constructor() {
this.models = this.initializeExpandedModels();
@ -958,88 +960,12 @@ class ExpandedModelsDatabase {
}
calculateRealisticTokensPerSecond(model, hardware) {
// Extract model parameters from name or size
const modelParams = this.extractModelParams(model);
// Get hardware specifics
const cpuModel = hardware.cpu?.brand || hardware.cpu?.model || '';
const gpuModel = hardware.gpu?.model || '';
const cores = hardware.cpu?.physicalCores || hardware.cpu?.cores || 1;
const baseSpeed = hardware.cpu?.speed || 2.4;
const vramGB = hardware.gpu?.vram || 0;
// Check hardware type
const isAppleSilicon = process.platform === 'darwin' && (
gpuModel.toLowerCase().includes('apple') ||
gpuModel.toLowerCase().includes('m1') ||
gpuModel.toLowerCase().includes('m2') ||
gpuModel.toLowerCase().includes('m3') ||
gpuModel.toLowerCase().includes('m4')
);
const isIntegratedGPU = /iris.*xe|iris.*graphics|uhd.*graphics|vega.*integrated|radeon.*graphics/i.test(gpuModel);
const hasDedicatedGPU = vramGB > 0 && !isIntegratedGPU && !isAppleSilicon;
let tokensPerSecond;
if (isAppleSilicon) {
// Apple Silicon unified memory - more optimistic but realistic
let baseTPS = 25;
if (gpuModel.toLowerCase().includes('m4 pro')) baseTPS = 35;
else if (gpuModel.toLowerCase().includes('m4')) baseTPS = 30;
else if (gpuModel.toLowerCase().includes('m3 pro')) baseTPS = 30;
else if (gpuModel.toLowerCase().includes('m3')) baseTPS = 25;
else if (gpuModel.toLowerCase().includes('m2 pro')) baseTPS = 28;
else if (gpuModel.toLowerCase().includes('m2')) baseTPS = 22;
else if (gpuModel.toLowerCase().includes('m1 pro')) baseTPS = 25;
else if (gpuModel.toLowerCase().includes('m1')) baseTPS = 20;
// Scale by model size (Apple Silicon handles larger models better)
tokensPerSecond = Math.max(8, Math.round(baseTPS / Math.max(0.8, modelParams)));
} else if (hasDedicatedGPU) {
// Dedicated GPU - much better performance
let gpuTPS = 30;
if (gpuModel.toLowerCase().includes('gb10') ||
gpuModel.toLowerCase().includes('grace blackwell') ||
gpuModel.toLowerCase().includes('dgx spark')) gpuTPS = 90;
else if (gpuModel.toLowerCase().includes('h100')) gpuTPS = 120;
else if (gpuModel.toLowerCase().includes('a100')) gpuTPS = 95;
else if (gpuModel.toLowerCase().includes('rtx 50')) gpuTPS = 65;
else if (gpuModel.toLowerCase().includes('rtx 40')) gpuTPS = 50;
else if (gpuModel.toLowerCase().includes('rtx 30')) gpuTPS = 40;
else if (gpuModel.toLowerCase().includes('rtx 20')) gpuTPS = 30;
else if (gpuModel.toLowerCase().includes('p100')) gpuTPS = 32;
else if (vramGB >= 16) gpuTPS = 45;
else if (vramGB >= 8) gpuTPS = 35;
else if (vramGB >= 4) gpuTPS = 25;
// Scale by model size for GPU
tokensPerSecond = Math.max(10, Math.round(gpuTPS / Math.max(0.5, modelParams)));
} else {
// CPU-only or integrated GPU - most realistic and conservative
const hasAVX512 = cpuModel.toLowerCase().includes('intel') &&
(cpuModel.includes('12th') || cpuModel.includes('13th') || cpuModel.includes('14th'));
const hasAVX2 = cpuModel.toLowerCase().includes('intel') || cpuModel.toLowerCase().includes('amd');
// Base CPU coefficient - much more conservative
let cpuK = 1.8; // Conservative baseline
if (hasAVX512) cpuK = 2.6;
else if (hasAVX2) cpuK = 2.2;
// iGPU boost (small)
const iGpuMultiplier = isIntegratedGPU ? 1.3 : 1.0;
// Calculate with realistic threading limits
const effectiveThreads = Math.min(cores, 8); // Diminishing returns after 8 threads
const baseTPS = (cpuK * baseSpeed * effectiveThreads * iGpuMultiplier) / Math.max(1.5, modelParams);
// Apply realistic CPU limits
const maxCPUTPS = hasAVX512 ? 25 : (isIntegratedGPU ? 20 : 15);
tokensPerSecond = Math.max(2, Math.min(maxCPUTPS, Math.round(baseTPS)));
}
return tokensPerSecond;
const speedProfile = estimateTokenSpeedFromHardware(hardware, {
modelSizeB: modelParams,
modelName: model.name
});
return speedProfile.tokensPerSecond;
}
extractModelParams(model) {
@ -1047,11 +973,12 @@ class ExpandedModelsDatabase {
const name = model.name.toLowerCase();
// Look for patterns like "7b", "3.8b", "0.5b", etc.
const paramMatch = name.match(/(\d+\.?\d*)[bm](?:\s|$)/);
const paramMatch = name.match(/(\d+\.?\d*)\s*([bm])(?:\s|$)/);
if (paramMatch) {
const value = parseFloat(paramMatch[1]);
const unit = paramMatch[2].toLowerCase();
// Convert millions to billions if needed
return paramMatch[1].includes('m') ? value / 1000 : value;
return unit === 'm' ? value / 1000 : value;
}
// Fallback to size-based estimation

View file

@ -1,4 +1,4 @@
const fetch = require('node-fetch');
const fetch = require('../utils/fetch');
class OllamaClient {
constructor(baseURL = null) {
@ -377,6 +377,29 @@ class OllamaClient {
}
}
calculateTokensPerSecond(data, totalTimeMs) {
const evalCount = Number(data?.eval_count) || 0;
const evalDurationNs = Number(data?.eval_duration) || 0;
const totalSeconds = Math.max(0, Number(totalTimeMs) || 0) / 1000;
const evalTokensPerSecond = evalDurationNs > 0 && evalCount > 0
? (evalCount / (evalDurationNs / 1_000_000_000))
: 0;
const endToEndTokensPerSecond = totalSeconds > 0 && evalCount > 0
? (evalCount / totalSeconds)
: 0;
// Prefer eval-only throughput when available because it excludes load/setup overhead.
const preferred = evalTokensPerSecond > 0 ? evalTokensPerSecond : endToEndTokensPerSecond;
return {
tokensPerSecond: Math.round(preferred * 10) / 10,
evalTokensPerSecond: Math.round(evalTokensPerSecond * 10) / 10,
endToEndTokensPerSecond: Math.round(endToEndTokensPerSecond * 10) / 10
};
}
async testModelPerformance(modelName, testPrompt = "Hello, how are you?") {
const availability = await this.checkOllamaAvailability();
if (!availability.available) {
@ -413,13 +436,15 @@ class OllamaClient {
const endTime = Date.now();
const totalTime = endTime - startTime;
const tokensGenerated = data.eval_count || 50;
const tokensPerSecond = Math.round((tokensGenerated / (totalTime / 1000)) * 10) / 10;
const tokensGenerated = Number(data.eval_count) || 0;
const speed = this.calculateTokensPerSecond(data, totalTime);
return {
success: true,
responseTime: totalTime,
tokensPerSecond,
tokensPerSecond: speed.tokensPerSecond,
evalTokensPerSecond: speed.evalTokensPerSecond,
endToEndTokensPerSecond: speed.endToEndTokensPerSecond,
tokensGenerated,
loadTime: data.load_duration ? Math.round(data.load_duration / 1000000) : null,
evalTime: data.eval_duration ? Math.round(data.eval_duration / 1000000) : null,

View file

@ -9,6 +9,7 @@ const { animateBanner, renderPersistentBanner } = require('./cli-theme');
const PRIMARY_COMMAND_PRIORITY = [
'check',
'help',
'mcp-setup',
'recommend',
'ai-run',
'ollama-plan',
@ -89,6 +90,90 @@ function tokenizeArgString(rawInput = '') {
return tokens;
}
function getRequiredOptionPrompts(commandMeta) {
const commandOptions = Array.isArray(commandMeta?.command?.options)
? commandMeta.command.options
: [];
const isMandatoryOption = (option) => {
if (!option) return false;
if (typeof option.mandatory === 'boolean') {
return option.mandatory;
}
return Boolean(option.required);
};
return commandOptions
.filter((option) => isMandatoryOption(option))
.map((option) => {
const flag = option.long || option.short || option.flags.split(/[,\s|]+/).find((token) => token.startsWith('-'));
const attributeName =
typeof option.attributeName === 'function'
? option.attributeName()
: String(flag || 'required').replace(/^-+/, '').replace(/[^a-zA-Z0-9]+/g, '_');
const optionType = typeof option.isBoolean === 'function' && option.isBoolean() ? 'boolean' : 'value';
const promptName = `required_${attributeName}`;
const optionFlags = option.flags || flag || attributeName;
const message =
optionType === 'boolean'
? `Required flag ${optionFlags} (enable?):`
: `Required option ${optionFlags}:`;
return {
promptName,
flag,
optionFlags,
variadic: Boolean(option.variadic),
optionType,
message
};
})
.filter((entry) => Boolean(entry.flag));
}
function normalizeVariadicValue(rawValue) {
const tokens = tokenizeArgString(rawValue);
const values = [];
tokens.forEach((token) => {
String(token)
.split(',')
.map((piece) => piece.trim())
.filter(Boolean)
.forEach((piece) => values.push(piece));
});
return values;
}
function buildRequiredOptionArgs(requiredOptionPrompts, answers) {
const args = [];
requiredOptionPrompts.forEach((entry) => {
const answer = answers ? answers[entry.promptName] : undefined;
if (entry.optionType === 'boolean') {
if (answer) args.push(entry.flag);
return;
}
const rawValue = String(answer || '').trim();
if (!rawValue) return;
if (entry.variadic) {
const values = normalizeVariadicValue(rawValue);
if (values.length > 0) {
args.push(entry.flag, ...values);
}
return;
}
args.push(entry.flag, rawValue);
});
return args;
}
function buildCommandCatalog(program) {
return (program.commands || [])
.map((command) => ({
@ -227,6 +312,7 @@ function renderPanel(state, catalog, primaryCommands, options = {}) {
async function collectCommandArgs(commandMeta) {
const prompts = [];
const requiredPrompts = REQUIRED_ARG_PROMPTS[commandMeta.name] || [];
const requiredOptionPrompts = getRequiredOptionPrompts(commandMeta);
const promptOptionalArgs = commandMeta.name !== 'help';
for (const requiredPrompt of requiredPrompts) {
@ -234,15 +320,45 @@ async function collectCommandArgs(commandMeta) {
type: 'input',
name: requiredPrompt.name,
message: requiredPrompt.message,
prefix: ' ',
validate: requiredPrompt.validate
});
}
for (const requiredOptionPrompt of requiredOptionPrompts) {
prompts.push({
type: requiredOptionPrompt.optionType === 'boolean' ? 'confirm' : 'input',
name: requiredOptionPrompt.promptName,
message: requiredOptionPrompt.message,
prefix: ' ',
default: requiredOptionPrompt.optionType === 'boolean' ? true : '',
validate:
requiredOptionPrompt.optionType === 'boolean'
? undefined
: (value) => {
const trimmed = String(value || '').trim();
if (!trimmed) {
return `Provide a value for ${requiredOptionPrompt.optionFlags}`;
}
if (requiredOptionPrompt.variadic) {
const values = normalizeVariadicValue(trimmed);
if (values.length === 0) {
return `Provide at least one value for ${requiredOptionPrompt.optionFlags}`;
}
}
return true;
}
});
}
if (promptOptionalArgs) {
prompts.push({
type: 'input',
name: 'extraArgs',
message: 'Optional extra params (example: --json --limit 5):',
prefix: ' ',
default: ''
});
}
@ -259,6 +375,8 @@ async function collectCommandArgs(commandMeta) {
if (value) args.push(value);
}
args.push(...buildRequiredOptionArgs(requiredOptionPrompts, answers));
if (promptOptionalArgs) {
const extraArgs = String(answers.extraArgs || '').trim();
if (extraArgs) {
@ -371,7 +489,8 @@ async function launchInteractivePanel(options) {
{
type: 'input',
name: 'continue',
message: 'Press Enter to return to the panel'
message: 'Press Enter to return to the panel',
prefix: ' '
}
]);
} catch (error) {
@ -379,7 +498,8 @@ async function launchInteractivePanel(options) {
{
type: 'input',
name: 'continue',
message: `Command failed (${error.message}). Press Enter to continue`
message: `Command failed (${error.message}). Press Enter to continue`,
prefix: ' '
}
]);
} finally {
@ -468,6 +588,9 @@ module.exports = {
launchInteractivePanel,
__private: {
tokenizeArgString,
getRequiredOptionPrompts,
buildRequiredOptionArgs,
normalizeVariadicValue,
buildCommandCatalog,
buildPrimaryCommands,
getVisibleCommands,

17
src/utils/fetch.js Normal file
View file

@ -0,0 +1,17 @@
let nodeFetchPromise = null;
function getNodeFetch() {
if (!nodeFetchPromise) {
nodeFetchPromise = import('node-fetch').then((mod) => mod.default || mod);
}
return nodeFetchPromise;
}
function fetchWithFallback(...args) {
if (typeof globalThis.fetch === 'function') {
return globalThis.fetch(...args);
}
return getNodeFetch().then((fetchImpl) => fetchImpl(...args));
}
module.exports = fetchWithFallback;

View file

@ -0,0 +1,207 @@
function toNumber(value, fallback = 0) {
if (typeof value === 'number' && Number.isFinite(value)) return value;
if (typeof value === 'string') {
const trimmed = value.trim();
if (trimmed) {
const parsed = Number(trimmed);
if (Number.isFinite(parsed)) return parsed;
}
}
return fallback;
}
function normalizeModelSizeB(rawModelSizeB) {
const value = toNumber(rawModelSizeB, 1);
return Math.max(0.7, value);
}
function detectAppleSilicon(architecture, cpuModel, gpuModel) {
const signal = `${architecture} ${cpuModel} ${gpuModel}`.toLowerCase();
return (
signal.includes('apple silicon') ||
/(^|\s)m[1-4](\s|$)/.test(signal) ||
signal.includes('apple m1') ||
signal.includes('apple m2') ||
signal.includes('apple m3') ||
signal.includes('apple m4')
);
}
function isIntegratedGPU(gpuModel = '') {
return /iris.*xe|iris.*graphics|uhd.*graphics|vega.*integrated|radeon.*graphics|intel.*integrated|integrated/i.test(gpuModel);
}
function getAppleSiliconBaseline(cpuModel, gpuModel) {
const signal = `${cpuModel} ${gpuModel}`.toLowerCase();
const profiles = [
{ pattern: /m4 ultra/, tps7b: 95 },
{ pattern: /m4 max/, tps7b: 65 },
{ pattern: /m4 pro/, tps7b: 43 },
{ pattern: /m4/, tps7b: 30 },
{ pattern: /m3 ultra/, tps7b: 88 },
{ pattern: /m3 max/, tps7b: 58 },
{ pattern: /m3 pro/, tps7b: 34 },
{ pattern: /m3/, tps7b: 27 },
{ pattern: /m2 ultra/, tps7b: 80 },
{ pattern: /m2 max/, tps7b: 52 },
{ pattern: /m2 pro/, tps7b: 32 },
{ pattern: /m2/, tps7b: 24 },
{ pattern: /m1 ultra/, tps7b: 72 },
{ pattern: /m1 max/, tps7b: 48 },
{ pattern: /m1 pro/, tps7b: 30 },
{ pattern: /m1/, tps7b: 22 }
];
for (const profile of profiles) {
if (profile.pattern.test(signal)) {
return profile.tps7b;
}
}
return 24;
}
function getDedicatedGPUBaseline(gpuModel, vramGB) {
const signal = (gpuModel || '').toLowerCase();
const profiles = [
{ pattern: /h100/, tps7b: 170 },
{ pattern: /a100/, tps7b: 130 },
{ pattern: /rtx 50|rtx50|blackwell/, tps7b: 105 },
{ pattern: /rtx 4090/, tps7b: 80 },
{ pattern: /rtx 4080/, tps7b: 65 },
{ pattern: /rtx 3090/, tps7b: 62 },
{ pattern: /rtx 3080/, tps7b: 50 },
{ pattern: /rtx 3070/, tps7b: 42 },
{ pattern: /rtx 3060/, tps7b: 34 },
{ pattern: /rtx 20/, tps7b: 26 },
{ pattern: /rx 79|7900|7800/, tps7b: 52 },
{ pattern: /rx 69|6800/, tps7b: 42 }
];
for (const profile of profiles) {
if (profile.pattern.test(signal)) {
return profile.tps7b;
}
}
if (vramGB >= 24) return 60;
if (vramGB >= 16) return 48;
if (vramGB >= 12) return 40;
if (vramGB >= 8) return 30;
if (vramGB >= 4) return 18;
return 14;
}
function hasAVX512(cpuModel = '') {
const signal = cpuModel.toLowerCase();
return signal.includes('avx512') ||
(signal.includes('intel') &&
(signal.includes('12th') || signal.includes('13th') || signal.includes('14th')));
}
function hasAVX2(cpuModel = '') {
const signal = cpuModel.toLowerCase();
return signal.includes('avx2') || signal.includes('intel') || signal.includes('amd');
}
function getCPUBaseline(cpuModel, cores, baseSpeedGHz, integratedGpuAssist) {
let baseline = 4 + (cores * 0.6) + ((Math.max(1.5, baseSpeedGHz) - 2.0) * 2.5);
if (hasAVX512(cpuModel)) baseline += 3;
else if (hasAVX2(cpuModel)) baseline += 1.5;
if (integratedGpuAssist) baseline += 1;
const maxBaseline = integratedGpuAssist ? 22 : 18;
return Math.max(3, Math.min(maxBaseline, baseline));
}
function calculateSizeScale(modelSizeB) {
const scale = Math.pow(7 / modelSizeB, 0.72);
return Math.max(0.18, Math.min(2.2, scale));
}
function calculateMemoryFactor(modelSizeB, availableInferenceMemoryGB) {
const estimatedWorkingSetGB = (modelSizeB * 0.75) + 2;
const ratio = availableInferenceMemoryGB / Math.max(1, estimatedWorkingSetGB);
if (ratio >= 1.2) return 1.05;
if (ratio >= 1.0) return 1.0;
if (ratio >= 0.75) return 0.85;
if (ratio >= 0.6) return 0.65;
return 0.45;
}
function estimateTokenSpeedFromHardware(hardware = {}, options = {}) {
const cpuModel = String(hardware.cpu?.brand || hardware.cpu?.model || '');
const gpuModel = String(hardware.gpu?.model || '');
const architecture = String(hardware.cpu?.architecture || '');
const modelSizeB = normalizeModelSizeB(options.modelSizeB);
const cores = Math.max(1, toNumber(hardware.cpu?.physicalCores || hardware.cpu?.cores, 1));
const baseSpeedGHz = Math.max(1.5, toNumber(hardware.cpu?.speed || hardware.cpu?.speedMax, 2.4));
const memoryTotalGB = Math.max(
2,
toNumber(
hardware.memory?.total ||
hardware.memory?.totalGB ||
hardware.memory_gb,
8
)
);
const vramGB = Math.max(
0,
toNumber(
hardware.gpu?.vram ||
hardware.gpu?.vramGB ||
hardware.gpu?.totalVRAM ||
hardware.gpu?.memory?.total,
0
)
);
const appleSilicon = detectAppleSilicon(architecture, cpuModel, gpuModel);
const integrated = isIntegratedGPU(gpuModel);
const dedicatedGPU = vramGB > 0 && !integrated && !appleSilicon;
let baselineTPS7B;
let backend;
let availableInferenceMemoryGB;
if (appleSilicon) {
backend = 'metal';
baselineTPS7B = getAppleSiliconBaseline(cpuModel, gpuModel);
availableInferenceMemoryGB = memoryTotalGB * 0.82;
} else if (dedicatedGPU) {
backend = 'gpu';
baselineTPS7B = getDedicatedGPUBaseline(gpuModel, vramGB);
availableInferenceMemoryGB = vramGB + Math.min(memoryTotalGB * 0.15, 8);
} else {
backend = integrated ? 'integrated' : 'cpu';
baselineTPS7B = getCPUBaseline(cpuModel, cores, baseSpeedGHz, integrated);
availableInferenceMemoryGB = memoryTotalGB * 0.65;
}
const sizeScale = calculateSizeScale(modelSizeB);
const memoryFactor = calculateMemoryFactor(modelSizeB, availableInferenceMemoryGB);
const maxTPS = backend === 'metal' ? 140 : backend === 'gpu' ? 220 : 35;
const minTPS = backend === 'cpu' ? 1 : 2;
const estimated = baselineTPS7B * sizeScale * memoryFactor;
const tokensPerSecond = Math.max(minTPS, Math.min(maxTPS, Math.round(estimated)));
return {
tokensPerSecond,
backend,
baselineTPS7B: Math.round(baselineTPS7B * 10) / 10,
sizeScale: Math.round(sizeScale * 1000) / 1000,
memoryFactor: Math.round(memoryFactor * 1000) / 1000,
modelSizeB
};
}
module.exports = {
estimateTokenSpeedFromHardware
};

View file

@ -1,7 +1,11 @@
const assert = require('assert');
const { Command } = require('commander');
const {
__private: {
tokenizeArgString,
getRequiredOptionPrompts,
buildRequiredOptionArgs,
normalizeVariadicValue,
buildCommandCatalog,
buildPrimaryCommands,
getVisibleCommands,
@ -30,6 +34,43 @@ function run() {
assert.strictEqual(truncateText('short', 10), 'short');
assert.strictEqual(truncateText('abcdefghij', 6), 'abc...');
assert.deepStrictEqual(
normalizeVariadicValue('qwen2.5-coder:7b llama3.2:3b,mistral:7b'),
['qwen2.5-coder:7b', 'llama3.2:3b', 'mistral:7b']
);
const calibrateCommand = new Command('calibrate');
calibrateCommand.requiredOption('--suite <file>', 'Prompt suite path');
calibrateCommand.requiredOption('--models <identifiers...>', 'Models');
calibrateCommand.requiredOption('--output <file>', 'Output path');
calibrateCommand.option('--runtime <runtime>', 'Runtime backend', 'ollama');
calibrateCommand.option('--dry-run', 'Dry run mode');
const requiredOptionPrompts = getRequiredOptionPrompts({ command: calibrateCommand });
assert.deepStrictEqual(
requiredOptionPrompts.map((entry) => entry.optionFlags),
['--suite <file>', '--models <identifiers...>', '--output <file>'],
'only mandatory options should be discovered for interactive prompts'
);
const requiredOptionArgs = buildRequiredOptionArgs(requiredOptionPrompts, {
required_suite: './suite.jsonl',
required_models: 'qwen2.5-coder:7b, llama3.2:3b',
required_output: './calibration.json'
});
assert.deepStrictEqual(
requiredOptionArgs,
[
'--suite',
'./suite.jsonl',
'--models',
'qwen2.5-coder:7b',
'llama3.2:3b',
'--output',
'./calibration.json'
],
'required prompt answers should translate to runnable CLI args'
);
const mockProgram = {
commands: [

View file

@ -0,0 +1,136 @@
const assert = require('assert');
const fs = require('fs');
const os = require('os');
const path = require('path');
const { spawnSync } = require('child_process');
const BIN_PATH = path.resolve(__dirname, '..', 'bin', 'enhanced_cli.js');
const SAMPLE_SUITE = path.resolve(
__dirname,
'..',
'docs',
'fixtures',
'calibration',
'sample-suite.jsonl'
);
function stripAnsi(text = '') {
return String(text).replace(/\u001b\[[0-9;]*m/g, '');
}
function runCli(args, options = {}) {
const result = spawnSync(process.execPath, [BIN_PATH, ...args], {
cwd: options.cwd || process.cwd(),
encoding: 'utf8',
timeout: options.timeoutMs || 120000,
env: {
...process.env,
NO_COLOR: '1'
}
});
return {
...result,
stdout: stripAnsi(result.stdout || ''),
stderr: stripAnsi(result.stderr || '')
};
}
function assertExitOk(result, context) {
const combined = `${result.stdout}\n${result.stderr}`.trim();
assert.strictEqual(result.status, 0, `${context}\n${combined}`);
}
function run() {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'llm-checker-local-e2e-'));
try {
const calibrationOutput = path.join(tempDir, 'calibration-result.json');
const policyPath = path.join(tempDir, 'policy.yaml');
const auditPath = path.join(tempDir, 'audit.json');
const mcpSetupResult = runCli(['mcp-setup', '--json']);
assertExitOk(mcpSetupResult, 'mcp-setup --json failed');
const mcpSetup = JSON.parse(mcpSetupResult.stdout);
assert.strictEqual(mcpSetup.recommended.command, 'claude');
assert.ok(Array.isArray(mcpSetup.recommended.args) && mcpSetup.recommended.args.length >= 4);
const hwDetectResult = runCli(['hw-detect', '--json']);
assertExitOk(hwDetectResult, 'hw-detect --json failed');
const hardware = JSON.parse(hwDetectResult.stdout);
assert.ok(hardware && hardware.summary, 'hardware summary should exist');
const checkResult = runCli(['check', '--use-case', 'general', '--limit', '1', '--runtime', 'ollama', '--no-verbose']);
assertExitOk(checkResult, 'check command failed');
assert.ok(checkResult.stdout.includes('RECOMMENDED MODEL'), 'check output should include recommendation block');
const recommendResult = runCli(['recommend', '--no-verbose']);
assertExitOk(recommendResult, 'recommend command failed');
assert.ok(
recommendResult.stdout.includes('INTELLIGENT RECOMMENDATIONS BY CATEGORY'),
'recommend output should include recommendations section'
);
const calibrateResult = runCli([
'calibrate',
'--suite',
SAMPLE_SUITE,
'--models',
'qwen2.5-coder:7b,llama3.2:3b',
'--output',
calibrationOutput,
'--dry-run'
]);
assertExitOk(calibrateResult, 'calibrate command failed');
assert.ok(fs.existsSync(calibrationOutput), 'calibration output should exist');
const calibrationJson = JSON.parse(fs.readFileSync(calibrationOutput, 'utf8'));
assert.strictEqual(calibrationJson.execution_mode, 'dry-run');
const policyInitResult = runCli(['policy', 'init', '--file', policyPath, '--force']);
assertExitOk(policyInitResult, 'policy init failed');
assert.ok(fs.existsSync(policyPath), 'policy file should exist');
let policyContent = fs.readFileSync(policyPath, 'utf8');
policyContent = policyContent.replace(/mode:\s*enforce/g, 'mode: audit');
fs.writeFileSync(policyPath, policyContent, 'utf8');
const policyValidateResult = runCli(['policy', 'validate', '--file', policyPath]);
assertExitOk(policyValidateResult, 'policy validate failed');
const auditResult = runCli([
'audit',
'export',
'--policy',
policyPath,
'--command',
'recommend',
'--format',
'json',
'--out',
auditPath,
'--runtime',
'ollama',
'--optimize',
'balanced',
'--no-verbose'
]);
assertExitOk(auditResult, 'audit export failed');
assert.ok(fs.existsSync(auditPath), 'audit output should exist');
const auditJson = JSON.parse(fs.readFileSync(auditPath, 'utf8'));
assert.strictEqual(auditJson.command, 'recommend');
console.log('local-features-e2e.test.js: OK');
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
}
if (require.main === module) {
try {
run();
} catch (error) {
console.error('local-features-e2e.test.js: FAILED');
console.error(error);
process.exit(1);
}
}
module.exports = { run };

View file

@ -0,0 +1,246 @@
const assert = require('assert');
const path = require('path');
const { spawnSync } = require('child_process');
const OllamaClient = require('../src/ollama/client');
const fetch = require('../src/utils/fetch');
const DeterministicModelSelector = require('../src/models/deterministic-selector');
const AICheckSelector = require('../src/models/ai-check-selector');
const ROOT = path.resolve(__dirname, '..');
const CLI = path.resolve(ROOT, 'bin', 'enhanced_cli.js');
function stripAnsi(text = '') {
return String(text).replace(/\u001b\[[0-9;]*m/g, '');
}
function roundToSingleDecimal(value) {
return Math.round(Number(value || 0) * 10) / 10;
}
function isCloudPlaceholder(model) {
const name = String(model?.name || '').toLowerCase();
const source = String(model?.source || '').toLowerCase();
const type = String(model?.type || model?.model_type || '').toLowerCase();
const fileSizeGB = Number(model?.fileSizeGB) || 0;
const cloudTagged = (
name.includes('-cloud') ||
name.endsWith(':cloud') ||
source.includes('cloud') ||
type === 'cloud' ||
type === 'remote' ||
type === 'hosted'
);
return cloudTagged && fileSizeGB <= 0;
}
function parseModelSizeB(sizeLabel = '') {
const match = String(sizeLabel).trim().toUpperCase().match(/^(\d+\.?\d*)\s*([BM])$/);
if (!match) return 1;
const value = parseFloat(match[1]);
return match[2] === 'M' ? value / 1000 : value;
}
function runCli(args) {
return spawnSync(process.execPath, [CLI, ...args], {
cwd: ROOT,
encoding: 'utf8',
env: { ...process.env, NO_COLOR: '1' }
});
}
async function run() {
const client = new OllamaClient('http://localhost:11434');
const availability = await client.checkOllamaAvailability();
assert.strictEqual(
availability.available,
true,
`Ollama must be available for real integration test. Details: ${JSON.stringify(availability)}`
);
const versionResponse = await fetch('http://localhost:11434/api/version', {
headers: { 'Content-Type': 'application/json' }
});
assert(versionResponse.ok, `Expected /api/version to return HTTP 200, got ${versionResponse.status}`);
const versionBody = await versionResponse.json();
assert(versionBody.version, 'Expected version payload from Ollama');
const fallbackFetchScript = `
globalThis.fetch = undefined;
const fetch = require('./src/utils/fetch');
fetch('http://localhost:11434/api/version', { headers: { 'Content-Type': 'application/json' } })
.then(async (res) => {
if (!res.ok) throw new Error('HTTP ' + res.status);
const body = await res.json();
if (!body.version) throw new Error('Missing version');
process.stdout.write(body.version);
})
.catch((error) => {
console.error(error.message);
process.exit(1);
});
`;
const fallbackFetchResult = spawnSync(process.execPath, ['-e', fallbackFetchScript], {
cwd: ROOT,
encoding: 'utf8',
env: { ...process.env, NO_COLOR: '1' }
});
assert.strictEqual(
fallbackFetchResult.status,
0,
`node-fetch fallback path failed: ${fallbackFetchResult.stderr || fallbackFetchResult.stdout}`
);
const localModels = await client.getLocalModels();
assert(localModels.length > 0, 'Expected at least one local Ollama model');
const runnableModels = localModels.filter((model) => !isCloudPlaceholder(model) && Number(model.fileSizeGB) > 0);
assert(runnableModels.length > 0, 'Expected at least one runnable local model (non-cloud)');
const probeModel = [...runnableModels].sort((a, b) => a.fileSizeGB - b.fileSizeGB)[0];
const probeModelSizeB = parseModelSizeB(probeModel.size);
const generationPrompt = 'Explica en 80 palabras por que las pruebas de integracion son utiles.';
const generationRequest = {
model: probeModel.name,
prompt: generationPrompt,
stream: false,
options: { num_predict: 120, temperature: 0.2 }
};
const generationStartMs = Date.now();
const generationResponse = await fetch('http://localhost:11434/api/generate', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(generationRequest)
});
assert(generationResponse.ok, `Expected /api/generate to return 200, got ${generationResponse.status}`);
const generationData = await generationResponse.json();
const totalTimeMs = Date.now() - generationStartMs;
assert((Number(generationData.eval_count) || 0) > 0, 'Expected eval_count > 0 from real generation');
const speed = client.calculateTokensPerSecond(generationData, totalTimeMs);
const expectedEvalTokensPerSecond = Number(generationData.eval_duration) > 0
? roundToSingleDecimal(Number(generationData.eval_count) / (Number(generationData.eval_duration) / 1_000_000_000))
: 0;
const expectedEndToEndTokensPerSecond = roundToSingleDecimal(
Number(generationData.eval_count) / Math.max(totalTimeMs / 1000, 0.001)
);
const expectedPreferred = expectedEvalTokensPerSecond > 0
? expectedEvalTokensPerSecond
: expectedEndToEndTokensPerSecond;
assert.strictEqual(
speed.evalTokensPerSecond,
expectedEvalTokensPerSecond,
`Eval throughput mismatch. Expected ${expectedEvalTokensPerSecond}, got ${speed.evalTokensPerSecond}`
);
assert.strictEqual(
speed.endToEndTokensPerSecond,
expectedEndToEndTokensPerSecond,
`End-to-end throughput mismatch. Expected ${expectedEndToEndTokensPerSecond}, got ${speed.endToEndTokensPerSecond}`
);
assert.strictEqual(
speed.tokensPerSecond,
expectedPreferred,
`Preferred throughput mismatch. Expected ${expectedPreferred}, got ${speed.tokensPerSecond}`
);
const performance = await client.testModelPerformance(probeModel.name, 'Responde solo: ok');
assert.strictEqual(performance.success, true, 'Expected testModelPerformance success');
assert(performance.tokensPerSecond > 0, `Expected positive tokensPerSecond, got ${performance.tokensPerSecond}`);
assert(
Number.isFinite(performance.evalTokensPerSecond) && Number.isFinite(performance.endToEndTokensPerSecond),
'Expected evalTokensPerSecond and endToEndTokensPerSecond in performance result'
);
if (performance.evalTokensPerSecond > 0) {
assert.strictEqual(
performance.tokensPerSecond,
performance.evalTokensPerSecond,
'When eval throughput exists, tokensPerSecond must match eval throughput'
);
}
const deterministicSelector = new DeterministicModelSelector();
const probeTPS = await deterministicSelector.runSingleProbe(probeModel.name, 'general');
assert(Number.isFinite(probeTPS) && probeTPS > 0, `Expected positive real probe TPS, got ${probeTPS}`);
const aiCheckSelector = new AICheckSelector();
const aiResult = await aiCheckSelector.callOllamaEvaluator(probeModel.name, {
hardware: { category: 'general' },
candidates: [
{
name: probeModel.name,
paramsB: probeModelSizeB,
quant: String(probeModel.quantization || 'Q4'),
requiredGB: Math.max(1, Number(probeModel.fileSizeGB) || 1),
budgetGB: 24,
installed: true
}
]
});
assert.strictEqual(aiResult.winner, probeModel.name, `Expected evaluator winner to be ${probeModel.name}`);
assert(Array.isArray(aiResult.ranking), 'Expected ranking array in evaluator response');
assert.strictEqual(aiResult.ranking.length, 1, 'Expected single ranking entry with one candidate');
assert.strictEqual(aiResult.ranking[0].name, probeModel.name, 'Expected ranking to include selected probe model');
const planResult = runCli(['ollama-plan', '--json']);
assert.strictEqual(
planResult.status,
0,
`ollama-plan --json failed: ${stripAnsi(planResult.stderr || planResult.stdout)}`
);
const planPayload = JSON.parse(planResult.stdout);
const selectedModels = planPayload.selection?.selected || [];
assert(selectedModels.length > 0, 'Expected selected models from ollama-plan');
assert(
selectedModels.every((name) => !/-cloud|:cloud$/i.test(String(name))),
`Cloud placeholder models should be excluded from default plan selection: ${selectedModels.join(', ')}`
);
const cloudPlaceholder = localModels.find((model) => isCloudPlaceholder(model));
if (cloudPlaceholder) {
assert(
!selectedModels.includes(cloudPlaceholder.name),
`Cloud placeholder model ${cloudPlaceholder.name} must not be in default selected models`
);
const cloudOnlyResult = runCli(['ollama-plan', '--models', cloudPlaceholder.name, '--json']);
assert.notStrictEqual(
cloudOnlyResult.status,
0,
'Expected non-zero exit when planning only a cloud placeholder model'
);
const cloudOutput = stripAnsi(`${cloudOnlyResult.stdout}\n${cloudOnlyResult.stderr}`);
assert(
cloudOutput.includes('No matching local models found'),
`Expected cloud-only selection to fail with local model message. Output: ${cloudOutput}`
);
}
const singleModelPlanResult = runCli(['ollama-plan', '--models', probeModel.name, '--json']);
assert.strictEqual(
singleModelPlanResult.status,
0,
`ollama-plan for ${probeModel.name} failed: ${stripAnsi(singleModelPlanResult.stderr || singleModelPlanResult.stdout)}`
);
const singleModelPlan = JSON.parse(singleModelPlanResult.stdout);
assert.deepStrictEqual(singleModelPlan.selection.selected, [probeModel.name]);
assert(
singleModelPlan.plan?.memory?.recommendedEstimatedGB <= (singleModelPlan.plan?.memory?.budgetGB + 0.2),
'Expected recommended memory to fit within budget for single-model plan'
);
console.log('ollama-client-speed-metrics.test.js: OK');
}
if (require.main === module) {
run().catch((error) => {
console.error('ollama-client-speed-metrics.test.js: FAILED');
console.error(error);
process.exit(1);
});
}
module.exports = { run };

View file

@ -11,11 +11,14 @@ const TESTS = [
{ name: 'CUDA Jetson detection', file: 'cuda-jetson-detection.test.js', category: 'Hardware' },
{ name: 'Hardware simulation scoring', file: 'hardware-simulation-tests.js', category: 'Hardware' },
{ name: 'Hardware detector regression', file: 'hardware-detector-regression.js', category: 'Hardware' },
{ name: 'Token speed estimation', file: 'token-speed-estimation.test.js', category: 'Performance' },
{ name: 'Deterministic model pool', file: 'deterministic-model-pool-check.js', category: 'Recommendations' },
{ name: 'Ollama capacity planner', file: 'ollama-capacity-planner.test.js', category: 'Ollama' },
{ name: 'Ollama speed metrics', file: 'ollama-client-speed-metrics.test.js', category: 'Ollama' },
{ name: 'Runtime + speculative decoding', file: 'runtime-specdec-tests.js', category: 'Runtime' },
{ name: 'CLI interface smoke', file: 'ui-cli-smoke.test.js', category: 'UI' },
{ name: 'CLI interactive panel helpers', file: 'cli-interactive-panel.test.js', category: 'UI' },
{ name: 'Local features end-to-end', file: 'local-features-e2e.test.js', category: 'E2E' },
{ name: 'Calibration schema', file: 'calibration-schema.test.js', category: 'Calibration' },
{ name: 'Calibration command', file: 'calibrate-command.test.js', category: 'Calibration' },
{ name: 'Calibration end-to-end', file: 'calibration-e2e-integration.test.js', category: 'Calibration' },

View file

@ -0,0 +1,98 @@
const assert = require('assert');
const HardwareDetector = require('../src/hardware/detector');
const PerformanceAnalyzer = require('../analyzer/performance');
const ExpandedModelsDatabase = require('../src/models/expanded_database');
const { estimateTokenSpeedFromHardware } = require('../src/utils/token-speed-estimator');
function isM4ProMac(hardware) {
const cpuBrand = String(hardware?.cpu?.brand || '').toLowerCase();
const gpuModel = String(hardware?.gpu?.model || '').toLowerCase();
return (
String(hardware?.os?.platform || '').toLowerCase() === 'darwin' &&
String(hardware?.cpu?.architecture || '').toLowerCase().includes('apple') &&
(cpuBrand.includes('m4 pro') || gpuModel.includes('m4 pro'))
);
}
async function run() {
const detector = new HardwareDetector();
const hardware = await detector.getSystemInfo(true);
assert.strictEqual(
String(hardware.os?.platform || '').toLowerCase(),
'darwin',
`Expected macOS platform, got ${hardware.os?.platform}`
);
assert(
String(hardware.cpu?.architecture || '').toLowerCase().includes('apple'),
`Expected Apple Silicon architecture, got ${hardware.cpu?.architecture}`
);
assert(
Number(hardware.memory?.total) >= 16,
`Expected at least 16GB RAM on reference Mac, got ${hardware.memory?.total}GB`
);
const speed3B = estimateTokenSpeedFromHardware(hardware, { modelSizeB: 3, modelName: 'llama3.2:3b' });
const speed8B = estimateTokenSpeedFromHardware(hardware, { modelSizeB: 8, modelName: 'llama3.1:8b' });
const speed13B = estimateTokenSpeedFromHardware(hardware, { modelSizeB: 13, modelName: 'llama3.1:13b' });
assert.strictEqual(speed3B.backend, 'metal', `Expected metal backend on Apple Silicon, got ${speed3B.backend}`);
assert(
speed3B.tokensPerSecond > speed8B.tokensPerSecond && speed8B.tokensPerSecond > speed13B.tokensPerSecond,
`Expected monotonic scaling 3B > 8B > 13B, got ${speed3B.tokensPerSecond} > ${speed8B.tokensPerSecond} > ${speed13B.tokensPerSecond}`
);
assert(speed8B.tokensPerSecond > 0, `Expected positive TPS for 8B, got ${speed8B.tokensPerSecond}`);
if (isM4ProMac(hardware)) {
assert(
speed8B.tokensPerSecond >= 30 && speed8B.tokensPerSecond <= 70,
`M4 Pro 8B estimate out of expected band (30-70 t/s), got ${speed8B.tokensPerSecond}`
);
}
const analyzer = new PerformanceAnalyzer();
const parsedSizeFromGB = analyzer.parseModelSize('4.9GB');
assert(
parsedSizeFromGB >= 7 && parsedSizeFromGB <= 9.5,
`parseModelSize should infer params from GB notation, got ${parsedSizeFromGB}`
);
const perfEstimate = analyzer.calculateRealisticPerformance(
{ name: 'llama3.1:8b', size: '8B', requirements: { ram: 10 } },
hardware
);
assert.strictEqual(
perfEstimate.estimatedTokensPerSecond,
speed8B.tokensPerSecond,
`Performance analyzer should use shared estimator (expected ${speed8B.tokensPerSecond}, got ${perfEstimate.estimatedTokensPerSecond})`
);
const database = new ExpandedModelsDatabase();
const dbSpeed = database.calculateRealisticTokensPerSecond(
{ name: 'llama3.1:8b', size: '8B' },
hardware
);
assert.strictEqual(
dbSpeed,
speed8B.tokensPerSecond,
`Expanded database estimator should align with shared estimator (expected ${speed8B.tokensPerSecond}, got ${dbSpeed})`
);
const tinyParams = database.extractModelParams({ name: 'all-minilm:22m', size: '22M' });
assert(
tinyParams > 0 && tinyParams < 0.1,
`extractModelParams should convert M suffix to billions, got ${tinyParams}`
);
console.log('token-speed-estimation.test.js: OK');
}
if (require.main === module) {
run().catch((error) => {
console.error('token-speed-estimation.test.js: FAILED');
console.error(error);
process.exit(1);
});
}
module.exports = { run };

View file

@ -25,6 +25,7 @@ function run() {
assert.ok(stripAnsi(help.stdout).includes('recommend'), 'top-level help should list recommend command');
assert.ok(stripAnsi(help.stdout).includes('calibrate'), 'top-level help should list calibrate command');
assert.ok(stripAnsi(help.stdout).includes('ollama-plan'), 'top-level help should list ollama-plan command');
assert.ok(stripAnsi(help.stdout).includes('mcp-setup'), 'top-level help should list mcp-setup command');
const commandHelp = runCli(['help']);
assert.strictEqual(commandHelp.status, 0, stripAnsi(commandHelp.stderr || commandHelp.stdout));
@ -84,6 +85,23 @@ function run() {
'ollama-plan help should expose objective option'
);
const mcpSetupHelp = runCli(['mcp-setup', '--help']);
assert.strictEqual(mcpSetupHelp.status, 0, stripAnsi(mcpSetupHelp.stderr || mcpSetupHelp.stdout));
assert.ok(
stripAnsi(mcpSetupHelp.stdout).includes('Show or apply Claude MCP setup for llm-checker'),
'mcp-setup help should describe command purpose'
);
const mcpSetupJson = runCli(['mcp-setup', '--json']);
assert.strictEqual(mcpSetupJson.status, 0, stripAnsi(mcpSetupJson.stderr || mcpSetupJson.stdout));
const parsedMcpSetup = JSON.parse(stripAnsi(mcpSetupJson.stdout));
assert.strictEqual(parsedMcpSetup.recommended.command, 'claude', 'mcp-setup json should target claude CLI');
assert.ok(
Array.isArray(parsedMcpSetup.recommended.args) &&
parsedMcpSetup.recommended.args.slice(0, 2).join(' ') === 'mcp add',
'mcp-setup json should include claude mcp add args'
);
console.log('ui-cli-smoke.test.js: OK');
}