vrr/llm-checker
2
0
Fork 0
mirror of https://github.com/signerless/llm-checker.git synced 2026-10-19 01:49:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(calibration): add routing policy flow and docs hub

Browse source
This commit is contained in:
signerless 2026-02-17 18:49:45 +01:00
parent c967465f45
commit d7a70acf25
25 changed files with 2613 additions and 1393 deletions
Download patch file Download diff file Expand all files Collapse all files

389
ADVANCED_USAGE.md
View file

@ -1,388 +1,5 @@
# Advanced Usage Guide – LLM Checker 2.0 🎛️
# Moved
This document goes beyond the quick‑start, showing how to **tweak, automate and extend LLM Checker for professional workflows**.
This document has moved to:
---
## 🎛️ Advanced Configuration
### Custom configuration file
Create `~/.llm-checker.json` to override defaults:
```json
{
"analysis": {
"defaultUseCase": "code",
"performanceTesting": true
},
"display": {
"maxModelsPerTable": 15,
"compactMode": true
},
"filters": {
"minCompatibilityScore": 75,
"excludeModels": ["very-large-model"]
}
}
```
### Environment variables
```bash
# Disable GPU detection (handy for unit tests)
export LLM_CHECKER_NO_GPU=true
# Force VRAM if auto‑detection fails
export LLM_CHECKER_VRAM_GB=8
# Point to a remote Ollama daemon
export OLLAMA_BASE_URL=http://remote-server:11434
# Verbose logs
export LLM_CHECKER_LOG_LEVEL=debug
# Custom cache directory
export LLM_CHECKER_CACHE_DIR=/custom/cache/path
# Disable coloured output (for scripts/CI)
export NO_COLOR=1
```
---
## 🔧 Advanced Workflows
### 1 — Automated analysis for CI/CD
```bash
#!/usr/bin/env bash
# ci-llm-check.sh – Continuous Integration helper
# Silent JSON output
llm-checker check --format json --quiet > hardware-analysis.json
# Fail the job if no compatible models
COMPATIBLE_MODELS=$(jq '.compatible | length' < hardware-analysis.json)
if [ "$COMPATIBLE_MODELS" -gt 0 ]; then
echo "✅ Hardware supports $COMPATIBLE_MODELS model(s)"
exit 0
else
echo "❌ Hardware not powerful enough for local LLMs"
exit 1
fi
```
### 2 — Automated developer setup
```bash
#!/usr/bin/env bash
# setup-dev-environment.sh
echo "🔍 Analysing hardware…"
llm-checker check --use-case code --filter medium --ollama-only > analysis.txt
# Extract recommended installation commands
INSTALL_COMMANDS=$(grep "ollama pull" analysis.txt)
echo "📦 Installing recommended models…"
echo "$INSTALL_COMMANDS" | head -3 | while read cmd; do
echo "Running: $cmd"
$cmd
done
echo "✅ Development environment ready"
```
### 3 — Continuous performance monitoring
```bash
#!/usr/bin/env bash
# monitor-performance.sh
while true; do
echo "$(date): System health check…"
# Benchmark
llm-checker check --performance-test --quiet | grep "tokens/sec"
# Running Ollama models
llm-checker ollama --running
sleep 300 # every 5 min
done
```
---
## 📊 Comparative Analysis
### Compare multiple RAM configs
```bash
echo "RAM sensitivity test:"
for ram in 8 16 32 64; do
echo "=== ${ram} GB RAM ==="
LLM_CHECKER_RAM_GB=$ram llm-checker check --quiet | grep "Compatible:"
done
```
### Benchmark every installed model
```bash
echo "# Performance Report – $(date)" > performance-report.md
ollama list | grep -v NAME | awk '{print $1}' | while read model; do
echo "## $model" >> performance-report.md
llm-checker ollama --test "$model" >> performance-report.md
echo "" >> performance-report.md
done
```
---
## 🛠️ IDE & Editor Integration
### VS Code tasks
```jsonc
// .vscode/tasks.json
{
"version": "2.0.1",
"tasks": [
{
"label": "Check LLM Compatibility",
"type": "shell",
"command": "llm-checker",
"args": ["check", "--use-case", "code", "--detailed"],
"group": "build",
"presentation": {
"echo": true,
"reveal": "always",
"focus": false,
"panel": "shared"
}
},
{
"label": "Install Best Code Model",
"type": "shell",
"command": "bash",
"args": [
"-c",
"llm-checker check --use-case code --ollama-only | grep 'ollama pull' | head -1 | bash"
],
"group": "build"
}
]
}
```
### Vim / Neovim Lua helper
```lua
-- lua/llm-checker.lua
local M = {}
function M.check_compatibility()
local output = vim.fn.system('llm-checker check --use-case code --quiet')
vim.api.nvim_echo({{output, 'Normal'}}, false, {})
end
function M.install_best_model()
local cmd = vim.fn.system("llm-checker check --use-case code --ollama-only | grep 'ollama pull' | head -1")
if cmd ~= '' then
vim.fn.system(cmd)
print('Installed model: ' .. cmd)
end
end
return M
```
---
## 🔍 Advanced Troubleshooting
### Full debug mode
```bash
export DEBUG=1
export LLM_CHECKER_LOG_LEVEL=debug
llm-checker check --detailed 2>&1 | tee debug.log
```
### Common issues & fixes
#### 1 — Wrong hardware detection
```bash
llm-checker check --detailed | grep -A10 "Hardware"
# Override manually
export LLM_CHECKER_RAM_GB=16
export LLM_CHECKER_VRAM_GB=8
export LLM_CHECKER_CPU_CORES=8
```
#### 2 — Ollama not detected
```bash
# Check service
systemctl status ollama # Linux
brew services list | grep ollama # macOS
# Test API
curl http://localhost:11434/api/version
# Custom URL
export OLLAMA_BASE_URL=http://192.168.1.100:11434
```
#### 3 — Models not marked compatible
```bash
# Remove score filter
llm-checker check --min-score 0 --include-all
# Model‑level diagnostics
llm-checker analyze-model "TinyLlama 1.1B"
```
---
## 📈 Performance Optimisation
### Architecture‑specific flags
#### Apple Silicon (M1/M2/M3)
```bash
export LLAMA_METAL=1 # Use Metal backend
llm-checker check --filter small,medium --detailed | grep -A5 "Apple Silicon"
```
#### NVIDIA GPUs
```bash
nvidia-smi # Verify CUDA
llm-checker check --gpu-acceleration cuda
```
#### AMD GPUs
```bash
rocm-smi # Verify ROCm
export LLAMA_OPENCL=1 # OpenCL fallback
```
### Quantisation tuning
```bash
for quant in Q2_K Q4_0 Q4_K_M Q5_0 Q8_0; do
echo "=== $quant ==="
llm-checker analyze --quantization $quant
done
```
---
## 🚀 Specific Use Cases
### 1 — Building AI apps
```bash
llm-checker check --use-case chat --filter small,medium
ollama pull $(llm-checker check --use-case chat | grep "ollama pull" | head -1 | cut -d' ' -f3)
curl -X POST http://localhost:11434/api/generate -H "Content-Type: application/json" -d '{"model":"llama3.2:3b","prompt":"Hello!","stream":false}'
```
### 2 — Large‑scale code analysis
```bash
ollama pull codellama:7b
echo 'def fibonacci(n): pass' | ollama run codellama:7b "Complete this function:"
```
### 3 — Semantic search
```bash
llm-checker check --filter embeddings
ollama pull all-minilm
echo "machine learning" | ollama run all-minilm "Generate embedding:"
```
---
## 📊 Reporting & Metrics
### Automated reports
```bash
llm-checker check --detailed --export json > report.json
llm-checker check --detailed --export html > report.html
llm-checker check --detailed --export csv > report.csv
# Trend tracking
echo "$(date),$(llm-checker check --quiet | grep 'Compatible:' | grep -o '[0-9]*')" >> compatibility-trends.csv
```
### Adoption metrics
```bash
llm-checker ollama --list --format json | jq '.[] | {name, sizeGB: .fileSizeGB, lastUsed: .modified}'
find ~/.llm-checker/benchmarks -name "*.json" | xargs jq '.tokensPerSecond' | sort -n
```
---
## 🔐 Security & Privacy
### Fully offline mode
```bash
export LLM_CHECKER_OFFLINE=1
export LLM_CHECKER_NO_UPDATE_CHECK=1
llm-checker check --offline --no-cloud
```
### Sanitise logs
```bash
llm-checker check --anonymize-hardware > safe-report.txt
```
---
## 🤝 Contribution & Extensibility
### Adding custom models
```javascript
// custom-models.js
module.exports = [
{
name: "MyCustom Model 7B",
size: "7B",
type: "local",
category: "medium",
requirements: { ram: 8, vram: 4, cpu_cores: 4, storage: 7 },
frameworks: ["ollama"],
installation: { ollama: "ollama pull mycustom:7b" }
}
];
```
### Plugin system
```bash
export LLM_CHECKER_PLUGINS_DIR=~/.llm-checker/plugins
llm-checker check --load-plugins
```
---
Made with ❤️ by **Pavelevich**.
- [docs/guides/advanced-usage.md](docs/guides/advanced-usage.md)

60
CHANGELOG.md
View file

@ -1,59 +1,5 @@
Changelog
=========
# Moved
3.2.8 — Multimodal Classification Hotfix (2026-02-17)
-----------------------------------------------------
This document has moved to:
- Fixed false multimodal recommendations caused by noisy `input_types` metadata (for example, coding models incorrectly marked as image-capable by upstream scraping noise).
- Hardened modality inference: `input_types=image` alone is no longer enough; recommendation logic now also requires explicit multimodal metadata or strong vision naming/context hints.
- Added deterministic regression coverage to ensure coding-only models are excluded from multimodal picks when metadata is ambiguous.
3.2.7 — License Update: No Paid Distribution (2026-02-17)
----------------------------------------------------------
- Replaced MIT license with **NPDL-1.0** (No Paid Distribution License).
- New license terms allow free use/modification/redistribution but prohibit paid distribution or paid hosted/API delivery without a separate commercial license.
- Updated package metadata (`license: SEE LICENSE IN LICENSE`) and README license badges/section.
3.2.6 — Recommendation & Detection Regression Hardening (2026-02-17)
--------------------------------------------------------------------
- Recommend: enforce feasible 30B-class coverage for capable discrete multi-GPU profiles (non-speed objectives).
- Recommend: add deterministic regression for dual-GPU 36GB aggregate VRAM scenarios.
- Hardware detection: preserve heterogeneous multi-GPU inventory summaries (e.g. mixed V100/P40/M40).
- Hardware mapping/fallbacks:
- Added AMD Radeon AI PRO R9700 (PCI ID `7551`) support path.
- Added NVIDIA GTX 1070 Ti (`1b82`) fallback mapping.
- Re-verified Linux RX 7900 XTX non-ROCm fallback detection path.
- Docs: updated distribution/install notes and recommend optimization profile examples.
3.2.5 — Deterministic Selector Memory Modeling Fixes (2026-02-17)
------------------------------------------------------------------
- MoE memory estimation: fixed active-parameter memory path for deterministic model selection.
- Added deterministic regression coverage for MoE active/fallback parameter handling.
- Improved deterministic recommendation stability for memory-fit edge cases.
3.0.7 — Fix TPS Estimation (2025-12-31)
---------------------------------------
- Fix: TPS was overestimated by 2-10x across all hardware
- Updated speed coefficients to match real Ollama benchmarks:
- H100: 120 TPS (was 400), RTX 4090: 70 TPS (was 260)
- M4 Pro: 45 TPS (was 270), CPU: 5 TPS (was 50)
- Changed quantization baseline from FP16 to Q4_K_M (the most common format)
- Added diminishing returns for small models (1-3B don't scale linearly)
- Added comprehensive hardware simulation test suite (17 test cases)
2.7.2 — Security & Robustness (2025-09-08)
------------------------------------------
- Security: Removed insecure “curl | sh” install instructions from CLI messages and setup script. Now we reference official docs/package managers.
- Network hardening: Added request timeouts and a 5MB response size limit in the Ollama native scraper to prevent hanging connections and excessive memory use.
- Safer caching: Moved Ollama cache to `~/.llm-checker/cache/ollama` with backward-compatible reads from the legacy `src/ollama/.cache` folder.
- CLI updates: Adjusted CLI to read the new cache location with fallback to legacy path.
- No breaking changes: Functionality remains the same; legacy cache is still read. On write, new cache path is used.
2.7.1
------
- Previous version in repository.
- [docs/reference/changelog.md](docs/reference/changelog.md)

81
README.md
View file

@ -24,6 +24,8 @@
<p align="center">
<a href="#installation">Installation</a> &bull;
<a href="#quick-start">Quick Start</a> &bull;
<a href="#calibration-quick-start-10-minutes">Calibration Quick Start</a> &bull;
<a href="docs/README.md">Docs</a> &bull;
<a href="#claude-code-mcp">Claude MCP</a> &bull;
<a href="#commands">Commands</a> &bull;
<a href="#scoring-system">Scoring</a> &bull;
@ -54,6 +56,17 @@ Choosing the right LLM for your hardware is complex. With thousands of model var
---
## Documentation
- [Docs Hub](docs/README.md)
- [Usage Guide](docs/guides/usage-guide.md)
- [Advanced Usage](docs/guides/advanced-usage.md)
- [Technical Reference](docs/reference/technical-docs.md)
- [Changelog](docs/reference/changelog.md)
- [Calibration Fixtures](docs/fixtures/calibration/README.md)
---
## Comparison with Other Tooling (e.g. `llmfit`)
LLM Checker and `llmfit` solve related but different problems:
@ -147,6 +160,51 @@ llm-checker search qwen --use-case coding
---
## Calibration Quick Start (10 Minutes)
This path produces both calibration artifacts and verifies calibrated routing in one pass.
### 1) Use the sample prompt suite
```bash
cp ./docs/fixtures/calibration/sample-suite.jsonl ./sample-suite.jsonl
```
### 2) Generate calibration artifacts (dry-run)
```bash
mkdir -p ./artifacts
llm-checker calibrate \
--suite ./sample-suite.jsonl \
--models qwen2.5-coder:7b llama3.2:3b \
--runtime ollama \
--objective balanced \
--dry-run \
--output ./artifacts/calibration-result.json \
--policy-out ./artifacts/calibration-policy.yaml
```
Artifacts created:
- `./artifacts/calibration-result.json` (calibration contract)
- `./artifacts/calibration-policy.yaml` (routing policy for runtime commands)
### 3) Apply calibrated routing
```bash
llm-checker recommend --calibrated ./artifacts/calibration-policy.yaml --category coding
llm-checker ai-run --calibrated ./artifacts/calibration-policy.yaml --category coding --prompt "Refactor this function"
```
Notes:
- `--policy <file>` has precedence over `--calibrated [file]`.
- If `--calibrated` has no path, discovery uses `~/.llm-checker/calibration-policy.{yaml,yml,json}`.
- `--mode full` currently requires `--runtime ollama`.
- `./docs/fixtures/calibration/sample-generated-policy.yaml` shows the expected policy structure.
---
## Claude Code MCP
LLM Checker includes a built-in [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) server, allowing **Claude Code** and other MCP-compatible AI assistants to analyze your hardware and manage local models directly.
@ -229,6 +287,7 @@ Claude will automatically call the right tools and give you actionable results.
| `hw-detect` | Detect GPU/CPU capabilities, memory, backends |
| `check` | Full system analysis with compatible models and recommendations |
| `recommend` | Intelligent recommendations by category (coding, reasoning, multimodal, etc.) |
| `calibrate` | Generate calibration result + routing policy artifacts from a JSONL prompt suite |
| `installed` | Rank your installed Ollama models by compatibility |
### Advanced Commands (require `sql.js`)
@ -263,6 +322,28 @@ llm-checker check --policy ./policy.yaml --use-case coding --runtime vllm
llm-checker recommend --policy ./policy.yaml --category coding
```
### Calibrated Routing in `recommend` and `ai-run`
`recommend` and `ai-run` now support calibration routing policies generated by `calibrate --policy-out`.
- `--calibrated [file]`:
- If `file` is omitted, discovery defaults to `~/.llm-checker/calibration-policy.{yaml,yml,json}`.
- `--policy <file>` takes precedence over `--calibrated` for routing resolution.
- Resolution precedence:
- `--policy` (explicit)
- `--calibrated` (explicit file or default discovery)
- deterministic selector fallback
- CLI output includes routing provenance (`--policy`, `--calibrated`, or default discovery) and the selected route/model.
Examples:
```bash
llm-checker recommend --calibrated --category coding
llm-checker recommend --calibrated ./calibration-policy.yaml --category reasoning
llm-checker ai-run --calibrated --category coding --prompt "Refactor this function"
llm-checker ai-run --policy ./calibration-policy.yaml --prompt "Summarize this report"
```
### Policy Audit Export
Use `audit export` when you need machine-readable compliance evidence for CI/CD gates, governance reviews, or security tooling.

605
TECHNICAL_DOCS.md
View file

@ -1,604 +1,5 @@
# 🔧 LLM Checker - Technical Documentation
# Moved
Detailed technical documentation for developers, contributors, and advanced users.
This document has moved to:
---
## 🏗️ **Architecture Overview**
### **System Design**
```
┌─────────────────────────────────────────────────┐
│ CLI Interface │
│ (enhanced_cli.js) │
└─────────────────────┬───────────────────────────┘
│
┌─────────────────────▼───────────────────────────┐
│ Main LLM Checker │
│ (src/index.js) │
└─────────┬───────────────────────────┬───────────┘
│ │
┌─────────▼─────────┐ ┌─────────▼─────────┐
│ Hardware │ │ AI Model │
│ Detection │ │ Selector │
│ │ │ │
│ • CPU/RAM/GPU │ │ • Mathematical │
│ • Tier Analysis │ │ Algorithm │
│ • Performance │ │ • 5-Factor │
│ Calculation │ │ Scoring │
└─────────┬─────────┘ └─────────┬─────────┘
│ │
└─────────┬───────────────┬─┘
│ │
┌─────────▼─────────┐ ┌─▼─────────────┐
│ Model Database │ │ Ollama │
│ │ │ Integration │
│ • 15+ Models │ │ │
│ • Characteristics │ │ • Client │
│ • Scoring Data │ │ • Scraper │
└───────────────────┘ └───────────────┘
```
---
## 🧠 **Intelligent Selector Algorithm**
### **Core Algorithm Flow**
```javascript
function selectBestModel(hardware, availableModels, userPreference) {
// 1. Hardware Analysis
const analysis = analyzeHardware(hardware);
// 2. Model Scoring
const scores = availableModels.map(model => ({
model,
score: calculateModelScore(model, hardware, analysis, userPreference)
}));
// 3. Ranking & Selection
const ranked = scores.sort((a, b) => b.score - a.score);
// 4. Confidence Calculation
const confidence = calculateConfidence(ranked[0].score, analysis);
// 5. Reasoning Generation
const reasoning = generateReasoning(ranked[0].model, hardware, analysis);
return {
bestModel: ranked[0].model,
confidence,
reasoning,
allPredictions: ranked
};
}
```
### **5-Factor Scoring System**
#### **1. Memory Efficiency Score (35% weight)**
```javascript
calculateMemoryEfficiencyScore(model, analysis) {
const requiredMemory = model.memory_requirement;
const availableMemory = analysis.available_memory.total;
if (requiredMemory > availableMemory) {
return 0.1; // Severe penalty for overflow
}
const utilizationRatio = requiredMemory / availableMemory;
// Optimal utilization curve
if (utilizationRatio <= 0.3) return 0.6; // Under-utilizing
if (utilizationRatio <= 0.5) return 0.8; // Good utilization
if (utilizationRatio <= 0.7) return 1.0; // Optimal utilization
if (utilizationRatio <= 0.9) return 0.9; // High utilization
return 0.7; // Very tight fit
}
```
#### **2. Performance Match Score (25% weight)**
```javascript
calculatePerformanceMatchScore(model, analysis) {
// CPU capability vs model requirements
const cpuMatch = Math.min(1.0,
analysis.cpu_tier.multiplier * model.cpu_intensive
);
// Memory tier bonus
const memoryMatch = analysis.memory_tier.multiplier / 2.0;
// GPU acceleration bonus
const gpuBoost = analysis.gpu_tier.multiplier > 1.0 ? 0.2 : 0;
return Math.min(1.0, cpuMatch + memoryMatch + gpuBoost);
}
```
#### **3. Task Optimization Score (20% weight)**
```javascript
calculateTaskOptimizationScore(model, userPreference) {
if (model.specialization.includes(userPreference)) {
return 1.0; // Perfect match
}
// Check for compatible tasks
const compatibleTasks = {
'coding': ['programming', 'debugging'],
'general': ['chat', 'reasoning'],
'chat': ['general', 'reasoning']
};
const compatible = compatibleTasks[userPreference] || [];
const hasCompatible = compatible.some(task =>
model.specialization.includes(task)
);
return hasCompatible ? 0.7 : 0.5;
}
```
#### **4. Quality/Popularity Score (15% weight)**
```javascript
calculatePopularityQualityScore(model) {
const qualityNormalized = model.quality_score / 10.0;
const popularityNormalized = model.popularity_score / 10.0;
// Quality is more important than popularity
return (qualityNormalized * 0.6) + (popularityNormalized * 0.4);
}
```
#### **5. Resource Efficiency Score (5% weight)**
```javascript
calculateResourceEfficiencyScore(model, analysis) {
const efficiencyFactors = {
inference_speed: {
'very_fast': 1.0,
'fast': 0.9,
'medium': 0.8,
'slow': 0.6,
'very_slow': 0.4
}
};
const speedScore = efficiencyFactors.inference_speed[model.inference_speed];
const thermalScore = analysis.thermal_constraint;
return (speedScore * 0.7) + (thermalScore * 0.3);
}
```
---
## 💾 **Model Database Schema**
### **Model Entry Structure**
```javascript
{
// Basic Information
name: "Human-readable name",
size_gb: 3.8, // Disk space requirement
parameters: 7, // Model parameters in billions
memory_requirement: 8, // RAM needed in GB
// Performance Requirements
cpu_cores_min: 4, // Minimum CPU cores
cpu_intensive: 0.7, // CPU usage factor (0-1)
// Specialization & Quality
specialization: ['general', 'chat', 'reasoning'],
quality_score: 9.2, // Community quality rating (0-10)
popularity_score: 9.8, // Adoption/usage score (0-10)
// Technical Specifications
context_length: 4096, // Maximum context tokens
quantization: 'Q4_0', // Quantization method
inference_speed: 'medium' // Speed category
}
```
### **Hardware Tier Definitions**
```javascript
const hardwareTiers = {
memory: {
ultra_low: { min: 0, max: 4, multiplier: 0.3 },
low: { min: 4, max: 8, multiplier: 0.6 },
medium: { min: 8, max: 16, multiplier: 1.0 },
high: { min: 16, max: 32, multiplier: 1.4 },
very_high: { min: 32, max: 64, multiplier: 1.8 },
extreme: { min: 64, max: 128, multiplier: 2.2 }
},
cpu: {
ultra_low: { min: 0, max: 2, multiplier: 0.4 },
low: { min: 2, max: 4, multiplier: 0.7 },
medium: { min: 4, max: 8, multiplier: 1.0 },
high: { min: 8, max: 16, multiplier: 1.3 },
very_high: { min: 16, max: 32, multiplier: 1.6 },
extreme: { min: 32, max: 64, multiplier: 2.0 }
}
};
```
---
## 🔍 **Hardware Detection System**
### **Multi-Platform Detection**
```javascript
async getSystemSpecs() {
// Try systeminformation first (most detailed)
if (si) {
try {
const [cpu, mem, graphics, osInfo] = await Promise.all([
si.cpu(), si.mem(), si.graphics(), si.osInfo()
]);
return this.processDetailedSpecs(cpu, mem, graphics, osInfo);
} catch (error) {
console.warn('Detailed detection failed, using fallback');
}
}
// Fallback to Node.js built-ins
const os = require('os');
return {
cpu_cores: os.cpus().length,
cpu_freq_max: 3.0, // Estimated
total_ram_gb: os.totalmem() / (1024 ** 3),
gpu_model_normalized: os.platform() === 'darwin' ? 'apple_silicon' : 'cpu_only',
gpu_vram_gb: os.platform() === 'darwin' ? os.totalmem() / (1024 ** 3) * 0.75 : 0,
platform: os.platform()
};
}
```
### **Apple Silicon Optimization**
```javascript
handleAppleSilicon(totalRAM) {
// Apple Silicon uses unified memory architecture
const unifiedMemory = totalRAM;
// Estimate GPU-accessible memory (60% of total)
const gpuMemory = unifiedMemory * 0.6;
// System overhead (30% reserved for macOS)
const availableMemory = unifiedMemory * 0.7;
return {
total_ram_gb: unifiedMemory,
gpu_vram_gb: gpuMemory,
available_memory: availableMemory,
gpu_model_normalized: 'apple_silicon',
architecture_boost: 1.15 // 15% performance bonus
};
}
```
---
## 🚀 **Performance Optimizations**
### **Caching Strategy**
```javascript
class ModelSelector {
constructor() {
this.hardwareCache = null;
this.hardwareCacheExpiry = 5 * 60 * 1000; // 5 minutes
this.modelCache = new Map();
}
async getHardwareSpecs() {
if (this.hardwareCache &&
Date.now() - this.hardwareCache.timestamp < this.hardwareCacheExpiry) {
return this.hardwareCache.data;
}
const specs = await this.detectHardware();
this.hardwareCache = {
data: specs,
timestamp: Date.now()
};
return specs;
}
}
```
### **Lazy Loading**
```javascript
class IntelligentSelector {
constructor() {
this._modelDatabase = null;
this._hardwareTiers = null;
}
get modelDatabase() {
if (!this._modelDatabase) {
this._modelDatabase = this.initializeModelDatabase();
}
return this._modelDatabase;
}
}
```
---
## 🔌 **API Integration Points**
### **Adding New Models**
```javascript
// 1. Add to model database
const newModel = {
'new-model:7b': {
name: 'New Model 7B',
size_gb: 4.0,
parameters: 7,
memory_requirement: 8,
cpu_cores_min: 4,
cpu_intensive: 0.7,
specialization: ['general', 'chat'],
quality_score: 8.5,
popularity_score: 7.0,
context_length: 4096,
quantization: 'Q4_0',
inference_speed: 'medium'
}
};
// 2. The system automatically includes it in selection
```
### **Custom Hardware Profiles**
```javascript
// Override hardware detection for testing
const customHardware = {
cpu_cores: 16,
cpu_freq_max: 3.8,
total_ram_gb: 32,
gpu_model_normalized: 'rtx_4090',
gpu_vram_gb: 24
};
const result = await aiSelector.selectBestModel(
candidateModels,
customHardware,
'coding'
);
```
---
## 🧪 **Testing Framework**
### **Unit Tests Structure**
```javascript
// Test hardware analysis
describe('Hardware Analysis', () => {
test('should classify high-end system correctly', () => {
const hardware = { cpu_cores: 12, total_ram_gb: 24 };
const analysis = selector.analyzeHardware(hardware);
expect(analysis.overall_tier).toBe('high');
});
test('should handle Apple Silicon correctly', () => {
const hardware = {
gpu_model_normalized: 'apple_silicon',
total_ram_gb: 16
};
const analysis = selector.analyzeHardware(hardware);
expect(analysis.gpu_tier.multiplier).toBeGreaterThan(1.0);
});
});
// Test scoring algorithm
describe('Model Scoring', () => {
test('should penalize oversized models', () => {
const smallHardware = { total_ram_gb: 8 };
const largeModel = { memory_requirement: 16 };
const score = selector.calculateModelScore(largeModel, smallHardware);
expect(score).toBeLessThan(50);
});
});
```
### **Integration Tests**
```javascript
// Test full selection pipeline
describe('Model Selection Integration', () => {
test('should select appropriate model for given hardware', async () => {
const hardware = { cpu_cores: 8, total_ram_gb: 16 };
const models = ['llama2:7b', 'llama2:13b', 'phi3:mini'];
const result = await selector.selectBestModel(models, hardware);
expect(result.bestModel).toBe('llama2:7b');
expect(result.confidence).toBeGreaterThan(0.8);
expect(result.method).toBe('intelligent_mathematical');
});
});
```
---
## 📊 **Monitoring & Analytics**
### **Performance Metrics**
```javascript
class PerformanceMonitor {
static trackSelection(hardware, selectedModel, confidence) {
const metrics = {
timestamp: Date.now(),
hardware_tier: this.getHardwareTier(hardware),
model_size: this.getModelSize(selectedModel),
confidence: confidence,
selection_time: process.hrtime.bigint() - startTime
};
// Log or send to analytics service
console.log('Selection metrics:', metrics);
}
}
```
### **Error Tracking**
```javascript
class ErrorTracker {
static logSelectionError(error, context) {
const errorReport = {
error: error.message,
stack: error.stack,
context: {
hardware: context.hardware,
models: context.models,
timestamp: new Date().toISOString()
}
};
// Send to error tracking service
console.error('Selection error:', errorReport);
}
}
```
---
## 🔧 **Configuration Options**
### **Environment Variables**
```bash
# Enable debug mode
DEBUG=1 llm-checker ai-check
# Disable hardware caching
NO_CACHE=1 llm-checker ai-check
# Custom model database path
MODEL_DB_PATH=/path/to/custom/models.json llm-checker ai-check
```
### **Configuration File**
```javascript
// ~/.llm-checker/config.json
{
"selector": {
"weights": {
"memory_efficiency": 0.35,
"performance_match": 0.25,
"task_optimization": 0.20,
"popularity_quality": 0.15,
"resource_efficiency": 0.05
},
"cache_ttl": 300000,
"fallback_enabled": true
},
"hardware": {
"force_detection": false,
"custom_multipliers": {
"apple_silicon": 1.15,
"rtx_40_series": 1.3
}
}
}
```
---
## 🚀 **Deployment & Distribution**
### **Build Process**
```bash
# Development
npm run dev
# Testing
npm test
# Build for distribution
npm run build
# Package for npm
npm pack
```
### **CI/CD Pipeline**
```yaml
# .github/workflows/test.yml
name: Test
on: [push, pull_request]
jobs:
test:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
node-version: [16, 18, 20]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- run: npm ci
- run: npm test
```
---
## 🔍 **Debugging Guide**
### **Common Issues**
1. **Hardware Detection Fails**
```bash
DEBUG=1 llm-checker ai-check
# Check output for detection errors
```
2. **Low Confidence Scores**
```javascript
// Check if models are too large for hardware
const analysis = selector.analyzeHardware(hardware);
console.log('Available memory:', analysis.available_memory.total);
```
3. **Unexpected Model Selection**
```javascript
// Enable verbose scoring
const result = selector.selectBestModel(models, hardware, 'general');
console.log('All scores:', result.allPredictions);
```
### **Debug Mode Output**
```bash
DEBUG=1 llm-checker ai-check --models llama2:7b mistral:7b
# Outputs:
# [DEBUG] Hardware analysis: { cpu_cores: 12, total_ram_gb: 24 }
# [DEBUG] Model scores: [{ model: 'llama2:7b', score: 99 }, ...]
# [DEBUG] Selection reasoning: Optimal memory utilization...
```
---
**This technical documentation covers the core implementation details. For usage examples, see [USAGE_GUIDE.md](USAGE_GUIDE.md).**
- [docs/reference/technical-docs.md](docs/reference/technical-docs.md)

343
USAGE_GUIDE.md
View file

@ -1,342 +1,5 @@
# 📚 LLM Checker - Complete Usage Guide
# Moved
This guide provides detailed examples and use cases for the LLM Checker intelligent model selector.
This document has moved to:
---
## 🎯 **Common Use Cases**
### **1. First-Time User - "What model should I use?"**
**Scenario**: You're new to LLMs and want to know what works best on your system.
```bash
# Get intelligent recommendation for your hardware
llm-checker ai-check
# See what models are available locally
llm-checker list-models
# Get broader system analysis
llm-checker check
```
**Expected Output**:
```
🧠 INTELLIGENT MODEL SELECTION
│ 🏆 Selected Model: llama2:7b
│ 📊 Confidence: 100%
│ 🔢 Intelligence Score: 99/100
│ 💡 AI Analysis: Excellent fit for your high hardware configuration...
```
**What to do next**: Run `ollama pull llama2:7b` to install the recommended model.
---
### **2. Developer - "I need a coding assistant"**
**Scenario**: You want to find the best model for programming tasks.
```bash
# Compare coding-specific models
llm-checker ai-check --models codellama:7b deepseek-coder:6.7b starcoder:7b
# Browse all coding models
llm-checker list-models --category coding
# Quick coding session
llm-checker ai-run --prompt "Help me write a Python function to parse JSON"
```
**Why it works**: The intelligent selector considers:
- Model specialization in coding tasks
- Context length for large code files
- Performance on programming benchmarks
---
### **3. Researcher - "I need reasoning capabilities"**
**Scenario**: You're doing research and need models good at complex reasoning.
```bash
# Compare reasoning models
llm-checker ai-check --models llama2:13b qwen2:7b mistral:7b
# Get category-specific recommendations
llm-checker recommend --category reasoning
# Check what large models you can run
llm-checker ai-check --models llama2:70b claude:20b gpt4:175b
```
**Intelligence in action**: Prioritizes models with:
- Higher parameter counts for complex reasoning
- Strong performance on reasoning benchmarks
- Sufficient context length for multi-step problems
---
### **4. Hardware Upgrader - "What can I run with better hardware?"**
**Scenario**: You're considering a hardware upgrade and want to see potential.
```bash
# Check current capabilities
llm-checker check --detailed
# See what models require more resources
llm-checker list-models --full | grep "16.*GB\|32.*GB"
# Compare small vs large models
llm-checker ai-check --models phi3:mini llama2:7b llama2:13b llama2:70b
```
**Insight**: The hardware tier analysis shows exactly what upgrade would unlock new model tiers.
---
### **5. Performance Optimizer - "What's the fastest model for my needs?"**
**Scenario**: You need the best performance/speed ratio.
```bash
# Find efficient models
llm-checker list-models --popular --limit 20
# Compare performance profiles
llm-checker ai-check --models phi3:mini mistral:7b gemma:7b
# Test multiple sizes of same family
llm-checker ai-check --models llama2:7b llama2:13b
```
**Intelligence factors**: Considers inference speed, memory efficiency, and quality balance.
---
## 🛠️ **Advanced Usage Patterns**
### **Command Chaining for Workflows**
```bash
# Complete discovery workflow
llm-checker check && \
llm-checker list-models --category coding && \
llm-checker ai-check --models codellama:7b deepseek-coder:6.7b
# Automated model testing
for model in llama2:7b mistral:7b phi3:mini; do
echo "Testing $model:"
llm-checker ai-check --models $model
done
```
### **JSON Output for Integration**
```bash
# Export model database
llm-checker list-models --json > my-models.json
# Integrate with scripts
MODEL=$(llm-checker ai-check --models llama2:7b mistral:7b --json | jq -r '.bestModel')
echo "Best model: $MODEL"
```
### **Custom Model Comparison**
```bash
# Compare your installed models
MODELS=$(ollama list | tail -n +2 | awk '{print $1}' | tr '\n' ' ')
llm-checker ai-check --models $MODELS
```
---
## 🎨 **Understanding the Output**
### **Intelligence Score Breakdown**
| Score Range | Meaning | Action |
|-------------|---------|---------|
| **90-100** | Perfect match | Use immediately |
| **70-89** | Great fit | Recommended choice |
| **50-69** | Good option | Consider if you like the model |
| **30-49** | Suboptimal | Better alternatives exist |
| **0-29** | Poor fit | Likely won't work well |
### **Confidence Levels**
| Confidence | Interpretation |
|------------|----------------|
| **95-100%** | Algorithm is certain about hardware compatibility |
| **80-94%** | High confidence, good match expected |
| **60-79%** | Moderate confidence, should work well |
| **40-59%** | Lower confidence, may have limitations |
| **<40%** | Uncertain, proceed with caution |
### **Hardware Tier Impact**
```
Your Hardware Tier: HIGH
├── Can run: 7B-13B models optimally
├── Possible: 30B models with limitations
├── Recommended: llama2:7b, mistral:7b, codellama:7b
└── Avoid: 70B+ models (insufficient memory)
```
---
## 🚨 **Troubleshooting**
### **"No models found"**
```bash
# Check if Ollama is running
ollama list
# Install some models first
ollama pull llama2:7b
ollama pull phi3:mini
# Then rerun
llm-checker ai-check
```
### **"Low confidence scores"**
This usually means:
- **Limited hardware** - Consider smaller models
- **Unusual configuration** - Algorithm less certain
- **Memory constraints** - Most models too large
**Solution**: Try with smaller models:
```bash
llm-checker ai-check --models phi3:mini gemma:2b qwen2:1.5b
```
### **"Model doesn't perform as expected"**
The algorithm optimizes for compatibility, not task-specific performance. For specialized needs:
```bash
# Check task-specific models
llm-checker list-models --category coding # For programming
llm-checker list-models --category creative # For writing
```
---
## 🎓 **Tips & Best Practices**
### **1. Start Small, Scale Up**
```bash
# Begin with efficient models
llm-checker ai-check --models phi3:mini gemma:2b
# Then try larger ones
llm-checker ai-check --models llama2:7b mistral:7b
```
### **2. Use Specific Comparisons**
```bash
# Instead of checking everything
llm-checker list-models --limit 50
# Compare specific candidates
llm-checker ai-check --models model1 model2 model3
```
### **3. Monitor Resource Usage**
```bash
# Check system impact
llm-checker check --detailed
# Monitor during model execution
htop # or Activity Monitor on macOS
```
### **4. Keep Database Updated**
```bash
# Regular updates for new models
llm-checker update-db
# Force refresh if needed
llm-checker update-db --force
```
### **5. Leverage Categories**
```bash
llm-checker list-models --category coding # Programming
llm-checker list-models --category reasoning # Research
llm-checker list-models --category creative # Writing
```
---
## 🔬 **Advanced Configuration**
### **For Power Users**
The intelligent selector can be customized by editing `src/ai/intelligent-selector.js`:
```javascript
// Adjust scoring weights
this.performanceWeights = {
memory_efficiency: 0.40, // Increase memory priority
performance_match: 0.30, // Boost performance importance
task_optimization: 0.15, // Reduce task specificity
popularity_quality: 0.10, // Less weight on popularity
resource_efficiency: 0.05 // Keep efficiency low
};
```
### **Custom Model Profiles**
Add your own models to the database:
```javascript
'custom-model:7b': {
name: 'Custom Model 7B',
size_gb: 4.2,
parameters: 7,
memory_requirement: 8,
cpu_cores_min: 4,
cpu_intensive: 0.8,
specialization: ['custom', 'specialized'],
quality_score: 9.0,
popularity_score: 5.0,
context_length: 8192,
quantization: 'Q4_0',
inference_speed: 'medium'
}
```
---
## 📊 **Performance Benchmarking**
### **Optional ML Training Pipeline**
For users who want to contribute benchmarking data:
```bash
# Collect performance data
npm run benchmark
# Process and label data
cd ml-model && python python/dataset_aggregator.py
# Train TabTransformer model
npm run train-ai
# Use trained model for predictions
llm-checker ai-check --status
```
This creates an ONNX model that learns from real performance data across different hardware configurations.
---
**💡 Pro Tip**: Use `llm-checker ai-check --help` and `llm-checker ai-run --help` to see all available options for any command!
- [docs/guides/usage-guide.md](docs/guides/usage-guide.md)

187
bin/enhanced_cli.js
View file

@ -25,6 +25,14 @@ const {
} = require('../src/runtime/runtime-support');
const { CalibrationManager } = require('../src/calibration/calibration-manager');
const { SUPPORTED_CALIBRATION_OBJECTIVES } = require('../src/calibration/schemas');
const {
resolveRoutingPolicyPreference,
normalizeTaskName,
inferTaskFromPrompt,
resolveCalibrationRoute,
getRouteModelCandidates,
selectModelFromRoute
} = require('../src/calibration/policy-routing');
const SpeculativeDecodingEstimator = require('../src/models/speculative-decoding-estimator');
const PolicyManager = require('../src/policy/policy-manager');
const PolicyEngine = require('../src/policy/policy-engine');
@ -1076,6 +1084,119 @@ function displayIntelligentRecommendations(intelligentData) {
console.log(chalk.red('╰'));
}
function toCalibrationSourceLabel(source) {
if (source === 'default-discovery') {
return '~/.llm-checker/calibration-policy.{yaml,yml,json}';
}
return source || 'unknown';
}
function collectRecommendationModelIdentifiers(intelligentData) {
const identifiers = new Set();
const summary = intelligentData?.summary || {};
if (summary.best_overall?.identifier) {
identifiers.add(summary.best_overall.identifier);
}
if (summary.by_category && typeof summary.by_category === 'object') {
Object.values(summary.by_category).forEach((entry) => {
if (entry?.identifier) {
identifiers.add(entry.identifier);
}
});
}
const recommendationGroups = intelligentData?.recommendations || {};
Object.values(recommendationGroups).forEach((group) => {
const models = Array.isArray(group?.bestModels) ? group.bestModels : [];
models.forEach((model) => {
if (model?.model_identifier) {
identifiers.add(model.model_identifier);
}
});
});
return Array.from(identifiers);
}
function resolveCalibratedRouteDecision(calibratedPolicy, requestedTask, availableModels = []) {
if (!calibratedPolicy?.policy) return null;
const resolvedRoute = resolveCalibrationRoute(calibratedPolicy.policy, requestedTask);
if (!resolvedRoute?.route) return null;
const routeCandidates = getRouteModelCandidates(resolvedRoute.route);
const routeSelection = selectModelFromRoute(resolvedRoute.route, availableModels);
const selectedModel = routeSelection?.selectedModel || routeCandidates[0] || null;
return {
requestedTask: resolvedRoute.requestedTask,
resolvedTask: resolvedRoute.resolvedTask,
usedTaskFallback: Boolean(resolvedRoute.usedTaskFallback),
primary: resolvedRoute.route.primary,
fallbacks: Array.isArray(resolvedRoute.route.fallbacks) ? resolvedRoute.route.fallbacks : [],
routeCandidates,
selectedModel,
matchedRouteModel: routeSelection?.matchedRouteModel || (routeCandidates[0] || null),
matchedAvailableModel: Boolean(routeSelection),
usedRouteFallbackModel: Boolean(routeSelection?.usedFallback)
};
}
function displayCalibratedRoutingDecision(commandName, calibratedPolicy, routeDecision, warnings = []) {
if (!calibratedPolicy && (!warnings || warnings.length === 0)) {
return;
}
console.log('\n' + chalk.bgBlue.white.bold(' CALIBRATED ROUTING '));
console.log(chalk.blue('╭' + '─'.repeat(78)));
console.log(chalk.blue('│') + ` Command: ${chalk.cyan(commandName)}`);
if (calibratedPolicy) {
console.log(chalk.blue('│') + ` Policy: ${chalk.green(calibratedPolicy.policyPath)}`);
console.log(chalk.blue('│') + ` Source: ${chalk.magenta(toCalibrationSourceLabel(calibratedPolicy.source))}`);
} else {
console.log(chalk.blue('│') + chalk.yellow(' Policy: not active (deterministic fallback)'));
}
if (routeDecision) {
const requestedTask = routeDecision.requestedTask || 'general';
const resolvedTask = routeDecision.resolvedTask || requestedTask;
const taskDisplay = routeDecision.usedTaskFallback
? `${requestedTask} → ${resolvedTask}`
: requestedTask;
const selectedModel = routeDecision.selectedModel || routeDecision.primary || 'N/A';
const selectedLabel = routeDecision.usedRouteFallbackModel
? `${selectedModel} (fallback)`
: selectedModel;
console.log(chalk.blue('│') + ` Task: ${chalk.white(taskDisplay)}`);
console.log(chalk.blue('│') + ` Route primary: ${chalk.green(routeDecision.primary || 'N/A')}`);
if (routeDecision.fallbacks && routeDecision.fallbacks.length > 0) {
console.log(chalk.blue('│') + ` Route fallbacks: ${chalk.gray(routeDecision.fallbacks.join(', '))}`);
}
console.log(chalk.blue('│') + ` Selected model: ${chalk.green.bold(selectedLabel)}`);
if (!routeDecision.matchedAvailableModel) {
console.log(
chalk.blue('│') +
chalk.yellow(' Route did not match local/recommended models; using route primary for visibility.')
);
}
}
if (warnings && warnings.length > 0) {
warnings.forEach((warning) => {
console.log(chalk.blue('│') + chalk.yellow(` Warning: ${warning}`));
});
}
console.log(chalk.blue('╰'));
}
function displayModelsStats(originalCount, filteredCount, options) {
console.log('\n' + chalk.bgGreen.white.bold(' DATABASE STATS '));
console.log(chalk.green('╭' + '─'.repeat(60)));
@ -2928,6 +3049,10 @@ program
.option('--optimize <profile>', 'Optimization profile (balanced|speed|quality|context|coding)', 'balanced')
.option('--no-verbose', 'Disable step-by-step progress display')
.option('--policy <file>', 'Evaluate recommendations against a policy file')
.option(
'--calibrated [file]',
'Use calibrated routing policy (optional file path; defaults to ~/.llm-checker/calibration-policy.{yaml,yml,json})'
)
.addHelpText(
'after',
`
@ -2935,6 +3060,11 @@ Enterprise policy examples:
$ llm-checker recommend --policy ./policy.yaml
$ llm-checker recommend --policy ./policy.yaml --category coding
$ llm-checker recommend --policy ./policy.yaml --no-verbose
Calibrated routing examples:
$ llm-checker recommend --calibrated --category coding
$ llm-checker recommend --calibrated ./calibration-policy.yaml --category reasoning
$ llm-checker recommend --policy ./calibration-policy.yaml --category coding
`
)
.action(async (options) => {
@ -2942,7 +3072,13 @@ Enterprise policy examples:
try {
const verboseEnabled = options.verbose !== false;
const checker = new (getLLMChecker())({ verbose: verboseEnabled });
const policyConfig = options.policy ? loadPolicyConfiguration(options.policy) : null;
const routingPreference = resolveRoutingPolicyPreference({
policyOption: options.policy,
calibratedOption: options.calibrated,
loadEnterprisePolicy: loadPolicyConfiguration
});
const policyConfig = routingPreference.enterprisePolicy;
const calibratedPolicy = routingPreference.calibratedPolicy;
if (!verboseEnabled) {
process.stdout.write(chalk.gray('Generating recommendations...'));
@ -2979,11 +3115,18 @@ Enterprise policy examples:
policyEnforcement = resolvePolicyEnforcement(policyConfig.policy, policyEvaluation);
}
const routingTask = normalizeTaskName(options.category || 'general');
const recommendationIdentifiers = collectRecommendationModelIdentifiers(intelligentRecommendations);
const routeDecision = calibratedPolicy
? resolveCalibratedRouteDecision(calibratedPolicy, routingTask, recommendationIdentifiers)
: null;
// Mostrar información del sistema
displaySystemInfo(hardware, { summary: { hardwareTier: intelligentRecommendations.summary.hardware_tier } });
// Mostrar recomendaciones
displayIntelligentRecommendations(intelligentRecommendations);
displayCalibratedRoutingDecision('recommend', calibratedPolicy, routeDecision, routingPreference.warnings);
if (policyConfig && policyEvaluation && policyEnforcement) {
displayPolicySummary('recommend', policyConfig, policyEvaluation, policyEnforcement);
@ -3243,7 +3386,13 @@ program
.command('ai-run')
.description('AI-powered model selection and execution')
.option('-m, --models <models...>', 'Specific models to choose from')
.option('-c, --category <category>', 'Task category hint (coding, reasoning, multimodal, general, etc.)')
.option('--prompt <prompt>', 'Prompt to run with selected model')
.option('--policy <file>', 'Explicit calibrated routing policy file (takes precedence over --calibrated)')
.option(
'--calibrated [file]',
'Enable calibrated routing policy (optional file path; defaults to ~/.llm-checker/calibration-policy.{yaml,yml,json})'
)
.action(async (options) => {
showAsciiArt('ai-run');
// Check if Ollama is installed first
@ -3257,6 +3406,11 @@ program
const aiSelector = new AIModelSelector();
const checker = new (getLLMChecker())();
const systemInfo = await checker.getSystemInfo();
const routingPreference = resolveRoutingPolicyPreference({
policyOption: options.policy,
calibratedOption: options.calibrated
});
const calibratedPolicy = routingPreference.calibratedPolicy;
// Get available models or use provided ones
let candidateModels = options.models;
@ -3284,6 +3438,10 @@ program
return;
}
}
candidateModels = Array.isArray(candidateModels)
? candidateModels.filter((model) => typeof model === 'string' && model.trim().length > 0)
: [];
// AI selection
const systemSpecs = {
@ -3294,10 +3452,33 @@ program
gpu_model_normalized: systemInfo.gpu?.model ||
(systemInfo.cpu?.manufacturer === 'Apple' ? 'apple_silicon' : 'cpu_only')
};
const result = await aiSelector.selectBestModel(candidateModels, systemSpecs);
const taskHint = normalizeTaskName(options.category || inferTaskFromPrompt(options.prompt));
const routeDecision = calibratedPolicy
? resolveCalibratedRouteDecision(calibratedPolicy, taskHint, candidateModels)
: null;
let result;
if (routeDecision && routeDecision.matchedAvailableModel && routeDecision.selectedModel) {
result = {
bestModel: routeDecision.selectedModel,
confidence: routeDecision.usedRouteFallbackModel ? 0.82 : 0.94,
method: 'calibrated-policy-route',
reasoning: `Selected from calibrated policy route for ${routeDecision.resolvedTask}`
};
} else {
if (routeDecision && routeDecision.routeCandidates.length > 0) {
routingPreference.warnings.push(
`Calibrated route candidates (${routeDecision.routeCandidates.join(
', '
)}) are not installed locally. Falling back to AI selector.`
);
}
result = await aiSelector.selectBestModel(candidateModels, systemSpecs, taskHint);
}
spinner.succeed(`Selected ${chalk.green.bold(result.bestModel)} (${result.method}, ${Math.round(result.confidence * 100)}% confidence)`);
displayCalibratedRoutingDecision('ai-run', calibratedPolicy, routeDecision, routingPreference.warnings);
// Execute the selected model
console.log(chalk.magenta.bold(`\nLaunching ${result.bestModel}...`));

17
docs/README.md Normal file
View file

@ -0,0 +1,17 @@
# LLM Checker Docs
Central documentation index for this project.
## Guides
- [Usage Guide](guides/usage-guide.md)
- [Advanced Usage](guides/advanced-usage.md)
## Reference
- [Technical Docs](reference/technical-docs.md)
- [Changelog](reference/changelog.md)
## Fixtures
- [Calibration Fixtures](fixtures/calibration/README.md)

12
docs/fixtures/calibration/README.md Normal file
View file

@ -0,0 +1,12 @@
# Calibration Docs Fixtures
This folder contains deterministic examples used in documentation and tests.
- `sample-suite.jsonl`:
- Minimal prompt suite for first-time calibration runs.
- Includes `coding` and `general` tasks.
- `sample-generated-policy.yaml`:
- Example routing policy artifact generated from calibration output.
- Compatible with `recommend --calibrated` and `ai-run --calibrated`.
Use these fixtures to run the calibration quick start without creating files from scratch.

20
docs/fixtures/calibration/sample-generated-policy.yaml Normal file
View file

@ -0,0 +1,20 @@
schema_version: "1.0"
generated_at: "2026-02-17T00:00:00.000Z"
objective: balanced
source:
calibration_version: "contract-2026-02-17T00-00-00-000Z"
calibration_result_path: "./artifacts/calibration-result.json"
routing:
coding:
primary: "qwen2.5-coder:7b"
fallbacks:
- "llama3.2:3b"
rationale: "Draft routing generated from calibration contract output."
general:
primary: "qwen2.5-coder:7b"
fallbacks:
- "llama3.2:3b"
rationale: "Draft routing generated from calibration contract output."
metadata:
runtime: "ollama"
hardware_fingerprint: "darwin-arm64-24gb"

3
docs/fixtures/calibration/sample-suite.jsonl Normal file
View file

@ -0,0 +1,3 @@
{"id":"coding-1","task":"coding","prompt":"Write a Python function that validates email addresses and returns True or False."}
{"id":"coding-2","task":"coding","prompt":"Refactor this callback-based JavaScript snippet into async/await style."}
{"id":"general-1","task":"general","prompt":"Summarize the tradeoffs of edge deployment versus centralized inference in five bullet points."}

388
docs/guides/advanced-usage.md Normal file
View file

@ -0,0 +1,388 @@
# Advanced Usage Guide – LLM Checker 2.0 🎛️
This document goes beyond the quick‑start, showing how to **tweak, automate and extend LLM Checker for professional workflows**.
---
## 🎛️ Advanced Configuration
### Custom configuration file
Create `~/.llm-checker.json` to override defaults:
```json
{
"analysis": {
"defaultUseCase": "code",
"performanceTesting": true
},
"display": {
"maxModelsPerTable": 15,
"compactMode": true
},
"filters": {
"minCompatibilityScore": 75,
"excludeModels": ["very-large-model"]
}
}
```
### Environment variables
```bash
# Disable GPU detection (handy for unit tests)
export LLM_CHECKER_NO_GPU=true
# Force VRAM if auto‑detection fails
export LLM_CHECKER_VRAM_GB=8
# Point to a remote Ollama daemon
export OLLAMA_BASE_URL=http://remote-server:11434
# Verbose logs
export LLM_CHECKER_LOG_LEVEL=debug
# Custom cache directory
export LLM_CHECKER_CACHE_DIR=/custom/cache/path
# Disable coloured output (for scripts/CI)
export NO_COLOR=1
```
---
## 🔧 Advanced Workflows
### 1 — Automated analysis for CI/CD
```bash
#!/usr/bin/env bash
# ci-llm-check.sh – Continuous Integration helper
# Silent JSON output
llm-checker check --format json --quiet > hardware-analysis.json
# Fail the job if no compatible models
COMPATIBLE_MODELS=$(jq '.compatible | length' < hardware-analysis.json)
if [ "$COMPATIBLE_MODELS" -gt 0 ]; then
echo "✅ Hardware supports $COMPATIBLE_MODELS model(s)"
exit 0
else
echo "❌ Hardware not powerful enough for local LLMs"
exit 1
fi
```
### 2 — Automated developer setup
```bash
#!/usr/bin/env bash
# setup-dev-environment.sh
echo "🔍 Analysing hardware…"
llm-checker check --use-case code --filter medium --ollama-only > analysis.txt
# Extract recommended installation commands
INSTALL_COMMANDS=$(grep "ollama pull" analysis.txt)
echo "📦 Installing recommended models…"
echo "$INSTALL_COMMANDS" | head -3 | while read cmd; do
echo "Running: $cmd"
$cmd
done
echo "✅ Development environment ready"
```
### 3 — Continuous performance monitoring
```bash
#!/usr/bin/env bash
# monitor-performance.sh
while true; do
echo "$(date): System health check…"
# Benchmark
llm-checker check --performance-test --quiet | grep "tokens/sec"
# Running Ollama models
llm-checker ollama --running
sleep 300 # every 5 min
done
```
---
## 📊 Comparative Analysis
### Compare multiple RAM configs
```bash
echo "RAM sensitivity test:"
for ram in 8 16 32 64; do
echo "=== ${ram} GB RAM ==="
LLM_CHECKER_RAM_GB=$ram llm-checker check --quiet | grep "Compatible:"
done
```
### Benchmark every installed model
```bash
echo "# Performance Report – $(date)" > performance-report.md
ollama list | grep -v NAME | awk '{print $1}' | while read model; do
echo "## $model" >> performance-report.md
llm-checker ollama --test "$model" >> performance-report.md
echo "" >> performance-report.md
done
```
---
## 🛠️ IDE & Editor Integration
### VS Code tasks
```jsonc
// .vscode/tasks.json
{
"version": "2.0.1",
"tasks": [
{
"label": "Check LLM Compatibility",
"type": "shell",
"command": "llm-checker",
"args": ["check", "--use-case", "code", "--detailed"],
"group": "build",
"presentation": {
"echo": true,
"reveal": "always",
"focus": false,
"panel": "shared"
}
},
{
"label": "Install Best Code Model",
"type": "shell",
"command": "bash",
"args": [
"-c",
"llm-checker check --use-case code --ollama-only | grep 'ollama pull' | head -1 | bash"
],
"group": "build"
}
]
}
```
### Vim / Neovim Lua helper
```lua
-- lua/llm-checker.lua
local M = {}
function M.check_compatibility()
local output = vim.fn.system('llm-checker check --use-case code --quiet')
vim.api.nvim_echo({{output, 'Normal'}}, false, {})
end
function M.install_best_model()
local cmd = vim.fn.system("llm-checker check --use-case code --ollama-only | grep 'ollama pull' | head -1")
if cmd ~= '' then
vim.fn.system(cmd)
print('Installed model: ' .. cmd)
end
end
return M
```
---
## 🔍 Advanced Troubleshooting
### Full debug mode
```bash
export DEBUG=1
export LLM_CHECKER_LOG_LEVEL=debug
llm-checker check --detailed 2>&1 | tee debug.log
```
### Common issues & fixes
#### 1 — Wrong hardware detection
```bash
llm-checker check --detailed | grep -A10 "Hardware"
# Override manually
export LLM_CHECKER_RAM_GB=16
export LLM_CHECKER_VRAM_GB=8
export LLM_CHECKER_CPU_CORES=8
```
#### 2 — Ollama not detected
```bash
# Check service
systemctl status ollama # Linux
brew services list | grep ollama # macOS
# Test API
curl http://localhost:11434/api/version
# Custom URL
export OLLAMA_BASE_URL=http://192.168.1.100:11434
```
#### 3 — Models not marked compatible
```bash
# Remove score filter
llm-checker check --min-score 0 --include-all
# Model‑level diagnostics
llm-checker analyze-model "TinyLlama 1.1B"
```
---
## 📈 Performance Optimisation
### Architecture‑specific flags
#### Apple Silicon (M1/M2/M3)
```bash
export LLAMA_METAL=1 # Use Metal backend
llm-checker check --filter small,medium --detailed | grep -A5 "Apple Silicon"
```
#### NVIDIA GPUs
```bash
nvidia-smi # Verify CUDA
llm-checker check --gpu-acceleration cuda
```
#### AMD GPUs
```bash
rocm-smi # Verify ROCm
export LLAMA_OPENCL=1 # OpenCL fallback
```
### Quantisation tuning
```bash
for quant in Q2_K Q4_0 Q4_K_M Q5_0 Q8_0; do
echo "=== $quant ==="
llm-checker analyze --quantization $quant
done
```
---
## 🚀 Specific Use Cases
### 1 — Building AI apps
```bash
llm-checker check --use-case chat --filter small,medium
ollama pull $(llm-checker check --use-case chat | grep "ollama pull" | head -1 | cut -d' ' -f3)
curl -X POST http://localhost:11434/api/generate -H "Content-Type: application/json" -d '{"model":"llama3.2:3b","prompt":"Hello!","stream":false}'
```
### 2 — Large‑scale code analysis
```bash
ollama pull codellama:7b
echo 'def fibonacci(n): pass' | ollama run codellama:7b "Complete this function:"
```
### 3 — Semantic search
```bash
llm-checker check --filter embeddings
ollama pull all-minilm
echo "machine learning" | ollama run all-minilm "Generate embedding:"
```
---
## 📊 Reporting & Metrics
### Automated reports
```bash
llm-checker check --detailed --export json > report.json
llm-checker check --detailed --export html > report.html
llm-checker check --detailed --export csv > report.csv
# Trend tracking
echo "$(date),$(llm-checker check --quiet | grep 'Compatible:' | grep -o '[0-9]*')" >> compatibility-trends.csv
```
### Adoption metrics
```bash
llm-checker ollama --list --format json | jq '.[] | {name, sizeGB: .fileSizeGB, lastUsed: .modified}'
find ~/.llm-checker/benchmarks -name "*.json" | xargs jq '.tokensPerSecond' | sort -n
```
---
## 🔐 Security & Privacy
### Fully offline mode
```bash
export LLM_CHECKER_OFFLINE=1
export LLM_CHECKER_NO_UPDATE_CHECK=1
llm-checker check --offline --no-cloud
```
### Sanitise logs
```bash
llm-checker check --anonymize-hardware > safe-report.txt
```
---
## 🤝 Contribution & Extensibility
### Adding custom models
```javascript
// custom-models.js
module.exports = [
{
name: "MyCustom Model 7B",
size: "7B",
type: "local",
category: "medium",
requirements: { ram: 8, vram: 4, cpu_cores: 4, storage: 7 },
frameworks: ["ollama"],
installation: { ollama: "ollama pull mycustom:7b" }
}
];
```
### Plugin system
```bash
export LLM_CHECKER_PLUGINS_DIR=~/.llm-checker/plugins
llm-checker check --load-plugins
```
---
Made with ❤️ by **Pavelevich**.

374
docs/guides/usage-guide.md Normal file
View file

@ -0,0 +1,374 @@
# 📚 LLM Checker - Complete Usage Guide
This guide provides detailed examples and use cases for the LLM Checker intelligent model selector.
---
## 🎯 **Common Use Cases**
### **1. First-Time User - "What model should I use?"**
**Scenario**: You're new to LLMs and want to know what works best on your system.
```bash
# Get intelligent recommendation for your hardware
llm-checker ai-check
# See what models are available locally
llm-checker list-models
# Get broader system analysis
llm-checker check
```
**Expected Output**:
```
🧠 INTELLIGENT MODEL SELECTION
│ 🏆 Selected Model: llama2:7b
│ 📊 Confidence: 100%
│ 🔢 Intelligence Score: 99/100
│ 💡 AI Analysis: Excellent fit for your high hardware configuration...
```
**What to do next**: Run `ollama pull llama2:7b` to install the recommended model.
---
### **2. Developer - "I need a coding assistant"**
**Scenario**: You want to find the best model for programming tasks.
```bash
# Compare coding-specific models
llm-checker ai-check --models codellama:7b deepseek-coder:6.7b starcoder:7b
# Browse all coding models
llm-checker list-models --category coding
# Quick coding session
llm-checker ai-run --prompt "Help me write a Python function to parse JSON"
# Quick coding session with calibrated routing policy (auto-discovery)
llm-checker ai-run --calibrated --category coding --prompt "Help me write a Python function to parse JSON"
```
**Why it works**: The intelligent selector considers:
- Model specialization in coding tasks
- Context length for large code files
- Performance on programming benchmarks
---
### **2.5. Calibration Workflow - "Route by measured policy"**
**Scenario**: You want deterministic routing based on your own calibration artifacts.
```bash
# Use repository fixture (or your own JSONL suite)
cp ./docs/fixtures/calibration/sample-suite.jsonl ./sample-suite.jsonl
# Generate calibration contract + routing policy
llm-checker calibrate \
--suite ./sample-suite.jsonl \
--models qwen2.5-coder:7b llama3.2:3b \
--runtime ollama \
--objective balanced \
--dry-run \
--output ./artifacts/calibration-result.json \
--policy-out ./artifacts/calibration-policy.yaml
# Route recommend/ai-run through calibrated policy
llm-checker recommend --calibrated ./artifacts/calibration-policy.yaml --category coding
llm-checker ai-run --calibrated ./artifacts/calibration-policy.yaml --category coding --prompt "Refactor this function"
```
**Flag precedence**:
- `--policy <file>` overrides `--calibrated [file]`
- `--calibrated` (without path) loads `~/.llm-checker/calibration-policy.{yaml,yml,json}`
---
### **3. Researcher - "I need reasoning capabilities"**
**Scenario**: You're doing research and need models good at complex reasoning.
```bash
# Compare reasoning models
llm-checker ai-check --models llama2:13b qwen2:7b mistral:7b
# Get category-specific recommendations
llm-checker recommend --category reasoning
# Check what large models you can run
llm-checker ai-check --models llama2:70b claude:20b gpt4:175b
```
**Intelligence in action**: Prioritizes models with:
- Higher parameter counts for complex reasoning
- Strong performance on reasoning benchmarks
- Sufficient context length for multi-step problems
---
### **4. Hardware Upgrader - "What can I run with better hardware?"**
**Scenario**: You're considering a hardware upgrade and want to see potential.
```bash
# Check current capabilities
llm-checker check --detailed
# See what models require more resources
llm-checker list-models --full | grep "16.*GB\|32.*GB"
# Compare small vs large models
llm-checker ai-check --models phi3:mini llama2:7b llama2:13b llama2:70b
```
**Insight**: The hardware tier analysis shows exactly what upgrade would unlock new model tiers.
---
### **5. Performance Optimizer - "What's the fastest model for my needs?"**
**Scenario**: You need the best performance/speed ratio.
```bash
# Find efficient models
llm-checker list-models --popular --limit 20
# Compare performance profiles
llm-checker ai-check --models phi3:mini mistral:7b gemma:7b
# Test multiple sizes of same family
llm-checker ai-check --models llama2:7b llama2:13b
```
**Intelligence factors**: Considers inference speed, memory efficiency, and quality balance.
---
## 🛠️ **Advanced Usage Patterns**
### **Command Chaining for Workflows**
```bash
# Complete discovery workflow
llm-checker check && \
llm-checker list-models --category coding && \
llm-checker ai-check --models codellama:7b deepseek-coder:6.7b
# Automated model testing
for model in llama2:7b mistral:7b phi3:mini; do
echo "Testing $model:"
llm-checker ai-check --models $model
done
```
### **JSON Output for Integration**
```bash
# Export model database
llm-checker list-models --json > my-models.json
# Integrate with scripts
MODEL=$(llm-checker ai-check --models llama2:7b mistral:7b --json | jq -r '.bestModel')
echo "Best model: $MODEL"
```
### **Custom Model Comparison**
```bash
# Compare your installed models
MODELS=$(ollama list | tail -n +2 | awk '{print $1}' | tr '\n' ' ')
llm-checker ai-check --models $MODELS
```
---
## 🎨 **Understanding the Output**
### **Intelligence Score Breakdown**
| Score Range | Meaning | Action |
|-------------|---------|---------|
| **90-100** | Perfect match | Use immediately |
| **70-89** | Great fit | Recommended choice |
| **50-69** | Good option | Consider if you like the model |
| **30-49** | Suboptimal | Better alternatives exist |
| **0-29** | Poor fit | Likely won't work well |
### **Confidence Levels**
| Confidence | Interpretation |
|------------|----------------|
| **95-100%** | Algorithm is certain about hardware compatibility |
| **80-94%** | High confidence, good match expected |
| **60-79%** | Moderate confidence, should work well |
| **40-59%** | Lower confidence, may have limitations |
| **<40%** | Uncertain, proceed with caution |
### **Hardware Tier Impact**
```
Your Hardware Tier: HIGH
├── Can run: 7B-13B models optimally
├── Possible: 30B models with limitations
├── Recommended: llama2:7b, mistral:7b, codellama:7b
└── Avoid: 70B+ models (insufficient memory)
```
---
## 🚨 **Troubleshooting**
### **"No models found"**
```bash
# Check if Ollama is running
ollama list
# Install some models first
ollama pull llama2:7b
ollama pull phi3:mini
# Then rerun
llm-checker ai-check
```
### **"Low confidence scores"**
This usually means:
- **Limited hardware** - Consider smaller models
- **Unusual configuration** - Algorithm less certain
- **Memory constraints** - Most models too large
**Solution**: Try with smaller models:
```bash
llm-checker ai-check --models phi3:mini gemma:2b qwen2:1.5b
```
### **"Model doesn't perform as expected"**
The algorithm optimizes for compatibility, not task-specific performance. For specialized needs:
```bash
# Check task-specific models
llm-checker list-models --category coding # For programming
llm-checker list-models --category creative # For writing
```
---
## 🎓 **Tips & Best Practices**
### **1. Start Small, Scale Up**
```bash
# Begin with efficient models
llm-checker ai-check --models phi3:mini gemma:2b
# Then try larger ones
llm-checker ai-check --models llama2:7b mistral:7b
```
### **2. Use Specific Comparisons**
```bash
# Instead of checking everything
llm-checker list-models --limit 50
# Compare specific candidates
llm-checker ai-check --models model1 model2 model3
```
### **3. Monitor Resource Usage**
```bash
# Check system impact
llm-checker check --detailed
# Monitor during model execution
htop # or Activity Monitor on macOS
```
### **4. Keep Database Updated**
```bash
# Regular updates for new models
llm-checker update-db
# Force refresh if needed
llm-checker update-db --force
```
### **5. Leverage Categories**
```bash
llm-checker list-models --category coding # Programming
llm-checker list-models --category reasoning # Research
llm-checker list-models --category creative # Writing
```
---
## 🔬 **Advanced Configuration**
### **For Power Users**
The intelligent selector can be customized by editing `src/ai/intelligent-selector.js`:
```javascript
// Adjust scoring weights
this.performanceWeights = {
memory_efficiency: 0.40, // Increase memory priority
performance_match: 0.30, // Boost performance importance
task_optimization: 0.15, // Reduce task specificity
popularity_quality: 0.10, // Less weight on popularity
resource_efficiency: 0.05 // Keep efficiency low
};
```
### **Custom Model Profiles**
Add your own models to the database:
```javascript
'custom-model:7b': {
name: 'Custom Model 7B',
size_gb: 4.2,
parameters: 7,
memory_requirement: 8,
cpu_cores_min: 4,
cpu_intensive: 0.8,
specialization: ['custom', 'specialized'],
quality_score: 9.0,
popularity_score: 5.0,
context_length: 8192,
quantization: 'Q4_0',
inference_speed: 'medium'
}
```
---
## 📊 **Performance Benchmarking**
### **Optional ML Training Pipeline**
For users who want to contribute benchmarking data:
```bash
# Collect performance data
npm run benchmark
# Process and label data
cd ml-model && python python/dataset_aggregator.py
# Train TabTransformer model
npm run train-ai
# Use trained model for predictions
llm-checker ai-check --status
```
This creates an ONNX model that learns from real performance data across different hardware configurations.
---
**💡 Pro Tip**: Use `llm-checker ai-check --help` and `llm-checker ai-run --help` to see all available options for any command!

108
docs/reference/changelog.md Normal file
View file

@ -0,0 +1,108 @@
Changelog
=========
3.3.0 — Calibration Docs + E2E Coverage (2026-02-17)
----------------------------------------------------
- Added a calibration quick-start flow in `README.md` designed for first-time setup in under 10 minutes.
- Added docs fixtures for calibration onboarding:
- `docs/fixtures/calibration/sample-suite.jsonl`
- `docs/fixtures/calibration/sample-generated-policy.yaml`
- `docs/fixtures/calibration/README.md`
- Added deterministic end-to-end test coverage for the path:
- `calibrate --policy-out ...` → `recommend --calibrated ...`
- New test: `tests/calibration-e2e-integration.test.js`
- Expanded usage docs to include calibration routing workflow and precedence behavior:
- `--policy` precedence over `--calibrated`
- default calibrated discovery path at `~/.llm-checker/calibration-policy.{yaml,yml,json}`
- Added command documentation updates for calibration artifacts:
- `calibration-result.json`
- `calibration-policy.yaml`
- Hardened Jetson CUDA detection to prevent CPU-only fallback on valid Jetson/L4T systems:
- Expanded Jetson platform markers (`/etc/nv_tegra_release`, device-tree compatible IDs, kernel/utility hints).
- Expanded Jetson CUDA runtime hints (`/etc/nv_tegra_release`, tegra runtime paths/tools).
- Added regression coverage for Jetson marker-based detection paths:
- `tests/cuda-jetson-detection.test.js`
Known limitations:
- `calibrate --mode full` currently supports `--runtime ollama` only.
- Routing selection in `recommend`/`ai-run` still falls back to deterministic selection when calibrated policy is missing/invalid or when route models are unavailable.
- Calibration suite quality checks (`checks`) are optional in `dry-run` and `contract-only` modes and do not execute runtime validation.
3.2.9 — Calibrated Routing for Recommend/AI-Run (2026-02-17)
-------------------------------------------------------------
- Added calibrated routing integration to `recommend` and `ai-run`:
- new `--calibrated [file]` option (with default discovery at `~/.llm-checker/calibration-policy.{yaml,yml,json}`).
- `--policy` precedence over `--calibrated` for route resolution.
- deterministic selector fallback when calibrated routing is unavailable.
- `recommend` now supports dual policy behavior:
- enterprise governance policy (`policy.yaml`) remains supported.
- calibration routing policy can be provided via `--policy` or `--calibrated`.
- `ai-run` now accepts calibrated routing options and can select an installed model directly from calibrated primary/fallback routes before AI selector fallback.
- Added calibrated routing provenance output (policy source + resolved task/route/selected model) to `recommend` and `ai-run`.
- Added calibration routing integration tests and fixtures:
- `tests/calibration-routing-policy.test.js`
- `tests/calibration-fixtures/calibration-policy-valid.yaml`
- Updated CLI smoke coverage for new `--calibrated`/`--policy` help surfaces in `recommend` and `ai-run`.
- Documentation updates:
- README calibrated routing guide and precedence examples.
- USAGE_GUIDE calibrated `ai-run` example.
3.2.8 — Multimodal Classification Hotfix (2026-02-17)
-----------------------------------------------------
- Fixed false multimodal recommendations caused by noisy `input_types` metadata (for example, coding models incorrectly marked as image-capable by upstream scraping noise).
- Hardened modality inference: `input_types=image` alone is no longer enough; recommendation logic now also requires explicit multimodal metadata or strong vision naming/context hints.
- Added deterministic regression coverage to ensure coding-only models are excluded from multimodal picks when metadata is ambiguous.
3.2.7 — License Update: No Paid Distribution (2026-02-17)
----------------------------------------------------------
- Replaced MIT license with **NPDL-1.0** (No Paid Distribution License).
- New license terms allow free use/modification/redistribution but prohibit paid distribution or paid hosted/API delivery without a separate commercial license.
- Updated package metadata (`license: SEE LICENSE IN LICENSE`) and README license badges/section.
3.2.6 — Recommendation & Detection Regression Hardening (2026-02-17)
--------------------------------------------------------------------
- Recommend: enforce feasible 30B-class coverage for capable discrete multi-GPU profiles (non-speed objectives).
- Recommend: add deterministic regression for dual-GPU 36GB aggregate VRAM scenarios.
- Hardware detection: preserve heterogeneous multi-GPU inventory summaries (e.g. mixed V100/P40/M40).
- Hardware mapping/fallbacks:
- Added AMD Radeon AI PRO R9700 (PCI ID `7551`) support path.
- Added NVIDIA GTX 1070 Ti (`1b82`) fallback mapping.
- Re-verified Linux RX 7900 XTX non-ROCm fallback detection path.
- Docs: updated distribution/install notes and recommend optimization profile examples.
3.2.5 — Deterministic Selector Memory Modeling Fixes (2026-02-17)
------------------------------------------------------------------
- MoE memory estimation: fixed active-parameter memory path for deterministic model selection.
- Added deterministic regression coverage for MoE active/fallback parameter handling.
- Improved deterministic recommendation stability for memory-fit edge cases.
3.0.7 — Fix TPS Estimation (2025-12-31)
---------------------------------------
- Fix: TPS was overestimated by 2-10x across all hardware
- Updated speed coefficients to match real Ollama benchmarks:
- H100: 120 TPS (was 400), RTX 4090: 70 TPS (was 260)
- M4 Pro: 45 TPS (was 270), CPU: 5 TPS (was 50)
- Changed quantization baseline from FP16 to Q4_K_M (the most common format)
- Added diminishing returns for small models (1-3B don't scale linearly)
- Added comprehensive hardware simulation test suite (17 test cases)
2.7.2 — Security & Robustness (2025-09-08)
------------------------------------------
- Security: Removed insecure “curl | sh” install instructions from CLI messages and setup script. Now we reference official docs/package managers.
- Network hardening: Added request timeouts and a 5MB response size limit in the Ollama native scraper to prevent hanging connections and excessive memory use.
- Safer caching: Moved Ollama cache to `~/.llm-checker/cache/ollama` with backward-compatible reads from the legacy `src/ollama/.cache` folder.
- CLI updates: Adjusted CLI to read the new cache location with fallback to legacy path.
- No breaking changes: Functionality remains the same; legacy cache is still read. On write, new cache path is used.
2.7.1
------
- Previous version in repository.

604
docs/reference/technical-docs.md Normal file
View file

@ -0,0 +1,604 @@
# 🔧 LLM Checker - Technical Documentation
Detailed technical documentation for developers, contributors, and advanced users.
---
## 🏗️ **Architecture Overview**
### **System Design**
```
┌─────────────────────────────────────────────────┐
│ CLI Interface │
│ (enhanced_cli.js) │
└─────────────────────┬───────────────────────────┘
│
┌─────────────────────▼───────────────────────────┐
│ Main LLM Checker │
│ (src/index.js) │
└─────────┬───────────────────────────┬───────────┘
│ │
┌─────────▼─────────┐ ┌─────────▼─────────┐
│ Hardware │ │ AI Model │
│ Detection │ │ Selector │
│ │ │ │
│ • CPU/RAM/GPU │ │ • Mathematical │
│ • Tier Analysis │ │ Algorithm │
│ • Performance │ │ • 5-Factor │
│ Calculation │ │ Scoring │
└─────────┬─────────┘ └─────────┬─────────┘
│ │
└─────────┬───────────────┬─┘
│ │
┌─────────▼─────────┐ ┌─▼─────────────┐
│ Model Database │ │ Ollama │
│ │ │ Integration │
│ • 15+ Models │ │ │
│ • Characteristics │ │ • Client │
│ • Scoring Data │ │ • Scraper │
└───────────────────┘ └───────────────┘
```
---
## 🧠 **Intelligent Selector Algorithm**
### **Core Algorithm Flow**
```javascript
function selectBestModel(hardware, availableModels, userPreference) {
// 1. Hardware Analysis
const analysis = analyzeHardware(hardware);
// 2. Model Scoring
const scores = availableModels.map(model => ({
model,
score: calculateModelScore(model, hardware, analysis, userPreference)
}));
// 3. Ranking & Selection
const ranked = scores.sort((a, b) => b.score - a.score);
// 4. Confidence Calculation
const confidence = calculateConfidence(ranked[0].score, analysis);
// 5. Reasoning Generation
const reasoning = generateReasoning(ranked[0].model, hardware, analysis);
return {
bestModel: ranked[0].model,
confidence,
reasoning,
allPredictions: ranked
};
}
```
### **5-Factor Scoring System**
#### **1. Memory Efficiency Score (35% weight)**
```javascript
calculateMemoryEfficiencyScore(model, analysis) {
const requiredMemory = model.memory_requirement;
const availableMemory = analysis.available_memory.total;
if (requiredMemory > availableMemory) {
return 0.1; // Severe penalty for overflow
}
const utilizationRatio = requiredMemory / availableMemory;
// Optimal utilization curve
if (utilizationRatio <= 0.3) return 0.6; // Under-utilizing
if (utilizationRatio <= 0.5) return 0.8; // Good utilization
if (utilizationRatio <= 0.7) return 1.0; // Optimal utilization
if (utilizationRatio <= 0.9) return 0.9; // High utilization
return 0.7; // Very tight fit
}
```
#### **2. Performance Match Score (25% weight)**
```javascript
calculatePerformanceMatchScore(model, analysis) {
// CPU capability vs model requirements
const cpuMatch = Math.min(1.0,
analysis.cpu_tier.multiplier * model.cpu_intensive
);
// Memory tier bonus
const memoryMatch = analysis.memory_tier.multiplier / 2.0;
// GPU acceleration bonus
const gpuBoost = analysis.gpu_tier.multiplier > 1.0 ? 0.2 : 0;
return Math.min(1.0, cpuMatch + memoryMatch + gpuBoost);
}
```
#### **3. Task Optimization Score (20% weight)**
```javascript
calculateTaskOptimizationScore(model, userPreference) {
if (model.specialization.includes(userPreference)) {
return 1.0; // Perfect match
}
// Check for compatible tasks
const compatibleTasks = {
'coding': ['programming', 'debugging'],
'general': ['chat', 'reasoning'],
'chat': ['general', 'reasoning']
};
const compatible = compatibleTasks[userPreference] || [];
const hasCompatible = compatible.some(task =>
model.specialization.includes(task)
);
return hasCompatible ? 0.7 : 0.5;
}
```
#### **4. Quality/Popularity Score (15% weight)**
```javascript
calculatePopularityQualityScore(model) {
const qualityNormalized = model.quality_score / 10.0;
const popularityNormalized = model.popularity_score / 10.0;
// Quality is more important than popularity
return (qualityNormalized * 0.6) + (popularityNormalized * 0.4);
}
```
#### **5. Resource Efficiency Score (5% weight)**
```javascript
calculateResourceEfficiencyScore(model, analysis) {
const efficiencyFactors = {
inference_speed: {
'very_fast': 1.0,
'fast': 0.9,
'medium': 0.8,
'slow': 0.6,
'very_slow': 0.4
}
};
const speedScore = efficiencyFactors.inference_speed[model.inference_speed];
const thermalScore = analysis.thermal_constraint;
return (speedScore * 0.7) + (thermalScore * 0.3);
}
```
---
## 💾 **Model Database Schema**
### **Model Entry Structure**
```javascript
{
// Basic Information
name: "Human-readable name",
size_gb: 3.8, // Disk space requirement
parameters: 7, // Model parameters in billions
memory_requirement: 8, // RAM needed in GB
// Performance Requirements
cpu_cores_min: 4, // Minimum CPU cores
cpu_intensive: 0.7, // CPU usage factor (0-1)
// Specialization & Quality
specialization: ['general', 'chat', 'reasoning'],
quality_score: 9.2, // Community quality rating (0-10)
popularity_score: 9.8, // Adoption/usage score (0-10)
// Technical Specifications
context_length: 4096, // Maximum context tokens
quantization: 'Q4_0', // Quantization method
inference_speed: 'medium' // Speed category
}
```
### **Hardware Tier Definitions**
```javascript
const hardwareTiers = {
memory: {
ultra_low: { min: 0, max: 4, multiplier: 0.3 },
low: { min: 4, max: 8, multiplier: 0.6 },
medium: { min: 8, max: 16, multiplier: 1.0 },
high: { min: 16, max: 32, multiplier: 1.4 },
very_high: { min: 32, max: 64, multiplier: 1.8 },
extreme: { min: 64, max: 128, multiplier: 2.2 }
},
cpu: {
ultra_low: { min: 0, max: 2, multiplier: 0.4 },
low: { min: 2, max: 4, multiplier: 0.7 },
medium: { min: 4, max: 8, multiplier: 1.0 },
high: { min: 8, max: 16, multiplier: 1.3 },
very_high: { min: 16, max: 32, multiplier: 1.6 },
extreme: { min: 32, max: 64, multiplier: 2.0 }
}
};
```
---
## 🔍 **Hardware Detection System**
### **Multi-Platform Detection**
```javascript
async getSystemSpecs() {
// Try systeminformation first (most detailed)
if (si) {
try {
const [cpu, mem, graphics, osInfo] = await Promise.all([
si.cpu(), si.mem(), si.graphics(), si.osInfo()
]);
return this.processDetailedSpecs(cpu, mem, graphics, osInfo);
} catch (error) {
console.warn('Detailed detection failed, using fallback');
}
}
// Fallback to Node.js built-ins
const os = require('os');
return {
cpu_cores: os.cpus().length,
cpu_freq_max: 3.0, // Estimated
total_ram_gb: os.totalmem() / (1024 ** 3),
gpu_model_normalized: os.platform() === 'darwin' ? 'apple_silicon' : 'cpu_only',
gpu_vram_gb: os.platform() === 'darwin' ? os.totalmem() / (1024 ** 3) * 0.75 : 0,
platform: os.platform()
};
}
```
### **Apple Silicon Optimization**
```javascript
handleAppleSilicon(totalRAM) {
// Apple Silicon uses unified memory architecture
const unifiedMemory = totalRAM;
// Estimate GPU-accessible memory (60% of total)
const gpuMemory = unifiedMemory * 0.6;
// System overhead (30% reserved for macOS)
const availableMemory = unifiedMemory * 0.7;
return {
total_ram_gb: unifiedMemory,
gpu_vram_gb: gpuMemory,
available_memory: availableMemory,
gpu_model_normalized: 'apple_silicon',
architecture_boost: 1.15 // 15% performance bonus
};
}
```
---
## 🚀 **Performance Optimizations**
### **Caching Strategy**
```javascript
class ModelSelector {
constructor() {
this.hardwareCache = null;
this.hardwareCacheExpiry = 5 * 60 * 1000; // 5 minutes
this.modelCache = new Map();
}
async getHardwareSpecs() {
if (this.hardwareCache &&
Date.now() - this.hardwareCache.timestamp < this.hardwareCacheExpiry) {
return this.hardwareCache.data;
}
const specs = await this.detectHardware();
this.hardwareCache = {
data: specs,
timestamp: Date.now()
};
return specs;
}
}
```
### **Lazy Loading**
```javascript
class IntelligentSelector {
constructor() {
this._modelDatabase = null;
this._hardwareTiers = null;
}
get modelDatabase() {
if (!this._modelDatabase) {
this._modelDatabase = this.initializeModelDatabase();
}
return this._modelDatabase;
}
}
```
---
## 🔌 **API Integration Points**
### **Adding New Models**
```javascript
// 1. Add to model database
const newModel = {
'new-model:7b': {
name: 'New Model 7B',
size_gb: 4.0,
parameters: 7,
memory_requirement: 8,
cpu_cores_min: 4,
cpu_intensive: 0.7,
specialization: ['general', 'chat'],
quality_score: 8.5,
popularity_score: 7.0,
context_length: 4096,
quantization: 'Q4_0',
inference_speed: 'medium'
}
};
// 2. The system automatically includes it in selection
```
### **Custom Hardware Profiles**
```javascript
// Override hardware detection for testing
const customHardware = {
cpu_cores: 16,
cpu_freq_max: 3.8,
total_ram_gb: 32,
gpu_model_normalized: 'rtx_4090',
gpu_vram_gb: 24
};
const result = await aiSelector.selectBestModel(
candidateModels,
customHardware,
'coding'
);
```
---
## 🧪 **Testing Framework**
### **Unit Tests Structure**
```javascript
// Test hardware analysis
describe('Hardware Analysis', () => {
test('should classify high-end system correctly', () => {
const hardware = { cpu_cores: 12, total_ram_gb: 24 };
const analysis = selector.analyzeHardware(hardware);
expect(analysis.overall_tier).toBe('high');
});
test('should handle Apple Silicon correctly', () => {
const hardware = {
gpu_model_normalized: 'apple_silicon',
total_ram_gb: 16
};
const analysis = selector.analyzeHardware(hardware);
expect(analysis.gpu_tier.multiplier).toBeGreaterThan(1.0);
});
});
// Test scoring algorithm
describe('Model Scoring', () => {
test('should penalize oversized models', () => {
const smallHardware = { total_ram_gb: 8 };
const largeModel = { memory_requirement: 16 };
const score = selector.calculateModelScore(largeModel, smallHardware);
expect(score).toBeLessThan(50);
});
});
```
### **Integration Tests**
```javascript
// Test full selection pipeline
describe('Model Selection Integration', () => {
test('should select appropriate model for given hardware', async () => {
const hardware = { cpu_cores: 8, total_ram_gb: 16 };
const models = ['llama2:7b', 'llama2:13b', 'phi3:mini'];
const result = await selector.selectBestModel(models, hardware);
expect(result.bestModel).toBe('llama2:7b');
expect(result.confidence).toBeGreaterThan(0.8);
expect(result.method).toBe('intelligent_mathematical');
});
});
```
---
## 📊 **Monitoring & Analytics**
### **Performance Metrics**
```javascript
class PerformanceMonitor {
static trackSelection(hardware, selectedModel, confidence) {
const metrics = {
timestamp: Date.now(),
hardware_tier: this.getHardwareTier(hardware),
model_size: this.getModelSize(selectedModel),
confidence: confidence,
selection_time: process.hrtime.bigint() - startTime
};
// Log or send to analytics service
console.log('Selection metrics:', metrics);
}
}
```
### **Error Tracking**
```javascript
class ErrorTracker {
static logSelectionError(error, context) {
const errorReport = {
error: error.message,
stack: error.stack,
context: {
hardware: context.hardware,
models: context.models,
timestamp: new Date().toISOString()
}
};
// Send to error tracking service
console.error('Selection error:', errorReport);
}
}
```
---
## 🔧 **Configuration Options**
### **Environment Variables**
```bash
# Enable debug mode
DEBUG=1 llm-checker ai-check
# Disable hardware caching
NO_CACHE=1 llm-checker ai-check
# Custom model database path
MODEL_DB_PATH=/path/to/custom/models.json llm-checker ai-check
```
### **Configuration File**
```javascript
// ~/.llm-checker/config.json
{
"selector": {
"weights": {
"memory_efficiency": 0.35,
"performance_match": 0.25,
"task_optimization": 0.20,
"popularity_quality": 0.15,
"resource_efficiency": 0.05
},
"cache_ttl": 300000,
"fallback_enabled": true
},
"hardware": {
"force_detection": false,
"custom_multipliers": {
"apple_silicon": 1.15,
"rtx_40_series": 1.3
}
}
}
```
---
## 🚀 **Deployment & Distribution**
### **Build Process**
```bash
# Development
npm run dev
# Testing
npm test
# Build for distribution
npm run build
# Package for npm
npm pack
```
### **CI/CD Pipeline**
```yaml
# .github/workflows/test.yml
name: Test
on: [push, pull_request]
jobs:
test:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
node-version: [16, 18, 20]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- run: npm ci
- run: npm test
```
---
## 🔍 **Debugging Guide**
### **Common Issues**
1. **Hardware Detection Fails**
```bash
DEBUG=1 llm-checker ai-check
# Check output for detection errors
```
2. **Low Confidence Scores**
```javascript
// Check if models are too large for hardware
const analysis = selector.analyzeHardware(hardware);
console.log('Available memory:', analysis.available_memory.total);
```
3. **Unexpected Model Selection**
```javascript
// Enable verbose scoring
const result = selector.selectBestModel(models, hardware, 'general');
console.log('All scores:', result.allPredictions);
```
### **Debug Mode Output**
```bash
DEBUG=1 llm-checker ai-check --models llama2:7b mistral:7b
# Outputs:
# [DEBUG] Hardware analysis: { cpu_cores: 12, total_ram_gb: 24 }
# [DEBUG] Model scores: [{ model: 'llama2:7b', score: 99 }, ...]
# [DEBUG] Selection reasoning: Optimal memory utilization...
```
---
**This technical documentation covers the core implementation details. For usage examples, see [Usage Guide](../guides/usage-guide.md).**

4
package-lock.json generated
View file

@ -1,12 +1,12 @@
{
"name": "llm-checker",
"version": "3.2.8",
"version": "3.2.9",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "llm-checker",
"version": "3.2.8",
"version": "3.2.9",
"hasInstallScript": true,
"license": "SEE LICENSE IN LICENSE",
"os": [

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "llm-checker",
"version": "3.2.8",
"version": "3.2.9",
"description": "Intelligent CLI tool with AI-powered model selection that analyzes your hardware and recommends optimal LLM models for your system",
"bin": {
"llm-checker": "bin/cli.js",

376
src/calibration/policy-routing.js Normal file
View file

@ -0,0 +1,376 @@
const fs = require('fs');
const os = require('os');
const path = require('path');
const YAML = require('yaml');
const { calibrationPolicySchema } = require('./schemas');
const DEFAULT_CALIBRATION_POLICY_FILENAMES = [
'calibration-policy.yaml',
'calibration-policy.yml',
'calibration-policy.json'
];
const TASK_ALIASES = {
code: 'coding',
coder: 'coding',
programming: 'coding',
chat: 'talking',
conversation: 'talking',
talk: 'talking',
summarize: 'reading',
summary: 'reading',
summarization: 'reading',
vision: 'multimodal',
image: 'multimodal'
};
function isNonEmptyString(value) {
return typeof value === 'string' && value.trim().length > 0;
}
function formatValidationError(error) {
if (!error || !Array.isArray(error.issues)) {
return String(error?.message || 'validation failed');
}
return error.issues
.map((issue) => {
const location = Array.isArray(issue.path) && issue.path.length > 0
? issue.path.join('.')
: 'root';
return `${location}: ${issue.message}`;
})
.join('; ');
}
function resolvePolicyPath(policyPath, cwd = process.cwd()) {
if (!isNonEmptyString(policyPath)) {
throw new Error('Calibration policy path must be a non-empty string.');
}
return path.isAbsolute(policyPath)
? policyPath
: path.resolve(cwd, policyPath);
}
function parseCalibrationPolicyPayload(payloadText, policyPath) {
const extension = path.extname(policyPath).toLowerCase();
let parsed;
try {
if (extension === '.json') {
parsed = JSON.parse(payloadText);
} else {
parsed = YAML.parse(payloadText);
}
} catch (error) {
throw new Error(`Failed to parse calibration policy file: ${error.message}`);
}
try {
return calibrationPolicySchema.parse(parsed);
} catch (error) {
throw new Error(`Invalid calibration policy payload: ${formatValidationError(error)}`);
}
}
function loadCalibrationPolicyFile(policyPath, options = {}) {
const resolvedPath = resolvePolicyPath(policyPath, options.cwd);
if (!fs.existsSync(resolvedPath)) {
throw new Error(`Calibration policy file not found: ${resolvedPath}`);
}
const stats = fs.statSync(resolvedPath);
if (!stats.isFile()) {
throw new Error(`Calibration policy path must be a file: ${resolvedPath}`);
}
const payloadText = fs.readFileSync(resolvedPath, 'utf8');
const policy = parseCalibrationPolicyPayload(payloadText, resolvedPath);
return {
policyPath: resolvedPath,
policy
};
}
function tryLoadCalibrationPolicy(policyPath, options = {}) {
const resolvedPath = resolvePolicyPath(policyPath, options.cwd);
try {
const loaded = loadCalibrationPolicyFile(resolvedPath, options);
return {
ok: true,
...loaded
};
} catch (error) {
return {
ok: false,
resolvedPath,
error
};
}
}
function getDefaultCalibrationPolicyCandidates(homeDir = os.homedir()) {
const baseDir = path.join(homeDir, '.llm-checker');
return DEFAULT_CALIBRATION_POLICY_FILENAMES.map((fileName) => path.join(baseDir, fileName));
}
function discoverDefaultCalibrationPolicyPath(homeDir = os.homedir()) {
const candidates = getDefaultCalibrationPolicyCandidates(homeDir);
for (const candidate of candidates) {
if (fs.existsSync(candidate) && fs.statSync(candidate).isFile()) {
return candidate;
}
}
return null;
}
function normalizeTaskName(task) {
const normalized = String(task || 'general').trim().toLowerCase();
if (!normalized) return 'general';
return TASK_ALIASES[normalized] || normalized;
}
function inferTaskFromPrompt(prompt) {
const text = String(prompt || '').toLowerCase();
if (!text.trim()) return 'general';
if (/\b(code|coding|refactor|function|bug|debug|typescript|javascript|python|java|rust|go)\b/.test(text)) {
return 'coding';
}
if (/\b(reason|reasoning|analy[sz]e|logic|prove|derive|step by step)\b/.test(text)) {
return 'reasoning';
}
if (/\b(vision|image|photo|diagram|screenshot)\b/.test(text)) {
return 'multimodal';
}
if (/\b(summarize|summary|read|reading|article|document)\b/.test(text)) {
return 'reading';
}
if (/\b(creative|story|poem|brainstorm|marketing copy)\b/.test(text)) {
return 'creative';
}
if (/\b(chat|talk|conversation|assistant)\b/.test(text)) {
return 'talking';
}
return 'general';
}
function buildTaskCandidates(requestedTask) {
const normalized = normalizeTaskName(requestedTask);
const candidates = [normalized];
if (normalized === 'talking') {
candidates.push('chat');
} else if (normalized === 'chat') {
candidates.push('talking');
}
if (!candidates.includes('general')) {
candidates.push('general');
}
return [...new Set(candidates)];
}
function resolveCalibrationRoute(policy, requestedTask) {
const routing = policy && typeof policy.routing === 'object' ? policy.routing : null;
if (!routing) return null;
const routeKeys = Object.keys(routing);
if (routeKeys.length === 0) return null;
const normalizedTask = normalizeTaskName(requestedTask);
const taskCandidates = buildTaskCandidates(normalizedTask);
for (const taskName of taskCandidates) {
if (Object.prototype.hasOwnProperty.call(routing, taskName)) {
return {
requestedTask: normalizedTask,
resolvedTask: taskName,
usedTaskFallback: taskName !== normalizedTask,
route: routing[taskName]
};
}
}
const fallbackTask = routeKeys[0];
return {
requestedTask: normalizedTask,
resolvedTask: fallbackTask,
usedTaskFallback: true,
route: routing[fallbackTask]
};
}
function getRouteModelCandidates(route) {
if (!route || !isNonEmptyString(route.primary)) return [];
const merged = [route.primary, ...(Array.isArray(route.fallbacks) ? route.fallbacks : [])];
const unique = [];
for (const item of merged) {
if (!isNonEmptyString(item)) continue;
const trimmed = item.trim();
if (!unique.includes(trimmed)) {
unique.push(trimmed);
}
}
return unique;
}
function normalizeModelIdentifier(value) {
return String(value || '').trim().toLowerCase();
}
function splitModelIdentifier(value) {
const normalized = normalizeModelIdentifier(value);
if (!normalized) return { full: '', base: '' };
const [base] = normalized.split(':');
return { full: normalized, base: base || normalized };
}
function modelIdentifiersMatch(left, right) {
const leftId = splitModelIdentifier(left);
const rightId = splitModelIdentifier(right);
if (!leftId.full || !rightId.full) return false;
if (leftId.full === rightId.full) return true;
if (leftId.base === rightId.base) return true;
if (leftId.full.startsWith(`${rightId.base}:`)) return true;
if (rightId.full.startsWith(`${leftId.base}:`)) return true;
return false;
}
function selectModelFromRoute(route, availableModels = []) {
const routeCandidates = getRouteModelCandidates(route);
if (routeCandidates.length === 0) return null;
if (!Array.isArray(availableModels) || availableModels.length === 0) {
return {
selectedModel: routeCandidates[0],
matchedRouteModel: routeCandidates[0],
usedFallback: false,
routeCandidates
};
}
for (const routeModel of routeCandidates) {
const matched = availableModels.find((candidate) =>
modelIdentifiersMatch(routeModel, candidate)
);
if (matched) {
return {
selectedModel: matched,
matchedRouteModel: routeModel,
usedFallback: routeModel !== routeCandidates[0],
routeCandidates
};
}
}
return null;
}
function resolveRoutingPolicyPreference({
policyOption,
calibratedOption,
loadEnterprisePolicy,
cwd = process.cwd(),
homeDir = os.homedir()
} = {}) {
const result = {
enterprisePolicy: null,
calibratedPolicy: null,
warnings: []
};
const calibratedRequested = calibratedOption !== undefined && calibratedOption !== false;
if (isNonEmptyString(policyOption)) {
const calibrationAttempt = tryLoadCalibrationPolicy(policyOption, { cwd });
if (calibrationAttempt.ok) {
result.calibratedPolicy = {
policyPath: calibrationAttempt.policyPath,
policy: calibrationAttempt.policy,
source: '--policy'
};
} else {
if (typeof loadEnterprisePolicy !== 'function') {
throw calibrationAttempt.error;
}
result.enterprisePolicy = loadEnterprisePolicy(policyOption);
}
if (calibratedRequested) {
result.warnings.push('Ignoring --calibrated because --policy takes precedence.');
}
return result;
}
if (!calibratedRequested) {
return result;
}
if (isNonEmptyString(calibratedOption)) {
const attempt = tryLoadCalibrationPolicy(calibratedOption, { cwd });
if (attempt.ok) {
result.calibratedPolicy = {
policyPath: attempt.policyPath,
policy: attempt.policy,
source: '--calibrated'
};
return result;
}
result.warnings.push(
`Unable to load calibrated policy from ${attempt.resolvedPath}: ${attempt.error.message}. Falling back to deterministic selector.`
);
return result;
}
const discoveredPath = discoverDefaultCalibrationPolicyPath(homeDir);
if (!discoveredPath) {
result.warnings.push(
'No default calibrated policy found at ~/.llm-checker/calibration-policy.{yaml,yml,json}. Falling back to deterministic selector.'
);
return result;
}
const defaultAttempt = tryLoadCalibrationPolicy(discoveredPath, { cwd });
if (defaultAttempt.ok) {
result.calibratedPolicy = {
policyPath: defaultAttempt.policyPath,
policy: defaultAttempt.policy,
source: 'default-discovery'
};
return result;
}
result.warnings.push(
`Unable to load discovered calibrated policy ${defaultAttempt.resolvedPath}: ${defaultAttempt.error.message}. Falling back to deterministic selector.`
);
return result;
}
module.exports = {
DEFAULT_CALIBRATION_POLICY_FILENAMES,
getDefaultCalibrationPolicyCandidates,
discoverDefaultCalibrationPolicyPath,
loadCalibrationPolicyFile,
tryLoadCalibrationPolicy,
normalizeTaskName,
inferTaskFromPrompt,
resolveCalibrationRoute,
getRouteModelCandidates,
modelIdentifiersMatch,
selectModelFromRoute,
resolveRoutingPolicyPreference
};

40
src/hardware/backends/cuda-detector.js
View file

@ -66,9 +66,28 @@ class CUDADetector {
isJetsonPlatform() {
if (process.platform !== 'linux') return false;
// Strong L4T marker present on Jetson devices
if (this.readFileIfExists('/etc/nv_tegra_release')) {
return true;
}
const modelPaths = [
'/proc/device-tree/model',
'/sys/firmware/devicetree/base/model'
'/sys/firmware/devicetree/base/model',
'/proc/device-tree/compatible',
'/sys/firmware/devicetree/base/compatible'
];
const jetsonMarkers = [
'jetson',
'tegra',
'orin',
'xavier',
'p3701',
'p3767',
'p2888',
'p3668',
'p3448'
];
for (const modelPath of modelPaths) {
@ -76,11 +95,25 @@ class CUDADetector {
if (!model) continue;
const modelLower = model.toLowerCase();
if (modelLower.includes('jetson') || modelLower.includes('tegra')) {
if (jetsonMarkers.some((marker) => modelLower.includes(marker))) {
return true;
}
}
// Jetson kernels often include tegra in release string
const kernelRelease = (os.release() || '').toLowerCase();
if (kernelRelease.includes('tegra')) {
return true;
}
// Last-resort utility-based detection for minimal installs
if (process.arch === 'arm64' && (
fs.existsSync('/usr/bin/tegrastats') ||
fs.existsSync('/usr/sbin/nvpmodel')
)) {
return true;
}
const cpuInfo = this.readFileIfExists('/proc/cpuinfo');
if (cpuInfo) {
const cpuLower = cpuInfo.toLowerCase();
@ -96,6 +129,9 @@ class CUDADetector {
const runtimeHints = [
'/usr/local/cuda',
'/usr/bin/tegrastats',
'/usr/sbin/nvpmodel',
'/usr/lib/aarch64-linux-gnu/tegra',
'/etc/nv_tegra_release',
'/dev/nvhost-gpu',
'/dev/nvmap',
'/proc/driver/nvidia/version'

193
tests/calibration-e2e-integration.test.js Normal file
View file

@ -0,0 +1,193 @@
const assert = require('assert');
const fs = require('fs');
const os = require('os');
const path = require('path');
const YAML = require('yaml');
const { spawnSync } = require('child_process');
const BIN_PATH = path.resolve(__dirname, '..', 'bin', 'enhanced_cli.js');
const DOC_FIXTURES_DIR = path.resolve(__dirname, '..', 'docs', 'fixtures', 'calibration');
function stripAnsi(text = '') {
return String(text).replace(/\u001b\[[0-9;]*m/g, '');
}
function runCli(args, cwd, homeDir) {
return spawnSync(process.execPath, [BIN_PATH, ...args], {
cwd,
encoding: 'utf8',
env: {
...process.env,
HOME: homeDir,
USERPROFILE: homeDir,
NO_COLOR: '1'
}
});
}
function writeDeterministicOllamaCache(homeDir) {
const cacheDir = path.join(homeDir, '.llm-checker', 'cache', 'ollama');
fs.mkdirSync(cacheDir, { recursive: true });
const models = [
{
model_identifier: 'qwen2.5-coder',
model_name: 'qwen2.5-coder',
description: 'Synthetic coding model for calibration E2E',
primary_category: 'coding',
context_length: '32K',
variants: [
{
tag: 'qwen2.5-coder:7b',
size: '7b',
quantization: 'Q4_0',
real_size_gb: 4.9,
categories: ['coding'],
family: 'qwen2.5'
}
],
tags: ['qwen2.5-coder:7b'],
use_cases: ['coding', 'programming']
},
{
model_identifier: 'llama3.2',
model_name: 'llama3.2',
description: 'Synthetic general model for calibration E2E',
primary_category: 'general',
context_length: '8K',
variants: [
{
tag: 'llama3.2:3b',
size: '3b',
quantization: 'Q4_0',
real_size_gb: 2.2,
categories: ['general'],
family: 'llama3.2'
}
],
tags: ['llama3.2:3b'],
use_cases: ['general', 'chat']
}
];
const payload = {
models,
total_count: models.length,
cached_at: new Date().toISOString(),
expires_at: new Date(Date.now() + 6 * 60 * 60 * 1000).toISOString()
};
fs.writeFileSync(
path.join(cacheDir, 'ollama-detailed-models.json'),
JSON.stringify(payload, null, 2),
'utf8'
);
}
function copyFixtureSuite(tempDir) {
const fixturePath = path.join(DOC_FIXTURES_DIR, 'sample-suite.jsonl');
const suitePath = path.join(tempDir, 'sample-suite.jsonl');
fs.copyFileSync(fixturePath, suitePath);
return suitePath;
}
function run() {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'llm-checker-calibration-e2e-'));
const homeDir = path.join(tempDir, 'home');
fs.mkdirSync(homeDir, { recursive: true });
try {
const suitePath = copyFixtureSuite(tempDir);
writeDeterministicOllamaCache(homeDir);
const resultPath = path.join(tempDir, 'calibration-result.json');
const policyPath = path.join(tempDir, 'calibration-policy.yaml');
const calibrateResult = runCli(
[
'calibrate',
'--suite',
suitePath,
'--models',
'qwen2.5-coder:7b,llama3.2:3b',
'--runtime',
'ollama',
'--objective',
'balanced',
'--dry-run',
'--output',
resultPath,
'--policy-out',
policyPath
],
tempDir,
homeDir
);
assert.strictEqual(
calibrateResult.status,
0,
`calibrate command should succeed\nSTDOUT:\n${stripAnsi(calibrateResult.stdout)}\nSTDERR:\n${stripAnsi(calibrateResult.stderr)}`
);
assert.ok(fs.existsSync(resultPath), 'calibration result artifact should exist');
assert.ok(fs.existsSync(policyPath), 'calibration policy artifact should exist');
const generatedPolicy = YAML.parse(fs.readFileSync(policyPath, 'utf8'));
assert.strictEqual(generatedPolicy.schema_version, '1.0');
assert.ok(generatedPolicy.routing.coding, 'generated policy should include coding route');
assert.ok(generatedPolicy.routing.general, 'generated policy should include general route');
const recommendResult = runCli(
[
'recommend',
'--calibrated',
policyPath,
'--category',
'coding',
'--no-verbose'
],
tempDir,
homeDir
);
assert.strictEqual(
recommendResult.status,
0,
`recommend with calibrated policy should succeed\nSTDOUT:\n${stripAnsi(recommendResult.stdout)}\nSTDERR:\n${stripAnsi(recommendResult.stderr)}`
);
const recommendOutput = stripAnsi(`${recommendResult.stdout}\n${recommendResult.stderr}`);
assert.ok(
recommendOutput.includes('CALIBRATED ROUTING'),
'recommend output should include calibrated routing block'
);
assert.ok(
recommendOutput.includes('Source: --calibrated'),
'recommend output should report calibrated source'
);
assert.ok(
/Route primary:\s+qwen2\.5-coder:7b/.test(recommendOutput),
'recommend output should show policy route primary model'
);
assert.ok(
/Selected model:\s+qwen2\.5-coder:7b/.test(recommendOutput),
'recommend output should resolve selected model from calibrated route'
);
console.log('calibration-e2e-integration.test.js: OK');
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
}
if (require.main === module) {
try {
run();
} catch (error) {
console.error('calibration-e2e-integration.test.js: FAILED');
console.error(error);
process.exit(1);
}
}
module.exports = { run };

5
tests/calibration-fixtures/README.md Normal file
View file

@ -0,0 +1,5 @@
# Calibration Fixtures
- `calibration-policy-valid.yaml`:
- Valid calibration routing policy used by `calibration-routing-policy.test.js`.
- Covers task routing (`coding`, `general`) with primary/fallback chains.

19
tests/calibration-fixtures/calibration-policy-valid.yaml Normal file
View file

@ -0,0 +1,19 @@
schema_version: "1.0"
generated_at: "2026-02-17T00:00:00.000Z"
objective: balanced
source:
calibration_version: "contract-2026-02-17"
calibration_result_path: "/tmp/calibration-result.json"
routing:
coding:
primary: "qwen2.5-coder:7b"
fallbacks:
- "llama3.2:3b"
- "phi3:mini"
general:
primary: "llama3.2:3b"
fallbacks:
- "phi3:mini"
metadata:
runtime: "ollama"
hardware_fingerprint: "darwin-arm64-24gb"

122
tests/calibration-routing-policy.test.js Normal file
View file

@ -0,0 +1,122 @@
const assert = require('assert');
const fs = require('fs');
const os = require('os');
const path = require('path');
const {
resolveRoutingPolicyPreference,
resolveCalibrationRoute,
selectModelFromRoute
} = require('../src/calibration/policy-routing');
const FIXTURES_DIR = path.resolve(__dirname, 'calibration-fixtures');
function copyCalibrationFixture(tempDir, targetFileName = 'calibration-policy.yaml') {
const fixturePath = path.join(FIXTURES_DIR, 'calibration-policy-valid.yaml');
const targetPath = path.join(tempDir, targetFileName);
fs.copyFileSync(fixturePath, targetPath);
return targetPath;
}
function run() {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'llm-checker-calibration-routing-'));
const tempHome = path.join(tempDir, 'home');
fs.mkdirSync(tempHome, { recursive: true });
try {
const calibrationPolicyPath = copyCalibrationFixture(tempDir, 'calibration-policy.yaml');
const preferenceFromPolicy = resolveRoutingPolicyPreference({
policyOption: calibrationPolicyPath,
calibratedOption: true,
cwd: tempDir,
homeDir: tempHome
});
assert.ok(preferenceFromPolicy.calibratedPolicy, 'policy option should resolve calibrated policy');
assert.strictEqual(preferenceFromPolicy.calibratedPolicy.source, '--policy');
assert.strictEqual(preferenceFromPolicy.calibratedPolicy.policyPath, calibrationPolicyPath);
assert.ok(
preferenceFromPolicy.warnings.some((warning) => warning.includes('Ignoring --calibrated')),
'policy option should take precedence over --calibrated'
);
const enterprisePolicyPath = path.join(tempDir, 'enterprise-policy.yaml');
fs.writeFileSync(enterprisePolicyPath, 'mode: enforce\n', 'utf8');
let enterpriseLoadCalled = false;
const preferenceEnterprise = resolveRoutingPolicyPreference({
policyOption: enterprisePolicyPath,
calibratedOption: calibrationPolicyPath,
cwd: tempDir,
homeDir: tempHome,
loadEnterprisePolicy: (filePath) => {
enterpriseLoadCalled = true;
return { policyPath: filePath, policy: { mode: 'enforce' } };
}
});
assert.ok(enterpriseLoadCalled, 'enterprise loader should run when policy is not a calibration policy');
assert.ok(preferenceEnterprise.enterprisePolicy, 'enterprise policy should be returned');
assert.ok(!preferenceEnterprise.calibratedPolicy, 'enterprise policy fallback should not set calibrated policy');
assert.ok(
preferenceEnterprise.warnings.some((warning) => warning.includes('Ignoring --calibrated')),
'policy precedence warning should still be emitted'
);
const defaultPolicyDir = path.join(tempHome, '.llm-checker');
fs.mkdirSync(defaultPolicyDir, { recursive: true });
const discoveredPath = copyCalibrationFixture(defaultPolicyDir, 'calibration-policy.yaml');
const preferenceDiscovered = resolveRoutingPolicyPreference({
calibratedOption: true,
cwd: tempDir,
homeDir: tempHome
});
assert.ok(preferenceDiscovered.calibratedPolicy, 'default discovery should load calibrated policy');
assert.strictEqual(preferenceDiscovered.calibratedPolicy.source, 'default-discovery');
assert.strictEqual(preferenceDiscovered.calibratedPolicy.policyPath, discoveredPath);
const preferenceMissingExplicit = resolveRoutingPolicyPreference({
calibratedOption: path.join(tempDir, 'missing-policy.yaml'),
cwd: tempDir,
homeDir: tempHome
});
assert.ok(!preferenceMissingExplicit.calibratedPolicy, 'missing explicit calibrated policy should not resolve');
assert.ok(
preferenceMissingExplicit.warnings.some((warning) => warning.includes('Unable to load calibrated policy')),
'missing explicit policy should emit fallback warning'
);
const routeCoding = resolveCalibrationRoute(preferenceFromPolicy.calibratedPolicy.policy, 'coding');
assert.ok(routeCoding, 'coding route should resolve');
assert.strictEqual(routeCoding.resolvedTask, 'coding');
assert.ok(!routeCoding.usedTaskFallback, 'coding should not need task fallback');
const fallbackModelSelection = selectModelFromRoute(routeCoding.route, [
'llama3.2:3b',
'mistral:7b'
]);
assert.ok(fallbackModelSelection, 'fallback model should be selected from available models');
assert.strictEqual(fallbackModelSelection.selectedModel, 'llama3.2:3b');
assert.strictEqual(fallbackModelSelection.matchedRouteModel, 'llama3.2:3b');
assert.strictEqual(fallbackModelSelection.usedFallback, true);
const routeCreative = resolveCalibrationRoute(preferenceFromPolicy.calibratedPolicy.policy, 'creative');
assert.ok(routeCreative, 'route fallback should still return a route');
assert.strictEqual(routeCreative.resolvedTask, 'general');
assert.strictEqual(routeCreative.usedTaskFallback, true);
console.log('calibration-routing-policy.test.js: OK');
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
}
if (require.main === module) {
try {
run();
} catch (error) {
console.error('calibration-routing-policy.test.js: FAILED');
console.error(error);
process.exit(1);
}
}
module.exports = { run };

37
tests/cuda-jetson-detection.test.js
View file

@ -4,6 +4,7 @@
*/
const assert = require('assert');
const fs = require('fs');
const CUDADetector = require('../src/hardware/backends/cuda-detector');
function testJetsonFallbackWithoutNvidiaSMI() {
@ -53,9 +54,45 @@ function testNoFalsePositiveWithoutJetsonHints() {
assert.strictEqual(detector.detect(), null, 'detect() should return null when CUDA is unavailable');
}
function testJetsonPlatformMarkerFromNvTegraRelease() {
if (process.platform !== 'linux') {
return;
}
const detector = new CUDADetector();
detector.readFileIfExists = (path) => (
path === '/etc/nv_tegra_release' ? '# R35 (release), REVISION: 4.1' : null
);
assert.strictEqual(
detector.isJetsonPlatform(),
true,
'Jetson platform should be detected via /etc/nv_tegra_release marker'
);
}
function testJetsonCudaSupportMarkerFromNvTegraRelease() {
const detector = new CUDADetector();
const originalExistsSync = fs.existsSync;
try {
fs.existsSync = (path) => path === '/etc/nv_tegra_release';
assert.strictEqual(
detector.hasJetsonCudaSupport(),
true,
'Jetson CUDA support should be detected via /etc/nv_tegra_release marker'
);
} finally {
fs.existsSync = originalExistsSync;
}
}
function run() {
testJetsonFallbackWithoutNvidiaSMI();
testNoFalsePositiveWithoutJetsonHints();
testJetsonPlatformMarkerFromNvTegraRelease();
testJetsonCudaSupportMarkerFromNvTegraRelease();
console.log('✅ cuda-jetson-detection.test.js passed');
}

2
tests/run-all-tests.js
View file

@ -16,7 +16,9 @@ const TESTS = [
{ name: 'CLI interface smoke', file: 'ui-cli-smoke.test.js', category: 'UI' },
{ name: 'Calibration schema', file: 'calibration-schema.test.js', category: 'Calibration' },
{ name: 'Calibration command', file: 'calibrate-command.test.js', category: 'Calibration' },
{ name: 'Calibration end-to-end', file: 'calibration-e2e-integration.test.js', category: 'Calibration' },
{ name: 'Calibration full mode', file: 'calibration-full-mode.test.js', category: 'Calibration' },
{ name: 'Calibration routing policy', file: 'calibration-routing-policy.test.js', category: 'Calibration' },
{ name: 'Policy commands', file: 'policy-commands.test.js', category: 'Policy' },
{ name: 'Policy CLI enforcement', file: 'policy-cli-enforcement.js', category: 'Policy' },
{ name: 'Policy engine', file: 'policy-engine.test.js', category: 'Policy' },

15
tests/ui-cli-smoke.test.js
View file

@ -31,6 +31,10 @@ function run() {
stripAnsi(recommendHelp.stdout).includes('Get intelligent model recommendations for your hardware'),
'recommend help should describe command purpose'
);
assert.ok(
stripAnsi(recommendHelp.stdout).includes('--calibrated [file]'),
'recommend help should expose calibrated routing option'
);
const calibrateHelp = runCli(['calibrate', '--help']);
assert.strictEqual(calibrateHelp.status, 0, stripAnsi(calibrateHelp.stderr || calibrateHelp.stdout));
@ -39,6 +43,17 @@ function run() {
'calibrate help should describe command purpose'
);
const aiRunHelp = runCli(['ai-run', '--help']);
assert.strictEqual(aiRunHelp.status, 0, stripAnsi(aiRunHelp.stderr || aiRunHelp.stdout));
assert.ok(
stripAnsi(aiRunHelp.stdout).includes('--policy <file>'),
'ai-run help should expose policy option'
);
assert.ok(
stripAnsi(aiRunHelp.stdout).includes('--calibrated [file]'),
'ai-run help should expose calibrated routing option'
);
console.log('ui-cli-smoke.test.js: OK');
}