diff --git a/.gitignore b/.gitignore index 0850ce1..d9db564 100644 --- a/.gitignore +++ b/.gitignore @@ -87,4 +87,4 @@ dmypy.json Thumbs.db # Project-specific -.oct/ \ No newline at end of file +.oct/.venv-test/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..09eb137 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,177 @@ +# Changelog + +All notable changes to Guanaco will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +--- + +## [0.5.1] - 2026-06-10 + +### Fixed +- **Updater now stashes + hard-resets instead of merge-pulling.** Previously if the working tree had any local modifications (e.g. leftover version-string edits from a prior partial update), `git pull` would abort with a merge conflict and the update silently failed, leaving the old code running. The updater now unconditionally stashes any local changes (including untracked files) and resets to `origin/{branch}` before reinstalling. This makes the **Apply Update** button work reliably on every install, even if the repo is dirty. + +### Changed +- Bumped version to 0.5.1 to ensure existing 0.4.2 → 0.5.x update paths hit the new updater logic immediately. + +--- + +## [0.5.0] - 2026-06-10 + +### Major Release — Usage Tracking, ROI Dashboard, and Multi-Account Infrastructure + +This release represents a significant milestone: Guanaco now tracks every token accurately, displays real cost analytics via a web dashboard, rotates multiple Ollama Cloud accounts, and scrapes live usage tiers from ollama.com instead of guessing. + +--- + +### New Features + +#### Token Estimation & Accurate Usage Tracking +- **skimtoken estimation fallback** (`analytics.py`): When the upstream API (OpenRouter, Ollama Cloud) omits `usage` data in the response, Guanaco now falls back to `skimtoken` for token estimation instead of logging zero tokens. Estimates are ~15% accurate — dramatically better than silently losing usage data. +- **Proper total_tokens calculation**: Fixed `total_tokens = prompt_tokens + completion_tokens` in analytics logging. Previously some code paths double-counted or omitted totals. +- **`fallback_reason` audit column**: Added to `request_log` schema. When token estimation is used instead of API-reported usage, the reason is recorded (e.g. `"api_omitted_usage"`, `"stream_missing_usage"`) for later audit. +- **Input cache-read pricing** (`roi.py`): Tracks `input_cache_read` tokens separately from `input_cache_write`, applying the correct Ollama Cloud discount rate (typically 0.25× of input price for cache hits). + +#### ROI Dashboard & Per-Model Analytics +- **Web dashboard** (`dashboard/`): New FastAPI-mounted dashboard at `/dashboard/` showing: + - Total tokens consumed (last 24h, 7d, 30d, all-time) + - Per-model token distribution with visual bars + - Cost estimates in USD using live OpenRouter pricing + - **ROI configuration panel**: Slider for `cache_hit_pct` (default 70%), editable price multipliers per model + - **Per-model value scoring**: Each model gets a "value score" based on (capability / cost) ratio, helping users pick the cheapest model for a given task +- **OpenRouter price fetcher** (`roi.py`): Scrapes current model pricing from `https://openrouter.ai/api/v1/models` with 24h caching. Falls back to hardcoded prices if fetch fails. +- **Cache-hit discount logic**: ROI calculations apply the user-configured `cache_hit_pct` to reduce effective input costs, reflecting real-world Ollama Cloud behavior where repeated prompts are cached. + +#### Real Usage-Level Scraping from ollama.com +- **`_fetch_usage_level_sync()`** (`client.py`): New synchronous HTML scraper that parses `ollama.com/library/{model}` pages to determine actual GPU usage tiers: + - Handles **top-level model badges** (`x-test-model-cost-slot-active`) for unified-tier models + - Handles **per-tag listings** (`x-test-model-tag-cost` + `x-test-model-tag-usage-slot-active`) for models with multiple size variants + - Returns usage level 1-4, which maps to multiplier 0.25×, 0.50×, 0.75×, 1.00× +- **`fetch_usage_levels()`**: Async parallel fetcher with **1-hour global cache**. Fetches all library pages concurrently using `asyncio.gather()` with thread-pool execution for the blocking HTTP requests. +- **Wired into API responses**: Both `/v1/models` (OpenAI-compatible) and `/api/ollama/models` (internal) now return `usage_multiplier` and `usage_level` fields based on scraped live data. +- **Fixes major heuristic errors**: + - `gemma3:4b` and `gemma3:12b`: was 1.00×, now correctly **0.25×** (1 slot) + - `gemma3:27b`: was 1.00×, now correctly **0.50×** (2 slots) + - `ministral-3`: was 0.75×, now correctly **0.25×** (1 slot) + - `qwen3-vl`: stays **0.75×** (3 slots) — heuristic accidentally got this one right + - `deepseek-v4-pro`: stays **1.00×** (4 slots) + +#### Multi-Account Ollama Cloud Rotation +- **`accounts.py`**: New module managing multiple Ollama Cloud accounts: + - Each account has its own API key + session cookie + - Load-balanced request routing based on usage and subscription tier + - Quota-aware selection: least-loaded accounts preferred; new/untested accounts get priority for immediate validation +- **Premium model routing**: Models `kimi-k2.6` and `glm-5.1` restricted to Pro/Max accounts only. Free-tier accounts are skipped for these models. +- **Per-account usage tracking**: Analytics DB records which account handled each request, enabling per-account cost breakdowns. + +#### Model Catalog Expansion +- Added `minimax-m3` (new MiniMax flagship) +- Added `nemotron-3-ultra` (NVIDIA enterprise model) +- Added `kimi-k2.6` with 200k context window support + +#### Web Search / Scrape Emulation +- **Search provider plugins** (`search/providers/`): Modular search backend support: + - Brave Search API + - Cohere RAG API + - Exa (formerly Metaphor) + - Firecrawl (web scraping) + - Jina AI (neural search) + - SearXNG (self-hosted meta-search) + - Serper (Google Search API) + - Tavily (AI-native search) +- **Search router** (`search/base.py`): Unified interface — Guanaco presents a single `/search` endpoint regardless of which provider is configured. + +### Fixes + +#### Config & Install Robustness +- **Missing `UsageConfig` fields**: Added `last_plan`, `redirect_on_full`, `last_session_reset`, `last_weekly_reset`, `last_checked` to prevent `AttributeError` crashes on configs from v0.4.2 and earlier. +- **Config migration layer**: `load_config()` now auto-migrates v0.4.2 configs to v0.4.3+ schema on first load. No manual intervention needed. +- **Package rename**: Renamed PyPI package from `guanaco` → `guanaco-llm-proxy` to avoid collision with an existing `guanaco` package on PyPI. +- **Install script fixes** (`install.sh`): + - Ollama API key validation now uses the correct env var name + - Fixed `.env` file write pattern (was writing malformed key=value pairs) + - Fixed `grep` pattern for detecting existing config +- **Startup version sanity check**: Detects repo/venv version mismatch on boot and logs a warning. Prevents confusing "why is `/health` showing the old version?" issues. +- **systemd service**: Fixed `WorkingDirectory` to point at the actual repo checkout. Added `GUANACO_CONFIG_DIR` env var to service file. + +#### Dashboard & Analytics Fixes +- **Removed broken `usage_multiplier` column**: The analytics DB no longer tracks `usage_multiplier` per request (it was always wrong due to heuristic mismatch). Model-level multipliers are now fetched live from ollama.com. +- **Backward compat for `SearchConfig`**: Older installs missing search configuration no longer crash on startup. + +### Infrastructure + +#### CI/CD +- **GitHub Actions CI** (`.github/workflows/ci.yml`): Runs on every push — lint, type-check, unit tests. +- **GitHub Actions Release** (`.github/workflows/release.yml`): Automated PyPI publish on tag push. + +#### Docker +- **`Dockerfile.test`**: Containerized smoke-test environment for CI. +- **`test-local.sh`**: One-command local smoke test — builds Docker image, starts server, hits `/health`, validates version string. + +#### Project Hygiene +- Added `CODE_OF_CONDUCT.md`, `CONTRIBUTING.md`, `LICENSE` (MIT) +- Added `.gitignore` with Python/venv patterns +- Added macOS launch agent plist (`com.guanaco.start.plist`) +- Added systemd service templates (`guanaco.service`, `oct.service`) + +### API Changes + +#### Added Fields +- `/v1/models` response now includes: + - `usage_multiplier` (float): cost multiplier 0.25-1.00 + - `usage_level` (int): raw level 1-4, 0 = unknown +- `/api/ollama/models` response now includes: + - `usage_multiplier` (float) + - `usage_level` (int) + +#### Schema Changes +- `request_log` table: added `fallback_reason TEXT` column +- `request_log` table: removed `usage_multiplier` column (was unreliable) +- New `roi_config` table: stores `cache_hit_pct`, `price_multiplier`, per-model overrides + +### Performance +- **Parallel library scraping**: All ollama.com library pages are fetched concurrently. For a catalog of ~50 models, total scrape time is ~3-5 seconds vs. ~60 seconds sequential. +- **1-hour cache**: Scraped usage levels are cached globally, so the 3-5 second penalty only hits once per hour. +- **ROI price cache**: OpenRouter prices cached for 24 hours. Dashboard loads instantly after first visit. + +### Deprecated / Removed +- **Heuristic `_get_model_multiplier()`**: Still exists as fallback when ollama.com scraping fails, but is no longer the primary source. Returns `0.25` for ≤8B, `0.50` for ≤70B, `0.75` for ≤200B, `1.00` for larger. +- **`usage_multiplier` column in analytics DB**: Dropped. Use `/v1/models` or `/api/ollama/models` to get live multipliers. + +### Known Issues +- **Dev server restart unreliable on isolated instance**: The `uvicorn` process sometimes starts without producing logs. Production (`systemctl restart guanaco.service`) is unaffected. +- **Library scraper depends on ollama.com DOM**: If ollama.com changes their HTML test attributes (`x-test-model-*`), the scraper will fall back to heuristic. Monitor `/api/ollama/models` for sudden multiplier shifts. + +--- + +## [0.4.2] - 2026-05-15 + +### New Features +- Multi-account Ollama Cloud rotation with quota-aware selection +- Premium model routing (`kimi-k2.6`, `glm-5.1` → Pro/Max only) +- Per-account usage tracking + +--- + +## [0.4.1] - 2026-05-01 + +### Fixes +- Rate-limit retry logic for Ollama Cloud 429 responses +- SSE streaming stability improvements + +--- + +## [0.4.0] - 2026-04-20 + +### New Features +- Initial Ollama Cloud proxy support +- OpenAI-compatible `/v1/chat/completions` endpoint +- Token usage tracking with SQLite analytics DB +- Basic web dashboard + +--- + +## [0.3.9] and earlier + +See [GitHub releases](https://github.com/evangit2/guanaco/releases) for earlier versions. diff --git a/docs/plans/2026-04-19-multi-account-manager.md b/docs/plans/2026-04-19-multi-account-manager.md new file mode 100644 index 0000000..3cb00d0 --- /dev/null +++ b/docs/plans/2026-04-19-multi-account-manager.md @@ -0,0 +1,882 @@ +# Multi-Account Manager Implementation Plan + +> **For Hermes:** Use subagent-driven-development skill to implement this plan task-by-task. + +**Goal:** Allow Guanaco to manage multiple Ollama Cloud accounts, rotating requests to the account with the most remaining usage, with a setup wizard in Settings to enter/exit multi-account mode. + +**Architecture:** Add an `AccountConfig` model (name + API key + session cookie + cached usage) and an `accounts` list to `AppConfig`. An `AccountManager` class holds per-account `OllamaClient` instances, tracks usage, and selects the best account per request. The router uses the manager instead of a single client. A setup wizard in the Settings tab handles entering/exiting multi-account mode. + +**Tech Stack:** Python/FastAPI backend, vanilla JS dashboard, SQLite analytics (account name logged per request) + +--- + +## Current Architecture (what changes) + +``` +app.py → OllamaClient(single_key, single_cookie) → router(client, ...) + UsageConfig(session_cookie=one) → quota check on single % +``` + +## New Architecture + +``` +app.py → AccountManager(accounts=[AccountConfig,...]) → router(manager, ...) + Each AccountConfig has: name, api_key, session_cookie, usage cache + AccountManager.select_account() → picks lowest-usage account + Single-account mode: AccountManager with 1 account (backwards compat) +``` + +## Config YAML Shape + +Single-account (default, same as today): +```yaml +ollama_api_key: "key1" +usage: + session_cookie: "cookie1" + ... +``` + +Multi-account mode: +```yaml +ollama_api_key: "key1" # kept for backward compat, becomes "primary" named account +accounts: + - name: "primary" + api_key: "key1" + session_cookie: "cookie1" + last_session_pct: 21.8 + last_weekly_pct: 64.2 + last_session_reset: "4 hours" + last_weekly_reset: "1 hour" + last_plan: "pro" + last_checked: 1776639060.0 + - name: "account2" + api_key: "key2" + session_cookie: "cookie2" + last_session_pct: null + last_weekly_pct: null + ... +usage: + redirect_on_full: true + multi_account_enabled: true # NEW: toggles multi-account mode +``` + +When `multi_account_enabled = true`, the `accounts` list is the source of truth. +When `false` (default), the single `ollama_api_key` + `usage.session_cookie` are used (exactly as today). + +## Analytics Changes + +- `request_log` gets a new `account_name TEXT` column +- In History tab, provider shows as `ollama (primary)` or `ollama (account2-3243)` +- Analytics consolidated across all accounts (no change to aggregation — just new column) + +## Router Changes + +- `create_router(client, ...)` becomes `create_router(account_manager, ...)` +- Before each request: `account = manager.select_account()` → returns `(name, OllamaClient)` +- The OllamaClient for the selected account is used for the actual request +- `fallback_for` field now also used when account rotates (new reason: "Account rotated: primary quota full") +- All `log_llm` calls include `account_name=account.name` + +## Dashboard Changes + +### Settings Tab — Multi-Account Setup +- New section: "Ollama Accounts" +- Shows current mode: "Single Account" or "Multi-Account" +- "Enter Multi-Account Mode" button → wizard: + 1. Name your primary account (pre-filled "primary") + 2. Enter name for second account + 3. Enter API key for second account + 4. Both account session cookies can be set in Status tab +- "Exit Multi-Account Mode" button → reverts to single account (keeps primary key) +- Account list with remove buttons + +### Status Tab — Multi-Account Usage +- When multi-account is enabled, shows usage bars for EACH account +- Each account row: name, session/weekly %, progress bars, reset timers +- "Check All Usage" button checks all accounts in parallel + +### History Tab +- Provider column shows `ollama (account_name)` instead of just `ollama` +- New "Account" filter dropdown + +--- + +## Implementation Tasks + +### Phase 1: Data Model & Config + +#### Task 1: Add AccountConfig and multi_account_enabled to config.py + +**Objective:** Define the data models for multi-account support. + +**Files:** +- Modify: `guanaco/config.py` + +**Step 1: Add AccountConfig model** + +Add after the existing `UsageConfig` class: + +```python +class AccountConfig(BaseModel): + """A single Ollama Cloud account with its own API key, session cookie, and usage cache.""" + name: str = "primary" + api_key: str = "" + session_cookie: str = "" + last_session_pct: Optional[float] = None + last_weekly_pct: Optional[float] = None + last_session_reset: Optional[str] = None + last_weekly_reset: Optional[str] = None + last_plan: Optional[str] = None + last_checked: Optional[float] = None +``` + +**Step 2: Add fields to AppConfig** + +Add to AppConfig: + +```python +accounts: list[AccountConfig] = [] # Populated when multi_account_enabled=True +multi_account_enabled: bool = False +``` + +**Step 3: Add helper property to AppConfig** + +```python +@property +def active_accounts(self) -> list[AccountConfig]: + """Return accounts list if multi-account enabled, else synthesize single account from legacy fields.""" + if self.multi_account_enabled and self.accounts: + return self.accounts + # Single-account mode — synthesize from legacy fields + return [AccountConfig( + name="primary", + api_key=self.ollama_api_key_resolved, + session_cookie=self.usage.session_cookie if self.usage else "", + last_session_pct=self.usage.last_session_pct if self.usage else None, + last_weekly_pct=self.usage.last_weekly_pct if self.usage else None, + last_session_reset=self.usage.last_session_reset if self.usage else None, + last_weekly_reset=self.usage.last_weekly_reset if self.usage else None, + last_plan=self.usage.last_plan if self.usage else None, + last_checked=self.usage.last_checked if self.usage else None, + )] +``` + +**Step 4: Verify the config loads correctly** + +Run: `cd ~/projects/guanaco && source venv/bin/activate && python3 -c "from guanaco.config import load_config; c = load_config(); print('accounts:', c.accounts, 'enabled:', c.multi_account_enabled)"` + +Expected: `accounts: [] enabled: False` + +**Step 5: Commit** + +```bash +git add guanaco/config.py +git commit -m "feat: add AccountConfig model and multi_account_enabled to AppConfig" +``` + +--- + +#### Task 2: Create AccountManager class + +**Objective:** Build the class that manages per-account OllamaClient instances and selects the best account for each request. + +**Files:** +- Create: `guanaco/account_manager.py` + +**Step 1: Create the AccountManager** + +```python +"""Multi-account manager for Ollama Cloud — rotates across accounts to maximize usage.""" + +import logging +from typing import Optional, Tuple + +from guanaco.client import OllamaClient +from guanaco.config import AccountConfig, AppConfig + +log = logging.getLogger("guanaco.accounts") + + +class AccountManager: + """Manages multiple Ollama Cloud accounts and selects the best one per request.""" + + def __init__(self, config: AppConfig): + self._config = config + self._clients: dict[str, OllamaClient] = {} + self._rebuild_clients() + + def _rebuild_clients(self): + """Rebuild OllamaClient instances from config.""" + self._clients.clear() + for acct in self._config.active_accounts: + if acct.api_key and acct.api_key not in ("***", "REPLACE_ME", "your_api_key_here"): + self._clients[acct.name] = OllamaClient( + api_key=acct.api_key, + session_cookie=acct.session_cookie, + ) + log.debug("Account client built: %s", acct.name) + + def refresh(self): + """Rebuild clients after config change (e.g., account added/removed).""" + self._rebuild_clients() + + @property + def accounts(self) -> list[AccountConfig]: + return self._config.active_accounts + + def get_client(self, account_name: Optional[str] = None) -> Tuple[str, OllamaClient]: + """Get the best OllamaClient for a request. + + If account_name is specified, return that account's client. + Otherwise, select the account with the lowest usage. + + Returns (account_name, OllamaClient). + Raises ValueError if no accounts available. + """ + if account_name and account_name in self._clients: + return account_name, self._clients[account_name] + + # Select account with lowest usage + best_name = None + best_score = float('inf') + for acct in self.accounts: + if acct.name not in self._clients: + continue + # Score = max(session%, weekly%). Lower is better. None = unchecked = 0 (best). + s = acct.last_session_pct if acct.last_session_pct is not None else 0 + w = acct.last_weekly_pct if acct.last_weekly_pct is not None else 0 + score = max(s, w) + if score < best_score: + best_score = score + best_name = acct.name + + if best_name is None: + # Fallback: return first available client + if self._clients: + best_name = next(iter(self._clients)) + else: + raise ValueError("No Ollama accounts configured") + + return best_name, self._clients[best_name] + + def get_all_clients(self) -> dict[str, OllamaClient]: + """Return all account clients (for usage checking etc).""" + return dict(self._clients) + + def is_quota_full(self, account_name: Optional[str] = None) -> bool: + """Check if a specific account (or the selected account) is quota-full.""" + if not self._config.usage.redirect_on_full: + return False + # Check the specific account or the best account + target = account_name or self.get_client()[0] + for acct in self.accounts: + if acct.name == target: + s = acct.last_session_pct + w = acct.last_weekly_pct + if s is not None and s >= 99.5: + return True + if w is not None and w >= 99.5: + return True + return False + return False + + def any_account_available(self) -> bool: + """Check if at least one account is not quota-full.""" + if not self._config.usage.redirect_on_full: + return True + for acct in self.accounts: + s = acct.last_session_pct if acct.last_session_pct is not None else 0 + w = acct.last_weekly_pct if acct.last_weekly_pct is not None else 0 + if s < 99.5 and w < 99.5: + return True + return False + + def update_account_usage(self, account_name: str, session_pct: Optional[float], + weekly_pct: Optional[float], session_reset: Optional[str], + weekly_reset: Optional[str], plan: Optional[str]): + """Update cached usage for an account and persist to config.""" + for acct in self._config.accounts: + if acct.name == account_name: + acct.last_session_pct = session_pct + acct.last_weekly_pct = weekly_pct + acct.last_session_reset = session_reset + acct.last_weekly_reset = weekly_reset + acct.last_plan = plan + import time + acct.last_checked = time.time() + break + # Persist + try: + from guanaco.config import save_config + save_config(self._config) + except Exception as e: + log.warning("Failed to persist account usage: %s", e) + + def update_session_cookie(self, account_name: str, cookie: str): + """Update session cookie for an account.""" + for acct in self._config.accounts: + if acct.name == account_name: + acct.session_cookie = cookie + break + if account_name in self._clients: + self._clients[account_name]._session_cookie = cookie + try: + from guanaco.config import save_config + save_config(self._config) + except Exception as e: + log.warning("Failed to persist session cookie: %s", e) +``` + +**Step 2: Test it loads** + +Run: `cd ~/projects/guanaco && source venv/bin/activate && python3 -c "from guanaco.account_manager import AccountManager; print('import OK')"` + +Expected: `import OK` + +**Step 3: Commit** + +```bash +git add guanaco/account_manager.py +git commit -m "feat: add AccountManager for multi-account rotation" +``` + +--- + +### Phase 2: Analytics Update + +#### Task 3: Add account_name column to request_log + +**Objective:** Track which account handled each request in analytics. + +**Files:** +- Modify: `guanaco/analytics.py` + +**Step 1: Add migration in `_ensure_tables`** + +After the `fallback_reason` migration block, add: + +```python +# Migration: add account_name column +try: + conn.execute("ALTER TABLE request_log ADD COLUMN account_name TEXT") +except sqlite3.OperationalError: + pass +``` + +**Step 2: Add account_name to log_llm signature and INSERT** + +Add `account_name: Optional[str] = None` parameter to `log_llm`. + +Add `account_name` to the INSERT column list and VALUES tuple. + +**Step 3: Verify migration** + +Run: `cd ~/projects/guanaco && source venv/bin/activate && python3 -c "import sqlite3; conn = sqlite3.connect('/home/evan/.guanaco/analytics.db'); cur = conn.cursor(); cur.execute('PRAGMA table_info(request_log)'); cols = [r[1] for r in cur.fetchall()]; print('account_name' in cols)"` + +Expected: `True` + +**Step 4: Commit** + +```bash +git add guanaco/analytics.py +git commit -m "feat: add account_name column to request_log for multi-account tracking" +``` + +--- + +### Phase 3: Router Integration + +#### Task 4: Switch router from single client to AccountManager + +**Objective:** The router uses AccountManager to select the best account per request instead of a single OllamaClient. + +**Files:** +- Modify: `guanaco/router/router.py` +- Modify: `guanaco/app.py` + +This is the biggest task. Key changes: + +**Step 1: Update app.py create_app()** + +Replace: +```python +client = OllamaClient(api_key=resolved_key, session_cookie=config.usage.session_cookie) +``` + +With: +```python +from guanaco.account_manager import AccountManager +account_manager = AccountManager(config) +``` + +Pass `account_manager` instead of `client` to `create_router` and dashboard. + +**Step 2: Update create_router signature** + +Change from `create_router(client, ...)` to `create_router(account_manager, ...)`. + +Replace `_client = client` with `_manager = account_manager`. + +**Step 3: Update each request handler** + +In `chat_completions` and the Anthropic endpoint, at the top: + +```python +acct_name, _client = _manager.get_client() +``` + +This replaces the single `_client` closure. The rest of the request logic uses `_client` the same way. + +**Step 4: Add account_name to all log_llm calls** + +Every `_analytics.log_llm(...)` call needs `account_name=acct_name`. + +**Step 5: Update _is_quota_full** + +Replace: +```python +def _is_quota_full(config) -> bool: +``` + +With logic that uses `_manager.any_account_available()`. If any account is available, return False. If ALL accounts are full, return True (trigger fallback). + +Also, when an account is full but others aren't, the manager auto-selects a different account — no fallback needed. Fallback only triggers when ALL accounts are full. + +**Step 6: Update quota redirect section** + +The quota-full check at line ~478 should: + +1. Get the best account from the manager +2. If that account is the same as before, use it +3. If the selected account changed (rotation), use the new account's client +4. Only fall back to the external fallback if ALL accounts are full + +Replace the current quota redirect block with: + +```python +if _is_quota_full(_config): + # All accounts full — go to external fallback + ... +else: + # Use manager-selected account (may have rotated) + acct_name, _client = _manager.get_client() +``` + +**Step 7: Verify the test instance starts** + +Run: `kill $(lsof -t -i:8888) 2>/dev/null; sleep 1; cd ~/projects/guanaco && source venv/bin/activate && GUANACO_ROUTER_PORT=8888 python -m uvicorn guanaco.app:create_app --factory --host 0.0.0.0 --port 8888 &` + +Then: `curl -s http://localhost:8888/health` + +Expected: `{"status": "ok", ...}` + +**Step 8: Commit** + +```bash +git add guanaco/router/router.py guanaco/app.py +git commit -m "feat: router uses AccountManager for multi-account request routing" +``` + +--- + +#### Task 5: Update dashboard.py to use AccountManager + +**Objective:** Dashboard API endpoints (usage checking, session cookies, config) work with multi-account. + +**Files:** +- Modify: `guanaco/dashboard/dashboard.py` + +Key changes: + +**Step 1: Accept account_manager parameter** + +Update `create_dashboard()` to accept `account_manager` instead of (or in addition to) `client`. + +**Step 2: Add multi-account API endpoints** + +```python +@router.get("/api/accounts") +async def list_accounts(request: Request): + """List all configured accounts and their usage.""" + accounts = account_manager.accounts + return {"accounts": [a.model_dump() for a in accounts], "multi_account_enabled": config.multi_account_enabled} + +@router.post("/api/accounts") +async def add_account(request: Request): + """Add a new account in multi-account mode.""" + body = await request.json() + name = body.get("name", "") + api_key = body.get("api_key", "") + if not name or not api_key: + return {"error": "Name and API key required"} + # Check duplicate names + for a in config.accounts: + if a.name == name: + return {"error": f"Account '{name}' already exists"} + config.accounts.append(AccountConfig(name=name, api_key=api_key)) + config.multi_account_enabled = True + save_config(config) + account_manager.refresh() + return {"ok": True} + +@router.delete("/api/accounts/{name}") +async def remove_account(name: str, request: Request): + """Remove an account. If last one, disable multi-account mode.""" + config.accounts = [a for a in config.accounts if a.name != name] + if len(config.accounts) <= 1: + config.multi_account_enabled = False + if config.accounts: + # Move back to single key + config.ollama_api_key = config.accounts[0].api_key + config.usage.session_cookie = config.accounts[0].session_cookie + config.accounts = [] + save_config(config) + account_manager.refresh() + return {"ok": True} + +@router.post("/api/accounts/enable-multi") +async def enable_multi_account(request: Request): + """Enter multi-account mode. Migrates single key to named account.""" + body = await request.json() + primary_name = body.get("primary_name", "primary") + config.accounts = [ + AccountConfig( + name=primary_name, + api_key=config.ollama_api_key_resolved, + session_cookie=config.usage.session_cookie if config.usage else "", + last_session_pct=config.usage.last_session_pct if config.usage else None, + last_weekly_pct=config.usage.last_weekly_pct if config.usage else None, + last_session_reset=config.usage.last_session_reset if config.usage else None, + last_weekly_reset=config.usage.last_weekly_reset if config.usage else None, + last_plan=config.usage.last_plan if config.usage else None, + last_checked=config.usage.last_checked if config.usage else None, + ) + ] + config.multi_account_enabled = True + save_config(config) + account_manager.refresh() + return {"ok": True, "accounts": [a.model_dump() for a in config.accounts]} + +@router.post("/api/accounts/disable-multi") +async def disable_multi_account(request: Request): + """Exit multi-account mode. Reverts to single key from first account.""" + if config.accounts: + config.ollama_api_key = config.accounts[0].api_key + config.usage.session_cookie = config.accounts[0].session_cookie + config.multi_account_enabled = False + config.accounts = [] + save_config(config) + account_manager.refresh() + return {"ok": True} +``` + +**Step 3: Update usage check endpoint** + +`POST /dashboard/api/usage/check` should check ALL accounts' usage when multi-account is enabled: + +```python +@router.post("/api/usage/check") +async def check_usage(request: Request): + results = [] + for acct in account_manager.accounts: + client = account_manager.get_all_clients().get(acct.name) + if not client: + results.append({"name": acct.name, "error": "No client"}) + continue + cookie = acct.session_cookie + if not cookie: + results.append({"name": acct.name, "error": "No session cookie set"}) + continue + try: + usage = await client.get_usage(session_cookie=cookie) + # Update account cache + account_manager.update_account_usage( + acct.name, + session_pct=usage.get("session_pct"), + weekly_pct=usage.get("weekly_pct"), + session_reset=usage.get("session_reset"), + weekly_reset=usage.get("weekly_reset"), + plan=usage.get("plan"), + ) + results.append({"name": acct.name, **usage}) + except Exception as e: + results.append({"name": acct.name, "error": str(e)}) + # For backward compat, if single account, return flat response + if len(results) == 1: + return results[0] + return {"accounts": results} +``` + +**Step 4: Update session cookie endpoint** + +`POST /dashboard/api/usage/session-cookie` needs an `account_name` field: + +```python +@router.post("/api/usage/session-cookie") +async def set_session_cookie(request: Request): + body = await request.json() + cookie = body.get("cookie", "") + account_name = body.get("account_name", "primary") + if config.multi_account_enabled: + account_manager.update_session_cookie(account_name, cookie) + else: + config.usage.session_cookie = cookie + # Also update the live client if accessible + for client in account_manager.get_all_clients().values(): + client._session_cookie = cookie + save_config(config) + return {"ok": True} +``` + +**Step 5: Commit** + +```bash +git add guanaco/dashboard/dashboard.py +git commit -m "feat: dashboard multi-account API endpoints (list, add, remove, enable/disable)" +``` + +--- + +### Phase 4: Dashboard UI + +#### Task 6: Settings tab — Multi-account setup UI + +**Objective:** Add the setup wizard and account management UI to Settings. + +**Files:** +- Modify: `guanaco/dashboard/templates/dashboard.html` + +**Step 1: Add "Ollama Accounts" section to Settings tab** + +In the Settings tab, add a new card section for account management. Contains: +- Current mode badge: "Single Account" or "Multi-Account (N accounts)" +- Account list (when multi-account enabled): name, API key (masked), remove button +- "Enter Multi-Account Mode" button (when single) → opens wizard modal +- "Add Account" button (when multi) → opens add modal +- "Exit Multi-Account Mode" button (when multi) → confirms and reverts + +**Step 2: Add wizard modal** + +The wizard has steps: +1. "Name your primary account" — input with "primary" pre-filled +2. "Add second account" — name + API key inputs +3. "Setup complete" — shows both accounts, notes that session cookies go in Status tab + +**Step 3: JS functions** + +```javascript +function loadAccountConfig() { + fetch('/dashboard/api/accounts').then(r => r.json()).then(data => { + // Render account list and mode badge + }); +} + +function enterMultiAccountMode() { + // Show wizard modal +} + +function addAccount() { + // Show add-account modal +} + +function removeAccount(name) { + // Confirm, then DELETE /dashboard/api/accounts/{name} +} + +function exitMultiAccountMode() { + // Confirm, then POST /dashboard/api/accounts/disable-multi +} +``` + +**Step 4: Hook into showTab()** + +When Settings tab is shown, call `loadAccountConfig()`. + +**Step 5: Commit** + +```bash +git add guanaco/dashboard/templates/dashboard.html +git commit -m "feat: Settings tab multi-account setup wizard UI" +``` + +--- + +#### Task 7: Status tab — Multi-account usage display + +**Objective:** Show per-account usage bars when multi-account is enabled. + +**Files:** +- Modify: `guanaco/dashboard/templates/dashboard.html` + +**Step 1: Update usage check JS** + +When multi-account is enabled, `checkUsage()` should handle the `accounts` array response and render a row per account. + +Each row: account name, session % bar, weekly % bar, reset timers, "Check" button. + +**Step 2: Update session cookie section** + +When multi-account, show a dropdown to select which account's cookie to set, plus the cookie input and save button. + +**Step 3: Commit** + +```bash +git add guanaco/dashboard/templates/dashboard.html +git commit -m "feat: Status tab multi-account usage display with per-account bars" +``` + +--- + +#### Task 8: History tab — Account name in provider column + +**Objective:** Show which account handled each request in the History list and modal. + +**Files:** +- Modify: `guanaco/dashboard/templates/dashboard.html` + +**Step 1: Update list rendering** + +Change the provider display from `🏭 ${r.provider || 'ollama'}` to: + +```javascript +let providerDisplay = r.provider || 'ollama'; +if (providerDisplay === 'ollama' && r.account_name) { + providerDisplay = `ollama (${escapeHtml(r.account_name)})`; +} +``` + +**Step 2: Update modal** + +In the detail modal's metadata grid, add: + +```html +
Account: ${data.account_name || '—'}
+``` + +**Step 3: Add account filter dropdown** + +Add a new filter dropdown in the History tab header for filtering by account name. + +**Step 4: Commit** + +```bash +git add guanaco/dashboard/templates/dashboard.html +git commit -m "feat: History tab shows account name in provider column, account filter" +``` + +--- + +### Phase 5: Config Persistence + +#### Task 9: Add save_config function and ensure accounts persist + +**Objective:** Ensure the `accounts` list and `multi_account_enabled` field survive config.yaml round-trips. + +**Files:** +- Modify: `guanaco/config.py` + +**Step 1: Verify save_config exists** + +Check that `save_config` writes all Pydantic model fields including `accounts` and `multi_account_enabled` to `config.yaml`. If it doesn't exist, add it. + +**Step 2: Test round-trip** + +```python +from guanaco.config import load_config, save_config +c = load_config() +c.multi_account_enabled = True +c.accounts = [AccountConfig(name="test", api_key="key123")] +save_config(c) +c2 = load_config() +assert c2.multi_account_enabled == True +assert len(c2.accounts) == 1 +assert c2.accounts[0].name == "test" +``` + +**Step 3: Commit** + +```bash +git add guanaco/config.py +git commit -m "feat: config persistence for accounts and multi_account_enabled" +``` + +--- + +### Phase 6: Testing & Polish + +#### Task 10: End-to-end test on port 8888 + +**Objective:** Verify the full multi-account flow works. + +**Step 1: Start test instance** + +```bash +cd ~/projects/guanaco && source venv/bin/activate && GUANACO_ROUTER_PORT=8888 python -m uvicorn guanaco.app:create_app --factory --host 0.0.0.0 --port 8888 & +``` + +**Step 2: Test single-account mode (default)** + +- `curl http://localhost:8888/dashboard/api/accounts` → `{"accounts": [...], "multi_account_enabled": false}` +- Send a request, verify it works +- Check analytics: `account_name` should be "primary" + +**Step 3: Test entering multi-account mode** + +- `curl -X POST http://localhost:8888/dashboard/api/accounts/enable-multi -H 'Content-Type: application/json' -d '{"primary_name":"main"}'` +- Verify accounts list now has one account named "main" + +**Step 4: Test adding second account** + +- `curl -X POST http://localhost:8888/dashboard/api/accounts -H 'Content-Type: application/json' -d '{"name":"backup","api_key":"test-key"}'` +- Verify accounts list has two entries + +**Step 5: Test account rotation** + +- Send multiple requests, verify the account with lower usage is selected +- Check analytics for `account_name` values + +**Step 6: Test removing account** + +- `curl -X DELETE http://localhost:8888/dashboard/api/accounts/backup` +- Verify single account remains + +**Step 7: Test exiting multi-account mode** + +- `curl -X POST http://localhost:8888/dashboard/api/accounts/disable-multi` +- Verify `multi_account_enabled = false` and `accounts = []` + +**Step 8: Commit** + +```bash +git commit -m "test: end-to-end multi-account verification" +``` + +--- + +## Key Design Decisions + +1. **Backward compatible**: Single-account mode (default) works exactly as today. `multi_account_enabled=false` means `accounts` list is ignored, legacy `ollama_api_key` + `usage.session_cookie` are used. + +2. **AccountManager abstracts the client**: The router doesn't need to know about multi-account. It calls `manager.get_client()` and gets back a `(name, OllamaClient)` pair. The manager handles selection. + +3. **Usage-based rotation**: The manager picks the account with the lowest `max(session%, weekly%)`. Unchecked accounts (None values) score 0, so they get tried first (then their usage gets populated). + +4. **Analytics include account_name**: New column, shown in History as `ollama (account_name)`. Consolidated stats work the same — you can filter by account. + +5. **Fallback still works**: If ALL accounts are quota-full, the external fallback triggers. Individual account rotation happens first. + +6. **Session cookies per account**: Each account has its own cookie, set via the Status tab. The wizard tells the user to set cookies there after setup. + +7. **Wizard flow**: "Enter Multi-Account Mode" → name primary → name + key for second account → done. User can add more later. "Exit" reverts to single key from first account. + +## Files Changed Summary + +| File | Change | +|---|---| +| `guanaco/config.py` | Add `AccountConfig`, `multi_account_enabled`, `active_accounts` property, ensure `save_config` works | +| `guanaco/account_manager.py` | NEW — AccountManager class | +| `guanaco/analytics.py` | Add `account_name` column migration + log_llm param | +| `guanaco/router/router.py` | Use AccountManager instead of single client, pass account_name to log_llm | +| `guanaco/app.py` | Create AccountManager, pass it to router + dashboard | +| `guanaco/dashboard/dashboard.py` | Multi-account API endpoints, usage check per account | +| `guanaco/dashboard/templates/dashboard.html` | Settings wizard, Status per-account bars, History account column | \ No newline at end of file diff --git a/guanaco/__init__.py b/guanaco/__init__.py index b885581..c7e78f9 100644 --- a/guanaco/__init__.py +++ b/guanaco/__init__.py @@ -1,7 +1,21 @@ """guanaco — maximize your Ollama Cloud subscription.""" +# Single source of truth for version. +# importlib.metadata can return stale values after git-pull without re-pip-install, +# so we always use the hardcoded fallback and only override if metadata matches. +__version__ = "0.5.1" + try: from importlib.metadata import version as _version - __version__ = _version("guanaco") + _pkg_ver = _version("guanaco-llm-proxy") + # Only override hardcoded if installed metadata is *newer or same* — + # prevents stale metadata from git-pull without re-pip-install from + # reverting the version to an old value. + import re + _m = re.match(r"^(\d+)\.(\d+)\.(\d+)$", _pkg_ver or "") + if _m: + _hardcoded = tuple(int(x) for x in __version__.split(".")) + if tuple(int(x) for x in _m.groups()) >= _hardcoded: + __version__ = _pkg_ver except Exception: - __version__ = "0.3.6" # fallback when not installed via pip \ No newline at end of file + pass \ No newline at end of file diff --git a/guanaco/accounts.py b/guanaco/accounts.py new file mode 100644 index 0000000..f909ab1 --- /dev/null +++ b/guanaco/accounts.py @@ -0,0 +1,146 @@ +"""Multi-account Ollama key rotation with quota-aware selection.""" + +import logging +import time +from typing import Optional + +from guanaco.config import OllamaAccount + +logger = logging.getLogger(__name__) + +# Models that require a paid Ollama plan (not available on free tier). +PREMIUM_MODELS = {"kimi-k2.6", "glm-5.1"} + + +def model_requires_premium(model: str) -> bool: + """Check if a model requires a paid Ollama plan (pro/max). + + Matches case-insensitively against model name substrings. + E.g. 'kimi-k2.6-0915' matches 'k2.6'. + """ + model_lower = model.lower().strip() + for pm in PREMIUM_MODELS: + if pm in model_lower: + return True + return False + + +class AccountPool: + """Manages a pool of Ollama accounts and selects the best one for each request. + + Selection strategy: + 1. Among accounts with usage data, pick the one with the lowest session_pct + 2. Among accounts without usage data, round-robin + 3. If only one account, always use it + """ + + def __init__(self, accounts: list[OllamaAccount]): + self._accounts: list[OllamaAccount] = accounts + self._rr_index: int = 0 + + @property + def accounts(self) -> list[OllamaAccount]: + return self._accounts + + def update_accounts(self, accounts: list[OllamaAccount]) -> None: + """Replace the account list (e.g., after config save).""" + self._accounts = accounts + + def get_account(self, preferred: Optional[str] = None, model: Optional[str] = None) -> OllamaAccount: + """Select the best account for the next request. + + Strategy: prefer accounts with the most available quota. + - If model is premium, skip free-tier accounts. + - Accounts with no usage data are assumed fresh (0%) — preferred. + - Among accounts with usage data, pick lowest session_pct. + - Tie-break with round-robin for equal-priority accounts. + + Args: + preferred: If set, try this account name first. + model: If set, check if model requires premium and filter accordingly. + + Returns: + The selected OllamaAccount. + """ + active = [a for a in self._accounts if a.api_key and a.api_key not in ("***", "REPLACE_ME")] + if not active: + return self._accounts[0] if self._accounts else OllamaAccount(name="ollama") + + # If model requires premium, filter out free-tier accounts + premium_needed = model and model_requires_premium(model) + if premium_needed: + eligible = [a for a in active if a.last_plan and a.last_plan.lower() != "free"] + if eligible: + active = eligible + logger.info(f"Model '{model}' requires premium plan, filtered to {len(eligible)} eligible accounts") + else: + logger.warning(f"Model '{model}' requires premium but no paid accounts available, trying all") + + if len(active) == 1: + return active[0] + + if preferred: + for a in active: + if a.name == preferred: + return a + + # Split by whether we have usage data + without_usage = [a for a in active if a.last_session_pct is None] + with_usage = [a for a in active if a.last_session_pct is not None] + + # Prefer accounts with no usage data (fresh/unknown quota) over known-heavy ones + if without_usage: + idx = self._rr_index % len(without_usage) + self._rr_index += 1 + account = without_usage[idx] + logger.debug(f"Selected account '{account.name}' (no usage data, round-robin)") + return account + + # All have usage data — pick the one with lowest usage + best = min(with_usage, key=lambda a: a.last_session_pct or 100) + logger.debug(f"Selected account '{best.name}' (session: {best.last_session_pct}%)") + return best + + def update_usage(self, account_name: str, session_pct: Optional[float], + weekly_pct: Optional[float], plan: Optional[str] = None, + session_reset: Optional[str] = None, weekly_reset: Optional[str] = None) -> None: + """Update usage data for a specific account.""" + for a in self._accounts: + if a.name == account_name: + a.last_session_pct = session_pct + a.last_weekly_pct = weekly_pct + a.last_plan = plan + a.last_session_reset = session_reset + a.last_weekly_reset = weekly_reset + a.last_checked = time.time() + break + + def mark_429(self, account_name: str) -> None: + """Mark that an account hit a 429. Temporarily deprioritize it.""" + for a in self._accounts: + if a.name == account_name: + if a.last_session_pct is not None: + a.last_session_pct = min(a.last_session_pct + 25, 100) + else: + a.last_session_pct = 75 + logger.info(f"Account '{account_name}' hit 429, deprioritizing (session est: {a.last_session_pct}%)") + break + + def account_names(self) -> list[str]: + """List all account names.""" + return [a.name for a in self._accounts] + + def get_by_name(self, name: str) -> Optional[OllamaAccount]: + """Find account by name.""" + for a in self._accounts: + if a.name == name: + return a + return None + + def name_taken(self, name: str) -> bool: + """Check if an account name is already used.""" + return any(a.name == name for a in self._accounts) + + def is_reserved_name(self, name: str) -> bool: + """Check if a name is reserved (case-insensitive).""" + return name.lower() in ("ollama", "primary", "default") \ No newline at end of file diff --git a/guanaco/analytics.py b/guanaco/analytics.py index 1e97bcf..b57f8c5 100644 --- a/guanaco/analytics.py +++ b/guanaco/analytics.py @@ -15,6 +15,11 @@ import uuid from pathlib import Path from typing import Optional +try: + from skimtoken.multilingual_simple import estimate_tokens as _estimate_tokens +except Exception: + _estimate_tokens = None + def _default_db_path() -> Path: from guanaco.config import get_default_config_dir @@ -72,6 +77,22 @@ class AnalyticsLogger: conn.execute("ALTER TABLE request_log ADD COLUMN fallback_for TEXT") except sqlite3.OperationalError: pass # column already exists + # Migration: add caller info and content columns for full history + for col in ["source_ip TEXT", "source_port INTEGER", "user_agent TEXT", "input_text TEXT", "output_text TEXT"]: + try: + conn.execute(f"ALTER TABLE request_log ADD COLUMN {col}") + except sqlite3.OperationalError: + pass # column already exists + # Migration: add fallback_reason column + try: + conn.execute("ALTER TABLE request_log ADD COLUMN fallback_reason TEXT") + except sqlite3.OperationalError: + pass + # Migration: add account_name column for multi-account rotation + try: + conn.execute("ALTER TABLE request_log ADD COLUMN account_name TEXT") + except sqlite3.OperationalError: + pass conn.execute(""" CREATE TABLE IF NOT EXISTS status_events ( id TEXT PRIMARY KEY, @@ -130,25 +151,101 @@ class AnalyticsLogger: provider: Optional[str] = None, fallback_for: Optional[str] = None, extra: Optional[dict] = None, + # Full history fields (optional, requires opt-in) + source_ip: Optional[str] = None, + source_port: Optional[int] = None, + user_agent: Optional[str] = None, + input_text: Optional[str] = None, + output_text: Optional[str] = None, + fallback_reason: Optional[str] = None, + account_name: Optional[str] = None, ) -> str: """Log an LLM request. Returns the log entry ID.""" # Normalize model name so glm-5.1:cloud and glm-5.1 are grouped together model = _normalize_model_name(model) fallback_for = _normalize_model_name(fallback_for) if fallback_for else fallback_for + + # Fallback: if API returned zeros or None but we have text, estimate tokens + # using skimtoken for multilingual/CJK-aware approximation (~15% error). + # This prevents silently losing token data when providers omit the usage block. + if (not prompt_tokens or prompt_tokens == 0) and input_text: + if _estimate_tokens is not None: + prompt_tokens = max(1, _estimate_tokens(input_text)) + else: + prompt_tokens = max(1, len(input_text) // 3) + if (not completion_tokens or completion_tokens == 0) and output_text: + if _estimate_tokens is not None: + completion_tokens = max(1, _estimate_tokens(output_text)) + else: + completion_tokens = max(1, len(output_text) // 4) + total_tokens = prompt_tokens + completion_tokens + entry_id = str(uuid.uuid4()) with sqlite3.connect(self.db_path) as conn: conn.execute( """INSERT INTO request_log (id, ts, type, model, prompt_tokens, completion_tokens, total_tokens, tps, prompt_tps, ttft_seconds, total_duration_seconds, - load_duration_seconds, error, request_id, provider, fallback_for) - VALUES (?, ?, 'llm', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + load_duration_seconds, error, request_id, provider, fallback_for, + source_ip, source_port, user_agent, input_text, output_text, fallback_reason, account_name) + VALUES (?, ?, 'llm', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (entry_id, time.time(), model, prompt_tokens, completion_tokens, total_tokens, tps, prompt_tps, ttft_seconds, total_duration_seconds, - load_duration_seconds, error, request_id, provider, fallback_for), + load_duration_seconds, error, request_id, provider, fallback_for, + source_ip, source_port, user_agent, input_text, output_text, fallback_reason, account_name), ) + + # Write plaintext log file if configured + if input_text or output_text: + try: + from guanaco.config import get_config + _cfg = get_config() + if _cfg.history.log_to_files: + self._write_log_file(entry_id, model, provider, source_ip, input_text, output_text, error, _cfg.history) + except Exception: + pass # Don't break the request if log file writing fails + return entry_id + def _write_log_file(self, entry_id: str, model: str, provider: Optional[str], + source_ip: Optional[str], input_text: Optional[str], + output_text: Optional[str], error: Optional[str], + history_config=None): + """Write a plaintext log file for this request.""" + try: + log_dir = history_config.get_log_dir() if history_config else None + if not log_dir: + return + ts = time.strftime("%Y%m%d_%H%M%S", time.localtime()) + # One file per request: __.log + safe_model = model.replace("/", "_").replace(":", "_").replace(" ", "_") + filename = f"{ts}_{safe_model}_{entry_id[:8]}.log" + filepath = log_dir / filename + + lines = [] + lines.append(f"=== Guanaco Request Log ===") + lines.append(f"ID: {entry_id}") + lines.append(f"Time: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())}") + lines.append(f"Model: {model}") + lines.append(f"Provider: {provider or 'ollama'}") + lines.append(f"Caller: {source_ip or 'unknown'}") + if error: + lines.append(f"Error: {error}") + lines.append(f"") + if input_text: + lines.append(f"--- INPUT ---") + lines.append(input_text) + lines.append(f"") + if output_text: + lines.append(f"--- OUTPUT ---") + lines.append(output_text) + lines.append(f"") + lines.append(f"=== END ===") + + filepath.write_text("\n".join(lines), encoding="utf-8") + except Exception: + pass # Don't break the request if log file writing fails + def log_search( self, provider: str, @@ -413,6 +510,19 @@ class AnalyticsLogger: fallbacks.append({ "original_model": row[0], "fallback_count": row[1], "last_used": row[2], }) + + # Fallback rate (24h window) - percentage of requests routed to fallback + cutoff_24h = time.time() - (24 * 3600) + fb_24h = conn.execute( + "SELECT COUNT(*) FROM request_log WHERE type='llm' AND fallback_for IS NOT NULL AND ts > ?", + (cutoff_24h,) + ).fetchone()[0] + main_24h = conn.execute( + "SELECT COUNT(*) FROM request_log WHERE type='llm' AND (provider='ollama' OR provider IS NULL) AND fallback_for IS NULL AND ts > ?", + (cutoff_24h,) + ).fetchone()[0] + total_24h = fb_24h + main_24h + fallback_rate = round((fb_24h / total_24h) * 100, 1) if total_24h > 0 else 0.0 # Recent errors error_rows = conn.execute( @@ -457,6 +567,7 @@ class AnalyticsLogger: "recent_errors": recent_errors, "status_errors": status_error_count, "status_warnings": status_warning_count, + "fallback_rate": fallback_rate, "usage": { "session_pct": usage_row[0] if usage_row else None, "weekly_pct": usage_row[1] if usage_row else None, @@ -502,9 +613,165 @@ class AnalyticsLogger: ).fetchall() return [dict(r) for r in rows] + def get_fallback_rate(self, hours: int = 24) -> dict: + """Calculate fallback routing rate for the specified time window. + + Returns the percentage of requests that were routed to fallback provider + due to main provider failures (timeout, error, quota full). + """ + cutoff = time.time() - (hours * 3600) + + with sqlite3.connect(self.db_path) as conn: + fallback_count = conn.execute( + "SELECT COUNT(*) FROM request_log WHERE type='llm' AND fallback_for IS NOT NULL AND ts > ?", + (cutoff,) + ).fetchone()[0] + main_count = conn.execute( + "SELECT COUNT(*) FROM request_log WHERE type='llm' AND (provider='ollama' OR provider IS NULL) AND fallback_for IS NULL AND ts > ?", + (cutoff,) + ).fetchone()[0] + total = fallback_count + main_count + rate = round((fallback_count / total) * 100, 1) if total > 0 else 0.0 + return { + "rate": rate, + "fallback_count": fallback_count, + "main_count": main_count, + "total": total, + "hours": hours, + } + + + def get_history( + self, + limit: int = 100, + offset: int = 0, + model_filter: Optional[str] = None, + provider_filter: Optional[str] = None, + has_content: Optional[bool] = None, + errors_only: bool = False, + include_content: bool = False, + ) -> list[dict]: + """Get paginated request history with optional filters. + + Args: + limit: Max results to return + offset: Skip this many results (pagination) + model_filter: Filter by model name + provider_filter: Filter by provider + has_content: Filter to only requests with/without saved content + errors_only: Filter to only failed requests (error IS NOT NULL) + include_content: Include input_text/output_text in results + """ + query = "SELECT * FROM request_log WHERE type='llm'" + params = [] + + if model_filter: + query += " AND model = ?" + params.append(model_filter) + if provider_filter: + query += " AND provider = ?" + params.append(provider_filter) + if errors_only: + query += " AND error IS NOT NULL AND error != ''" + elif has_content is True: + query += " AND input_text IS NOT NULL" + elif has_content is False: + query += " AND input_text IS NULL" + + query += " ORDER BY ts DESC LIMIT ? OFFSET ?" + params.extend([limit, offset]) + + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + rows = conn.execute(query, params).fetchall() + results = [] + for row in rows: + d = dict(row) + # Add has_content flag for badge rendering without needing full text + has_input = bool(d.get("input_text")) + has_output = bool(d.get("output_text")) + d["has_content"] = has_input or has_output + # Don't include content unless requested (can be large) + if not include_content: + d.pop("input_text", None) + d.pop("output_text", None) + # Format timestamp + d["ts_formatted"] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(d["ts"])) + results.append(d) + return results + + def get_request_detail(self, request_id: str) -> Optional[dict]: + """Get full details of a single request including content.""" + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + row = conn.execute( + "SELECT * FROM request_log WHERE id = ?", + (request_id,) + ).fetchone() + if row: + d = dict(row) + d["ts_formatted"] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(d["ts"])) + return d + return None + + def get_history_stats(self) -> dict: + """Get stats about history logging.""" + with sqlite3.connect(self.db_path) as conn: + total = conn.execute("SELECT COUNT(*) FROM request_log WHERE type='llm'").fetchone()[0] + with_content = conn.execute( + "SELECT COUNT(*) FROM request_log WHERE type='llm' AND input_text IS NOT NULL" + ).fetchone()[0] + oldest = conn.execute( + "SELECT MIN(ts) FROM request_log WHERE type='llm'" + ).fetchone()[0] + newest = conn.execute( + "SELECT MAX(ts) FROM request_log WHERE type='llm'" + ).fetchone()[0] + + # Storage size estimate + content_size = conn.execute( + "SELECT COALESCE(SUM(LENGTH(input_text) + LENGTH(output_text)), 0) FROM request_log WHERE input_text IS NOT NULL OR output_text IS NOT NULL" + ).fetchone()[0] + + # Error count + error_count = conn.execute( + "SELECT COUNT(*) FROM request_log WHERE type='llm' AND error IS NOT NULL AND error != ''" + ).fetchone()[0] + + return { + "total_requests": total, + "requests_with_content": with_content, + "error_count": error_count, + "oldest_ts": oldest, + "newest_ts": newest, + "content_size_bytes": content_size, + "content_size_mb": round(content_size / (1024 * 1024), 2), + } + def clear(self): """Clear all analytics data.""" with sqlite3.connect(self.db_path) as conn: conn.execute("DELETE FROM request_log") conn.execute("DELETE FROM status_events") - conn.execute("DELETE FROM usage_snapshots") \ No newline at end of file + conn.execute("DELETE FROM usage_snapshots") + + def cleanup_old_log_files(self, history_config=None): + """Delete log files older than retention_days from the history_logs directory.""" + if not history_config or not history_config.log_to_files: + return 0 + retention_days = history_config.retention_days + if retention_days <= 0: + return 0 # 0 means keep forever + try: + log_dir = history_config.get_log_dir() + if not log_dir.exists(): + return 0 + cutoff = time.time() - (retention_days * 86400) + deleted = 0 + for f in log_dir.glob("*.log"): + if f.stat().st_mtime < cutoff: + f.unlink() + deleted += 1 + return deleted + except Exception: + return 0 \ No newline at end of file diff --git a/guanaco/app.py b/guanaco/app.py index 42b5dd9..ea554b7 100644 --- a/guanaco/app.py +++ b/guanaco/app.py @@ -13,11 +13,8 @@ from fastapi.middleware.cors import CORSMiddleware from guanaco.config import load_config, get_config, AppConfig, get_base_url, get_tailscale_ip from guanaco.client import OllamaClient -try: - from importlib.metadata import version as _pkg_version - __version__ = _pkg_version("guanaco") -except Exception: - __version__ = "0.3.6" +from guanaco.accounts import AccountPool +__version__ = "0.5.1" from guanaco.router.router import create_router as create_llm_router from guanaco.search.providers import ALL_PROVIDERS from guanaco.dashboard import create_dashboard_router @@ -36,6 +33,13 @@ def create_app(config: AppConfig | None = None) -> FastAPI: client = OllamaClient(api_key=resolved_key, session_cookie=config.usage.session_cookie) + # Ensure primary account exists in the accounts list + if not config.ollama_accounts: + config.ollama_accounts = [config.primary_account] + elif not any(a.name == "ollama" for a in config.ollama_accounts): + config.ollama_accounts.insert(0, config.primary_account) + account_pool = AccountPool(config.ollama_accounts) + from guanaco.config import get_default_config_dir key_manager = ApiKeyManager(get_default_config_dir()) analytics = AnalyticsLogger() @@ -133,7 +137,7 @@ def create_app(config: AppConfig | None = None) -> FastAPI: return {"status": "ok", "version": __version__} # ── LLM Router ── - app.include_router(create_llm_router(client, analytics=analytics, config=config)) + app.include_router(create_llm_router(client, analytics=analytics, config=config, account_pool=account_pool)) # ── Search Providers ── for provider_cls in ALL_PROVIDERS: @@ -263,7 +267,10 @@ def create_app(config: AppConfig | None = None) -> FastAPI: print(f" [WARN] Firecrawl SDK compat routes not loaded: {e}") # ── Dashboard ── - app.include_router(create_dashboard_router(key_manager, analytics, client), prefix="/dashboard") + app.include_router(create_dashboard_router(key_manager, analytics, client, account_pool=account_pool), prefix="/dashboard") + + # Store account_pool on app state for dashboard access + app.state.account_pool = account_pool # ── Ollama status & models (top-level API) ── diff --git a/guanaco/cli.py b/guanaco/cli.py index d30dea9..a088e3b 100644 --- a/guanaco/cli.py +++ b/guanaco/cli.py @@ -159,10 +159,10 @@ def _run_setup(): print("\n📡 LLM Configuration") print(" Available Ollama Cloud models: qwen3:480b, gpt-oss:120b, deepseek-v3.1, oss120b") print(" Also: qwen3.5:122b, glm-5.1, minimax-m2.7, llama4:109b, etc.") - reranker = input("Reranker model [oss120b]: ").strip() or "oss120b" - scraper = input("Scraper model [qwen3:480b]: ").strip() or "qwen3:480b" - summary = input("Summary model [qwen3:480b]: ").strip() or "qwen3:480b" - default_model = input("Default chat model [qwen3:480b]: ").strip() or "qwen3:480b" + reranker = input("Reranker model [nemotron-3-nano:30b]: ").strip() or "nemotron-3-nano:30b" + scraper = input("Scraper model [nemotron-3-nano:30b]: ").strip() or "nemotron-3-nano:30b" + summary = input("Summary model [nemotron-3-nano:30b]: ").strip() or "nemotron-3-nano:30b" + default_model = input("Default chat model [nemotron-3-nano:30b]: ").strip() or "nemotron-3-nano:30b" emulate_anthropic = input("Enable Anthropic /v1/messages emulation? [Y/n]: ").strip().lower() != "n" emulate_openai = input("Enable OpenAI /v1/chat/completions? [Y/n]: ").strip().lower() != "n" @@ -379,6 +379,9 @@ def _run_start(args): config = load_config() + # ── Version sanity check: repo vs installed package ── + _check_version_sanity() + if args.host: config.router.host = args.host if args.port: @@ -802,5 +805,64 @@ def _run_config(args): print(f" {en} {name} {key_status}") +def _check_version_sanity(): + """Warn if the installed package is out of sync with the repo checkout. + + Detects the common footgun where: + - install.sh clones to ~/.guanaco/repo and does `pip install -e .` + - But later someone edits the repo code without reinstalling + - Or installs a different version from PyPI over the editable install + """ + import importlib.util + from pathlib import Path + + try: + # Where does `guanaco` load from? + spec = importlib.util.find_spec("guanaco") + if spec is None or spec.origin is None: + return # can't determine, skip + installed_path = Path(spec.origin).resolve() + + # Check if it's an editable install (points into a repo checkout) + is_editable = False + repo_root = None + if installed_path.parts: + # Walk up to find .git + for parent in installed_path.parents: + if (parent / ".git").is_dir(): + is_editable = True + repo_root = parent + break + + # Read __version__ from the installed package + from guanaco import __version__ as installed_version + + if repo_root: + # Compare with repo __init__.py version + repo_init = repo_root / "guanaco" / "__init__.py" + if repo_init.exists(): + repo_version = "unknown" + for line in repo_init.read_text().splitlines(): + if '__version__' in line and '=' in line: + repo_version = line.split('=')[1].strip().strip('"').strip("'") + break + if repo_version != installed_version: + print(f"⚠️ VERSION MISMATCH DETECTED") + print(f" Installed package: v{installed_version} at {installed_path}") + print(f" Repo checkout: v{repo_version} at {repo_root}") + print(f" Fix: cd {repo_root} && pip install -e .") + print() + + # Also warn if installed from PyPI (site-packages) rather than editable + elif "site-packages" in str(installed_path): + print(f"⚠️ Installed from PyPI/site-packages, not editable install:") + print(f" {installed_path}") + print(f" If you're developing, use: pip install -e .") + print() + + except Exception: + pass # Don't crash startup for a sanity check + + if __name__ == "__main__": main() \ No newline at end of file diff --git a/guanaco/client.py b/guanaco/client.py index d12169a..ae7b9aa 100644 --- a/guanaco/client.py +++ b/guanaco/client.py @@ -5,6 +5,7 @@ from __future__ import annotations import json import time import logging +import re from typing import Optional import httpx @@ -20,39 +21,51 @@ OLLAMA_FETCH_URL = f"{OLLAMA_BASE}/api/web_fetch" OLLAMA_USAGE_URL = f"{OLLAMA_BASE}/api/account/usage" OLLAMA_SETTINGS_URL = f"{OLLAMA_BASE}/api/account/settings" +# Usage-level cache: maps model name → level (1-4) +_USAGE_LEVEL_CACHE: dict[str, int] = {} +_USAGE_LEVEL_CACHE_TIME: float = 0 +_USAGE_LEVEL_CACHE_TTL: float = 3600 # 1 hour + # Known cloud models (fallback + display info) # Names must match /v1/models response (e.g. "gemma4:31b", "qwen3.5:397b") +# usage_multiplier: relative GPU cost tier (0.25, 0.50, 0.75, 1.00) pulled from +# ollama.com model pages — used for weighted analytics + visual cost badges. KNOWN_CLOUD_MODELS = { - "gemma4": {"sizes": ["31b"], "family": "gemma", "capabilities": ["vision", "tools", "thinking", "cloud"]}, - "gemma3": {"sizes": ["4b", "12b", "27b"], "family": "gemma", "capabilities": ["vision", "tools", "thinking", "cloud"]}, - "qwen3.5": {"sizes": ["397b"], "family": "qwen", "capabilities": ["vision", "tools", "thinking", "cloud"]}, - "qwen3-vl": {"sizes": ["235b", "235b-instruct"], "family": "qwen", "capabilities": ["vision", "tools", "thinking", "cloud"]}, - "qwen3-coder": {"sizes": ["480b"], "family": "qwen", "capabilities": ["tools", "cloud"]}, - "qwen3-coder-next": {"sizes": [], "family": "qwen", "capabilities": ["tools", "cloud"]}, - "qwen3-next": {"sizes": ["80b"], "family": "qwen", "capabilities": ["tools", "thinking", "cloud"]}, - "minimax-m2": {"sizes": [], "family": "minimax", "capabilities": ["tools", "thinking", "cloud"]}, - "minimax-m2.7": {"sizes": [], "family": "minimax", "capabilities": ["tools", "thinking", "cloud"]}, - "minimax-m2.5": {"sizes": [], "family": "minimax", "capabilities": ["tools", "thinking", "cloud"]}, - "minimax-m2.1": {"sizes": [], "family": "minimax", "capabilities": ["tools", "thinking", "cloud"]}, - "glm-5.1": {"sizes": [], "family": "glm", "capabilities": ["tools", "thinking", "cloud"]}, - "glm-5": {"sizes": [], "family": "glm", "capabilities": ["tools", "thinking", "cloud"]}, - "glm-4.7": {"sizes": [], "family": "glm", "capabilities": ["tools", "thinking", "cloud"]}, - "glm-4.6": {"sizes": [], "family": "glm", "capabilities": ["tools", "thinking", "cloud"]}, - "gpt-oss": {"sizes": ["20b", "120b"], "family": "gpt-oss", "capabilities": ["tools", "thinking", "cloud"]}, - "deepseek-v3.1": {"sizes": ["671b"], "family": "deepseek", "capabilities": ["thinking", "cloud"]}, - "deepseek-v3.2": {"sizes": [], "family": "deepseek", "capabilities": ["thinking", "cloud"]}, - "devstral-small-2": {"sizes": ["24b"], "family": "devstral", "capabilities": ["tools", "cloud"]}, - "devstral-2": {"sizes": ["123b"], "family": "devstral", "capabilities": ["tools", "cloud"]}, - "nemotron-3-super": {"sizes": [], "family": "nemotron", "capabilities": ["tools", "thinking", "cloud"]}, - "nemotron-3-nano": {"sizes": ["30b"], "family": "nemotron", "capabilities": ["tools", "thinking", "cloud"]}, - "mistral-large-3": {"sizes": ["675b"], "family": "mistral", "capabilities": ["tools", "thinking", "cloud"]}, - "ministral-3": {"sizes": ["3b", "8b", "14b"], "family": "mistral", "capabilities": ["tools", "cloud"]}, - "kimi-k2.5": {"sizes": [], "family": "kimi", "capabilities": ["vision", "tools", "thinking", "cloud"]}, - "kimi-k2-thinking": {"sizes": [], "family": "kimi", "capabilities": ["thinking", "cloud"]}, - "kimi-k2": {"sizes": ["1t"], "family": "kimi", "capabilities": ["tools", "thinking", "cloud"]}, - "cogito-2.1": {"sizes": ["671b"], "family": "cogito", "capabilities": ["thinking", "cloud"]}, - "gemini-3-flash-preview": {"sizes": [], "family": "gemini", "capabilities": ["vision", "tools", "thinking", "cloud"]}, - "rnj-1": {"sizes": ["8b"], "family": "rnj", "capabilities": ["tools", "cloud"]}, + "gemma4": {"sizes": ["31b"], "family": "gemma", "capabilities": ["vision", "tools", "thinking", "cloud"], "usage_multiplier": 0.50}, + "gemma3": {"sizes": ["4b", "12b", "27b"], "family": "gemma", "capabilities": ["vision", "tools", "thinking", "cloud"], "usage_multiplier": 0.50}, + "qwen3.5": {"sizes": ["397b"], "family": "qwen", "capabilities": ["vision", "tools", "thinking", "cloud"], "usage_multiplier": 1.00}, + "qwen3-vl": {"sizes": ["235b", "235b-instruct"], "family": "qwen", "capabilities": ["vision", "tools", "thinking", "cloud"], "usage_multiplier": 0.75}, + "qwen3-coder": {"sizes": ["480b"], "family": "qwen", "capabilities": ["tools", "cloud"], "usage_multiplier": 0.75}, + "qwen3-coder-next": {"sizes": [], "family": "qwen", "capabilities": ["tools", "cloud"], "usage_multiplier": 0.75}, + "qwen3-next": {"sizes": ["80b"], "family": "qwen", "capabilities": ["tools", "thinking", "cloud"], "usage_multiplier": 0.75}, + "minimax-m2": {"sizes": [], "family": "minimax", "capabilities": ["tools", "thinking", "cloud"], "usage_multiplier": 0.50}, + "minimax-m2.7": {"sizes": [], "family": "minimax", "capabilities": ["tools", "thinking", "cloud"], "usage_multiplier": 0.50}, + "minimax-m2.5": {"sizes": [], "family": "minimax", "capabilities": ["tools", "thinking", "cloud"], "usage_multiplier": 0.50}, + "minimax-m2.1": {"sizes": [], "family": "minimax", "capabilities": ["tools", "thinking", "cloud"], "usage_multiplier": 0.50}, + "glm-5.1": {"sizes": [], "family": "glm", "capabilities": ["tools", "thinking", "cloud"], "usage_multiplier": 0.75}, + "glm-5": {"sizes": [], "family": "glm", "capabilities": ["tools", "thinking", "cloud"], "usage_multiplier": 0.75}, + "glm-4.7": {"sizes": [], "family": "glm", "capabilities": ["tools", "thinking", "cloud"], "usage_multiplier": 0.50}, + "glm-4.6": {"sizes": [], "family": "glm", "capabilities": ["tools", "thinking", "cloud"], "usage_multiplier": 0.50}, + "gpt-oss": {"sizes": ["20b", "120b"], "family": "gpt-oss", "capabilities": ["tools", "thinking", "cloud"], "usage_multiplier": 0.50}, + "deepseek-v3.1": {"sizes": ["671b"], "family": "deepseek", "capabilities": ["thinking", "cloud"], "usage_multiplier": 1.00}, + "deepseek-v3.2": {"sizes": [], "family": "deepseek", "capabilities": ["thinking", "cloud"], "usage_multiplier": 1.00}, + "deepseek-v4-pro": {"sizes": [], "family": "deepseek", "capabilities": ["vision", "tools", "thinking", "cloud"], "usage_multiplier": 1.00}, + "deepseek-v4-flash": {"sizes": [], "family": "deepseek", "capabilities": ["vision", "tools", "thinking", "cloud"], "usage_multiplier": 0.50}, + "devstral-small-2": {"sizes": ["24b"], "family": "devstral", "capabilities": ["tools", "cloud"], "usage_multiplier": 0.50}, + "devstral-2": {"sizes": ["123b"], "family": "devstral", "capabilities": ["tools", "cloud"], "usage_multiplier": 0.75}, + "nemotron-3-super": {"sizes": [], "family": "nemotron", "capabilities": ["tools", "thinking", "cloud"], "usage_multiplier": 0.50}, + "nemotron-3-nano": {"sizes": ["30b"], "family": "nemotron", "capabilities": ["tools", "thinking", "cloud"], "usage_multiplier": 0.25}, + "mistral-large-3": {"sizes": ["675b"], "family": "mistral", "capabilities": ["tools", "thinking", "cloud"], "usage_multiplier": 1.00}, + "ministral-3": {"sizes": ["3b", "8b", "14b"], "family": "mistral", "capabilities": ["tools", "cloud"], "usage_multiplier": 0.25}, + "kimi-k2.6": {"sizes": [], "family": "kimi", "capabilities": ["vision", "tools", "thinking", "cloud"], "usage_multiplier": 0.75, "context_length": 200000}, + "kimi-k2.5": {"sizes": [], "family": "kimi", "capabilities": ["vision", "tools", "thinking", "cloud"], "usage_multiplier": 0.75}, + "kimi-k2-thinking": {"sizes": [], "family": "kimi", "capabilities": ["thinking", "cloud"], "usage_multiplier": 0.75}, + "kimi-k2": {"sizes": ["1t"], "family": "kimi", "capabilities": ["tools", "thinking", "cloud"], "usage_multiplier": 1.00}, + "cogito-2.1": {"sizes": ["671b"], "family": "cogito", "capabilities": ["thinking", "cloud"], "usage_multiplier": 1.00}, + "gemini-3-flash-preview": {"sizes": [], "family": "gemini", "capabilities": ["vision", "tools", "thinking", "cloud"], "usage_multiplier": 0.50}, + "rnj-1": {"sizes": ["8b"], "family": "rnj", "capabilities": ["tools", "cloud"], "usage_multiplier": 0.25}, + "minimax-m3": {"sizes": [], "family": "minimax", "capabilities": ["vision", "tools", "thinking", "cloud"], "usage_multiplier": 0.75}, + "nemotron-3-ultra": {"sizes": [], "family": "nemotron", "capabilities": ["tools", "thinking", "cloud"], "usage_multiplier": 1.00}, } @@ -68,10 +81,120 @@ class OllamaClient: self._models_cache_time: float = 0 self._models_cache_ttl: float = 300.0 # 5 minutes - async def _get_client(self) -> httpx.AsyncClient: + @staticmethod + def _fetch_usage_level_sync(model_name: str) -> int: + """Scrape Ollama.com library page to count usage slots (1-4). + + Handles both top-level model badges and per-tag listings. + Returns 0 if the model page can't be found. + """ + import urllib.request + base = model_name.split(":")[0] + tag = model_name.split(":")[1] if ":" in model_name else None + url = f"https://ollama.com/library/{base}" + try: + req = urllib.request.Request(url, headers={"User-Agent": "Guanaco/1.0"}) + with urllib.request.urlopen(req, timeout=10) as resp: + html = resp.read().decode("utf-8", errors="replace") + + # 1) Top-level model badge (unified tier models) + top_active = len(re.findall(r'x-test-model-cost-slot-active', html)) + if top_active > 0: + return min(top_active, 4) + + # 2) Per-tag listing — parse each tag's usage slots + # The page shows tags in order; split by cost containers and + # match each cost section with the preceding tag name. + tag_levels: dict[str, int] = {} + sections = re.split(r'x-test-model-tag-cost', html) + for i in range(len(sections) - 1): + # Tag name is in the current section (last command input) + inputs = re.findall(r'value="' + re.escape(base) + r':([^"]+)"', sections[i]) + if not inputs: + continue + tag = inputs[-1].replace("-cloud", "") + # Cost slots are in the next section, before the next tag name + cost_part = re.split(r'value="' + re.escape(base) + r':', sections[i + 1])[0] + active = cost_part.count('x-test-model-tag-usage-slot-active') + if active > 0: + tag_levels[tag] = active + + # If we were asked for a specific tag, return its level + if tag and tag in tag_levels: + return min(tag_levels[tag], 4) + + # Otherwise return the max level across all tags (model's highest tier) + if tag_levels: + return min(max(tag_levels.values()), 4) + + # 3) Raw fallback: count all tag slots + raw_active = len(re.findall(r'x-test-model-tag-usage-slot-active', html)) + return min(raw_active, 4) if raw_active > 0 else 0 + except Exception: + return 0 + + async def fetch_usage_levels(self, model_names: list[str]) -> dict[str, int]: + """Fetch usage levels for multiple models in parallel. + + Results are cached globally for _USAGE_LEVEL_CACHE_TTL seconds. + Returns dict {model_name: level} where level is 1-4 (0 = unknown). + """ + global _USAGE_LEVEL_CACHE, _USAGE_LEVEL_CACHE_TIME + now = time.time() + # Refresh cache if stale + if now - _USAGE_LEVEL_CACHE_TIME > _USAGE_LEVEL_CACHE_TTL: + _USAGE_LEVEL_CACHE.clear() + _USAGE_LEVEL_CACHE_TIME = now + + # Deduplicate base names + to_fetch = [] + results: dict[str, int] = {} + for name in model_names: + base = name.split(":")[0] + if base in _USAGE_LEVEL_CACHE: + results[name] = _USAGE_LEVEL_CACHE[base] + elif base not in to_fetch: + to_fetch.append(base) + + if to_fetch: + import asyncio + loop = asyncio.get_event_loop() + # Run blocking scrapes in thread pool + tasks = [loop.run_in_executor(None, self._fetch_usage_level_sync, m) for m in to_fetch] + levels = await asyncio.gather(*tasks, return_exceptions=True) + for base, raw in zip(to_fetch, levels): + if isinstance(raw, Exception): + _USAGE_LEVEL_CACHE[base] = 0 + else: + _USAGE_LEVEL_CACHE[base] = raw # type: ignore[reportArgumentType] + + # Fill in results for all requested names + for name in model_names: + if name not in results: + base = name.split(":")[0] + results[name] = _USAGE_LEVEL_CACHE.get(base, 0) + return results + + async def _get_client(self, api_key_override: Optional[str] = None) -> httpx.AsyncClient: + """Get or create the httpx client, optionally with a different API key. + + When api_key_override is provided and differs from the default key, + creates a temporary client that must be closed by the caller. + Returns (client, is_temp) tuple so callers know whether to close it. + """ + key = api_key_override or self.api_key + use_temp = api_key_override is not None and api_key_override != self.api_key + + if use_temp: + # Per-request key override — create a temporary client + headers = {"Content-Type": "application/json"} + if key and key not in ("***", "REPLACE_ME", "your_api_key_here"): + headers["Authorization"] = f"Bearer {key}" + return httpx.AsyncClient(timeout=self.timeout, headers=headers) + + # Default behavior — cached singleton client if self._client is None or self._client.is_closed: headers = {"Content-Type": "application/json"} - # Only send Authorization if we have a real API key (not empty, placeholder, or masked) if self.api_key and self.api_key not in ("***", "REPLACE_ME", "your_api_key_here"): headers["Authorization"] = f"Bearer {self.api_key}" self._client = httpx.AsyncClient( @@ -82,35 +205,46 @@ class OllamaClient: # ── Search & Fetch ── - async def search(self, query: str, max_results: int = 10) -> dict: + async def search(self, query: str, max_results: int = 10, api_key: Optional[str] = None) -> dict: """Search the web using Ollama's web_search API.""" - client = await self._get_client() - payload = {"query": query, "max_results": max(min(max_results, 10), 1)} - resp = await client.post(OLLAMA_SEARCH_URL, json=payload) - resp.raise_for_status() - return resp.json() + client = await self._get_client(api_key_override=api_key) + is_temp = api_key is not None and api_key != self.api_key + try: + payload = {"query": query, "max_results": max(min(max_results, 10), 1)} + resp = await client.post(OLLAMA_SEARCH_URL, json=payload) + resp.raise_for_status() + return resp.json() + finally: + if is_temp and not client.is_closed: + await client.aclose() - async def fetch(self, url: str) -> dict: + async def fetch(self, url: str, api_key: Optional[str] = None) -> dict: """Fetch/scrape a URL using Ollama's web_fetch API.""" - client = await self._get_client() - payload = {"url": url} - resp = await client.post(OLLAMA_FETCH_URL, json=payload) - resp.raise_for_status() - return resp.json() + client = await self._get_client(api_key_override=api_key) + is_temp = api_key is not None and api_key != self.api_key + try: + payload = {"url": url} + resp = await client.post(OLLAMA_FETCH_URL, json=payload) + resp.raise_for_status() + return resp.json() + finally: + if is_temp and not client.is_closed: + await client.aclose() # ── Models ── - async def list_models(self, force_refresh: bool = False) -> list[dict]: + async def list_models(self, force_refresh: bool = False, api_key: Optional[str] = None) -> list[dict]: """List available Ollama Cloud models, with caching. Uses the OpenAI-compatible /v1/models endpoint which returns model IDs in standard format (e.g. 'gemma4:31b', 'qwen3.5:397b'). """ now = time.time() - if not force_refresh and self._models_cache and (now - self._models_cache_time) < self._models_cache_ttl: + if not force_refresh and not api_key and self._models_cache and (now - self._models_cache_time) < self._models_cache_ttl: return self._models_cache - client = await self._get_client() + client = await self._get_client(api_key_override=api_key) + is_temp = api_key is not None and api_key != self.api_key try: resp = await client.get(OLLAMA_MODELS_URL) if resp.status_code == 401: @@ -143,6 +277,9 @@ class OllamaClient: except Exception as e: logger.error(f"Error fetching models: {e}") raise + finally: + if is_temp and not client.is_closed: + await client.aclose() async def check_model_available(self, model_name: str) -> bool: """Check if a specific model is available on Ollama Cloud.""" @@ -152,38 +289,104 @@ class OllamaClient: return model_name in available_names or f"{model_name}-cloud" in available_names async def get_cloud_models(self) -> list[dict]: - """Get list of cloud-capable models with metadata.""" + """Get list of cloud-capable models with metadata. + + Fetches real usage levels from ollama.com library pages and includes + them as usage_multiplier (0.25-1.00) alongside capabilities. + """ models = await self.list_models() + # Fetch real usage levels from ollama.com + model_names = [m.get("name", m.get("model", "")) for m in models] + usage_levels = await self.fetch_usage_levels(model_names) + cloud_models = [] for m in models: name = m.get("name", m.get("model", "")) details = m.get("details", {}) - # Check if model has cloud capability (or is available via cloud API) - is_cloud = True # All models from /api/tags with auth are cloud-available - size_info = details.get("parameter_size", "") - family = details.get("family", "") - quant = details.get("quantization_level", "") + level = usage_levels.get(name, 0) + multiplier = level * 0.25 if level else self._get_model_multiplier(name) cloud_models.append({ "name": name, "display_name": name.replace("-cloud", ""), "size_bytes": m.get("size", 0), - "parameter_size": size_info, - "family": family, - "quantization": quant, + "parameter_size": details.get("parameter_size", ""), + "family": details.get("family", ""), + "quantization": details.get("quantization_level", ""), "capabilities": self._get_model_capabilities(name), + "usage_multiplier": multiplier, + "usage_level": level, # 1-4, 0 = unknown "modified_at": m.get("modified_at", ""), "digest": m.get("digest", "")[:12] if m.get("digest") else "", }) return cloud_models def _get_model_capabilities(self, model_name: str) -> list[str]: - """Get known capabilities for a model name.""" + """Get known capabilities for a model name. Falls back to name-based inference.""" base_name = model_name.split(":")[0].replace("-cloud", "") if base_name in KNOWN_CLOUD_MODELS: return KNOWN_CLOUD_MODELS[base_name].get("capabilities", ["cloud"]) - # Default capabilities for unknown models - return ["cloud"] + # ── Inference for unknown new models ── + lc = base_name.lower() + caps = ["cloud"] + # vision: VL models, gemma, gemini, kimi, deepseek (frontier), anything with "vision" in name + if any(k in lc for k in ("vl", "vision", "gemma", "gemini", "deepseek")) or lc.startswith("kimi-"): + caps.append("vision") + # tools: explicit coder/minimax/glm/mistral/gpt-oss/devstral/nemotron families, deepseek + if any(k in lc for k in ("coder", "minimax", "glm-", "mistral", "ministral", + "gpt-oss", "devstral", "nemotron", "deepseek", "rnj-1")): + caps.append("tools") + # thinking: deepseek, cogito, reasoning, think suffixes, kimi-k2* except k2.5/2.6, any kimi-k* with large sizes + if any(k in lc for k in ("deepseek", "cogito", "reason", "-thinking", "think")): + caps.append("thinking") + elif lc.startswith("kimi-k") and not ("k2.5" in lc or "k2.6" in lc): + # kimi-k2 (1t) and future kimi-k3, k4 etc are reasoning models + caps.append("thinking") + # Deduplicate and sort for consistency + return sorted(set(caps)) + + def _get_model_multiplier(self, model_name: str) -> float: + """Get usage multiplier (cost tier) for a model name. Falls back to size-based inference.""" + base_name = model_name.split(":")[0].replace("-cloud", "") + if base_name in KNOWN_CLOUD_MODELS: + return KNOWN_CLOUD_MODELS[base_name].get("usage_multiplier", 1.00) + # ── Inference from parameter size hints in the name ── + lc = base_name.lower() + # Extract size hint like ":30b" or "-30b" from the full model name + size_match = None + for part in model_name.replace("-cloud", "").split(":"): + m = __import__("re").search(r"(\d+)(b|t)", part, __import__("re").I) + if m: + num = int(m.group(1)) + unit = m.group(2).lower() + # If unit is 't' (trillion), treat as very large + if unit == "t": + return 1.00 + size_match = num + break + if size_match is not None: + if size_match <= 20: + return 0.25 + elif size_match <= 80: + return 0.50 + elif size_match <= 400: + return 0.75 + else: + return 1.00 + # Fallback: use name heuristics when no size hint + if any(k in lc for k in ("nano", "mini", "small", "rnj-1")): + return 0.25 + if any(k in lc for k in ("flash", "gemma", "gpt-oss", "minimax", "devstral-small", + "glm-4.", "super")): + return 0.50 + if any(k in lc for k in ("kimi-k", "qwen3-vl", "qwen3-coder", "qwen3-next", + "devstral-2", "glm-5")): + return 0.75 + if any(k in lc for k in ("pro", "qwen3.5", "deepseek-v3", "mistral-large", + "cogito", "kimi-k2:1t")): + return 1.00 + # Safest default — unknown might be expensive + return 1.00 # ── Usage / Quota ── @@ -228,6 +431,11 @@ class OllamaClient: Weekly usage 30.9% used ... Resets in 3 days + + Per-model breakdown (new feature): +
+
""" import re result = {} @@ -270,6 +478,45 @@ class OllamaClient: if plan_match: result["plan"] = plan_match.group(1).strip().lower() + # ── Per-model usage breakdown ── + # Find the two data-usage-track containers (session first, weekly second) + usage_tracks = re.findall( + r'data-usage-track[^\u003e]*aria-label="([^"]*usage[^"]*)"[^\u003e]*\u003e(.*?)\u003c/div\u003e\s*\u003c/div\u003e', + html, re.DOTALL | re.IGNORECASE + ) + + session_breakdown = [] + weekly_breakdown = [] + + for aria_label, track_html in usage_tracks: + # Extract segments within this track + # Each segment is a
-
Model
Requests
Prompt Tok
Comp Tok
Avg TPS
Avg TTFT
+
Model
Reqs
Prompt
Comp
Total
Avg TPS
Avg TTFT
@@ -362,8 +440,79 @@ - - `).join('') : '
No errors 🎉
'; - // Top stats + // Top stats — show raw tokens as primary count document.getElementById('stat-requests').textContent = (data.total_requests || 0).toLocaleString(); - document.getElementById('stat-tokens').textContent = ((data.prompt_tokens || 0) + (data.completion_tokens || 0)).toLocaleString(); + document.getElementById('stat-tokens').textContent = (data.total_tokens || 0).toLocaleString(); + document.getElementById('stat-tokens').nextElementSibling.textContent = 'Tokens'; document.getElementById('stat-tps').textContent = data.avg_tps || 0; document.getElementById('stat-ttft').textContent = data.avg_ttft ? (data.avg_ttft * 1000).toFixed(0) + 'ms' : '—'; document.getElementById('stat-keys').textContent = KEYS.length; @@ -923,6 +1480,255 @@ function clearAnalytics() { fetch('/dashboard/api/analytics/clear', {method: 'POST'}).then(() => loadAnalytics()); } + +// ─── History ─── + +let historyOffset = 0; +const HISTORY_LIMIT = 50; + +function loadHistoryConfig() { + fetch('/dashboard/api/history/config').then(r => r.json()).then(data => { + document.getElementById('history-enabled').checked = data.enabled || false; + document.getElementById('history-save-input').checked = data.save_input !== false; + document.getElementById('history-save-output').checked = data.save_output !== false; + document.getElementById('history-retention').value = data.retention_days || 30; + document.getElementById('history-log-to-files').checked = data.log_to_files || false; + document.getElementById('history-log-dir').value = data.log_dir || ''; + document.getElementById('history-logdir-row').style.display = data.log_to_files ? 'flex' : 'none'; + document.getElementById('history-config-panel').style.display = data.enabled ? 'block' : 'none'; + }); +} + +function saveHistoryConfig() { + const enabled = document.getElementById('history-enabled').checked; + document.getElementById('history-config-panel').style.display = enabled ? 'block' : 'none'; + const logToFiles = document.getElementById('history-log-to-files').checked; + document.getElementById('history-logdir-row').style.display = logToFiles ? 'flex' : 'none'; + + const data = { + enabled: enabled, + save_input: document.getElementById('history-save-input').checked, + save_output: document.getElementById('history-save-output').checked, + retention_days: parseInt(document.getElementById('history-retention').value) || 30, + log_to_files: logToFiles, + log_dir: document.getElementById('history-log-dir').value.trim(), + }; + + fetch('/dashboard/api/history/config', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify(data) + }).then(r => r.json()).then(resp => { + // Sync UI from server response to prevent drift + if (resp.history) { + document.getElementById('history-enabled').checked = resp.history.enabled || false; + document.getElementById('history-save-input').checked = resp.history.save_input !== false; + document.getElementById('history-save-output').checked = resp.history.save_output !== false; + document.getElementById('history-log-to-files').checked = resp.history.log_to_files || false; + document.getElementById('history-log-dir').value = resp.history.log_dir || ''; + document.getElementById('history-retention').value = resp.history.retention_days || 30; + document.getElementById('history-logdir-row').style.display = resp.history.log_to_files ? 'flex' : 'none'; + document.getElementById('history-config-panel').style.display = resp.history.enabled ? 'block' : 'none'; + } + // Reload stats and log files + loadHistoryStats(); + loadHistoryLogs(); + }); +} + +function loadHistoryStats() { + fetch('/dashboard/api/history/stats').then(r => r.json()).then(data => { + const statsEl = document.getElementById('history-stats'); + const errStr = data.error_count ? ` | ${data.error_count} failed` : ''; + statsEl.innerHTML = `${data.total_requests.toLocaleString()} requests, ${data.requests_with_content.toLocaleString()} with content, ${data.content_size_mb}MB${errStr}`; + }); +} + +function loadHistory() { + const model = document.getElementById('history-model-filter').value; + const provider = document.getElementById('history-provider-filter').value; + const contentFilter = document.getElementById('history-content-filter').value; + + let url = `/dashboard/api/history?limit=${HISTORY_LIMIT}&offset=${historyOffset}`; + if (model) url += `&model=${encodeURIComponent(model)}`; + if (provider) url += `&provider=${encodeURIComponent(provider)}`; + if (contentFilter === 'yes') url += '&has_content=true'; + if (contentFilter === 'no') url += '&has_content=false'; + if (contentFilter === 'errors') url += '&errors_only=true'; + + fetch(url).then(r => r.json()).then(data => { + const el = document.getElementById('history-rows'); + if (!data || data.length === 0) { + el.innerHTML = '
No requests found
'; + document.getElementById('history-pagination').innerHTML = ''; + return; + } + + el.innerHTML = data.map(r => { + const isErr = !!r.error; + const borderColor = isErr ? 'var(--red)' : 'transparent'; + const errSnippet = isErr ? `
⚠️ ${escapeHtml(r.error).substring(0, 120)}
` : ''; + return ` +
+
+ ${r.model || '—'} + ${r.ts_formatted} +
+
+ 📍 ${r.source_ip || '—'}:${r.source_port || ''} + 🏢 ${r.provider || 'ollama'} + 🔤 ${r.prompt_tokens || 0}/${r.completion_tokens || 0} + ${r.has_content ? '📝 content' : ''} + ${r.fallback_for ? `🔀 fallback` : ''} + ${isErr ? '❌ failed' : ''} +
+ ${errSnippet} + ${r.fallback_for && r.fallback_reason ? `
🔀 ${escapeHtml(r.fallback_reason).substring(0, 120)}
` : ''} +
+ `}).join(''); + + // Pagination + const pagEl = document.getElementById('history-pagination'); + if (historyOffset > 0 || data.length === HISTORY_LIMIT) { + pagEl.innerHTML = ` + + Row ${historyOffset + 1}+ + + `; + } else { + pagEl.innerHTML = ''; + } + }); + + loadHistoryStats(); +} + +function showHistoryDetail(id) { + fetch(`/dashboard/api/history/${id}`).then(r => r.json()).then(data => { + if (data.error) { + alert('Request not found'); + return; + } + + const hasContent = data.input_text || data.output_text; + const hasError = data.error; + + // Create modal + const modal = document.createElement('div'); + modal.className = 'history-modal'; + modal.style.cssText = 'position:fixed;inset:0;background:rgba(0,0,0,0.8);z-index:1000;display:flex;align-items:center;justify-content:center;padding:20px;'; + modal.innerHTML = ` +
+
+

Request Detail

+ +
+
+
+
Model: ${data.model || '—'}
+
Provider: ${data.provider || 'ollama'}
+
Time: ${data.ts_formatted}
+
Duration: ${data.total_duration_seconds ? data.total_duration_seconds.toFixed(2) + 's' : '—'}
+
Tokens: ${data.prompt_tokens || 0} in / ${data.completion_tokens || 0} out
+
TPS: ${data.tps ? data.tps.toFixed(2) : '—'}
+
Caller: ${data.source_ip || '—'}:${data.source_port || ''}
+
User Agent: ${data.user_agent || '—'}
+
+ ${data.fallback_for ? `
🔀 Fallback for: ${data.fallback_for}${data.fallback_reason ? `
Reason: ${escapeHtml(data.fallback_reason)}
` : ''}
` : ''} + ${data.error ? `
+
❌ Error
+
${escapeHtml(data.error)}
+ ${data.provider ? `
Provider: ${data.provider}${data.fallback_for ? ' (fallback also failed)' : ''}
` : ''} +
` : ''} + ${!hasContent && !hasError ? '
⚠️ Content not captured for this request. This usually happens when the request was handled by an instance running older code (before history was enabled), or the request had no input/output to save.
' : ''} + ${data.input_text ? ` +
+
📥 Input (${(data.input_text.length).toLocaleString()} chars)
+
${escapeHtml(data.input_text)}
+
+ ` : ''} + ${data.output_text ? ` +
+
📤 Output (${(data.output_text.length).toLocaleString()} chars)
+
${escapeHtml(data.output_text)}
+
+ ` : ''} +
+
+ `; + document.body.appendChild(modal); + modal.addEventListener('click', (e) => { + if (e.target === modal) modal.remove(); + }); + }); +} + +function escapeHtml(text) { + if (!text) return ''; + return text.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); +} + +// ─── History Log Files ─── + +function loadHistoryLogs() { + fetch('/dashboard/api/history/logs').then(r => r.json()).then(data => { + const section = document.getElementById('history-logs-section'); + const el = document.getElementById('history-log-files'); + + if (!data.enabled) { + section.style.display = 'none'; + return; + } + section.style.display = 'block'; + + if (!data.files || data.files.length === 0) { + el.innerHTML = '
No log files yet
'; + return; + } + + el.innerHTML = data.files.map(f => { + const sizeStr = f.size > 1024 ? (f.size / 1024).toFixed(1) + 'KB' : f.size + 'B'; + return `
+
+ 📄 ${f.name} + ${sizeStr} · ${f.modified_formatted} +
+ +
`; + }).join(''); + + if (data.total > 200) { + el.innerHTML += `
Showing 200 of ${data.total} files
`; + } + }); +} + +function viewLogFile(filename) { + fetch(`/dashboard/api/history/logs/${encodeURIComponent(filename)}`).then(r => r.json()).then(data => { + if (data.error) { alert(data.error); return; } + const modal = document.createElement('div'); + modal.className = 'history-modal'; + modal.style.cssText = 'position:fixed;inset:0;background:rgba(0,0,0,0.8);z-index:1000;display:flex;align-items:center;justify-content:center;padding:20px;'; + modal.innerHTML = ` +
+
+

📄 ${escapeHtml(filename)}

+ +
+
${escapeHtml(data.content)}
+
`; + document.body.appendChild(modal); + modal.addEventListener('click', (e) => { if (e.target === modal) modal.remove(); }); + }); +} + +function deleteLogFile(filename) { + if (!confirm(`Delete ${filename}?`)) return; + fetch(`/dashboard/api/history/logs/${encodeURIComponent(filename)}`, {method: 'DELETE'}).then(r => r.json()).then(() => { + loadHistoryLogs(); + }); +} + // ─── Status ─── function checkOllamaStatus() { @@ -955,6 +1761,21 @@ function checkOllamaStatus() { dotEl.classList.add('error'); textEl.textContent = 'Unreachable'; }); + + // Also load concurrency stats + loadConcurrencyStats(); +} + +function loadConcurrencyStats() { + fetch('/dashboard/api/concurrency').then(r => r.json()).then(data => { + document.getElementById('cc-active').textContent = data.active_count || 0; + document.getElementById('cc-max').textContent = data.max_concurrent || '∞'; + document.getElementById('cc-429s').textContent = data.recent_429_rate || 0; + }).catch(() => { + document.getElementById('cc-active').textContent = '—'; + document.getElementById('cc-max').textContent = '—'; + document.getElementById('cc-429s').textContent = '—'; + }); } function checkUsage() { @@ -965,6 +1786,8 @@ function checkUsage() { const weeklyBar = document.getElementById('usage-weekly-bar'); const sessionReset = document.getElementById('usage-session-reset'); const weeklyReset = document.getElementById('usage-weekly-reset'); + const sessionBreakdown = document.getElementById('usage-session-breakdown'); + const weeklyBreakdown = document.getElementById('usage-weekly-breakdown'); const lastCheckedRow = document.getElementById('usage-last-checked-row'); const lastCheckedEl = document.getElementById('usage-last-checked'); planEl.textContent = 'Loading...'; @@ -979,6 +1802,8 @@ function checkUsage() { weeklyBar.style.width = '0%'; sessionReset.textContent = ''; weeklyReset.textContent = ''; + sessionBreakdown.innerHTML = ''; + weeklyBreakdown.innerHTML = ''; lastCheckedRow.style.display = 'none'; } else { const plan = data.plan ? data.plan.charAt(0).toUpperCase() + data.plan.slice(1) : '—'; @@ -1001,6 +1826,72 @@ function checkUsage() { // Reset timers sessionReset.textContent = data.session_reset ? '⏱ Resets in ' + data.session_reset : ''; weeklyReset.textContent = data.weekly_reset ? '⏱ Resets in ' + data.weekly_reset : ''; + // Per-model breakdowns + function renderBreakdown(breakdown, container, idSuffix) { + if (!breakdown || breakdown.length === 0) { + container.innerHTML = ''; + return; + } + // Sort by percentage descending + const sorted = [...breakdown].sort((a, b) => b.pct - a.pct); + + // Separate significant vs tiny + const significant = []; + const tiny = []; + for (const b of sorted) { + if (b.pct >= 1.0) { + significant.push(b); + } else { + tiny.push(b); + } + } + + // Always show top 3 at minimum, even if tiny + while (significant.length < 3 && tiny.length > 0) { + significant.push(tiny.shift()); + } + + const hasMore = tiny.length > 0; + const all = [...significant, ...tiny]; + + function makeRow(b, hidden) { + const pctStr = b.pct >= 0.1 ? b.pct.toFixed(1) + '%' : '< 0.1%'; + return ` +
+ ${b.model} + ${b.requests.toLocaleString()} req · ${pctStr} +
+ `; + } + + const topHtml = significant.map(b => makeRow(b, false)).join(''); + const restHtml = tiny.map(b => makeRow(b, true)).join(''); + + const expandId = 'ub-expand-' + idSuffix; + const collapseId = 'ub-collapse-' + idSuffix; + + container.innerHTML = ` + +
+ ${topHtml} +
+ +
+ +
+ `; + } + renderBreakdown(data.session_breakdown, sessionBreakdown, 'session'); + renderBreakdown(data.weekly_breakdown, weeklyBreakdown, 'weekly'); // Last checked if (data.last_checked) { lastCheckedRow.style.display = ''; @@ -1058,6 +1949,8 @@ function clearSessionCookie() { document.getElementById('usage-weekly-bar').style.width = '0%'; document.getElementById('usage-session-reset').textContent = ''; document.getElementById('usage-weekly-reset').textContent = ''; + document.getElementById('usage-session-breakdown').innerHTML = ''; + document.getElementById('usage-weekly-breakdown').innerHTML = ''; } setTimeout(() => { statusEl.textContent = ''; statusEl.style.color = ''; }, 3000); }); @@ -1130,7 +2023,7 @@ function startUsageAutoRefresh(intervalSec) { return; } statusEl.textContent = 'every ' + formatInterval(intervalSec); - _usageAutoTimer = setInterval(() => { checkUsage(); }, intervalSec * 1000); + _usageAutoTimer = setInterval(() => { checkUsage(); checkAllAccountUsage(); }, intervalSec * 1000); } function formatInterval(sec) { @@ -1204,9 +2097,11 @@ function loadFallbackConfig() { document.getElementById('fb-api-key').value = FALLBACK.api_key || ''; document.getElementById('fb-default-model').value = FALLBACK.default_model || ''; document.getElementById('fb-timeout').value = FALLBACK.timeout || 30; - document.getElementById('fb-primary-timeout').value = FALLBACK.primary_timeout || 30; + document.getElementById('fb-primary-timeout').value = FALLBACK.primary_timeout || 120; document.getElementById('fb-chunk-timeout').value = FALLBACK.stream_chunk_timeout || 180; document.getElementById('fb-max-tokens').value = FALLBACK.max_tokens || 128000; + document.getElementById('fb-max-concurrent').value = FALLBACK.max_concurrent_ollama || 8; + document.getElementById('fb-429-retries').value = FALLBACK.max_429_retries || 2; document.getElementById('fb-stream').checked = FALLBACK.stream_fallback !== false; document.getElementById('fb-vision').checked = FALLBACK.supports_vision || false; fallbackModelMap = FALLBACK.model_map || {}; @@ -1222,11 +2117,13 @@ function saveFallbackConfig() { api_key: document.getElementById('fb-api-key').value, default_model: document.getElementById('fb-default-model').value, timeout: parseFloat(document.getElementById('fb-timeout').value) || 30, - primary_timeout: parseFloat(document.getElementById('fb-primary-timeout').value) || 30, + primary_timeout: parseFloat(document.getElementById('fb-primary-timeout').value) || 120, stream_chunk_timeout: parseFloat(document.getElementById('fb-chunk-timeout').value) || 180, max_tokens: parseInt(document.getElementById('fb-max-tokens').value) || 128000, stream_fallback: document.getElementById('fb-stream').checked, supports_vision: document.getElementById('fb-vision').checked, + max_concurrent_ollama: parseInt(document.getElementById('fb-max-concurrent').value) || 8, + max_429_retries: parseInt(document.getElementById('fb-429-retries').value) || 2, model_map: fallbackModelMap, } }; @@ -1317,20 +2214,82 @@ function _getCapabilities(modelName) { 'gpt-oss': ['cloud','tools','thinking'], 'deepseek-v3.1': ['cloud','thinking'], 'deepseek-v3.2': ['cloud','thinking'], + 'deepseek-v4-pro': ['cloud','tools','thinking','vision'], + 'deepseek-v4-flash': ['cloud','tools','thinking','vision'], + 'gemini-3-flash-preview': ['cloud','tools','thinking','vision'], 'glm-5.1': ['cloud','tools','thinking'], 'glm-5': ['cloud','tools','thinking'], 'minimax-m2.7': ['cloud','tools','thinking'], 'minimax-m2.5': ['cloud','tools','thinking'], 'minimax-m2.1': ['cloud','tools','thinking'], + 'minimax-m2': ['cloud','tools','thinking'], + 'minimax-m3': ['cloud','tools','thinking','vision'], 'devstral-small-2': ['cloud','tools'], 'devstral-2': ['cloud','tools'], 'nemotron-3-super': ['cloud','tools','thinking'], + 'nemotron-3-nano': ['cloud','tools','thinking'], + 'nemotron-3-ultra': ['cloud','tools','thinking'], 'mistral-large-3': ['cloud','tools','thinking'], 'ministral-3': ['cloud','tools'], + 'kimi-k2.6': ['cloud','tools','thinking','vision'], 'kimi-k2.5': ['cloud','tools','thinking','vision'], 'cogito-2.1': ['cloud','thinking'], }; - return known[base] || ['cloud']; + if (known[base]) return known[base]; + // ── Inference for unknown models ── + const lc = base.toLowerCase(); + const caps = ['cloud']; + if (lc.includes('vl') || lc.includes('vision') || lc.includes('gemma') || lc.includes('gemini') || lc.includes('deepseek') || lc.startsWith('kimi-')) + caps.push('vision'); + if (lc.includes('coder') || lc.includes('minimax') || lc.startsWith('glm-') || lc.includes('mistral') || + lc.includes('ministral') || lc.includes('gpt-oss') || lc.includes('devstral') || lc.includes('nemotron') || lc.includes('deepseek') || lc.includes('rnj-1')) + caps.push('tools'); + if (lc.includes('deepseek') || lc.includes('cogito') || lc.includes('reason') || lc.includes('-thinking') || lc.includes('think')) + caps.push('thinking'); + else if (lc.startsWith('kimi-k') && !(lc.includes('k2.5') || lc.includes('k2.6'))) + caps.push('thinking'); + // deduplicate + return [...new Set(caps)].sort(); +} + +function _getMultiplier(modelName) { + const base = modelName.split(':')[0]; + const mults = { + 'nemotron-3-nano': 0.25, 'ministral-3': 0.25, 'rnj-1': 0.25, + 'gemma4': 0.50, 'gemma3': 0.50, 'minimax-m2.7': 0.50, 'minimax-m2.5': 0.50, 'minimax-m2.1': 0.50, 'minimax-m2': 0.50, + 'gpt-oss': 0.50, 'devstral-small-2': 0.50, 'nemotron-3-super': 0.50, 'gemini-3-flash-preview': 0.50, + 'glm-4.7': 0.50, 'glm-4.6': 0.50, + 'qwen3-vl': 0.75, 'qwen3-coder': 0.75, 'qwen3-next': 0.75, 'devstral-2': 0.75, + 'glm-5.1': 0.75, 'glm-5': 0.75, + 'kimi-k2.6': 0.75, 'kimi-k2.5': 0.75, 'kimi-k2-thinking': 0.75, + 'qwen3.5': 1.00, 'deepseek-v3.1': 1.00, 'deepseek-v3.2': 1.00, 'deepseek-v4-pro': 1.00, + 'mistral-large-3': 1.00, 'cogito-2.1': 1.00, 'kimi-k2': 1.00, + 'deepseek-v4-flash': 0.50, 'gemini-3-flash-preview': 0.50, + 'minimax-m3': 0.75, + 'nemotron-3-ultra': 1.00, + }; + if (mults[base]) return mults[base]; + // ── Inference from size hint ── + const m = modelName.match(/(\d+)(b|t)/i); + if (m) { + const num = parseInt(m[1], 10); + const unit = m[2].toLowerCase(); + if (unit === 't') return 1.00; + if (num <= 20) return 0.25; + if (num <= 80) return 0.50; + if (num <= 400) return 0.75; + return 1.00; + } + // ── Fallback heuristics ── + const lc = base.toLowerCase(); + if (lc.includes('nano') || lc.includes('mini') || lc.includes('small') || lc.includes('rnj-1')) return 0.25; + if (lc.includes('flash') || lc.includes('gemma') || lc.includes('gpt-oss') || lc.includes('minimax') || + lc.includes('devstral-small') || lc.includes('glm-4.') || lc.includes('super')) return 0.50; + if (lc.includes('kimi-k') || lc.includes('qwen3-vl') || lc.includes('qwen3-coder') || lc.includes('qwen3-next') || + lc.includes('devstral-2') || lc.includes('glm-5')) return 0.75; + if (lc.includes('pro') || lc.includes('qwen3.5') || lc.includes('deepseek-v3') || lc.includes('mistral-large') || + lc.includes('cogito')) return 1.00; + return 0.75; } // ─── Init ─── @@ -1343,13 +2302,25 @@ document.addEventListener('DOMContentLoaded', () => { document.getElementById('anthropic-url').textContent = `${base}:${CONFIG.router_port}/v1/messages`; // Search endpoints — rendered dynamically from PROVIDERS data + // Hard fallback: if server didn't provide endpoint data, use known defaults + const KNOWN_ENDPOINTS = { + tavily: [{path:'/search',method:'POST'}], + exa: [{path:'/search',method:'POST'},{path:'/findSimilar',method:'POST'}], + searxng: [{path:'/search',method:'GET/POST'}], + firecrawl: [{path:'/scrape',method:'POST'},{path:'/search',method:'POST'},{path:'/crawl',method:'POST'},{path:'/extract',method:'POST'}], + serper: [{path:'/search',method:'POST'},{path:'/scrape',method:'POST'}], + jina: [{path:'/search',method:'POST'},{path:'/read',method:'POST'},{path:'/rerank',method:'POST'}], + cohere: [{path:'/rerank',method:'POST'}], + brave: [{path:'/search',method:'GET/POST'}] + }; const el = document.getElementById('search-endpoints'); let html = ''; Object.entries(PROVIDERS).forEach(([name, cfg]) => { - const eps = cfg.endpoints || []; + const eps = (cfg.endpoints && cfg.endpoints.length > 0) ? cfg.endpoints : (KNOWN_ENDPOINTS[name] || []); + const prefix = cfg.prefix || `/${name}`; eps.forEach((ep, i) => { const urlId = `ep-${name}-${i}`; - const fullUrl = `${base}:${CONFIG.router_port}${cfg.prefix}${ep.path}`; + const fullUrl = `${base}:${CONFIG.router_port}${prefix}${ep.path}`; html += `
${name} ${ep.method}
${fullUrl}
@@ -1365,6 +2336,7 @@ document.addEventListener('DOMContentLoaded', () => { `
${name}
` ).join(''); + // Stats const stats = USAGE; let totalReqs = 0, totalTokens = 0; @@ -1412,6 +2384,14 @@ document.addEventListener('DOMContentLoaded', () => { // Keys loadKeys(); + // History + loadHistoryConfig(); + loadHistory(); + loadHistoryLogs(); + + // Check for updates (auto-check on page load) + checkForUpdate(); + // Check connection checkOllamaStatus(); }); @@ -1585,42 +2565,57 @@ async function checkForUpdate() { btn.textContent = 'Checking...'; statusEl.textContent = ''; - try { - const resp = await fetch('/dashboard/api/update/check'); - const data = await resp.json(); + // Helper to actually perform the check + async function doCheck(attempt = 0) { + try { + const resp = await fetch('/dashboard/api/update/check'); + const data = await resp.json(); - document.getElementById('auto-update-toggle').checked = data.auto_update || false; + document.getElementById('auto-update-toggle').checked = data.auto_update || false; - if (data.error) { - statusEl.textContent = 'Error: ' + data.error; + if (data.error) { + // If service not ready yet, retry a couple times + if (attempt < 2 && data.error.includes('503')) { + await new Promise(r => setTimeout(r, 1500)); + return doCheck(attempt + 1); + } + statusEl.textContent = 'Error: ' + data.error; + statusEl.style.color = '#f87171'; + box.classList.add('hidden'); + return; + } + + statusEl.textContent = `v${data.current_version} installed`; + statusEl.style.color = '#94a3b8'; + + if (data.update_available) { + document.getElementById('update-title').textContent = `Update available: v${data.current_version} → v${data.latest_version}`; + document.getElementById('update-title').style.color = '#4ade80'; + document.getElementById('update-notes').textContent = data.release_notes ? data.release_notes.substring(0, 200) : 'No release notes available.'; + document.getElementById('update-link').innerHTML = data.release_url ? `View on GitHub →` : ''; + box.classList.remove('hidden'); + statusEl.textContent = `v${data.latest_version} available`; + statusEl.style.color = '#4ade80'; + } else { + box.classList.add('hidden'); + statusEl.textContent = `Up to date (v${data.current_version})`; + statusEl.style.color = '#4ade80'; + } + } catch (e) { + // Network error - service might be restarting + if (attempt < 2) { + await new Promise(r => setTimeout(r, 1500)); + return doCheck(attempt + 1); + } + statusEl.textContent = 'Failed to check: ' + e.message; statusEl.style.color = '#f87171'; - box.classList.add('hidden'); - return; } - - statusEl.textContent = `v${data.current_version} installed`; - statusEl.style.color = '#94a3b8'; - - if (data.update_available) { - document.getElementById('update-title').textContent = `Update available: v${data.current_version} → v${data.latest_version}`; - document.getElementById('update-title').style.color = '#4ade80'; - document.getElementById('update-notes').textContent = data.release_notes ? data.release_notes.substring(0, 200) : 'No release notes available.'; - document.getElementById('update-link').innerHTML = data.release_url ? `View on GitHub →` : ''; - box.classList.remove('hidden'); - statusEl.textContent = `v${data.latest_version} available`; - statusEl.style.color = '#4ade80'; - } else { - box.classList.add('hidden'); - statusEl.textContent = `Up to date (v${data.current_version})`; - statusEl.style.color = '#4ade80'; - } - } catch (e) { - statusEl.textContent = 'Failed to check: ' + e.message; - statusEl.style.color = '#f87171'; - } finally { - btn.disabled = false; - btn.textContent = 'Check for Update'; } + + await doCheck(); + + btn.disabled = false; + btn.textContent = 'Check for Update'; } async function applyUpdate() { @@ -1712,6 +2707,262 @@ async function toggleAutoUpdate() { toggle.checked = !enabled; } } + +// ─── ROI Calculator (Experimental) ─── + +let roiEnabled = false; +let roiPrice = 20.0; +let roiCacheHit = 0.0; + +function loadROIConfig() { + fetch('/dashboard/api/roi/config').then(r => r.json()).then(data => { + roiEnabled = data.enabled; + roiPrice = data.subscription_price || 20.0; + roiCacheHit = data.cache_hit_pct || 0.0; + document.getElementById('roi-enabled').checked = roiEnabled; + document.getElementById('roi-card').style.display = 'block'; + const planSel = document.getElementById('roi-plan'); + const customInput = document.getElementById('roi-custom-price'); + if (roiPrice == 20) planSel.value = '20'; + else if (roiPrice == 100) planSel.value = '100'; + else { + planSel.value = '0'; + customInput.value = roiPrice; + customInput.disabled = false; + } + + // Set cache hit slider + const cacheSlider = document.getElementById('cache-hit-slider'); + const cacheValue = document.getElementById('cache-hit-value'); + if (cacheSlider && cacheValue) { + cacheSlider.value = roiCacheHit; + cacheValue.textContent = roiCacheHit + '%'; + } + + if (roiEnabled) { + document.getElementById('roi-disabled-msg').style.display = 'none'; + calculateROI(); + } else { + document.getElementById('roi-results').style.display = 'none'; + } + }).catch(() => { + // ROI endpoints may not exist yet — hide the card + document.getElementById('roi-card').style.display = 'none'; + }); +} + +function toggleROI() { + const enabled = document.getElementById('roi-enabled').checked; + roiEnabled = enabled; + const price = parseFloat(document.getElementById('roi-plan').value) || parseFloat(document.getElementById('roi-custom-price').value) || 20.0; + const cacheHit = roiCacheHit; + fetch('/dashboard/api/roi/config', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ enabled, subscription_price: price, cache_hit_pct: cacheHit }) + }).then(r => r.json()).then(data => { + if (data.status === 'ok') { + if (enabled) { + document.getElementById('roi-disabled-msg').style.display = 'none'; + calculateROI(); + } else { + document.getElementById('roi-results').style.display = 'none'; + document.getElementById('roi-disabled-msg').style.display = 'block'; + } + } + }); +} + +function updateCacheHitDisplay() { + const val = document.getElementById('cache-hit-slider').value; + document.getElementById('cache-hit-value').textContent = val + '%'; +} + +function setCacheHit() { + const val = parseFloat(document.getElementById('cache-hit-slider').value) || 0; + roiCacheHit = val; + const price = parseFloat(document.getElementById('roi-plan').value) || parseFloat(document.getElementById('roi-custom-price').value) || 20.0; + fetch('/dashboard/api/roi/config', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ enabled: roiEnabled, subscription_price: price, cache_hit_pct: val }) + }).then(r => r.json()).then(data => { + if (data.status === 'ok' && roiEnabled) { + calculateROI(); + } + }); +} + +function setROIPlan() { + const val = document.getElementById('roi-plan').value; + const custom = document.getElementById('roi-custom-price'); + if (val === '0') { + custom.disabled = false; + custom.focus(); + } else { + custom.disabled = true; + custom.value = ''; + roiPrice = parseFloat(val); + } + if (roiEnabled) { + fetch('/dashboard/api/roi/config', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ enabled: true, subscription_price: roiPrice }) + }); + } +} + +async function calculateROI() { + if (!roiEnabled) return; + const loading = document.getElementById('roi-loading'); + const results = document.getElementById('roi-results'); + loading.style.display = 'block'; + results.style.display = 'none'; + try { + const resp = await fetch('/dashboard/api/roi/calculate'); + const data = await resp.json(); + loading.style.display = 'none'; + if (data.error) { + document.getElementById('roi-model-rows').innerHTML = '
' + data.error + '
'; + results.style.display = 'block'; + return; + } + renderROIResults(data); + loadROIComparison(); + results.style.display = 'block'; + } catch (e) { + loading.style.display = 'none'; + document.getElementById('roi-model-rows').innerHTML = '
Failed to calculate ROI
'; + results.style.display = 'block'; + } +} + +function renderROIResults(data) { + document.getElementById('roi-total-cost').textContent = '$' + (data.total_cost || 0).toLocaleString(undefined, {maximumFractionDigits: 2}); + document.getElementById('roi-weekly').textContent = '$' + (data.weekly_value || 0).toLocaleString(undefined, {maximumFractionDigits: 2}); + const monthly = data.monthly_value || 0; + document.getElementById('roi-monthly').textContent = '$' + monthly.toLocaleString(undefined, {maximumFractionDigits: 2}); + document.getElementById('roi-sub-cost').textContent = '$' + (data.subscription_monthly || 0).toLocaleString(undefined, {maximumFractionDigits: 0}); + document.getElementById('roi-est-value').textContent = '$' + monthly.toLocaleString(undefined, {maximumFractionDigits: 2}); + const savings = Math.max(0, monthly - (data.subscription_monthly || 0)); + document.getElementById('roi-savings').textContent = '$' + savings.toLocaleString(undefined, {maximumFractionDigits: 2}); + const mult = data.roi_multiplier || 0; + document.getElementById('roi-mult').textContent = mult.toFixed(1) + 'x'; + document.getElementById('roi-mult').style.color = mult > 1 ? 'var(--success)' : (mult > 0.5 ? '#facc15' : 'var(--danger)'); + // Use total_raw_tokens if available (new calc), otherwise sum from by_model (cached) + let rawTotal = data.total_raw_tokens || 0; + if (!rawTotal && data.by_model) { + rawTotal = data.by_model.reduce((s, m) => s + (m.prompt_tokens || 0) + (m.completion_tokens || 0), 0); + } + document.getElementById('roi-total-tokens').textContent = rawTotal.toLocaleString(); + + const rows = document.getElementById('roi-model-rows'); + if (data.by_model && data.by_model.length > 0) { + rows.innerHTML = data.by_model.map(m => { + const pt = m.prompt_tokens || 0; + const ct = m.completion_tokens || 0; + const totalToks = pt + ct; + return '
' + + '
' + m.model + '
' + + '
$' + (m.prompt_per_mt || 0).toFixed(4) + '
' + + '
$' + (m.completion_per_mt || 0).toFixed(4) + '
' + + '
$' + (m.cost || 0).toFixed(2) + '
' + + '
' + (m.pct_of_total || 0).toFixed(1) + '%
' + + '
' + totalToks.toLocaleString() + '
' + + '
'; + }).join(''); + } else { + rows.innerHTML = '
No usage data yet this week
'; + } + + // Weekly usage % + const weeklyPct = data.weekly_pct_used || 0; + document.getElementById('roi-weekly-pct').textContent = weeklyPct > 0 ? '· Based on ' + weeklyPct.toFixed(1) + '% of weekly quota used' : ''; + + // Unmatched models warning + const unmatchedEl = document.getElementById('roi-unmatched'); + if (data.unmatched_models && data.unmatched_models.length > 0) { + unmatchedEl.textContent = '⚠ Could not price: ' + data.unmatched_models.join(', '); + unmatchedEl.style.display = 'block'; + } else { + unmatchedEl.style.display = 'none'; + } + + // Stale prices warning + document.getElementById('roi-stale').style.display = data.prices_stale ? 'block' : 'none'; +} + +// ─── Model Value Comparison ─── + +async function loadROIComparison() { + if (!roiEnabled) return; + try { + const resp = await fetch('/dashboard/api/roi/comparison?period=weekly'); + const data = await resp.json(); + renderROIComparison(data); + } catch (e) { + document.getElementById('roi-comparison-rows').innerHTML = '
Failed to load comparison
'; + } +} + +function renderROIComparison(data) { + const rows = document.getElementById('roi-comparison-rows'); + const summary = document.getElementById('roi-comparison-summary'); + + if (data.error) { + rows.innerHTML = '
' + data.error + '
'; + summary.style.display = 'none'; + return; + } + + if (!data.models || data.models.length === 0) { + rows.innerHTML = '
No usage data for this period
'; + summary.style.display = 'none'; + return; + } + + rows.innerHTML = data.models.map(m => { + const score = m.score || 0; + const scoreColor = score > 0 ? 'var(--success)' : (score < 0 ? 'var(--danger)' : 'var(--text2)'); + const scoreSign = score > 0 ? '+' : ''; + return '
' + + '
' + m.model + '
' + + '
' + (m.requests || 0).toLocaleString() + '
' + + '
' + (m.total_tokens || 0).toLocaleString() + '
' + + '
$' + (m.actual_value || 0).toFixed(2) + '
' + + '
$' + (m.fair_share || 0).toFixed(2) + '
' + + '
' + scoreSign + score.toFixed(2) + '
' + + '
'; + }).join(''); + + // Summary bar + const net = data.net_score || 0; + const netColor = net > 0 ? 'var(--success)' : (net < 0 ? 'var(--danger)' : 'var(--text2)'); + const netSign = net > 0 ? '+' : ''; + summary.innerHTML = + '
' + + 'Period cost: $' + (data.period_sub_cost || 0).toFixed(2) + ' · Total value: $' + (data.total_actual_value || 0).toFixed(2) + '' + + 'Net: ' + netSign + net.toFixed(2) + '' + + '
'; + summary.style.display = 'block'; +} + +function loadLastROI() { + fetch('/dashboard/api/roi/last').then(r => r.json()).then(data => { + if (!data.error && (data.by_model || []).length > 0) { + renderROIResults(data); + document.getElementById('roi-results').style.display = 'block'; + } + }).catch(() => {}); +} + +function resetROIData() { + fetch('/dashboard/api/roi/reset', {method: 'POST'}).then(() => { + document.getElementById('roi-results').style.display = 'none'; + if (roiEnabled) calculateROI(); + }); +} \ No newline at end of file diff --git a/guanaco/roi.py b/guanaco/roi.py new file mode 100644 index 0000000..978b170 --- /dev/null +++ b/guanaco/roi.py @@ -0,0 +1,473 @@ +""" +OpenRouter price-based subscription value calculator. + +This module: +1. Fetches live model prices from OpenRouter's API +2. Maps Ollama Cloud model names to OpenRouter model IDs +3. Calculates "what would this usage have cost on OpenRouter?" +4. Compares against subscription price to show value multiplier + +Prices are cached for 1 hour to avoid rate limits. +""" +from __future__ import annotations + +import logging +import sqlite3 +import time +from pathlib import Path +from typing import Optional + +import httpx + +logger = logging.getLogger(__name__) + +# ── OpenRouter API ── +OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" +OPENROUTER_CACHE_TTL = 3600 # 1 hour + +# Model family mappings: ollama_name_fragment -> openrouter_id_fragment +# These are used when exact match fails +FAMILY_MAP = { + "gemma": ("google/gemma", "google/gemma"), + "gemma3": ("google/gemma", "google/gemma"), + "gemma4": ("google/gemma", "google/gemma"), + "qwen": ("qwen/qwen", "qwen/qwen"), + "qwen3": ("qwen/qwen3", "qwen/qwen"), + "qwen3.5": ("qwen/qwen3.5", "qwen/qwen"), + "qwen3-vl": ("qwen/qwen3-vl", "qwen/qwen3-vl"), + "qwen3-coder": ("qwen/qwen3-coder", "qwen/qwen"), + "qwen3-next": ("qwen/qwen3-next", "qwen/qwen"), + "deepseek": ("deepseek/deepseek", "deepseek/deepseek"), + "deepseek-v3": ("deepseek/deepseek", "deepseek/deepseek-v3"), + "deepseek-v4": ("deepseek/deepseek", "deepseek/deepseek-v4"), + "gpt-oss": ("openai/gpt-oss", "openai/gpt"), + "minimax": ("minimax/minimax", "minimax/minimax"), + "glm": ("zhipu/glm", "zhipu/glm"), + "glm-5": ("zhipu/glm-5", "zhipu/glm"), + "kimi": ("moonshot/kimi", "moonshot/kimi"), + "kimi-k2": ("moonshot/kimi", "moonshot/kimi"), + "devstral": ("mistral/devstral", "mistral/devstral"), + "mistral": ("mistral/mistral", "mistral/mistral"), + "ministral": ("mistral/ministral", "mistral/ministral"), + "nemotron": ("nvidia/nemotron", "nvidia/nemotron"), + "cogito": ("cogito/cogito", "cogito/"), + "gemini": ("google/gemini", "google/gemini"), + "rnj": ("", ""), +} + + +def _normalized(name: str) -> str: + """Strip provider prefix, ~leaderboard prefix, :cloud/:local suffixes, and lower-case.""" + base = name.split(":")[0].lower() + # Strip ~ prefix (leaderboard indicator on OpenRouter) + if base.startswith("~"): + base = base[1:] + # Strip provider/ prefix (e.g. moonshotai/kimi-k2.6 → kimi-k2.6) + if "/" in base: + base = base.split("/", 1)[1] + if base.endswith("-cloud"): + base = base[:-6] + return base + + +def _model_size(name: str) -> int: + """Extract parameter size in billions from model name, 0 if unknown.""" + import re + m = re.search(r"(\d+)(b|t)", name, re.I) + if not m: + return 0 + n = int(m.group(1)) + unit = m.group(2).lower() + return n * 1000 if unit == "t" else n + + +class PriceCache: + """Holds cached OpenRouter prices in memory with TTL.""" + def __init__(self): + self.prices: dict[str, dict] = {} + self.fetched_at: float = 0 + + def is_fresh(self) -> bool: + return self.prices and (time.time() - self.fetched_at) < OPENROUTER_CACHE_TTL + + def fetch(self) -> dict[str, dict]: + if self.is_fresh(): + logger.debug("Using cached OpenRouter prices") + return self.prices + + prices = {} + try: + logger.info("Fetching OpenRouter model prices...") + r = httpx.get(OPENROUTER_MODELS_URL, timeout=30) + r.raise_for_status() + data = r.json() + for model in data.get("data", []): + model_id = model.get("id", "") + pricing = model.get("pricing", {}) + prompt = float(pricing.get("prompt", 0) or 0) + completion = float(pricing.get("completion", 0) or 0) + cache_read = float(pricing.get("input_cache_read", 0) or 0) + if prompt > 0 or completion > 0: + # Convert from per-token ($/token) to per-million-tokens ($/Mt) + entry = { + "prompt": prompt * 1_000_000, + "completion": completion * 1_000_000, + } + if cache_read > 0: + entry["input_cache_read"] = cache_read * 1_000_000 + prices[model_id] = entry + self.prices = prices + self.fetched_at = time.time() + logger.info(f"Fetched {len(prices)} OpenRouter price entries") + except Exception as e: + logger.warning(f"Failed to fetch OpenRouter prices: {e}") + return self.prices + + +# Singleton cache +_price_cache = PriceCache() + + +def _find_best_price(prices: dict, ollama_name: str) -> dict: + """ + Given OpenRouter prices dict {model_id: {prompt, completion}} and an + Ollama model name, return best matching price dict. + """ + norm = _normalized(ollama_name) + size = _model_size(ollama_name) + + # 1. Exact normalized match (handles provider-prefixed OR IDs like moonshotai/kimi-k2.6) + for orouter_id, price_info in prices.items(): + if _normalized(orouter_id) == norm: + return price_info + + # 2. Family prefix match — use raw orouter_id so provider/ prefix matches + best_family_price = None + best_family_score = -9999 + for orouter_id, price_info in prices.items(): + for frag, (family_exact, family_prefix) in FAMILY_MAP.items(): + if frag in norm and family_prefix and family_prefix in orouter_id: + # Score by size closeness + o_size = _model_size(orouter_id) + score = -(abs(o_size - size)) # higher = closer size + if score > best_family_score: + best_family_score = score + best_family_price = price_info + if best_family_price: + return best_family_price + + # 3. Same parameter size match + if size > 0: + for orouter_id, price_info in prices.items(): + if _model_size(orouter_id) == size: + return price_info + + # 4. Size-window fallback + candidates = [] + for orouter_id, price_info in prices.items(): + o_size = _model_size(orouter_id) + if o_size == 0: + continue + window = max(20, size * 0.5) + if abs(o_size - size) <= window: + candidates.append(price_info) + if candidates: + candidates.sort(key=lambda p: p["completion"] + p["prompt"]) + return candidates[len(candidates) // 2] + + # 5. Global average + all_prices = [p for p in prices.values() if p["prompt"] > 0 or p["completion"] > 0] + if all_prices: + avg_prompt = sum(p["prompt"] for p in all_prices) / len(all_prices) + avg_comp = sum(p["completion"] for p in all_prices) / len(all_prices) + return {"prompt": avg_prompt, "completion": avg_comp} + + return {"prompt": 0.0, "completion": 0.0} + + +def _map_usage_to_prices(usage_by_model: dict, prices: dict, cache_hit_pct: float = 0.0) -> dict: + """ + Map usage to prices, optionally applying prompt-cache hit discount. + + For models with input_cache_read pricing (e.g. Claude Fable, Qwen, Minimax): + - uncached_prompt = prompt_tokens * (1 - cache_hit_pct) + - cached_prompt = prompt_tokens * cache_hit_pct + - prompt cost = uncached_prompt * prompt_price + cached_prompt * cache_read_price + """ + result = {} + cache_rate = max(0.0, min(100.0, cache_hit_pct)) / 100.0 + for model, usage in usage_by_model.items(): + price = _find_best_price(prices, model) + pt = usage.get("prompt_tokens", 0) + ct = usage.get("completion_tokens", 0) + + # Apply cache discount if model supports it + if "input_cache_read" in price and cache_rate > 0: + uncached_pt = pt * (1 - cache_rate) + cached_pt = pt * cache_rate + prompt_cost = (uncached_pt / 1_000_000 * price["prompt"]) + (cached_pt / 1_000_000 * price["input_cache_read"]) + # Store effective prompt price for display + effective_prompt = prompt_cost / (pt / 1_000_000) if pt > 0 else price["prompt"] + else: + prompt_cost = (pt / 1_000_000) * price["prompt"] + effective_prompt = price["prompt"] + + comp_cost = (ct / 1_000_000) * price["completion"] + cost = prompt_cost + comp_cost + + result[model] = { + "prompt_tokens": pt, + "completion_tokens": ct, + "prompt_per_mt": effective_prompt, + "completion_per_mt": price["completion"], + "cost": cost, + "matched_price_model": _find_best_price.__module__, + "cache_applied": "input_cache_read" in price and cache_rate > 0, + "cache_read_per_mt": price.get("input_cache_read"), + } + return result + + +def get_usage_from_analytics(db_path: Path | str, since: float = 0) -> tuple[dict, float]: + usage_by_model = {} + total_weighted = 0.0 + try: + conn = sqlite3.connect(str(db_path)) + rows = conn.execute( + """SELECT model, + IFNULL(SUM(prompt_tokens),0), + IFNULL(SUM(completion_tokens),0), + IFNULL(SUM(prompt_tokens * IFNULL(usage_multiplier,1.0)),0), + IFNULL(SUM(completion_tokens * IFNULL(usage_multiplier,1.0)),0), + COUNT(*) + FROM request_log WHERE type='llm' AND ts > ? GROUP BY model""", + (since,), + ).fetchall() + for row in rows: + model, pt, ct, w_pt, w_ct, req_count = row + usage_by_model[model] = { + "prompt_tokens": pt, + "completion_tokens": ct, + "weighted_prompt": w_pt, + "weighted_completion": w_ct, + "requests": req_count, + } + total_weighted += (w_pt + w_ct) + conn.close() + except Exception as e: + logger.warning(f"Failed to read analytics DB: {e}") + return usage_by_model, total_weighted + + +def _get_ollama_week_start() -> float: + """Return Unix timestamp of the most recent Sunday at 20:00 UTC. + + Ollama resets its weekly quota every Sunday at 8 PM UTC. + """ + from datetime import datetime, timedelta, timezone + now = datetime.now(timezone.utc) + days_since_sunday = now.weekday() + 1 if now.weekday() != 6 else 0 + sunday = now - timedelta(days=days_since_sunday) + reset = sunday.replace(hour=20, minute=0, second=0, microsecond=0) + if now < reset: + reset -= timedelta(days=7) + return reset.timestamp() + + +def calculate_roi( + db_path: Path | str, + subscription_monthly: float = 20.0, + weekly_pct_used: float = 0.0, + cache_hit_pct: float = 0.0, +) -> dict: + """ + Calculate subscription value vs OpenRouter pay-as-you-go. + + Args: + db_path: path to analytics SQLite DB + subscription_monthly: monthly sub cost (20 Pro, 100 Max) + weekly_pct_used: % of weekly quota consumed (from usage check) + cache_hit_pct: estimated % of prompt tokens hitting cache (0-100). + Used for models with input_cache_read pricing on OpenRouter. + + Returns dict with: + total_cost, total_prompt_tokens, total_completion_tokens, + total_weighted_tokens, weekly_value, monthly_value, + subscription_monthly, plan ("pro"|"max"), roi_multiplier, + weekly_pct_used, cache_hit_pct, prices_stale, + by_model[] with prompt_tokens, completion_tokens, prompt_per_mt, completion_per_mt, cost, + unmatched_models[] names with no price match + """ + # 1. Fetch prices + prices = _price_cache.fetch() + prices_stale = not prices or len(prices) < 10 + plan = "pro" if subscription_monthly <= 25 else "max" + + # 2. Get usage + since = _get_ollama_week_start() + usage_by_model, total_weighted = get_usage_from_analytics(db_path, since) + + # 3. Map usage to prices (with cache hit estimation) + priced = _map_usage_to_prices(usage_by_model, prices, cache_hit_pct) + + total_cost = sum(m["cost"] for m in priced.values()) + total_prompt = sum(m["prompt_tokens"] for m in priced.values()) + total_comp = sum(m["completion_tokens"] for m in priced.values()) + + # 4. Extrapolate to 100% weekly + if weekly_pct_used > 0: + weekly_value = total_cost / (weekly_pct_used / 100.0) + else: + weekly_value = total_cost + + monthly_value = weekly_value * 4 + roi_multiplier = (monthly_value / subscription_monthly) if subscription_monthly > 0 else 0 + + unmatched = [m for m in usage_by_model if priced.get(m, {}).get("cost", 0) == 0] + + # Per-model breakdown + by_model = [] + for model, detail in priced.items(): + pt = detail["prompt_tokens"] + ct = detail["completion_tokens"] + by_model.append({ + "model": model, + "prompt_tokens": pt, + "completion_tokens": ct, + "prompt_per_mt": round(detail["prompt_per_mt"], 6), + "completion_per_mt": round(detail["completion_per_mt"], 6), + "cost": round(detail["cost"], 4), + "pct_of_total": round((detail["cost"] / total_cost * 100), 2) if total_cost > 0 else 0, + "cache_applied": detail.get("cache_applied", False), + "cache_read_per_mt": round(detail.get("cache_read_per_mt"), 6) if detail.get("cache_read_per_mt") else None, + }) + by_model.sort(key=lambda x: x["cost"], reverse=True) + + return { + "total_cost": round(total_cost, 2), + "total_prompt_tokens": int(total_prompt), + "total_completion_tokens": int(total_comp), + "total_raw_tokens": int(total_prompt + total_comp), + "total_weighted_tokens": int(total_weighted), + "weekly_value": round(weekly_value, 2), + "monthly_value": round(monthly_value, 2), + "subscription_monthly": subscription_monthly, + "plan": plan, + "roi_multiplier": round(roi_multiplier, 2), + "weekly_pct_used": weekly_pct_used, + "cache_hit_pct": cache_hit_pct, + "prices_stale": prices_stale, + "by_model": by_model, + "unmatched_models": unmatched, + "price_models_available": len(prices), + } + + +def calculate_model_value_comparison( + db_path: Path | str, + subscription_monthly: float = 20.0, + weekly_pct_used: float = 0.0, + session_pct_used: float = 0.0, + period: str = "weekly", # "weekly" or "session" +) -> dict: + """ + Score each model: positive = gave more value than its fair share of sub. + + For each model actually used: + - actual_value = what those tokens would cost on OpenRouter + - fair_share = (model's weighted tokens / total weighted tokens) * subscription_cost_for_period + - score = actual_value - fair_share + positive = model punches above its weight (good deal) + negative = model is expensive for its token share (bad deal) + """ + prices = _price_cache.fetch() + if not prices: + return {"error": "No OpenRouter prices available", "models": []} + + # Determine time window and usage % + now = time.time() + if period == "session": + since = now - (5 * 3600) # 5-hour session window + pct_used = session_pct_used + else: + since = now - (7 * 24 * 3600) # 7-day weekly window + pct_used = weekly_pct_used + + usage_by_model, total_weighted = get_usage_from_analytics(db_path, since) + if not usage_by_model: + return {"models": [], "summary": {}} + + # Period subscription cost + weekly_sub_cost = subscription_monthly / 4.0 + if pct_used > 0: + period_sub_cost = weekly_sub_cost * (pct_used / 100.0) + else: + period_sub_cost = weekly_sub_cost + + # Map to prices + priced = _map_usage_to_prices(usage_by_model, prices) + + # Per-model scoring + models = [] + total_actual_value = 0.0 + + for model, usage in usage_by_model.items(): + detail = priced.get(model, {}) + actual_value = detail.get("cost", 0.0) + pt = usage.get("prompt_tokens", 0) + ct = usage.get("completion_tokens", 0) + model_raw = pt + ct + w_pt = usage.get("weighted_prompt", 0) + w_ct = usage.get("weighted_completion", 0) + model_weighted = w_pt + w_ct + + # Fair share of subscription based on WEIGHTED token proportion + # (subscription quota is weighted; actual value uses raw tokens at OpenRouter prices) + fair_share = (model_weighted / total_weighted * period_sub_cost) if total_weighted > 0 else 0 + score = actual_value - fair_share + score_pct = (score / fair_share * 100) if fair_share > 0 else 0 + + total_actual_value += actual_value + + models.append({ + "model": model, + "requests": usage.get("requests", 0), + "prompt_tokens": pt, + "completion_tokens": ct, + "total_tokens": int(model_raw), + "weighted_tokens": int(model_weighted), + "pct_of_total_tokens": round((model_weighted / total_weighted * 100), 2) if total_weighted > 0 else 0, + "actual_value": round(actual_value, 2), + "fair_share": round(fair_share, 2), + "score": round(score, 2), + "score_pct": round(score_pct, 1), + "prompt_per_mt": round(detail.get("prompt_per_mt", 0), 6), + "completion_per_mt": round(detail.get("completion_per_mt", 0), 6), + }) + + # Sort by score descending (best value first) + models.sort(key=lambda x: x["score"], reverse=True) + + # Summary + net_score = total_actual_value - period_sub_cost + + # Compute total raw tokens across all models for the summary + total_raw = sum(m.get("prompt_tokens", 0) + m.get("completion_tokens", 0) for m in models) + + return { + "period": period, + "subscription_monthly": subscription_monthly, + "period_sub_cost": round(period_sub_cost, 2), + "total_actual_value": round(total_actual_value, 2), + "total_raw_tokens": int(total_raw), + "total_weighted_tokens": int(total_weighted), + "net_score": round(net_score, 2), + "models": models, + "prices_stale": len(prices) < 10, + "price_models_available": len(prices), + } + + +def get_cached_roi() -> dict: + """Return last calculated ROI, or minimal default.""" + return _price_cache.prices # placeholder; real caching is in config diff --git a/guanaco/router/router.py b/guanaco/router/router.py index 4252e95..4494349 100644 --- a/guanaco/router/router.py +++ b/guanaco/router/router.py @@ -41,33 +41,68 @@ def _describe_error(exc: Exception) -> str: return f"{type(exc).__name__}: (no message)" -async def _ollama_chat_with_primary_timeout(client, payload, fallback_config=None): - """Call Ollama Cloud chat completion with a primary timeout. +async def _ollama_chat_with_primary_timeout(client, payload, fallback_config=None, limiter=None, api_key=None, account_name=None, account_pool=None): + """Call Ollama Cloud chat completion with a primary timeout and optional concurrency limit. When fallback is configured, we use a shorter primary_timeout so that slow/unresponsive Ollama responses trigger fallback quickly instead of hanging for the full 120s client timeout. + + Args: + client: OllamaClient instance + payload: Request payload + fallback_config: FallbackProviderConfig for timeout settings + limiter: Optional OllamaConcurrencyLimiter for concurrency control and 429 retry + api_key: Optional API key override for multi-account rotation + account_name: Optional account name for analytics logging + account_pool: Optional AccountPool for marking 429s against specific accounts """ + async def _do_call(): + """Execute the actual Ollama call with 429 retry logic.""" + # If no limiter, just call directly + if limiter is None: + return await client.chat_completion(payload, api_key=api_key) + + # Retry loop for 429s + for attempt in range(limiter.max_429_retries + 1): + try: + return await client.chat_completion(payload, api_key=api_key) + except Exception as e: + if attempt < limiter.max_429_retries and limiter.should_retry_429(e): + # Mark account as 429'd if we have an account pool + if account_name and account_pool: + account_pool.mark_429(account_name) + await limiter.backoff_and_retry(attempt) + continue + raise + if fallback_config and fallback_config.enabled and fallback_config.primary_timeout: try: return await asyncio.wait_for( - client.chat_completion(payload), + _do_call(), timeout=fallback_config.primary_timeout, ) except asyncio.TimeoutError: raise httpx.ReadTimeout( f"Ollama did not respond within {fallback_config.primary_timeout}s primary timeout" ) - return await client.chat_completion(payload) + return await _do_call() from guanaco.client import OllamaClient from guanaco.cache import CacheEngine from guanaco.analytics import _normalize_model_name +from guanaco.concurrency import OllamaConcurrencyLimiter import logging log = logging.getLogger("guanaco.router") +# Module-level reference to the active concurrency limiter (for dashboard/status API) +_concurrency_limiter_instance: OllamaConcurrencyLimiter = None + +def get_concurrency_limiter() -> Optional[OllamaConcurrencyLimiter]: + return _concurrency_limiter_instance + # ── Empty Response Retry ── MAX_EMPTY_RETRIES = 1 # How many times to retry on empty responses @@ -285,9 +320,13 @@ async def _call_fallback_provider(payload: dict, fallback_config, stream: bool = if fallback_config.api_key: headers["Authorization"] = f"Bearer {fallback_config.api_key}" + # Ensure the payload has the correct stream value — some providers (e.g. Fireworks) + # require "stream": true in the JSON body when max_tokens > 4096 + payload = dict(payload) + payload["stream"] = stream + # Inject fallback max_tokens if not already set in the payload if fallback_config.max_tokens and "max_tokens" not in payload: - payload = dict(payload) payload["max_tokens"] = fallback_config.max_tokens timeout = fallback_config.timeout or 60.0 @@ -320,12 +359,109 @@ async def _call_fallback_provider(payload: dict, fallback_config, stream: bool = # ── Provider creation ── -def create_router(client: OllamaClient, analytics=None, config=None) -> APIRouter: +def create_router(client: OllamaClient, analytics=None, config=None, account_pool=None) -> APIRouter: router = APIRouter(tags=["LLM Router"]) _analytics = analytics _config = config + _account_pool = account_pool _cache = CacheEngine(config.cache) if config else None + # Concurrency limiter: prevents 429 "too many concurrent requests" from Ollama Cloud + _max_concurrent = getattr(config.fallback, 'max_concurrent_ollama', 0) if config and config.fallback else 0 + _max_429_retries = getattr(config.fallback, 'max_429_retries', 2) if config and config.fallback else 2 + _backoff_base = getattr(config.fallback, 'backoff_base', 1.0) if config and config.fallback else 1.0 + _concurrency_limiter = OllamaConcurrencyLimiter( + max_concurrent=_max_concurrent, + max_429_retries=_max_429_retries, + base_backoff=_backoff_base, + ) + # Expose for dashboard API (module-level reference) + global _concurrency_limiter_instance + _concurrency_limiter_instance = _concurrency_limiter + + def _select_account(model: str = None): + """Select the best Ollama account for the next request. + + Returns (api_key, account_name) tuple. api_key is None for the primary account + (uses the default client key). account_name is 'ollama' for the primary account. + """ + if not _account_pool or len(_account_pool.accounts) <= 1: + return None, "ollama" + account = _account_pool.get_account(model=model) + return account.api_key, account.name + + def _history_kwargs(request: Request, messages=None, output_text=None) -> dict: + """Build history-related kwargs for analytics.log_llm() based on config. + + Returns a dict with source_ip, source_port, user_agent, and + conditionally input_text/output_text if history logging is enabled. + """ + kwargs = {} + # Always extract caller info + client_host = request.client + if client_host: + kwargs["source_ip"] = client_host.host + kwargs["source_port"] = client_host.port + kwargs["user_agent"] = request.headers.get("user-agent", "") + + # Only include content if history is enabled + hist = _config.history if _config else None + if hist and hist.enabled: + if messages and hist.save_input: + # Serialize the messages list to a JSON string + try: + if hasattr(messages, '__iter__') and not isinstance(messages, (str, dict)): + # It's a list or iterable of message objects + msgs = [] + for m in messages: + if hasattr(m, 'model_dump'): + msgs.append(m.model_dump(exclude_none=True)) + elif isinstance(m, dict): + msgs.append(m) + else: + msgs.append(str(m)) + elif isinstance(messages, str): + msgs = messages + else: + msgs = str(messages) + if isinstance(msgs, list): + input_str = json.dumps(msgs, ensure_ascii=False) + else: + input_str = str(msgs) + if len(input_str) > hist.max_content_size: + input_str = input_str[:hist.max_content_size] + "\n...[truncated]" + kwargs["input_text"] = input_str + log.debug("History: captured input_text (%d chars)", len(input_str)) + except Exception as e: + log.warning("History: failed to capture input_text: %s", e) + if output_text and hist.save_output: + if len(output_text) > hist.max_content_size: + output_text = output_text[:hist.max_content_size] + "\n...[truncated]" + kwargs["output_text"] = output_text + + return kwargs + + def _extract_output_text(resp: dict) -> Optional[str]: + """Extract the text content from an OpenAI-format chat completion response.""" + try: + choices = resp.get("choices", []) + if choices: + msg = choices[0].get("message", {}) + if isinstance(msg, dict): + parts = [] + content = msg.get("content", "") + if content: + parts.append(content) + # Also capture reasoning/thinking content from GLM etc + reasoning = msg.get("reasoning", "") or msg.get("reasoning_content", "") + if reasoning: + parts.append(f"\n{reasoning}\n") + if parts: + return "\n".join(parts) + except Exception: + pass + return None + # ── OpenAI-compatible endpoints ── @router.get("/v1/models") @@ -333,11 +469,17 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute """List available models by querying Ollama Cloud dynamically.""" try: models = await client.list_models() + # Fetch real usage levels from ollama.com library pages + model_names = [m.get("name", m.get("model", "")) for m in models] + usage_levels = await client.fetch_usage_levels(model_names) + data = [] for m in models: name = m.get("name", m.get("model", "")) display_name = name.replace("-cloud", "") if name.endswith("-cloud") else name details = m.get("details", {}) + level = usage_levels.get(name, 0) + multiplier = level * 0.25 if level else client._get_model_multiplier(name) data.append({ "id": display_name, "object": "model", @@ -347,6 +489,8 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute "root": display_name, "parent": None, "capabilities": client._get_model_capabilities(name), + "usage_multiplier": multiplier, + "usage_level": level, # 1-4, 0 = unknown "details": { "parameter_size": details.get("parameter_size", ""), "quantization": details.get("quantization_level", ""), @@ -385,6 +529,11 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute """OpenAI-compatible chat completions endpoint with fallback and smart caching (beta).""" start = time.time() resolved_model = _resolve_model(body.model, _config) if _config else body.model + _hist = _history_kwargs(request, messages=body.messages) + _request_key, _account_name = _select_account(model=resolved_model) + _ollama_provider = f"ollama:{_account_name}" if _account_name != "ollama" else "ollama" + # Include account_name in all analytics logging + _hist["account_name"] = _account_name # Convert image URLs to base64 for Ollama Cloud compatibility if _has_vision_content(body.messages): @@ -411,10 +560,45 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute payload_fb["model"] = fallback_model if body.stream: if _config.fallback.stream_fallback: - fallback_payload = dict(payload) - fallback_payload["model"] = fallback_model + fallback_payload2 = dict(payload) + fallback_payload2["model"] = fallback_model + + async def _quota_fallback_stream(): + acc_content = [] + fb_start = time.time() + fb_first = None + try: + async for fb_chunk in await _call_fallback_provider(fallback_payload2, _config.fallback, stream=True): + yield fb_chunk + txt = _extract_sse_content(fb_chunk) + if txt: + if fb_first is None: + fb_first = time.time() + acc_content.append(txt) + finally: + if _analytics: + _hist_kw2 = dict(_hist) + if acc_content and _config and _config.history.enabled and _config.history.save_output: + out_text = "".join(acc_content) + if len(out_text) > _config.history.max_content_size: + out_text = out_text[:_config.history.max_content_size] + "\n...[truncated]" + _hist_kw2["output_text"] = out_text + fb_chars = len("".join(acc_content)) + fb_tokens = max(1, fb_chars // 4) if fb_chars else 0 + fb_elapsed = time.time() - fb_start + _analytics.log_llm( + model=_normalize_model_name(fallback_model), + prompt_tokens=0, + completion_tokens=fb_tokens, + total_duration_seconds=fb_elapsed, + provider=_config.fallback.name, + fallback_for=_normalize_model_name(resolved_model), + fallback_reason=f"Quota full (session={_config.usage.last_session_pct or 0:.0f}%, weekly={_config.usage.last_weekly_pct or 0:.0f}%)", + **_hist_kw2, + ) + return StreamingResponse( - await _call_fallback_provider(fallback_payload, _config.fallback, stream=True), + _quota_fallback_stream(), media_type="text/event-stream", ) # Can't stream from fallback — do non-streaming @@ -437,12 +621,21 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute fallback_resp["_oct_fallback_provider"] = _config.fallback.name fallback_resp["_oct_original_model"] = _normalize_model_name(resolved_model) fallback_resp["_oct_quota_redirect"] = True + # Extract usage from fallback response when available + fb_usage = fallback_resp.get("usage", {}) + fb_prompt = fb_usage.get("prompt_tokens", 0) + fb_completion = fb_usage.get("completion_tokens", 0) if _analytics: _analytics.log_llm( model=_normalize_model_name(fallback_model), + prompt_tokens=fb_prompt, + completion_tokens=fb_completion, + total_tokens=fb_usage.get("total_tokens", fb_prompt + fb_completion), total_duration_seconds=time.time() - start, provider=_config.fallback.name, fallback_for=_normalize_model_name(resolved_model), + fallback_reason=f"Quota full (session={_config.usage.last_session_pct or 0:.0f}%, weekly={_config.usage.last_weekly_pct or 0:.0f}%)", + **_hist, ) return fallback_resp except Exception as e: @@ -456,7 +649,8 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute try: # ── Retry on empty response ── for attempt in range(MAX_EMPTY_RETRIES + 1): - resp = await _ollama_chat_with_primary_timeout(client, p, _config.fallback if _config else None) + async with _concurrency_limiter: + resp = await _ollama_chat_with_primary_timeout(client, p, _config.fallback if _config else None, _concurrency_limiter, api_key=_request_key, account_name=_account_name, account_pool=_account_pool) if not _is_empty_non_streaming_response(resp) or attempt == MAX_EMPTY_RETRIES: break log.warning("Empty cached-response from %s (attempt %d/%d), retrying...", resolved_model, attempt + 1, MAX_EMPTY_RETRIES + 1) @@ -466,6 +660,12 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute usage = resp.get("usage", {}) if _analytics: + hist_kw = dict(_hist) + out_txt = _extract_output_text(resp) + if out_txt and _config and _config.history.enabled and _config.history.save_output: + if len(out_txt) > _config.history.max_content_size: + out_txt = out_txt[:_config.history.max_content_size] + "\n...[truncated]" + hist_kw["output_text"] = out_txt _analytics.log_llm( model=resolved_model, prompt_tokens=usage.get("prompt_tokens", metrics.get("prompt_eval_count", 0)), @@ -476,7 +676,8 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute ttft_seconds=metrics.get("ttft_seconds"), total_duration_seconds=elapsed, load_duration_seconds=metrics.get("load_duration_ns", 0) / 1e9 if metrics.get("load_duration_ns") else None, - provider="ollama", + provider=_ollama_provider, + **hist_kw, ) return resp @@ -507,6 +708,8 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute total_duration_seconds=elapsed, provider=_config.fallback.name, fallback_for=_normalize_model_name(resolved_model), + fallback_reason=f"Ollama error: {_describe_error(ollama_error)}", + **_hist, ) fallback_resp["_oct_fallback"] = True @@ -517,11 +720,11 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute except Exception as fallback_err: log.warning("Fallback to %s failed for model %s (cached path): %s", _config.fallback.name, resolved_model, _describe_error(fallback_err)) if _analytics: - _analytics.log_llm(model=resolved_model, error=f"ollama: {_describe_error(ollama_error)}; fallback: {_describe_error(fallback_err)}", total_duration_seconds=time.time() - start) + _analytics.log_llm(model=resolved_model, error=f"ollama: {_describe_error(ollama_error)}; fallback: {_describe_error(fallback_err)}", total_duration_seconds=time.time() - start, **_hist) raise HTTPException(status_code=502, detail=f"Ollama Cloud error: {_describe_error(ollama_error)}; Fallback error: {_describe_error(fallback_err)}") if _analytics: - _analytics.log_llm(model=resolved_model, error=str(ollama_error), total_duration_seconds=time.time() - start) + _analytics.log_llm(model=resolved_model, error=str(ollama_error), total_duration_seconds=time.time() - start, **_hist) raise HTTPException(status_code=502, detail=f"Ollama Cloud error: {str(ollama_error)}") # Use cache for non-streaming @@ -530,7 +733,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute messages=[m.model_dump(exclude_none=True) for m in body.messages], params=payload, fetch_fn=_fetch_from_upstream, - provider="ollama", + provider=_ollama_provider, ) # Log cache metadata in analytics @@ -541,6 +744,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute model=resolved_model, total_duration_seconds=elapsed, provider=f"cache:{response.get('_oct_cache_type', 'unknown')}", + **_hist, ) return response @@ -549,11 +753,12 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute # Try Ollama Cloud first try: if body.stream: - return await _stream_completion_openai(client, payload, resolved_model, _analytics, start, _config) + return await _stream_completion_openai(client, payload, resolved_model, _analytics, start, _config, history_kwargs=_hist, limiter=_concurrency_limiter, api_key=_request_key, account_name=_account_name, account_pool=_account_pool) # ── Non-streaming: retry on empty response ── for attempt in range(MAX_EMPTY_RETRIES + 1): - resp = await _ollama_chat_with_primary_timeout(client, payload, _config.fallback if _config else None) + async with _concurrency_limiter: + resp = await _ollama_chat_with_primary_timeout(client, payload, _config.fallback if _config else None, _concurrency_limiter, api_key=_request_key, account_name=_account_name, account_pool=_account_pool) if not _is_empty_non_streaming_response(resp) or attempt == MAX_EMPTY_RETRIES: break log.warning("Empty response from %s (attempt %d/%d), retrying...", resolved_model, attempt + 1, MAX_EMPTY_RETRIES + 1) @@ -563,6 +768,12 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute usage = resp.get("usage", {}) if _analytics: + hist_kw = dict(_hist) + out_txt = _extract_output_text(resp) + if out_txt and _config and _config.history.enabled and _config.history.save_output: + if len(out_txt) > _config.history.max_content_size: + out_txt = out_txt[:_config.history.max_content_size] + "\n...[truncated]" + hist_kw["output_text"] = out_txt _analytics.log_llm( model=resolved_model, prompt_tokens=usage.get("prompt_tokens", metrics.get("prompt_eval_count", 0)), @@ -573,7 +784,8 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute ttft_seconds=metrics.get("ttft_seconds"), total_duration_seconds=elapsed, load_duration_seconds=metrics.get("load_duration_ns", 0) / 1e9 if metrics.get("load_duration_ns") else None, - provider="ollama", + provider=_ollama_provider, + **hist_kw, ) return resp @@ -588,16 +800,25 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute try: if body.stream and _config.fallback.stream_fallback: - return await _stream_fallback_openai(fallback_payload, _config, fallback_model, _analytics, start, "ollama_fallback") + return await _stream_fallback_openai(fallback_payload, _config, fallback_model, _analytics, start, "ollama_fallback", history_kwargs=_hist, fallback_for=resolved_model, fallback_reason=f"Ollama error: {_describe_error(ollama_error)}") fallback_resp = await _call_fallback_provider(fallback_payload, _config.fallback) elapsed = time.time() - start + # Extract usage from fallback response when available + fb_usage2 = fallback_resp.get("usage", {}) + fb_prompt2 = fb_usage2.get("prompt_tokens", 0) + fb_completion2 = fb_usage2.get("completion_tokens", 0) if _analytics: _analytics.log_llm( model=fallback_model, + prompt_tokens=fb_prompt2, + completion_tokens=fb_completion2, + total_tokens=fb_usage2.get("total_tokens", fb_prompt2 + fb_completion2), total_duration_seconds=elapsed, provider=_config.fallback.name, fallback_for=resolved_model, + fallback_reason=f"Ollama error: {_describe_error(ollama_error)}", + **_hist, ) # Tag response so dashboard can show it came from fallback @@ -609,11 +830,11 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute except Exception as fallback_err: log.warning("Fallback to %s failed for model %s: %s", _config.fallback.name, resolved_model, _describe_error(fallback_err)) if _analytics: - _analytics.log_llm(model=resolved_model, error=f"ollama: {_describe_error(ollama_error)}; fallback: {_describe_error(fallback_err)}", total_duration_seconds=time.time() - start) + _analytics.log_llm(model=resolved_model, error=f"ollama: {_describe_error(ollama_error)}; fallback: {_describe_error(fallback_err)}", total_duration_seconds=time.time() - start, **_hist) raise HTTPException(status_code=502, detail=f"Ollama Cloud error: {_describe_error(ollama_error)}; Fallback error: {_describe_error(fallback_err)}") if _analytics: - _analytics.log_llm(model=resolved_model, error=_describe_error(ollama_error), total_duration_seconds=time.time() - start) + _analytics.log_llm(model=resolved_model, error=_describe_error(ollama_error), total_duration_seconds=time.time() - start, **_hist) raise HTTPException(status_code=502, detail=f"Ollama Cloud error: {_describe_error(ollama_error)}") @router.post("/v1/chat/completions/refresh_models") @@ -666,6 +887,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute """Anthropic-compatible /v1/messages endpoint.""" start = time.time() resolved_model = _resolve_model(body.model, _config) if _config else body.model + _hist = _history_kwargs(request, messages=body.messages) # Convert Anthropic format to OpenAI format openai_messages = [] @@ -740,14 +962,22 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute try: if body.stream: - return await _stream_completion_anthropic(client, openai_payload, resolved_model, body.max_tokens, _analytics, start) + return await _stream_completion_anthropic(client, openai_payload, resolved_model, body.max_tokens, _analytics, start, history_kwargs=_hist, config=_config) resp = await client.chat_completion(openai_payload) elapsed = time.time() - start metrics = resp.pop("_oct_metrics", {}) usage = resp.get("usage", {}) + # Extract output text for history logging before conversion + _out_text = _extract_output_text(resp) + if _analytics: + hist_kw = dict(_hist) + if _out_text and _config and _config.history.enabled and _config.history.save_output: + if len(_out_text) > _config.history.max_content_size: + _out_text = _out_text[:_config.history.max_content_size] + "\n...[truncated]" + hist_kw["output_text"] = _out_text _analytics.log_llm( model=resolved_model, prompt_tokens=usage.get("prompt_tokens", metrics.get("prompt_eval_count", 0)), @@ -757,6 +987,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute ttft_seconds=metrics.get("ttft_seconds"), total_duration_seconds=elapsed, provider="ollama", + **hist_kw, ) # Convert OpenAI response to Anthropic format @@ -806,7 +1037,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute except Exception as e: if _analytics: - _analytics.log_llm(model=resolved_model, error=str(e), total_duration_seconds=time.time() - start) + _analytics.log_llm(model=resolved_model, error=str(e), total_duration_seconds=time.time() - start, **_hist) raise HTTPException(status_code=502, detail=f"Ollama Cloud error: {str(e)}") # ── Model selection endpoint ── @@ -1076,7 +1307,40 @@ async def _iter_stream_with_timeouts(aiter, first_chunk_timeout, inter_chunk_tim # If we never got any item, the aiter ended normally (empty stream) -async def _stream_completion_openai(client, payload, model, analytics, start_time, config=None): +def _extract_sse_content(chunk: str) -> str: + """Extract the content/reasoning text from an SSE data chunk for history logging.""" + try: + if not chunk.startswith("data: ") or "__oct_metrics__" in chunk: + return "" + data_str = chunk[6:].strip() + if data_str == "[DONE]": + return "" + data = json.loads(data_str) + choices = data.get("choices", []) + if choices: + delta = choices[0].get("delta", {}) + parts = [] + if delta.get("content"): + parts.append(delta["content"]) + reasoning = delta.get("reasoning", "") or delta.get("reasoning_content", "") + if reasoning: + parts.append(reasoning) + return "".join(parts) + except (json.JSONDecodeError, ValueError, KeyError, IndexError): + pass + return "" + + +def _accumulate_history_output(accumulated: list, chunk: str, history_kwargs: dict, config=None): + """Extract text from an SSE chunk and append to the accumulator if history is enabled.""" + if not history_kwargs or not config or not config.history.enabled or not config.history.save_output: + return + text = _extract_sse_content(chunk) + if text: + accumulated.append(text) + + +async def _stream_completion_openai(client, payload, model, analytics, start_time, config=None, history_kwargs=None, limiter=None, api_key=None, account_name=None, account_pool=None): """Stream OpenAI-format SSE responses, with fallback and timeout support. Key design: When fallback is configured with primary_timeout, we apply @@ -1091,42 +1355,88 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim use_timeouts = (fb and fb.enabled and fb.base_url and fb.primary_timeout and fb.primary_timeout > 0) + # Build account-aware provider name for analytics + _provider_name = f"ollama:{account_name}" if account_name and account_name != "ollama" else "ollama" + async def generate(): stream_metrics = {} used_fallback = False fallback_model = None original_error = None + accumulated_content = [] # For history: collect output text from stream try: if use_timeouts: chunk_timeout = fb.stream_chunk_timeout if fb.stream_chunk_timeout else 180.0 # ── Timed streaming: fail fast on first chunk, tolerate gaps after ── - ollama_stream = client.chat_completion_stream(payload) + # Acquire semaphore for the duration of the Ollama stream + sem_ctx = limiter.__aenter__() if limiter else None + if sem_ctx: + await sem_ctx + ollama_stream = None stream_closed = False # Wait for the first chunk with a strict timeout (triggers fallback fast) - try: - first_chunk = await asyncio.wait_for( - ollama_stream.__anext__(), timeout=fb.primary_timeout - ) - except asyncio.TimeoutError: - # First chunk timeout — no data sent to client yet, can still fallback + # Also retry on 429 with backoff + first_chunk = None + max_429 = limiter.max_429_retries if limiter else 0 + for attempt in range(max_429 + 1): try: - await ollama_stream.aclose() - except RuntimeError: - pass # Generator already cleaned up by cancellation - stream_closed = True - raise httpx.ReadTimeout( - f"Ollama did not produce first stream chunk within {fb.primary_timeout}s" - ) - except StopAsyncIteration: - # Empty stream — treat as error so fallback can handle it - try: - await ollama_stream.aclose() - except RuntimeError: - pass - stream_closed = True - raise httpx.ReadTimeout( - f"Ollama stream ended before producing any chunks" - ) + if ollama_stream is None: + ollama_stream = client.chat_completion_stream(payload, api_key=api_key) + first_chunk = await asyncio.wait_for( + ollama_stream.__anext__(), timeout=fb.primary_timeout + ) + break # Got a chunk, exit retry loop + except httpx.HTTPStatusError as e: + if attempt < max_429 and limiter and limiter.should_retry_429(e): + if account_name and account_pool: + account_pool.mark_429(account_name) + try: + await ollama_stream.aclose() + except (RuntimeError, Exception): + pass + ollama_stream = None + await limiter.backoff_and_retry(attempt) + continue + # Not a 429 or out of retries — release semaphore and re-raise + if sem_ctx: + try: + await limiter.__aexit__(None, None, None) + except Exception: + pass + sem_ctx = None + raise + except asyncio.TimeoutError: + # First chunk timeout — no data sent to client yet, can still fallback + try: + await ollama_stream.aclose() + except RuntimeError: + pass + stream_closed = True + if sem_ctx: + try: + await limiter.__aexit__(None, None, None) + except Exception: + pass + sem_ctx = None + raise httpx.ReadTimeout( + f"Ollama did not produce first stream chunk within {fb.primary_timeout}s" + ) + except StopAsyncIteration: + # Empty stream — treat as error so fallback can handle it + try: + await ollama_stream.aclose() + except RuntimeError: + pass + stream_closed = True + if sem_ctx: + try: + await limiter.__aexit__(None, None, None) + except Exception: + pass + sem_ctx = None + raise httpx.ReadTimeout( + f"Ollama stream ended before producing any chunks" + ) # Got first chunk — process it (metrics chunks are internal, not yield) if first_chunk.startswith("__oct_metrics__:"): @@ -1136,6 +1446,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim pass else: yield first_chunk + _accumulate_history_output(accumulated_content, first_chunk, history_kwargs, config) try: while True: try: @@ -1149,6 +1460,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim pass continue yield chunk + _accumulate_history_output(accumulated_content, chunk, history_kwargs, config) except StopAsyncIteration: break except asyncio.TimeoutError: @@ -1166,6 +1478,12 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim await ollama_stream.aclose() except RuntimeError: pass # Already closed + # Release concurrency semaphore + if sem_ctx and limiter: + try: + await limiter.__aexit__(None, None, None) + except Exception: + pass else: # ── No timeout wrapping: original buffered behavior ── for attempt in range(MAX_EMPTY_RETRIES + 1): @@ -1176,6 +1494,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim for chunk in chunks: yield chunk + _accumulate_history_output(accumulated_content, chunk, history_kwargs, config) except Exception as e: original_error = _describe_error(e) @@ -1189,6 +1508,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim async for chunk in await _call_fallback_provider(fallback_payload, config.fallback, stream=True): used_fallback = True yield chunk + _accumulate_history_output(accumulated_content, chunk, history_kwargs, config) except Exception as fallback_err: log.warning("Stream fallback to %s failed for model %s: %s", config.fallback.name, model, _describe_error(fallback_err)) error_data = json.dumps({"error": {"message": f"Ollama: {original_error}; Fallback: {_describe_error(fallback_err)}", "type": "server_error"}}) @@ -1200,24 +1520,41 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim yield "data: [DONE]\n\n" finally: elapsed = time.time() - start_time + # Build history output from accumulated stream content + _hist_kw = dict(history_kwargs) if history_kwargs else {} + if accumulated_content and config and config.history.enabled and config.history.save_output: + output_text = "".join(accumulated_content) + if len(output_text) > config.history.max_content_size: + output_text = output_text[:config.history.max_content_size] + "\n...[truncated]" + _hist_kw["output_text"] = output_text if analytics: if used_fallback and fallback_model: + # Fallback providers don't emit __oct_metrics__ — estimate from accumulated content + fb_stream_metrics = dict(stream_metrics) + if not fb_stream_metrics.get("eval_count") and accumulated_content: + fb_chars = len("".join(accumulated_content)) + fb_tokens = max(1, fb_chars // 4) if fb_chars else 0 + fb_stream_metrics["eval_count"] = fb_tokens + fb_stream_metrics.setdefault("elapsed_seconds", elapsed) analytics.log_llm( model=_normalize_model_name(fallback_model), - prompt_tokens=stream_metrics.get("prompt_eval_count", 0), - completion_tokens=stream_metrics.get("eval_count"), - tps=stream_metrics.get("tps"), - ttft_seconds=stream_metrics.get("ttft_seconds"), - total_duration_seconds=stream_metrics.get("elapsed_seconds", elapsed), + prompt_tokens=fb_stream_metrics.get("prompt_eval_count", 0), + completion_tokens=fb_stream_metrics.get("eval_count"), + tps=fb_stream_metrics.get("tps"), + ttft_seconds=fb_stream_metrics.get("ttft_seconds"), + total_duration_seconds=fb_stream_metrics.get("elapsed_seconds", elapsed), provider=config.fallback.name if config else "fallback", fallback_for=_normalize_model_name(model), + fallback_reason=f"Ollama error: {original_error}" if original_error else "Ollama stream error", + **_hist_kw, ) elif original_error: analytics.log_llm( model=_normalize_model_name(model), error=original_error, total_duration_seconds=elapsed, - provider="ollama", + provider=_provider_name, + **_hist_kw, ) else: analytics.log_llm( @@ -1227,77 +1564,107 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim tps=stream_metrics.get("tps"), ttft_seconds=stream_metrics.get("ttft_seconds"), total_duration_seconds=stream_metrics.get("elapsed_seconds", elapsed), - provider="ollama", + provider=_provider_name, + **_hist_kw, ) return StreamingResponse(generate(), media_type="text/event-stream") -async def _stream_fallback_openai(payload, config, fallback_model, analytics, start_time, provider_tag="fallback"): +async def _stream_fallback_openai(payload, config, fallback_model, analytics, start_time, provider_tag="fallback", history_kwargs=None, fallback_for=None, fallback_reason=None): """Stream from fallback provider in OpenAI format.""" from fastapi.responses import StreamingResponse async def generate(): + accumulated_content = [] try: async for chunk in await _call_fallback_provider(payload, config.fallback, stream=True): yield chunk + _accumulate_history_output(accumulated_content, chunk, history_kwargs, config) except Exception as e: error_data = json.dumps({"error": {"message": str(e), "type": "server_error"}}) yield f"data: {error_data}\n\n" yield "data: [DONE]\n\n" finally: elapsed = time.time() - start_time + _hist_kw = dict(history_kwargs) if history_kwargs else {} + if accumulated_content and config and config.history.enabled and config.history.save_output: + output_text = "".join(accumulated_content) + if len(output_text) > config.history.max_content_size: + output_text = output_text[:config.history.max_content_size] + "\n...[truncated]" + _hist_kw["output_text"] = output_text if analytics: - analytics.log_llm(model=_normalize_model_name(fallback_model), total_duration_seconds=elapsed, provider=provider_tag) - - return StreamingResponse(generate(), media_type="text/event-stream") + # Estimate tokens from accumulated content (fallbacks don't emit __oct_metrics__) + fb_chars = len("".join(accumulated_content)) + fb_tokens = max(1, fb_chars // 4) if fb_chars else 0 + analytics.log_llm( + model=_normalize_model_name(fallback_model), + completion_tokens=fb_tokens, + total_duration_seconds=elapsed, + provider=provider_tag, + fallback_for=_normalize_model_name(fallback_for) if fallback_for else None, + fallback_reason=fallback_reason, + **_hist_kw, + ) -async def _stream_completion_anthropic(client, payload, model, max_tokens, analytics, start_time): +async def _stream_completion_anthropic(client, payload, model, max_tokens, analytics, start_time, history_kwargs=None, config=None): """Stream Anthropic-format SSE responses, translating from Ollama's OpenAI format.""" from fastapi.responses import StreamingResponse msg_id = f"msg_{uuid.uuid4().hex[:24]}" async def generate(): + accumulated_content = [] + stream_metrics = {} yield f"event: message_start\ndata: {json.dumps({'type': 'message_start', 'message': {'id': msg_id, 'type': 'message', 'role': 'assistant', 'model': model, 'content': [], 'stop_reason': None, 'stop_sequence': None, 'usage': {'input_tokens': 0, 'output_tokens': 0}}})}\n\n" total_tokens = 0 first_token_time = None - async for chunk in client.chat_completion_stream(payload): - # Capture metrics from stream - if chunk.startswith("__oct_metrics__:"): - stream_metrics_raw = chunk.split(":", 1)[1] + try: + async for chunk in client.chat_completion_stream(payload): + # Capture metrics from stream + if chunk.startswith("__oct_metrics__:"): + stream_metrics_raw = chunk.split(":", 1)[1] + try: + stream_metrics = json.loads(stream_metrics_raw) + except (json.JSONDecodeError, ValueError): + pass + continue try: - stream_metrics = json.loads(stream_metrics_raw) - # Log with streaming metrics - if analytics: - analytics.log_llm( - model=model, - completion_tokens=stream_metrics.get("eval_count", total_tokens), - tps=stream_metrics.get("tps"), - ttft_seconds=stream_metrics.get("ttft_seconds") or (round(first_token_time - start_time, 3) if first_token_time else None), - total_duration_seconds=stream_metrics.get("elapsed_seconds", time.time() - start_time), - ) - except (json.JSONDecodeError, ValueError): - pass - continue - try: - if "data: " in chunk: - data_str = chunk[6:].strip() - if data_str == "[DONE]": - continue - data = json.loads(data_str) - choices = data.get("choices", []) - for choice in choices: - delta = choice.get("delta", {}) - content = delta.get("content", "") - if content: - if first_token_time is None: - first_token_time = time.time() - total_tokens += 1 - yield f"event: content_block_delta\ndata: {json.dumps({'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'text_delta', 'text': content}})}\n\n" - except (json.JSONDecodeError, KeyError): - continue + if "data: " in chunk: + data_str = chunk[6:].strip() + if data_str == "[DONE]": + continue + data = json.loads(data_str) + choices = data.get("choices", []) + for choice in choices: + delta = choice.get("delta", {}) + content = delta.get("content", "") + if content: + if first_token_time is None: + first_token_time = time.time() + total_tokens += 1 + accumulated_content.append(content) + yield f"event: content_block_delta\ndata: {json.dumps({'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'text_delta', 'text': content}})}\n\n" + except (json.JSONDecodeError, KeyError): + continue + finally: + # Log with streaming metrics and history + _hist_kw = dict(history_kwargs) if history_kwargs else {} + if accumulated_content and config and config.history.enabled and config.history.save_output: + output_text = "".join(accumulated_content) + if len(output_text) > config.history.max_content_size: + output_text = output_text[:config.history.max_content_size] + "\n...[truncated]" + _hist_kw["output_text"] = output_text + if analytics and stream_metrics: + analytics.log_llm( + model=model, + completion_tokens=stream_metrics.get("eval_count", total_tokens), + tps=stream_metrics.get("tps"), + ttft_seconds=stream_metrics.get("ttft_seconds") or (round(first_token_time - start_time, 3) if first_token_time else None), + total_duration_seconds=stream_metrics.get("elapsed_seconds", time.time() - start_time), + **_hist_kw, + ) yield f"event: message_delta\ndata: {json.dumps({'type': 'message_delta', 'delta': {'stop_reason': 'end_turn', 'stop_sequence': None}, 'usage': {'output_tokens': total_tokens}})}\n\n" yield f"event: message_stop\ndata: {json.dumps({'type': 'message_stop'})}\n\n" diff --git a/guanaco/search/base.py b/guanaco/search/base.py index 31a9aeb..4df8765 100644 --- a/guanaco/search/base.py +++ b/guanaco/search/base.py @@ -15,7 +15,7 @@ class ProviderEmulator(ABC): name: str = "" prefix: str = "" - endpoints: list[dict] = [] + endpoints: tuple[dict, ...] = () # Use tuple — mutable list default is a Python footgun def __init__(self, ollama_client: "OllamaClient", analytics: Optional["AnalyticsLogger"] = None): self.ollama = ollama_client diff --git a/guanaco/search/providers/brave.py b/guanaco/search/providers/brave.py index cad76b1..8f88251 100644 --- a/guanaco/search/providers/brave.py +++ b/guanaco/search/providers/brave.py @@ -29,7 +29,7 @@ class BraveSearchResponse(BaseModel): class BraveProvider(ProviderEmulator): name = "brave" prefix = "/brave" - endpoints = [{"path": "/search", "method": "GET/POST"}] + endpoints = ({"path": "/search", "method": "GET/POST"},) def register_routes(self, app): router = APIRouter(prefix=self.prefix, tags=["Brave"]) diff --git a/guanaco/search/providers/cohere.py b/guanaco/search/providers/cohere.py index 74c7297..6d439e0 100644 --- a/guanaco/search/providers/cohere.py +++ b/guanaco/search/providers/cohere.py @@ -36,7 +36,7 @@ class CohereRerankResponse(BaseModel): class CohereProvider(ProviderEmulator): name = "cohere" prefix = "/cohere" - endpoints = [{"path": "/rerank", "method": "POST"}] + endpoints = ({"path": "/rerank", "method": "POST"},) def register_routes(self, app): router = APIRouter(prefix=self.prefix, tags=["Cohere"]) diff --git a/guanaco/search/providers/exa.py b/guanaco/search/providers/exa.py index b99a4f7..23ece5d 100644 --- a/guanaco/search/providers/exa.py +++ b/guanaco/search/providers/exa.py @@ -64,7 +64,7 @@ class ExaFindSimilarResponse(BaseModel): class ExaProvider(ProviderEmulator): name = "exa" prefix = "/exa" - endpoints = [{"path": "/search", "method": "POST"}, {"path": "/findSimilar", "method": "POST"}] + endpoints = ({"path": "/search", "method": "POST"}, {"path": "/findSimilar", "method": "POST"}) def register_routes(self, app): router = APIRouter(prefix=self.prefix, tags=["Exa"]) diff --git a/guanaco/search/providers/firecrawl.py b/guanaco/search/providers/firecrawl.py index 792ee09..96a0826 100644 --- a/guanaco/search/providers/firecrawl.py +++ b/guanaco/search/providers/firecrawl.py @@ -107,12 +107,12 @@ class ExtractResponse(BaseModel): class FirecrawlProvider(ProviderEmulator): name = "firecrawl" prefix = "/firecrawl" - endpoints = [ + endpoints = ( {"path": "/scrape", "method": "POST"}, {"path": "/search", "method": "POST"}, {"path": "/crawl", "method": "POST"}, {"path": "/extract", "method": "POST"}, - ] + ) def register_routes(self, app): router = APIRouter(prefix=self.prefix, tags=["Firecrawl"]) diff --git a/guanaco/search/providers/jina.py b/guanaco/search/providers/jina.py index b3cc0f1..8414243 100644 --- a/guanaco/search/providers/jina.py +++ b/guanaco/search/providers/jina.py @@ -74,7 +74,7 @@ class JinaRerankResponse(BaseModel): class JinaProvider(ProviderEmulator): name = "jina" prefix = "/jina" - endpoints = [{"path": "/search", "method": "POST"}, {"path": "/read", "method": "POST"}, {"path": "/rerank", "method": "POST"}] + endpoints = ({"path": "/search", "method": "POST"}, {"path": "/read", "method": "POST"}, {"path": "/rerank", "method": "POST"}) def register_routes(self, app): router = APIRouter(prefix=self.prefix, tags=["Jina"]) diff --git a/guanaco/search/providers/searxng.py b/guanaco/search/providers/searxng.py index 0aff699..57fdc37 100644 --- a/guanaco/search/providers/searxng.py +++ b/guanaco/search/providers/searxng.py @@ -37,7 +37,7 @@ class SearXNGSearchResponse(BaseModel): class SearXNGProvider(ProviderEmulator): name = "searxng" prefix = "/searxng" - endpoints = [{"path": "/search", "method": "GET/POST"}] + endpoints = ({"path": "/search", "method": "GET/POST"},) def register_routes(self, app): router = APIRouter(prefix=self.prefix, tags=["SearXNG"]) diff --git a/guanaco/search/providers/serper.py b/guanaco/search/providers/serper.py index 61b704c..6dd5d7a 100644 --- a/guanaco/search/providers/serper.py +++ b/guanaco/search/providers/serper.py @@ -58,7 +58,7 @@ class SerperScrapeResponse(BaseModel): class SerperProvider(ProviderEmulator): name = "serper" prefix = "/serper" - endpoints = [{"path": "/search", "method": "POST"}, {"path": "/scrape", "method": "POST"}] + endpoints = ({"path": "/search", "method": "POST"}, {"path": "/scrape", "method": "POST"}) def register_routes(self, app): router = APIRouter(prefix=self.prefix, tags=["Serper"]) diff --git a/guanaco/search/providers/tavily.py b/guanaco/search/providers/tavily.py index 04adb84..c75f22b 100644 --- a/guanaco/search/providers/tavily.py +++ b/guanaco/search/providers/tavily.py @@ -45,7 +45,7 @@ class TavilySearchResponse(BaseModel): class TavilyProvider(ProviderEmulator): name = "tavily" prefix = "/tavily" - endpoints = [{"path": "/search", "method": "POST"}] + endpoints = ({"path": "/search", "method": "POST"},) def register_routes(self, app): router = APIRouter(prefix=self.prefix, tags=["Tavily"]) diff --git a/install.sh b/install.sh index 650ce40..2a0af80 100755 --- a/install.sh +++ b/install.sh @@ -188,6 +188,26 @@ echo "" prompt OLLAMA_API_KEY "Enter your Ollama API key" "" +if [ -n "$OLLAMA_API_KEY" ]; then + info "Validating API key with Ollama Cloud..." + VALIDATE_RESPONSE=$(curl -s -w "\n%{http_code}" "https://api.ollama.com/v1/models" \ + -H "Authorization: Bearer ${OLLAMA_API_KEY}" \ + -H "Accept: application/json" \ + --max-time 10 2>/dev/null || echo "timeout") + HTTP_CODE=$(echo "$VALIDATE_RESPONSE" | tail -1) + if [ "$HTTP_CODE" = "200" ]; then + success "API key validated (models endpoint responds OK)" + elif [ "$HTTP_CODE" = "401" ]; then + warn "API key returned 401 Unauthorized from Ollama Cloud" + warn "Key may be invalid or expired. Guanaco will still install but inference will fail." + warn "Fix with: guanaco setup (after install)" + elif [ "$HTTP_CODE" = "timeout" ]; then + warn "Could not reach Ollama Cloud (timeout). Skipping validation." + else + warn "API key validation returned HTTP $HTTP_CODE — key may not work" + fi +fi + if [ -z "$OLLAMA_API_KEY" ]; then echo "" warn "No API key provided. You can set it later with: guanaco setup" @@ -418,6 +438,8 @@ WorkingDirectory=${INSTALL_DIR} ExecStart=${VENV_DIR}/bin/python -m uvicorn guanaco.app:create_app --factory --host ${BIND_HOST} --port ${PORT} --log-level info Restart=on-failure RestartSec=5 +Environment=GUANACO_CONFIG_DIR=${CONFIG_DIR} +WorkingDirectory=${INSTALL_DIR}/repo [Install] WantedBy=multi-user.target diff --git a/pyproject.toml b/pyproject.toml index 404fe6b..02f60a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,8 +3,8 @@ requires = ["setuptools>=68.0", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "guanaco" -version = "0.3.6" +name = "guanaco-llm-proxy" +version = "0.5.1" description = "OpenAI-compatible LLM proxy that maximizes Ollama Cloud subscriptions — search/scrape API emulation, usage tracking, fallback provider support, and a web dashboard" readme = "README.md" license = {text = "MIT"} diff --git a/tests/test_fallback_stream.py b/tests/test_fallback_stream.py new file mode 100644 index 0000000..969d930 --- /dev/null +++ b/tests/test_fallback_stream.py @@ -0,0 +1,169 @@ +"""Test that fallback payloads always include the correct 'stream' value in the JSON body. + +Reproduction of the bug: +- Request with stream=true + max_tokens > 4096 +- Ollama fails, falls back to Fireworks/custom provider +- Old code: 'stream' was only passed as a function arg, not in the JSON body +- Fireworks rejects: "Requests with max_tokens > 4096 must have stream=true" +- Fixed: _call_fallback_provider now always injects payload["stream"] = stream +""" +import pytest +import copy +from unittest.mock import AsyncMock, MagicMock, patch +import sys +import os + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from guanaco.config import FallbackProviderConfig + + +def _make_config(): + return FallbackProviderConfig( + enabled=True, + name="fireworks", + base_url="https://api.fireworks.ai/inference/v1", + api_key="test-key", + default_model="accounts/fireworks/models/llama-v3p1-70b-instruct", + max_tokens=8192, + stream_fallback=True, + ) + + +@pytest.mark.asyncio +async def test_fallback_non_stream_payload_includes_stream_false(): + """Non-streaming fallback must have 'stream': false in the JSON body.""" + from guanaco.router.router import _call_fallback_provider + + config = _make_config() + payload = { + "model": "llama-v3p1-70b-instruct", + "messages": [{"role": "user", "content": "Hello"}], + "max_tokens": 8192, + } + + with patch("guanaco.router.router.httpx.AsyncClient") as mock_client_cls: + mock_instance = AsyncMock() + mock_client_cls.return_value.__aenter__ = AsyncMock(return_value=mock_instance) + mock_client_cls.return_value.__aexit__ = AsyncMock(return_value=False) + mock_response = MagicMock() + mock_response.json.return_value = { + "id": "test", + "object": "chat.completion", + "choices": [{"message": {"role": "assistant", "content": "Hi"}, "index": 0}], + } + mock_response.raise_for_status = MagicMock() + mock_instance.post = AsyncMock(return_value=mock_response) + + await _call_fallback_provider(payload, config, stream=False) + + sent_json = mock_instance.post.call_args[1]["json"] + assert sent_json["stream"] == False, f"Expected stream=False, got {sent_json.get('stream')}" + assert sent_json["max_tokens"] == 8192 + + +@pytest.mark.asyncio +async def test_fallback_stream_payload_includes_stream_true(): + """Streaming fallback must have 'stream': true in the JSON body. + + This is the critical fix for Fireworks' "max_tokens > 4096 requires stream=true" error. + + The function returns an async generator — we need to consume it to trigger + client.stream() which is inside the generator body. + """ + from guanaco.router.router import _call_fallback_provider + + config = _make_config() + payload = { + "model": "llama-v3p1-70b-instruct", + "messages": [{"role": "user", "content": "Hello"}], + "max_tokens": 8192, + } + + # Capture what gets passed to client.stream() + captured_payload = {} + + class FakeStreamResponse: + def raise_for_status(self): + pass + def aiter_lines(self): + return AsyncIter([]) + + class FakeStreamContext: + async def __aenter__(self): + return FakeStreamResponse() + async def __aexit__(self, *args): + pass + + class FakeAsyncClient: + def __init__(self, timeout=None): + pass + + def stream(self, method, url, json=None, headers=None): + captured_payload.update(json or {}) + return FakeStreamContext() + + async def aclose(self): + pass + + with patch("guanaco.router.router.httpx.AsyncClient", FakeAsyncClient): + gen = await _call_fallback_provider(payload, config, stream=True) + # Consume the generator to trigger the client.stream() call inside + async for _ in gen: + pass + + assert captured_payload.get("stream") == True, \ + f"CRITICAL: stream=true not in fallback JSON body! Got {captured_payload.get('stream')}. " \ + f"Fireworks will reject max_tokens={captured_payload.get('max_tokens')} > 4096 without stream=true." + assert captured_payload.get("max_tokens") == 8192 + + +@pytest.mark.asyncio +async def test_fallback_does_not_mutate_original_payload(): + """Ensure the original payload dict isn't mutated (should be safe to reuse).""" + from guanaco.router.router import _call_fallback_provider + + config = _make_config() + original = { + "model": "test-model", + "messages": [{"role": "user", "content": "Hi"}], + "max_tokens": 5000, + } + original_copy = copy.deepcopy(original) + + with patch("guanaco.router.router.httpx.AsyncClient") as mock_client_cls: + mock_instance = AsyncMock() + mock_client_cls.return_value.__aenter__ = AsyncMock(return_value=mock_instance) + mock_client_cls.return_value.__aexit__ = AsyncMock(return_value=False) + mock_response = MagicMock() + mock_response.json.return_value = {"id": "t", "object": "chat.completion", "choices": []} + mock_response.raise_for_status = MagicMock() + mock_instance.post = AsyncMock(return_value=mock_response) + + await _call_fallback_provider(original, config, stream=True) + + # Original should be unchanged — no "stream" key added + assert original == original_copy, f"Original payload was mutated! {original} != {original_copy}" + + +class AsyncIter: + def __init__(self, items): + self._items = iter(items) + def __aiter__(self): + return self + async def __anext__(self): + try: + return next(self._items) + except StopIteration: + raise StopAsyncIteration + + +if __name__ == "__main__": + import asyncio + asyncio.run(test_fallback_non_stream_payload_includes_stream_false()) + print("✅ Non-streaming fallback: stream=false in body") + asyncio.run(test_fallback_stream_payload_includes_stream_true()) + print("✅ Streaming fallback: stream=true in body (Fireworks fix)") + asyncio.run(test_fallback_does_not_mutate_original_payload()) + print("✅ Original payload not mutated") + print("\n✅ All fallback stream tests passed!") \ No newline at end of file