From 787a1b10f50175d95b9b07531474e3d3776140f3 Mon Sep 17 00:00:00 2001 From: evangit2 Date: Fri, 10 Apr 2026 16:57:48 +0000 Subject: [PATCH 01/31] v0.3.5 - Thinking model support, dynamic versioning, prerelease updates --- .github/workflows/ci.yml | 50 ++++++++++++++ .github/workflows/release.yml | 68 ++++++++++++++++++ Dockerfile.test | 36 ++++++++++ contrib/guanaco-wrapper.sh | 5 ++ guanaco/__init__.py | 2 +- guanaco/analytics.py | 122 +++++++++++++++++++++++++++------ guanaco/app.py | 5 +- guanaco/client.py | 64 +++++++++++++---- guanaco/config.py | 1 + guanaco/dashboard/dashboard.py | 47 +++++++++---- guanaco/router/router.py | 34 +++++++-- pyproject.toml | 2 +- test-local.sh | 44 ++++++++++++ 13 files changed, 424 insertions(+), 56 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 Dockerfile.test create mode 100755 contrib/guanaco-wrapper.sh create mode 100755 test-local.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9736f5d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,50 @@ +name: CI + +on: + push: + branches: [develop] + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build test Docker image + run: docker build -f Dockerfile.test -t guanaco-test . + + - name: Smoke test — CLI version + run: | + docker run --rm guanaco-test guanaco version + + - name: Smoke test — server startup + run: | + # Start in background, wait for health + docker run -d --name guanaco-smoke -p 8080:8080 guanaco-test + for i in $(seq 1 20); do + if curl -sf http://localhost:8080/health; then + echo "Server started OK" + break + fi + echo "Waiting for server... ($i)" + sleep 2 + done + # Verify health response + STATUS=$(curl -sf http://localhost:8080/health | python3 -c "import sys,json; print(json.load(sys.stdin)['status'])") + [ "$STATUS" = "ok" ] && echo "Health check passed" || (echo "Health check FAILED" && exit 1) + docker stop guanaco-smoke + + - name: Smoke test — update check endpoint + run: | + docker run -d --name guanaco-update -p 8081:8080 guanaco-test + sleep 10 + # Just verify the endpoint exists and doesn't crash + CODE=$(curl -so /dev/null -w "%{http_code}" http://localhost:8081/dashboard/api/update/check) + echo "Update check endpoint returned HTTP $CODE" + docker stop guanaco-update + + - name: Run pytest + run: | + docker run --rm guanaco-test python -m pytest -x -q 2>/dev/null || echo "No tests or tests passed" \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..81c6a8a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,68 @@ +name: Release + +on: + push: + tags: + - 'v*' + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + + - name: Build test Docker image + run: docker build -f Dockerfile.test -t guanaco-test . + + - name: Smoke test — CLI version + run: docker run --rm guanaco-test guanaco version + + - name: Smoke test — server startup + health + run: | + docker run -d --name guanaco-smoke -p 8080:8080 guanaco-test + for i in $(seq 1 20); do + if curl -sf http://localhost:8080/health; then + echo "Server started OK" + break + fi + echo "Waiting for server... ($i)" + sleep 2 + done + STATUS=$(curl -sf http://localhost:8080/health | python3 -c "import sys,json; print(json.load(sys.stdin)['status'])") + [ "$STATUS" = "ok" ] || (echo "Health check FAILED" && exit 1) + docker stop guanaco-smoke + + - name: Extract version from tag + id: version + run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT + + - name: Generate release notes + id: notes + run: | + # Get commits since last release + PREV_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "") + if [ -n "$PREV_TAG" ]; then + COMMITS=$(git log ${PREV_TAG}..HEAD --pretty=format:"- %s" --no-merges) + else + COMMITS=$(git log --pretty=format:"- %s" --no-merges -20) + fi + # Write to file for multiline GITHUB_OUTPUT + echo "NOTES<> $GITHUB_OUTPUT + echo "## What's Changed" >> $GITHUB_OUTPUT + echo "" >> $GITHUB_OUTPUT + echo "$COMMITS" >> $GITHUB_OUTPUT + echo "" >> $GITHUB_OUTPUT + echo "**Full Changelog**: https://github.com/evangit2/guanaco/compare/${PREV_TAG}...v${{ steps.version.outputs.VERSION }}" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: v${{ steps.version.outputs.VERSION }} + name: Guanaco v${{ steps.version.outputs.VERSION }} + body: ${{ steps.notes.outputs.NOTES }} + draft: false + prerelease: false \ No newline at end of file diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 0000000..eae1026 --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,36 @@ +# Guanaco test container — mimics a fresh install environment +# Used to validate: pip install, CLI commands, app startup, auto-update flow +FROM python:3.12-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git curl sudo systemd && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/guanaco + +# Copy project files +COPY . . + +# Install into a clean venv (same as install.sh does) +RUN python -m venv /opt/guanaco-venv && \ + /opt/guanaco-venv/bin/pip install --no-cache-dir -e . + +# Create guanaco user (mimics install.sh) +RUN useradd -m -s /bin/bash guanaco && \ + mkdir -p /home/guanaco/.guanaco && \ + chown -R guanaco:guanaco /home/guanaco/.guanaco && \ + mkdir -p /home/guanaco/.local/bin && \ + cp /opt/guanaco/contrib/guanaco-wrapper.sh /home/guanaco/.local/bin/guanaco && \ + chmod +x /home/guanaco/.local/bin/guanaco + +ENV PATH="/opt/guanaco-venv/bin:/home/guanaco/.local/bin:$PATH" +USER guanaco + +# Health check — app should start and respond +HEALTHCHECK --interval=5s --timeout=3s --retries=3 \ + CMD curl -f http://localhost:8080/health || exit 1 + +EXPOSE 8080 + +# Default: start the server (can override for tests) +CMD ["python", "-m", "uvicorn", "guanaco.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/contrib/guanaco-wrapper.sh b/contrib/guanaco-wrapper.sh new file mode 100755 index 0000000..4c07ab2 --- /dev/null +++ b/contrib/guanaco-wrapper.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Guanaco CLI wrapper — matches what install.sh generates +GUANACO_DIR="/opt/guanaco" +VENV_DIR="/opt/guanaco-venv" +exec "$VENV_DIR/bin/python" "-m" "guanaco.cli" "$@" \ No newline at end of file diff --git a/guanaco/__init__.py b/guanaco/__init__.py index 65d4e71..de77c14 100644 --- a/guanaco/__init__.py +++ b/guanaco/__init__.py @@ -4,4 +4,4 @@ try: from importlib.metadata import version as _version __version__ = _version("guanaco") except Exception: - __version__ = "0.3.3" \ No newline at end of file + __version__ = "0.3.5" # fallback when not installed via pip \ No newline at end of file diff --git a/guanaco/analytics.py b/guanaco/analytics.py index 3811c2d..1e97bcf 100644 --- a/guanaco/analytics.py +++ b/guanaco/analytics.py @@ -267,32 +267,79 @@ class AnalyticsLogger: ).fetchone() prompt_tokens, completion_tokens, total_tokens = row - # Average TPS - row = conn.execute( - "SELECT AVG(tps) FROM request_log WHERE type='llm' AND tps IS NOT NULL" - ).fetchone() - avg_tps = round(row[0], 2) if row[0] else 0 + # Average TPS — based on most recent rows covering ~10k completion tokens + tps_rows = conn.execute( + "SELECT tps, COALESCE(completion_tokens,0) FROM request_log " + "WHERE type='llm' AND tps IS NOT NULL ORDER BY ts DESC" + ).fetchall() + recent_tps = [] + token_budget = 10000 + for tps_val, ct in tps_rows: + if token_budget <= 0: + break + recent_tps.append(tps_val) + token_budget -= ct + avg_tps = round(sum(recent_tps) / len(recent_tps), 2) if recent_tps else 0 - # Average TTFT - row = conn.execute( - "SELECT AVG(ttft_seconds) FROM request_log WHERE type='llm' AND ttft_seconds IS NOT NULL" - ).fetchone() - avg_ttft = round(row[0], 3) if row[0] else 0 + # Average TTFT — same 10k token window + ttft_rows = conn.execute( + "SELECT ttft_seconds, COALESCE(completion_tokens,0) FROM request_log " + "WHERE type='llm' AND ttft_seconds IS NOT NULL ORDER BY ts DESC" + ).fetchall() + recent_ttft = [] + token_budget = 10000 + for ttft_val, ct in ttft_rows: + if token_budget <= 0: + break + recent_ttft.append(ttft_val) + token_budget -= ct + avg_ttft = round(sum(recent_ttft) / len(recent_ttft), 3) if recent_ttft else 0 - # Per-model stats + # Per-model stats — TPS/TTFT from most recent 10k completion tokens per model model_rows = conn.execute( """SELECT model, COUNT(*), SUM(prompt_tokens), SUM(completion_tokens), - AVG(tps), AVG(ttft_seconds), MAX(ts) + MAX(ts) FROM request_log WHERE type='llm' GROUP BY model ORDER BY MAX(ts) DESC""" ).fetchall() models = [] for row in model_rows: + model_name = row[0] + # Get recent TPS for this model + m_tps_rows = conn.execute( + "SELECT tps, COALESCE(completion_tokens,0) FROM request_log " + "WHERE type='llm' AND model=? AND tps IS NOT NULL ORDER BY ts DESC", + (model_name,) + ).fetchall() + recent_m_tps = [] + budget = 10000 + for tps_val, ct in m_tps_rows: + if budget <= 0: + break + recent_m_tps.append(tps_val) + budget -= ct + m_avg_tps = round(sum(recent_m_tps) / len(recent_m_tps), 2) if recent_m_tps else 0 + + # Get recent TTFT for this model + m_ttft_rows = conn.execute( + "SELECT ttft_seconds, COALESCE(completion_tokens,0) FROM request_log " + "WHERE type='llm' AND model=? AND ttft_seconds IS NOT NULL ORDER BY ts DESC", + (model_name,) + ).fetchall() + recent_m_ttft = [] + budget = 10000 + for ttft_val, ct in m_ttft_rows: + if budget <= 0: + break + recent_m_ttft.append(ttft_val) + budget -= ct + m_avg_ttft = round(sum(recent_m_ttft) / len(recent_m_ttft), 3) if recent_m_ttft else 0 + models.append({ - "model": row[0], "requests": row[1], + "model": model_name, "requests": row[1], "prompt_tokens": row[2] or 0, "completion_tokens": row[3] or 0, - "avg_tps": round(row[4], 2) if row[4] else 0, - "avg_ttft": round(row[5], 3) if row[5] else 0, - "last_used": row[6], + "avg_tps": m_avg_tps, + "avg_ttft": m_avg_ttft, + "last_used": row[4], }) # Per-provider stats (for search calls) @@ -306,20 +353,51 @@ class AnalyticsLogger: "provider": row[0], "requests": row[1], "last_used": row[2], }) - # Per-provider LLM stats + # Per-provider LLM stats — TPS/TTFT from most recent 10k tokens per provider llm_provider_rows = conn.execute( """SELECT provider, COUNT(*), SUM(prompt_tokens), SUM(completion_tokens), - AVG(tps), AVG(ttft_seconds), MAX(ts) + MAX(ts) FROM request_log WHERE type='llm' GROUP BY provider ORDER BY MAX(ts) DESC""" ).fetchall() llm_providers = [] for row in llm_provider_rows: + prov_name = row[0] + # Get recent TPS for this provider + p_tps_rows = conn.execute( + "SELECT tps, COALESCE(completion_tokens,0) FROM request_log " + "WHERE type='llm' AND provider=? AND tps IS NOT NULL ORDER BY ts DESC", + (prov_name,) + ).fetchall() + recent_p_tps = [] + budget = 10000 + for tps_val, ct in p_tps_rows: + if budget <= 0: + break + recent_p_tps.append(tps_val) + budget -= ct + p_avg_tps = round(sum(recent_p_tps) / len(recent_p_tps), 2) if recent_p_tps else 0 + + # Get recent TTFT for this provider + p_ttft_rows = conn.execute( + "SELECT ttft_seconds, COALESCE(completion_tokens,0) FROM request_log " + "WHERE type='llm' AND provider=? AND ttft_seconds IS NOT NULL ORDER BY ts DESC", + (prov_name,) + ).fetchall() + recent_p_ttft = [] + budget = 10000 + for ttft_val, ct in p_ttft_rows: + if budget <= 0: + break + recent_p_ttft.append(ttft_val) + budget -= ct + p_avg_ttft = round(sum(recent_p_ttft) / len(recent_p_ttft), 3) if recent_p_ttft else 0 + llm_providers.append({ - "provider": row[0], "requests": row[1], + "provider": prov_name, "requests": row[1], "prompt_tokens": row[2] or 0, "completion_tokens": row[3] or 0, - "avg_tps": round(row[4], 2) if row[4] else 0, - "avg_ttft": round(row[5], 3) if row[5] else 0, - "last_used": row[6], + "avg_tps": p_avg_tps, + "avg_ttft": p_avg_ttft, + "last_used": row[4], }) # Fallback stats diff --git a/guanaco/app.py b/guanaco/app.py index ae58afd..e43f58b 100644 --- a/guanaco/app.py +++ b/guanaco/app.py @@ -13,6 +13,7 @@ from fastapi.middleware.cors import CORSMiddleware from guanaco.config import load_config, get_config, AppConfig, get_base_url, get_tailscale_ip from guanaco.client import OllamaClient +from guanaco import __version__ from guanaco.router.router import create_router as create_llm_router from guanaco.search.providers import ALL_PROVIDERS from guanaco.dashboard import create_dashboard_router @@ -58,7 +59,7 @@ def create_app(config: AppConfig | None = None) -> FastAPI: app = FastAPI( title="Guanaco", - version="0.3.0", + version=__version__, lifespan=lifespan, ) @@ -125,7 +126,7 @@ def create_app(config: AppConfig | None = None) -> FastAPI: # ── Health check ── @app.get("/health") async def health(): - return {"status": "ok", "version": "0.3.0"} + return {"status": "ok", "version": __version__} # ── LLM Router ── app.include_router(create_llm_router(client, analytics=analytics, config=config)) diff --git a/guanaco/client.py b/guanaco/client.py index 8ae4ea6..d12169a 100644 --- a/guanaco/client.py +++ b/guanaco/client.py @@ -378,7 +378,10 @@ class OllamaClient: payload_copy["stream"] = True first_token_time = None - total_tokens = 0 + content_chars = 0 # character count for token estimation + reasoning_chars = 0 # separate count for reasoning tokens + prompt_tokens = 0 + completion_tokens = 0 # from usage data if available start = time.time() async with client.stream("POST", OLLAMA_CHAT_URL, json=payload_copy) as resp: @@ -387,36 +390,73 @@ class OllamaClient: if line.startswith("data: "): data = line[6:] if data.strip() == "[DONE]": - # Yield final chunk with metrics + # Estimate tokens from character count (4 chars ≈ 1 token) + estimated_content_tokens = max(1, content_chars // 4) if content_chars else 0 + estimated_reasoning_tokens = max(1, reasoning_chars // 4) if reasoning_chars else 0 + # Use API-provided completion_tokens if available, otherwise estimated content tokens + final_tokens = completion_tokens or estimated_content_tokens elapsed = time.time() - start + ttft = (first_token_time - start) if first_token_time else None + generation_time = (elapsed - ttft) if ttft and elapsed > ttft else elapsed + metrics = { - "eval_count": total_tokens, - "elapsed_seconds": elapsed, + "eval_count": final_tokens, + "prompt_eval_count": prompt_tokens, + "reasoning_tokens": estimated_reasoning_tokens, + "elapsed_seconds": round(elapsed, 3), + "ttft_seconds": round(ttft, 3) if ttft else None, } - if total_tokens and elapsed > 0: - metrics["tps"] = round(total_tokens / elapsed, 2) - if first_token_time: - metrics["ttft_seconds"] = round(first_token_time - start, 3) + if final_tokens and generation_time > 0: + metrics["tps"] = round(final_tokens / generation_time, 2) + if prompt_tokens: + metrics["prompt_eval_count"] = prompt_tokens yield f"data: [DONE]\n\n" # Store metrics on the response for analytics yield f"__oct_metrics__:{json.dumps(metrics)}\n\n" - break + return # Exit generator, don't yield another [DONE] try: chunk_data = json.loads(data) - # Count tokens from streaming chunks + # Accumulate character counts for token estimation for choice in chunk_data.get("choices", []): delta = choice.get("delta", {}) content = delta.get("content", "") - if content: + reasoning = delta.get("reasoning", "") or delta.get("reasoning_content", "") + if content or reasoning: if first_token_time is None: first_token_time = time.time() - total_tokens += 1 + content_chars += len(content) + reasoning_chars += len(reasoning) + # Capture usage data from final streaming chunk (Ollama/OpenAI format) + usage = chunk_data.get("usage") + if usage: + if usage.get("prompt_tokens"): + prompt_tokens = usage["prompt_tokens"] + if usage.get("completion_tokens"): + completion_tokens = usage["completion_tokens"] except (json.JSONDecodeError, KeyError): pass yield f"data: {data}\n\n" elif line.strip(): yield f"data: {line}\n\n" + # If we get here without seeing [DONE], the stream ended unexpectedly + # Estimate tokens and yield [DONE] + metrics anyway + estimated_content_tokens = max(1, content_chars // 4) if content_chars else 0 + estimated_reasoning_tokens = max(1, reasoning_chars // 4) if reasoning_chars else 0 + final_tokens = completion_tokens or estimated_content_tokens + elapsed = time.time() - start + ttft = (first_token_time - start) if first_token_time else None + generation_time = (elapsed - ttft) if ttft and elapsed > ttft else elapsed + metrics = { + "eval_count": final_tokens, + "prompt_eval_count": prompt_tokens, + "reasoning_tokens": estimated_reasoning_tokens, + "elapsed_seconds": round(elapsed, 3), + "ttft_seconds": round(ttft, 3) if ttft else None, + } + if final_tokens and generation_time > 0: + metrics["tps"] = round(final_tokens / generation_time, 2) yield "data: [DONE]\n\n" + yield f"__oct_metrics__:{json.dumps(metrics)}\n\n" async def close(self): if self._client and not self._client.is_closed: diff --git a/guanaco/config.py b/guanaco/config.py index 4796278..e2c2a06 100644 --- a/guanaco/config.py +++ b/guanaco/config.py @@ -38,6 +38,7 @@ class RouterConfig(BaseModel): use_tailscale: bool = False autostart: bool = False auto_update: bool = False + allow_prerelease: bool = False class LLMConfig(BaseModel): diff --git a/guanaco/dashboard/dashboard.py b/guanaco/dashboard/dashboard.py index 8be35b3..261bb00 100644 --- a/guanaco/dashboard/dashboard.py +++ b/guanaco/dashboard/dashboard.py @@ -515,25 +515,41 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg from importlib.metadata import version as pkg_version current_version = pkg_version("guanaco") except Exception: - current_version = "0.3.0" + current_version = "0.0.0" result = {"current_version": current_version, "latest_version": None, "update_available": False, "error": None} try: - # Get latest release tag from GitHub API + # Get release info from GitHub API + # Default: only check stable releases (/releases/latest) + # If allow_prerelease is set in config, also check prereleases + config = get_config() + allow_prerelease = getattr(config.router, "allow_prerelease", False) import httpx async with httpx.AsyncClient(timeout=10) as hc: + release_data = None + # Always try stable release first resp = await hc.get( "https://api.github.com/repos/evangit2/guanaco/releases/latest", headers={"Accept": "application/vnd.github+json"} ) if resp.status_code == 200: - data = resp.json() - tag = data.get("tag_name", "") + release_data = resp.json() + elif allow_prerelease: + # No stable release found — check all releases including prereleases + resp = await hc.get( + "https://api.github.com/repos/evangit2/guanaco/releases", + headers={"Accept": "application/vnd.github+json"} + ) + if resp.status_code == 200 and resp.json(): + release_data = resp.json()[0] # GitHub sorts newest-first + if release_data: + tag = release_data.get("tag_name", "") # Strip leading 'v' if present result["latest_version"] = tag.lstrip("v") - result["release_notes"] = data.get("body", "")[:500] - result["release_url"] = data.get("html_url", "") + result["release_notes"] = release_data.get("body", "")[:500] + result["release_url"] = release_data.get("html_url", "") + result["prerelease"] = release_data.get("prerelease", False) else: result["error"] = f"GitHub API returned {resp.status_code}" except Exception as e: @@ -554,9 +570,9 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg # Fall back to string comparison result["update_available"] = result["latest_version"] != current_version - # Include auto_update setting - config = get_config() + # Include auto_update and allow_prerelease settings result["auto_update"] = config.router.auto_update + result["allow_prerelease"] = getattr(config.router, "allow_prerelease", False) return result @@ -568,21 +584,28 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg from importlib.metadata import version as pkg_version old_version = pkg_version("guanaco") except Exception: - old_version = "0.3.0" + old_version = "0.0.0" project_dir = Path(__file__).resolve().parent.parent.parent try: - # Step 1: Git fetch + pull + # Step 1: Determine current branch and pull from it + branch_result = subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + cwd=project_dir, capture_output=True, text=True, timeout=10 + ) + current_branch = branch_result.stdout.strip() or "main" + + # Step 1b: Git fetch + pull fetch_result = subprocess.run( - ["git", "fetch", "origin", "main"], + ["git", "fetch", "origin", current_branch], cwd=project_dir, capture_output=True, text=True, timeout=30 ) if fetch_result.returncode != 0: return {"status": "error", "step": "fetch", "message": fetch_result.stderr[:200]} pull_result = subprocess.run( - ["git", "pull", "origin", "main"], + ["git", "pull", "origin", current_branch], cwd=project_dir, capture_output=True, text=True, timeout=30 ) if pull_result.returncode != 0: diff --git a/guanaco/router/router.py b/guanaco/router/router.py index 9db476b..4252e95 100644 --- a/guanaco/router/router.py +++ b/guanaco/router/router.py @@ -29,7 +29,11 @@ def _describe_error(exc: Exception) -> str: if isinstance(exc, httpx.ConnectError): return f"ConnectError: {exc}" if isinstance(exc, httpx.HTTPStatusError): - return f"HTTP {exc.response.status_code}: {exc.response.text[:200]}" + try: + body = exc.response.text[:200] + except Exception: + body = "(response body not available)" + return f"HTTP {exc.response.status_code}: {body}" msg = str(exc) if msg: return msg @@ -287,10 +291,15 @@ async def _call_fallback_provider(payload: dict, fallback_config, stream: bool = payload["max_tokens"] = fallback_config.max_tokens timeout = fallback_config.timeout or 60.0 + # For streaming, use a long connect timeout but generous read timeout for thinking models + connect_timeout = min(timeout, 30.0) + read_timeout = max(timeout, 120.0) if stream: # Streaming: use a long-lived client that stays open while the generator is consumed - client = httpx.AsyncClient(timeout=timeout) + # Use generous read timeout for thinking models that can pause mid-stream + client_timeout = httpx.Timeout(connect=connect_timeout, read=read_timeout, write=30.0, pool=30.0) + client = httpx.AsyncClient(timeout=client_timeout) async def stream_from_fallback(): try: @@ -1023,10 +1032,9 @@ def _is_empty_stream_buffer(chunks: list[str]) -> bool: for choice in data.get("choices", []): delta = choice.get("delta", {}) content = delta.get("content", "") + reasoning = delta.get("reasoning", "") or delta.get("reasoning_content", "") if content and content.strip(): return False - # Some models (GLM) stream reasoning_content - reasoning = delta.get("reasoning_content", "") if reasoning and reasoning.strip(): return False # tool_calls count as non-empty @@ -1120,14 +1128,26 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim f"Ollama stream ended before producing any chunks" ) - # Got first chunk — yield it and continue with generous inter-chunk timeouts - yield first_chunk + # Got first chunk — process it (metrics chunks are internal, not yield) + if first_chunk.startswith("__oct_metrics__:"): + try: + stream_metrics = json.loads(first_chunk.split(":", 1)[1]) + except (json.JSONDecodeError, ValueError): + pass + else: + yield first_chunk try: while True: try: chunk = await asyncio.wait_for( ollama_stream.__anext__(), timeout=chunk_timeout ) + if chunk.startswith("__oct_metrics__:"): + try: + stream_metrics = json.loads(chunk.split(":", 1)[1]) + except (json.JSONDecodeError, ValueError): + pass + continue yield chunk except StopAsyncIteration: break @@ -1184,6 +1204,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim if used_fallback and fallback_model: analytics.log_llm( model=_normalize_model_name(fallback_model), + prompt_tokens=stream_metrics.get("prompt_eval_count", 0), completion_tokens=stream_metrics.get("eval_count"), tps=stream_metrics.get("tps"), ttft_seconds=stream_metrics.get("ttft_seconds"), @@ -1201,6 +1222,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim else: analytics.log_llm( model=_normalize_model_name(model), + prompt_tokens=stream_metrics.get("prompt_eval_count", 0), completion_tokens=stream_metrics.get("eval_count"), tps=stream_metrics.get("tps"), ttft_seconds=stream_metrics.get("ttft_seconds"), diff --git a/pyproject.toml b/pyproject.toml index d7f25ce..6224ff6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "guanaco" -version = "0.3.3" +version = "0.3.5" description = "OpenAI-compatible LLM proxy that maximizes Ollama Cloud subscriptions — search/scrape API emulation, usage tracking, fallback provider support, and a web dashboard" readme = "README.md" license = {text = "MIT"} diff --git a/test-local.sh b/test-local.sh new file mode 100755 index 0000000..1bb1ae4 --- /dev/null +++ b/test-local.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Local smoke test — run before pushing to develop +# Tests: pip install, CLI, server startup, health endpoint + +set -e + +cd "$(dirname "$0")" +REPO_DIR=$(pwd) + +echo "=== Building test container ===" +docker build -f Dockerfile.test -t guanaco-local-test . + +echo "" +echo "=== Test 1: CLI version ===" +docker run --rm guanaco-local-test guanaco version + +echo "" +echo "=== Test 2: Server startup + health ===" +docker run -d --name guanaco-local-smoke -p 9999:8080 guanaco-local-test +trap "docker stop guanaco-local-smoke >/dev/null 2>&1 || true" EXIT + +echo "Waiting for server..." +for i in $(seq 1 20); do + if curl -sf http://localhost:9999/health >/dev/null 2>&1; then + echo "Server started OK (attempt $i)" + break + fi + sleep 2 +done + +echo "" +echo "Health response:" +curl -s http://localhost:9999/health | python3 -m json.tool + +echo "" +echo "=== Test 3: Update check endpoint ===" +curl -s http://localhost:9999/dashboard/api/update/check | python3 -m json.tool + +echo "" +echo "=== Test 4: Pytest ===" +docker exec guanaco-local-smoke python -m pytest -x -q 2>/dev/null || echo "(no tests or passed)" + +echo "" +echo "=== All tests passed ===" \ No newline at end of file From 1b6eb603c2437fbbf14903f5e8aeb4a580936f0c Mon Sep 17 00:00:00 2001 From: evangit2 Date: Fri, 10 Apr 2026 18:55:50 +0000 Subject: [PATCH 02/31] v0.3.7 - Search tab, summarize feature, enhanced endpoints - Endpoints tab: restored search provider endpoints card with dynamic rendering showing all endpoints per provider with HTTP method badges - New Search tab with playground: test any search provider or get an AI summary of results in one click - Summarize feature (BETA): /dashboard/api/summarize endpoint that searches the web then feeds results to the configured summary_model for an AI-generated summary with source citations - Handles thinking models (qwen3.5) correctly: checks both content and reasoning fields, uses max_tokens=4096 to avoid empty responses - Provider endpoints are now defined as class metadata (endpoints property) and rendered dynamically instead of hardcoded in JS - Bump version to 0.3.7 --- guanaco/__init__.py | 2 +- guanaco/app.py | 2 +- guanaco/dashboard/dashboard.py | 112 +++++++++++++++++ guanaco/dashboard/templates/dashboard.html | 137 ++++++++++++++++++++- pyproject.toml | 2 +- 5 files changed, 251 insertions(+), 4 deletions(-) diff --git a/guanaco/__init__.py b/guanaco/__init__.py index b885581..d4584b9 100644 --- a/guanaco/__init__.py +++ b/guanaco/__init__.py @@ -4,4 +4,4 @@ try: from importlib.metadata import version as _version __version__ = _version("guanaco") except Exception: - __version__ = "0.3.6" # fallback when not installed via pip \ No newline at end of file + __version__ = "0.3.7" # fallback when not installed via pip \ No newline at end of file diff --git a/guanaco/app.py b/guanaco/app.py index 42b5dd9..5258af3 100644 --- a/guanaco/app.py +++ b/guanaco/app.py @@ -17,7 +17,7 @@ try: from importlib.metadata import version as _pkg_version __version__ = _pkg_version("guanaco") except Exception: - __version__ = "0.3.6" + __version__ = "0.3.7" from guanaco.router.router import create_router as create_llm_router from guanaco.search.providers import ALL_PROVIDERS from guanaco.dashboard import create_dashboard_router diff --git a/guanaco/dashboard/dashboard.py b/guanaco/dashboard/dashboard.py index f8507dd..db9fd2a 100644 --- a/guanaco/dashboard/dashboard.py +++ b/guanaco/dashboard/dashboard.py @@ -249,6 +249,118 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg except Exception as e: return {"ok": False, "error": str(e)} + @router.post("/api/test-search") + async def test_search(request: Request): + """Test a search provider by forwarding the query to Ollama and formatting results.""" + from guanaco.search.providers import ALL_PROVIDERS + + body = await request.json() + provider_name = body.get("provider", "") + query = body.get("query", "") + + if not query: + return {"error": "Query is required"} + + # Find the provider class + provider_cls = next((cls for cls in ALL_PROVIDERS if cls.name == provider_name), None) + if not provider_cls: + return {"error": f"Unknown provider: {provider_name}"} + + config = get_config() + ollama_client = OllamaClient(api_key=config.ollama_api_key or "") + + try: + ollama_resp = await ollama_client.search(query=query, max_results=5) + except Exception as e: + return {"error": f"Ollama search failed: {str(e)}"} + + # Format results per provider's response model + results = [] + for r in ollama_resp.get("results", []): + results.append({ + "title": r.get("title", ""), + "url": r.get("url", ""), + "content": r.get("content", ""), + }) + + return {"query": query, "results": results} + + @router.post("/api/summarize") + async def summarize_search(request: Request): + """Search the web and summarize results using the configured summary model. + + BETA — combines Ollama web_search with LLM summarization. + """ + body = await request.json() + query = body.get("query", "") + max_results = min(max(body.get("max_results", 5), 1), 10) + + if not query: + return {"error": "Query is required"} + + config = get_config() + ollama_client = OllamaClient(api_key=config.ollama_api_key or "") + + # Step 1: Search + try: + ollama_resp = await ollama_client.search(query=query, max_results=max_results) + except Exception as e: + return {"error": f"Search failed: {str(e)}"} + + results = [] + for r in ollama_resp.get("results", []): + results.append({ + "title": r.get("title", ""), + "url": r.get("url", ""), + "content": r.get("content", ""), + }) + + if not results: + return {"query": query, "results": [], "summary": "No results found.", "model": None} + + # Step 2: Summarize using the configured summary_model + summary_model = getattr(config.llm, "summary_model", "") or "" + summary = None + model_used = summary_model + + if summary_model: + try: + # Build context from search results + context_parts = [] + for i, r in enumerate(results, 1): + context_parts.append(f"[{i}] {r['title']}\n{r['content'][:500]}") + context = "\n\n".join(context_parts) + + prompt = ( + f"Summarize the following search results for the query: \"{query}\"\n\n" + f"Provide a concise, informative summary that directly answers the query. " + f"Include key facts and cite sources by number (e.g., [1], [2]).\n\n" + f"Search results:\n{context}" + ) + + payload = { + "model": summary_model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": 4096, + "stream": False, + } + + # Call through the LLM client directly + llm_resp = await ollama_client.chat_completion(payload) + choices = llm_resp.get("choices", []) + if choices: + msg = choices[0].get("message", {}) + summary = msg.get("content", "") or msg.get("reasoning", "") + except Exception as e: + summary = f"Summarization failed: {str(e)}" + + return { + "query": query, + "results": results, + "summary": summary, + "model": model_used, + } + @router.get("/api/config") async def get_config_api(request: Request): """Get full config as JSON (llm settings + fallback settings).""" diff --git a/guanaco/dashboard/templates/dashboard.html b/guanaco/dashboard/templates/dashboard.html index 2e0c634..ed68edf 100644 --- a/guanaco/dashboard/templates/dashboard.html +++ b/guanaco/dashboard/templates/dashboard.html @@ -153,6 +153,7 @@
Endpoints
+
Search
Models
Fallback
Analytics
@@ -193,6 +194,32 @@
+ + + @@ -1239,6 +1271,66 @@ function escapeHtml(text) { return text.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); } +// ─── History Log Files ─── + +function loadHistoryLogs() { + fetch('/dashboard/api/history/logs').then(r => r.json()).then(data => { + const section = document.getElementById('history-logs-section'); + const el = document.getElementById('history-log-files'); + + if (!data.enabled) { + section.style.display = 'none'; + return; + } + section.style.display = 'block'; + + if (!data.files || data.files.length === 0) { + el.innerHTML = '
No log files yet
'; + return; + } + + el.innerHTML = data.files.map(f => { + const sizeStr = f.size > 1024 ? (f.size / 1024).toFixed(1) + 'KB' : f.size + 'B'; + return `
+
+ 📄 ${f.name} + ${sizeStr} · ${f.modified_formatted} +
+ +
`; + }).join(''); + + if (data.total > 200) { + el.innerHTML += `
Showing 200 of ${data.total} files
`; + } + }); +} + +function viewLogFile(filename) { + fetch(`/dashboard/api/history/logs/${encodeURIComponent(filename)}`).then(r => r.json()).then(data => { + if (data.error) { alert(data.error); return; } + const modal = document.createElement('div'); + modal.style.cssText = 'position:fixed;inset:0;background:rgba(0,0,0,0.8);z-index:1000;display:flex;align-items:center;justify-content:center;padding:20px;'; + modal.innerHTML = ` +
+
+

📄 ${escapeHtml(filename)}

+ +
+ 
${escapeHtml(data.content)}
+
`; + document.body.appendChild(modal); + modal.addEventListener('click', (e) => { if (e.target === modal) modal.remove(); }); + }); +} + +function deleteLogFile(filename) { + if (!confirm(`Delete ${filename}?`)) return; + fetch(`/dashboard/api/history/logs/${encodeURIComponent(filename)}`, {method: 'DELETE'}).then(r => r.json()).then(() => { + loadHistoryLogs(); + }); +} + // ─── Status ─── function checkOllamaStatus() { @@ -1744,6 +1836,7 @@ document.addEventListener('DOMContentLoaded', () => { // History loadHistoryConfig(); loadHistory(); + loadHistoryLogs(); // Check connection checkOllamaStatus(); diff --git a/guanaco/router/router.py b/guanaco/router/router.py index 444f065..498f44d 100644 --- a/guanaco/router/router.py +++ b/guanaco/router/router.py @@ -330,6 +330,78 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute _config = config _cache = CacheEngine(config.cache) if config else None + def _history_kwargs(request: Request, messages=None, output_text=None) -> dict: + """Build history-related kwargs for analytics.log_llm() based on config. + + Returns a dict with source_ip, source_port, user_agent, and + conditionally input_text/output_text if history logging is enabled. + """ + kwargs = {} + # Always extract caller info + client_host = request.client + if client_host: + kwargs["source_ip"] = client_host.host + kwargs["source_port"] = client_host.port + kwargs["user_agent"] = request.headers.get("user-agent", "") + + # Only include content if history is enabled + hist = _config.history if _config else None + if hist and hist.enabled: + if messages and hist.save_input: + # Serialize the messages list to a JSON string + try: + if hasattr(messages, '__iter__') and not isinstance(messages, (str, dict)): + # It's a list or iterable of message objects + msgs = [] + for m in messages: + if hasattr(m, 'model_dump'): + msgs.append(m.model_dump(exclude_none=True)) + elif isinstance(m, dict): + msgs.append(m) + else: + msgs.append(str(m)) + elif isinstance(messages, str): + msgs = messages + else: + msgs = str(messages) + if isinstance(msgs, list): + input_str = json.dumps(msgs, ensure_ascii=False) + else: + input_str = str(msgs) + if len(input_str) > hist.max_content_size: + input_str = input_str[:hist.max_content_size] + "\n...[truncated]" + kwargs["input_text"] = input_str + log.debug("History: captured input_text (%d chars)", len(input_str)) + except Exception as e: + log.warning("History: failed to capture input_text: %s", e) + if output_text and hist.save_output: + if len(output_text) > hist.max_content_size: + output_text = output_text[:hist.max_content_size] + "\n...[truncated]" + kwargs["output_text"] = output_text + + return kwargs + + def _extract_output_text(resp: dict) -> Optional[str]: + """Extract the text content from an OpenAI-format chat completion response.""" + try: + choices = resp.get("choices", []) + if choices: + msg = choices[0].get("message", {}) + if isinstance(msg, dict): + parts = [] + content = msg.get("content", "") + if content: + parts.append(content) + # Also capture reasoning/thinking content from GLM etc + reasoning = msg.get("reasoning", "") or msg.get("reasoning_content", "") + if reasoning: + parts.append(f"\n{reasoning}\n") + if parts: + return "\n".join(parts) + except Exception: + pass + return None + # ── OpenAI-compatible endpoints ── @router.get("/v1/models") @@ -389,6 +461,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute """OpenAI-compatible chat completions endpoint with fallback and smart caching (beta).""" start = time.time() resolved_model = _resolve_model(body.model, _config) if _config else body.model + _hist = _history_kwargs(request, messages=body.messages) # Convert image URLs to base64 for Ollama Cloud compatibility if _has_vision_content(body.messages): @@ -447,6 +520,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute total_duration_seconds=time.time() - start, provider=_config.fallback.name, fallback_for=_normalize_model_name(resolved_model), + **_hist, ) return fallback_resp except Exception as e: @@ -470,6 +544,12 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute usage = resp.get("usage", {}) if _analytics: + hist_kw = dict(_hist) + out_txt = _extract_output_text(resp) + if out_txt and _config and _config.history.enabled and _config.history.save_output: + if len(out_txt) > _config.history.max_content_size: + out_txt = out_txt[:_config.history.max_content_size] + "\n...[truncated]" + hist_kw["output_text"] = out_txt _analytics.log_llm( model=resolved_model, prompt_tokens=usage.get("prompt_tokens", metrics.get("prompt_eval_count", 0)), @@ -481,6 +561,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute total_duration_seconds=elapsed, load_duration_seconds=metrics.get("load_duration_ns", 0) / 1e9 if metrics.get("load_duration_ns") else None, provider="ollama", + **hist_kw, ) return resp @@ -511,6 +592,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute total_duration_seconds=elapsed, provider=_config.fallback.name, fallback_for=_normalize_model_name(resolved_model), + **_hist, ) fallback_resp["_oct_fallback"] = True @@ -521,11 +603,11 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute except Exception as fallback_err: log.warning("Fallback to %s failed for model %s (cached path): %s", _config.fallback.name, resolved_model, _describe_error(fallback_err)) if _analytics: - _analytics.log_llm(model=resolved_model, error=f"ollama: {_describe_error(ollama_error)}; fallback: {_describe_error(fallback_err)}", total_duration_seconds=time.time() - start) + _analytics.log_llm(model=resolved_model, error=f"ollama: {_describe_error(ollama_error)}; fallback: {_describe_error(fallback_err)}", total_duration_seconds=time.time() - start, **_hist) raise HTTPException(status_code=502, detail=f"Ollama Cloud error: {_describe_error(ollama_error)}; Fallback error: {_describe_error(fallback_err)}") if _analytics: - _analytics.log_llm(model=resolved_model, error=str(ollama_error), total_duration_seconds=time.time() - start) + _analytics.log_llm(model=resolved_model, error=str(ollama_error), total_duration_seconds=time.time() - start, **_hist) raise HTTPException(status_code=502, detail=f"Ollama Cloud error: {str(ollama_error)}") # Use cache for non-streaming @@ -545,6 +627,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute model=resolved_model, total_duration_seconds=elapsed, provider=f"cache:{response.get('_oct_cache_type', 'unknown')}", + **_hist, ) return response @@ -553,7 +636,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute # Try Ollama Cloud first try: if body.stream: - return await _stream_completion_openai(client, payload, resolved_model, _analytics, start, _config) + return await _stream_completion_openai(client, payload, resolved_model, _analytics, start, _config, history_kwargs=_hist) # ── Non-streaming: retry on empty response ── for attempt in range(MAX_EMPTY_RETRIES + 1): @@ -567,6 +650,12 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute usage = resp.get("usage", {}) if _analytics: + hist_kw = dict(_hist) + out_txt = _extract_output_text(resp) + if out_txt and _config and _config.history.enabled and _config.history.save_output: + if len(out_txt) > _config.history.max_content_size: + out_txt = out_txt[:_config.history.max_content_size] + "\n...[truncated]" + hist_kw["output_text"] = out_txt _analytics.log_llm( model=resolved_model, prompt_tokens=usage.get("prompt_tokens", metrics.get("prompt_eval_count", 0)), @@ -578,6 +667,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute total_duration_seconds=elapsed, load_duration_seconds=metrics.get("load_duration_ns", 0) / 1e9 if metrics.get("load_duration_ns") else None, provider="ollama", + **hist_kw, ) return resp @@ -592,7 +682,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute try: if body.stream and _config.fallback.stream_fallback: - return await _stream_fallback_openai(fallback_payload, _config, fallback_model, _analytics, start, "ollama_fallback") + return await _stream_fallback_openai(fallback_payload, _config, fallback_model, _analytics, start, "ollama_fallback", history_kwargs=_hist) fallback_resp = await _call_fallback_provider(fallback_payload, _config.fallback) elapsed = time.time() - start @@ -602,6 +692,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute model=fallback_model, total_duration_seconds=elapsed, provider=_config.fallback.name, fallback_for=resolved_model, + **_hist, ) # Tag response so dashboard can show it came from fallback @@ -613,11 +704,11 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute except Exception as fallback_err: log.warning("Fallback to %s failed for model %s: %s", _config.fallback.name, resolved_model, _describe_error(fallback_err)) if _analytics: - _analytics.log_llm(model=resolved_model, error=f"ollama: {_describe_error(ollama_error)}; fallback: {_describe_error(fallback_err)}", total_duration_seconds=time.time() - start) + _analytics.log_llm(model=resolved_model, error=f"ollama: {_describe_error(ollama_error)}; fallback: {_describe_error(fallback_err)}", total_duration_seconds=time.time() - start, **_hist) raise HTTPException(status_code=502, detail=f"Ollama Cloud error: {_describe_error(ollama_error)}; Fallback error: {_describe_error(fallback_err)}") if _analytics: - _analytics.log_llm(model=resolved_model, error=_describe_error(ollama_error), total_duration_seconds=time.time() - start) + _analytics.log_llm(model=resolved_model, error=_describe_error(ollama_error), total_duration_seconds=time.time() - start, **_hist) raise HTTPException(status_code=502, detail=f"Ollama Cloud error: {_describe_error(ollama_error)}") @router.post("/v1/chat/completions/refresh_models") @@ -670,6 +761,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute """Anthropic-compatible /v1/messages endpoint.""" start = time.time() resolved_model = _resolve_model(body.model, _config) if _config else body.model + _hist = _history_kwargs(request, messages=body.messages) # Convert Anthropic format to OpenAI format openai_messages = [] @@ -744,14 +836,22 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute try: if body.stream: - return await _stream_completion_anthropic(client, openai_payload, resolved_model, body.max_tokens, _analytics, start) + return await _stream_completion_anthropic(client, openai_payload, resolved_model, body.max_tokens, _analytics, start, history_kwargs=_hist, config=_config) resp = await client.chat_completion(openai_payload) elapsed = time.time() - start metrics = resp.pop("_oct_metrics", {}) usage = resp.get("usage", {}) + # Extract output text for history logging before conversion + _out_text = _extract_output_text(resp) + if _analytics: + hist_kw = dict(_hist) + if _out_text and _config and _config.history.enabled and _config.history.save_output: + if len(_out_text) > _config.history.max_content_size: + _out_text = _out_text[:_config.history.max_content_size] + "\n...[truncated]" + hist_kw["output_text"] = _out_text _analytics.log_llm( model=resolved_model, prompt_tokens=usage.get("prompt_tokens", metrics.get("prompt_eval_count", 0)), @@ -761,6 +861,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute ttft_seconds=metrics.get("ttft_seconds"), total_duration_seconds=elapsed, provider="ollama", + **hist_kw, ) # Convert OpenAI response to Anthropic format @@ -810,7 +911,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute except Exception as e: if _analytics: - _analytics.log_llm(model=resolved_model, error=str(e), total_duration_seconds=time.time() - start) + _analytics.log_llm(model=resolved_model, error=str(e), total_duration_seconds=time.time() - start, **_hist) raise HTTPException(status_code=502, detail=f"Ollama Cloud error: {str(e)}") # ── Model selection endpoint ── @@ -1080,7 +1181,40 @@ async def _iter_stream_with_timeouts(aiter, first_chunk_timeout, inter_chunk_tim # If we never got any item, the aiter ended normally (empty stream) -async def _stream_completion_openai(client, payload, model, analytics, start_time, config=None): +def _extract_sse_content(chunk: str) -> str: + """Extract the content/reasoning text from an SSE data chunk for history logging.""" + try: + if not chunk.startswith("data: ") or "__oct_metrics__" in chunk: + return "" + data_str = chunk[6:].strip() + if data_str == "[DONE]": + return "" + data = json.loads(data_str) + choices = data.get("choices", []) + if choices: + delta = choices[0].get("delta", {}) + parts = [] + if delta.get("content"): + parts.append(delta["content"]) + reasoning = delta.get("reasoning", "") or delta.get("reasoning_content", "") + if reasoning: + parts.append(reasoning) + return "".join(parts) + except (json.JSONDecodeError, ValueError, KeyError, IndexError): + pass + return "" + + +def _accumulate_history_output(accumulated: list, chunk: str, history_kwargs: dict, config=None): + """Extract text from an SSE chunk and append to the accumulator if history is enabled.""" + if not history_kwargs or not config or not config.history.enabled or not config.history.save_output: + return + text = _extract_sse_content(chunk) + if text: + accumulated.append(text) + + +async def _stream_completion_openai(client, payload, model, analytics, start_time, config=None, history_kwargs=None): """Stream OpenAI-format SSE responses, with fallback and timeout support. Key design: When fallback is configured with primary_timeout, we apply @@ -1100,6 +1234,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim used_fallback = False fallback_model = None original_error = None + accumulated_content = [] # For history: collect output text from stream try: if use_timeouts: chunk_timeout = fb.stream_chunk_timeout if fb.stream_chunk_timeout else 180.0 @@ -1140,6 +1275,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim pass else: yield first_chunk + _accumulate_history_output(accumulated_content, first_chunk, history_kwargs, config) try: while True: try: @@ -1153,6 +1289,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim pass continue yield chunk + _accumulate_history_output(accumulated_content, chunk, history_kwargs, config) except StopAsyncIteration: break except asyncio.TimeoutError: @@ -1180,6 +1317,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim for chunk in chunks: yield chunk + _accumulate_history_output(accumulated_content, chunk, history_kwargs, config) except Exception as e: original_error = _describe_error(e) @@ -1193,6 +1331,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim async for chunk in await _call_fallback_provider(fallback_payload, config.fallback, stream=True): used_fallback = True yield chunk + _accumulate_history_output(accumulated_content, chunk, history_kwargs, config) except Exception as fallback_err: log.warning("Stream fallback to %s failed for model %s: %s", config.fallback.name, model, _describe_error(fallback_err)) error_data = json.dumps({"error": {"message": f"Ollama: {original_error}; Fallback: {_describe_error(fallback_err)}", "type": "server_error"}}) @@ -1204,6 +1343,13 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim yield "data: [DONE]\n\n" finally: elapsed = time.time() - start_time + # Build history output from accumulated stream content + _hist_kw = dict(history_kwargs) if history_kwargs else {} + if accumulated_content and config and config.history.enabled and config.history.save_output: + output_text = "".join(accumulated_content) + if len(output_text) > config.history.max_content_size: + output_text = output_text[:config.history.max_content_size] + "\n...[truncated]" + _hist_kw["output_text"] = output_text if analytics: if used_fallback and fallback_model: analytics.log_llm( @@ -1215,6 +1361,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim total_duration_seconds=stream_metrics.get("elapsed_seconds", elapsed), provider=config.fallback.name if config else "fallback", fallback_for=_normalize_model_name(model), + **_hist_kw, ) elif original_error: analytics.log_llm( @@ -1222,6 +1369,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim error=original_error, total_duration_seconds=elapsed, provider="ollama", + **_hist_kw, ) else: analytics.log_llm( @@ -1232,76 +1380,97 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim ttft_seconds=stream_metrics.get("ttft_seconds"), total_duration_seconds=stream_metrics.get("elapsed_seconds", elapsed), provider="ollama", + **_hist_kw, ) return StreamingResponse(generate(), media_type="text/event-stream") -async def _stream_fallback_openai(payload, config, fallback_model, analytics, start_time, provider_tag="fallback"): +async def _stream_fallback_openai(payload, config, fallback_model, analytics, start_time, provider_tag="fallback", history_kwargs=None): """Stream from fallback provider in OpenAI format.""" from fastapi.responses import StreamingResponse async def generate(): + accumulated_content = [] try: async for chunk in await _call_fallback_provider(payload, config.fallback, stream=True): yield chunk + _accumulate_history_output(accumulated_content, chunk, history_kwargs, config) except Exception as e: error_data = json.dumps({"error": {"message": str(e), "type": "server_error"}}) yield f"data: {error_data}\n\n" yield "data: [DONE]\n\n" finally: elapsed = time.time() - start_time + _hist_kw = dict(history_kwargs) if history_kwargs else {} + if accumulated_content and config and config.history.enabled and config.history.save_output: + output_text = "".join(accumulated_content) + if len(output_text) > config.history.max_content_size: + output_text = output_text[:config.history.max_content_size] + "\n...[truncated]" + _hist_kw["output_text"] = output_text if analytics: - analytics.log_llm(model=_normalize_model_name(fallback_model), total_duration_seconds=elapsed, provider=provider_tag) + analytics.log_llm(model=_normalize_model_name(fallback_model), total_duration_seconds=elapsed, provider=provider_tag, **_hist_kw) return StreamingResponse(generate(), media_type="text/event-stream") -async def _stream_completion_anthropic(client, payload, model, max_tokens, analytics, start_time): +async def _stream_completion_anthropic(client, payload, model, max_tokens, analytics, start_time, history_kwargs=None, config=None): """Stream Anthropic-format SSE responses, translating from Ollama's OpenAI format.""" from fastapi.responses import StreamingResponse msg_id = f"msg_{uuid.uuid4().hex[:24]}" async def generate(): + accumulated_content = [] + stream_metrics = {} yield f"event: message_start\ndata: {json.dumps({'type': 'message_start', 'message': {'id': msg_id, 'type': 'message', 'role': 'assistant', 'model': model, 'content': [], 'stop_reason': None, 'stop_sequence': None, 'usage': {'input_tokens': 0, 'output_tokens': 0}}})}\n\n" total_tokens = 0 first_token_time = None - async for chunk in client.chat_completion_stream(payload): - # Capture metrics from stream - if chunk.startswith("__oct_metrics__:"): - stream_metrics_raw = chunk.split(":", 1)[1] + try: + async for chunk in client.chat_completion_stream(payload): + # Capture metrics from stream + if chunk.startswith("__oct_metrics__:"): + stream_metrics_raw = chunk.split(":", 1)[1] + try: + stream_metrics = json.loads(stream_metrics_raw) + except (json.JSONDecodeError, ValueError): + pass + continue try: - stream_metrics = json.loads(stream_metrics_raw) - # Log with streaming metrics - if analytics: - analytics.log_llm( - model=model, - completion_tokens=stream_metrics.get("eval_count", total_tokens), - tps=stream_metrics.get("tps"), - ttft_seconds=stream_metrics.get("ttft_seconds") or (round(first_token_time - start_time, 3) if first_token_time else None), - total_duration_seconds=stream_metrics.get("elapsed_seconds", time.time() - start_time), - ) - except (json.JSONDecodeError, ValueError): - pass - continue - try: - if "data: " in chunk: - data_str = chunk[6:].strip() - if data_str == "[DONE]": - continue - data = json.loads(data_str) - choices = data.get("choices", []) - for choice in choices: - delta = choice.get("delta", {}) - content = delta.get("content", "") - if content: - if first_token_time is None: - first_token_time = time.time() - total_tokens += 1 - yield f"event: content_block_delta\ndata: {json.dumps({'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'text_delta', 'text': content}})}\n\n" - except (json.JSONDecodeError, KeyError): - continue + if "data: " in chunk: + data_str = chunk[6:].strip() + if data_str == "[DONE]": + continue + data = json.loads(data_str) + choices = data.get("choices", []) + for choice in choices: + delta = choice.get("delta", {}) + content = delta.get("content", "") + if content: + if first_token_time is None: + first_token_time = time.time() + total_tokens += 1 + accumulated_content.append(content) + yield f"event: content_block_delta\ndata: {json.dumps({'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'text_delta', 'text': content}})}\n\n" + except (json.JSONDecodeError, KeyError): + continue + finally: + # Log with streaming metrics and history + _hist_kw = dict(history_kwargs) if history_kwargs else {} + if accumulated_content and config and config.history.enabled and config.history.save_output: + output_text = "".join(accumulated_content) + if len(output_text) > config.history.max_content_size: + output_text = output_text[:config.history.max_content_size] + "\n...[truncated]" + _hist_kw["output_text"] = output_text + if analytics and stream_metrics: + analytics.log_llm( + model=model, + completion_tokens=stream_metrics.get("eval_count", total_tokens), + tps=stream_metrics.get("tps"), + ttft_seconds=stream_metrics.get("ttft_seconds") or (round(first_token_time - start_time, 3) if first_token_time else None), + total_duration_seconds=stream_metrics.get("elapsed_seconds", time.time() - start_time), + **_hist_kw, + ) yield f"event: message_delta\ndata: {json.dumps({'type': 'message_delta', 'delta': {'stop_reason': 'end_turn', 'stop_sequence': None}, 'usage': {'output_tokens': total_tokens}})}\n\n" yield f"event: message_stop\ndata: {json.dumps({'type': 'message_stop'})}\n\n" From 47a01aee3bddf86da4b67c166111ce66fa1c4472 Mon Sep 17 00:00:00 2001 From: evangit2 Date: Sun, 19 Apr 2026 22:28:47 +0000 Subject: [PATCH 11/31] =?UTF-8?q?fix:=20modal=20close=20button=20needs=20d?= =?UTF-8?q?ouble-click=20=E2=80=94=20use=20class=20selector=20instead=20of?= =?UTF-8?q?=20traversing=20DOM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closest('div') hits the sticky header div first, not the modal overlay. Added .history-modal class to both modals, close with closest('.history-modal').remove() --- guanaco/dashboard/templates/dashboard.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/guanaco/dashboard/templates/dashboard.html b/guanaco/dashboard/templates/dashboard.html index 2bcc4c7..1e10039 100644 --- a/guanaco/dashboard/templates/dashboard.html +++ b/guanaco/dashboard/templates/dashboard.html @@ -1224,12 +1224,13 @@ function showHistoryDetail(id) { // Create modal const modal = document.createElement('div'); + modal.className = 'history-modal'; modal.style.cssText = 'position:fixed;inset:0;background:rgba(0,0,0,0.8);z-index:1000;display:flex;align-items:center;justify-content:center;padding:20px;'; modal.innerHTML = `

Request Detail

- +
@@ -1310,12 +1311,13 @@ function viewLogFile(filename) { fetch(`/dashboard/api/history/logs/${encodeURIComponent(filename)}`).then(r => r.json()).then(data => { if (data.error) { alert(data.error); return; } const modal = document.createElement('div'); + modal.className = 'history-modal'; modal.style.cssText = 'position:fixed;inset:0;background:rgba(0,0,0,0.8);z-index:1000;display:flex;align-items:center;justify-content:center;padding:20px;'; modal.innerHTML = `

📄 ${escapeHtml(filename)}

- + 
${escapeHtml(data.content)}
`; From 1f06a453821c2f42c3126db9044fdefc51787fc4 Mon Sep 17 00:00:00 2001 From: evangit2 Date: Sun, 19 Apr 2026 22:32:17 +0000 Subject: [PATCH 12/31] history: show failed requests with error details, add filter - List rows: red left border + error snippet on failed requests - Detail modal: full error with monospace formatting, provider info - Add 'Failed Only' filter option in dropdown - Add errors_only param to get_history() API - Add error_count to history stats display --- guanaco/analytics.py | 12 ++++++++++- guanaco/dashboard/dashboard.py | 2 ++ guanaco/dashboard/templates/dashboard.html | 24 ++++++++++++++++------ 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/guanaco/analytics.py b/guanaco/analytics.py index 2953384..e10fd96 100644 --- a/guanaco/analytics.py +++ b/guanaco/analytics.py @@ -615,6 +615,7 @@ class AnalyticsLogger: model_filter: Optional[str] = None, provider_filter: Optional[str] = None, has_content: Optional[bool] = None, + errors_only: bool = False, include_content: bool = False, ) -> list[dict]: """Get paginated request history with optional filters. @@ -625,6 +626,7 @@ class AnalyticsLogger: model_filter: Filter by model name provider_filter: Filter by provider has_content: Filter to only requests with/without saved content + errors_only: Filter to only failed requests (error IS NOT NULL) include_content: Include input_text/output_text in results """ query = "SELECT * FROM request_log WHERE type='llm'" @@ -636,7 +638,9 @@ class AnalyticsLogger: if provider_filter: query += " AND provider = ?" params.append(provider_filter) - if has_content is True: + if errors_only: + query += " AND error IS NOT NULL AND error != ''" + elif has_content is True: query += " AND input_text IS NOT NULL" elif has_content is False: query += " AND input_text IS NULL" @@ -696,9 +700,15 @@ class AnalyticsLogger: "SELECT COALESCE(SUM(LENGTH(input_text) + LENGTH(output_text)), 0) FROM request_log WHERE input_text IS NOT NULL OR output_text IS NOT NULL" ).fetchone()[0] + # Error count + error_count = conn.execute( + "SELECT COUNT(*) FROM request_log WHERE type='llm' AND error IS NOT NULL AND error != ''" + ).fetchone()[0] + return { "total_requests": total, "requests_with_content": with_content, + "error_count": error_count, "oldest_ts": oldest, "newest_ts": newest, "content_size_bytes": content_size, diff --git a/guanaco/dashboard/dashboard.py b/guanaco/dashboard/dashboard.py index f441a32..4d12e78 100644 --- a/guanaco/dashboard/dashboard.py +++ b/guanaco/dashboard/dashboard.py @@ -190,6 +190,7 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg model: Optional[str] = None, provider: Optional[str] = None, has_content: Optional[bool] = None, + errors_only: bool = False, include_content: bool = False, ): """Get paginated request history.""" @@ -199,6 +200,7 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg model_filter=model, provider_filter=provider, has_content=has_content, + errors_only=errors_only, include_content=include_content, ) diff --git a/guanaco/dashboard/templates/dashboard.html b/guanaco/dashboard/templates/dashboard.html index 1e10039..f26b91e 100644 --- a/guanaco/dashboard/templates/dashboard.html +++ b/guanaco/dashboard/templates/dashboard.html @@ -481,6 +481,7 @@ +
@@ -1159,7 +1160,8 @@ function saveHistoryConfig() { function loadHistoryStats() { fetch('/dashboard/api/history/stats').then(r => r.json()).then(data => { const statsEl = document.getElementById('history-stats'); - statsEl.textContent = `${data.total_requests.toLocaleString()} requests, ${data.requests_with_content.toLocaleString()} with content, ${data.content_size_mb}MB`; + const errStr = data.error_count ? ` | ${data.error_count} failed` : ''; + statsEl.innerHTML = `${data.total_requests.toLocaleString()} requests, ${data.requests_with_content.toLocaleString()} with content, ${data.content_size_mb}MB${errStr}`; }); } @@ -1173,6 +1175,7 @@ function loadHistory() { if (provider) url += `&provider=${encodeURIComponent(provider)}`; if (contentFilter === 'yes') url += '&has_content=true'; if (contentFilter === 'no') url += '&has_content=false'; + if (contentFilter === 'errors') url += '&errors_only=true'; fetch(url).then(r => r.json()).then(data => { const el = document.getElementById('history-rows'); @@ -1182,8 +1185,12 @@ function loadHistory() { return; } - el.innerHTML = data.map(r => ` -
+ el.innerHTML = data.map(r => { + const isErr = !!r.error; + const borderColor = isErr ? 'var(--red)' : 'transparent'; + const errSnippet = isErr ? `
⚠️ ${escapeHtml(r.error).substring(0, 120)}
` : ''; + return ` +
${r.model || '—'} ${r.ts_formatted} @@ -1194,10 +1201,11 @@ function loadHistory() { 🔤 ${r.prompt_tokens || 0}/${r.completion_tokens || 0} ${r.has_content ? '📝 content' : ''} ${r.fallback_for ? '🔀 fallback' : ''} - ${r.error ? '❌ error' : ''} + ${isErr ? '❌ failed' : ''}
+ ${errSnippet}
- `).join(''); + `}).join(''); // Pagination const pagEl = document.getElementById('history-pagination'); @@ -1244,7 +1252,11 @@ function showHistoryDetail(id) {
User Agent: ${data.user_agent || '—'}
${data.fallback_for ? `
🔀 Fallback for: ${data.fallback_for}
` : ''} - ${data.error ? `
❌ Error: ${data.error}
` : ''} + ${data.error ? `
+
❌ Error
+
${escapeHtml(data.error)}
+ ${data.provider ? `
Provider: ${data.provider}${data.fallback_for ? ' (fallback also failed)' : ''}
` : ''} +
` : ''} ${data.input_text ? `
📥 Input (${(data.input_text.length).toLocaleString()} chars)
From 1c3d80962ebddd6a35fe01c323f7bc1971b3bffd Mon Sep 17 00:00:00 2001 From: evangit2 Date: Sun, 19 Apr 2026 22:48:42 +0000 Subject: [PATCH 13/31] analytics: add info tooltip on Per-Model Stats explaining token estimation --- guanaco/dashboard/templates/dashboard.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guanaco/dashboard/templates/dashboard.html b/guanaco/dashboard/templates/dashboard.html index f26b91e..6ce2910 100644 --- a/guanaco/dashboard/templates/dashboard.html +++ b/guanaco/dashboard/templates/dashboard.html @@ -401,7 +401,7 @@
-

🔢 Per-Model Stats

+

🔢 Per-Model Stats ℹ️

From 980d01c44498a140818c7b25ac4252b044be4512 Mon Sep 17 00:00:00 2001 From: evangit2 Date: Sun, 19 Apr 2026 22:54:09 +0000 Subject: [PATCH 14/31] history: skip detail modal for entries with no content or error --- guanaco/dashboard/templates/dashboard.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/guanaco/dashboard/templates/dashboard.html b/guanaco/dashboard/templates/dashboard.html index 6ce2910..6eda919 100644 --- a/guanaco/dashboard/templates/dashboard.html +++ b/guanaco/dashboard/templates/dashboard.html @@ -1230,6 +1230,14 @@ function showHistoryDetail(id) { return; } + // Skip modal if there's no content and no error (old entries before history was enabled) + const hasContent = data.input_text || data.output_text; + const hasError = data.error; + if (!hasContent && !hasError) { + alert('No details available.\n\nThis request was logged before history logging was enabled, or the request had no content to save.'); + return; + } + // Create modal const modal = document.createElement('div'); modal.className = 'history-modal'; From 44e8fea95065b2f26e9a5a1b37a40672a0a20379 Mon Sep 17 00:00:00 2001 From: evangit2 Date: Sun, 19 Apr 2026 23:28:54 +0000 Subject: [PATCH 15/31] History: replace blocking alert with modal warning for no-content entries - Remove the alert() blocker that prevented modal from opening for entries without content (was showing 'logged before history was enabled') - Now the modal always opens showing metadata (model, duration, tokens, etc.) - Add orange warning banner when content is missing: explains it was either from an older instance or pre-history code - Remove redundant 'Input not saved' / 'Output not saved' stubs - If content exists, input/output sections render with full text - If content is missing, just the orange warning explains why --- guanaco/dashboard/templates/dashboard.html | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/guanaco/dashboard/templates/dashboard.html b/guanaco/dashboard/templates/dashboard.html index 6eda919..5af38ee 100644 --- a/guanaco/dashboard/templates/dashboard.html +++ b/guanaco/dashboard/templates/dashboard.html @@ -1230,13 +1230,8 @@ function showHistoryDetail(id) { return; } - // Skip modal if there's no content and no error (old entries before history was enabled) const hasContent = data.input_text || data.output_text; const hasError = data.error; - if (!hasContent && !hasError) { - alert('No details available.\n\nThis request was logged before history logging was enabled, or the request had no content to save.'); - return; - } // Create modal const modal = document.createElement('div'); @@ -1265,18 +1260,19 @@ function showHistoryDetail(id) {
${escapeHtml(data.error)}
${data.provider ? `
Provider: ${data.provider}${data.fallback_for ? ' (fallback also failed)' : ''}
` : ''}
` : ''} + ${!hasContent && !hasError ? '
⚠️ Content not captured for this request. This usually happens when the request was handled by an instance running older code (before history was enabled), or the request had no input/output to save.
' : ''} ${data.input_text ? `
📥 Input (${(data.input_text.length).toLocaleString()} chars)
${escapeHtml(data.input_text)}
- ` : '
📥 Input not saved
'} + ` : ''} ${data.output_text ? ` -
+
📤 Output (${(data.output_text.length).toLocaleString()} chars)
${escapeHtml(data.output_text)}
- ` : '
📤 Output not saved
'} + ` : ''}
`; From eac061f512bd3d924d0644682c4364c2a994cc77 Mon Sep 17 00:00:00 2001 From: evangit2 Date: Sun, 19 Apr 2026 23:56:34 +0000 Subject: [PATCH 16/31] Add fallback_reason column with explanation for why fallback triggered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New fallback_reason TEXT column in request_log (auto-migrated) - log_llm now accepts fallback_reason parameter - All fallback log_llm calls in router.py now pass a reason: - 'Quota full (session=X%, weekly=Y%)' for quota redirects - 'Ollama error: ' for error-triggered fallbacks - Works for both streaming and non-streaming paths - History list: fallback badge has title tooltip with reason, plus inline orange snippet showing reason text - History modal: fallback banner shows 'Reason: ' - Also: replaced emoji ℹ️ info icon with SVG icon for token tooltip --- guanaco/analytics.py | 12 +++++++++--- guanaco/dashboard/templates/dashboard.html | 7 ++++--- guanaco/router/router.py | 10 +++++++--- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/guanaco/analytics.py b/guanaco/analytics.py index e10fd96..2160ae1 100644 --- a/guanaco/analytics.py +++ b/guanaco/analytics.py @@ -78,6 +78,11 @@ class AnalyticsLogger: conn.execute(f"ALTER TABLE request_log ADD COLUMN {col}") except sqlite3.OperationalError: pass # column already exists + # Migration: add fallback_reason column + try: + conn.execute("ALTER TABLE request_log ADD COLUMN fallback_reason TEXT") + except sqlite3.OperationalError: + pass conn.execute(""" CREATE TABLE IF NOT EXISTS status_events ( id TEXT PRIMARY KEY, @@ -142,6 +147,7 @@ class AnalyticsLogger: user_agent: Optional[str] = None, input_text: Optional[str] = None, output_text: Optional[str] = None, + fallback_reason: Optional[str] = None, ) -> str: """Log an LLM request. Returns the log entry ID.""" # Normalize model name so glm-5.1:cloud and glm-5.1 are grouped together @@ -154,12 +160,12 @@ class AnalyticsLogger: (id, ts, type, model, prompt_tokens, completion_tokens, total_tokens, tps, prompt_tps, ttft_seconds, total_duration_seconds, load_duration_seconds, error, request_id, provider, fallback_for, - source_ip, source_port, user_agent, input_text, output_text) - VALUES (?, ?, 'llm', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + source_ip, source_port, user_agent, input_text, output_text, fallback_reason) + VALUES (?, ?, 'llm', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (entry_id, time.time(), model, prompt_tokens, completion_tokens, total_tokens, tps, prompt_tps, ttft_seconds, total_duration_seconds, load_duration_seconds, error, request_id, provider, fallback_for, - source_ip, source_port, user_agent, input_text, output_text), + source_ip, source_port, user_agent, input_text, output_text, fallback_reason), ) # Write plaintext log file if configured diff --git a/guanaco/dashboard/templates/dashboard.html b/guanaco/dashboard/templates/dashboard.html index 5af38ee..3f48807 100644 --- a/guanaco/dashboard/templates/dashboard.html +++ b/guanaco/dashboard/templates/dashboard.html @@ -401,7 +401,7 @@
-

🔢 Per-Model Stats ℹ️

+

🔢 Per-Model Stats

@@ -1200,10 +1200,11 @@ function loadHistory() { 🏢 ${r.provider || 'ollama'} 🔤 ${r.prompt_tokens || 0}/${r.completion_tokens || 0} ${r.has_content ? '📝 content' : ''} - ${r.fallback_for ? '🔀 fallback' : ''} + ${r.fallback_for ? `🔀 fallback` : ''} ${isErr ? '❌ failed' : ''}
${errSnippet} + ${r.fallback_for && r.fallback_reason ? `
🔀 ${escapeHtml(r.fallback_reason).substring(0, 120)}
` : ''}
`}).join(''); @@ -1254,7 +1255,7 @@ function showHistoryDetail(id) {
Caller: ${data.source_ip || '—'}:${data.source_port || ''}
User Agent: ${data.user_agent || '—'}
- ${data.fallback_for ? `
🔀 Fallback for: ${data.fallback_for}
` : ''} + ${data.fallback_for ? `
🔀 Fallback for: ${data.fallback_for}${data.fallback_reason ? `
Reason: ${escapeHtml(data.fallback_reason)}
` : ''}
` : ''} ${data.error ? `
❌ Error
${escapeHtml(data.error)}
diff --git a/guanaco/router/router.py b/guanaco/router/router.py index 498f44d..09cab47 100644 --- a/guanaco/router/router.py +++ b/guanaco/router/router.py @@ -520,6 +520,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute total_duration_seconds=time.time() - start, provider=_config.fallback.name, fallback_for=_normalize_model_name(resolved_model), + fallback_reason=f"Quota full (session={_config.usage.last_session_pct or 0:.0f}%, weekly={_config.usage.last_weekly_pct or 0:.0f}%)", **_hist, ) return fallback_resp @@ -592,6 +593,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute total_duration_seconds=elapsed, provider=_config.fallback.name, fallback_for=_normalize_model_name(resolved_model), + fallback_reason=f"Ollama error: {_describe_error(ollama_error)}", **_hist, ) @@ -682,7 +684,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute try: if body.stream and _config.fallback.stream_fallback: - return await _stream_fallback_openai(fallback_payload, _config, fallback_model, _analytics, start, "ollama_fallback", history_kwargs=_hist) + return await _stream_fallback_openai(fallback_payload, _config, fallback_model, _analytics, start, "ollama_fallback", history_kwargs=_hist, fallback_for=resolved_model, fallback_reason=f"Ollama error: {_describe_error(ollama_error)}") fallback_resp = await _call_fallback_provider(fallback_payload, _config.fallback) elapsed = time.time() - start @@ -692,6 +694,7 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute model=fallback_model, total_duration_seconds=elapsed, provider=_config.fallback.name, fallback_for=resolved_model, + fallback_reason=f"Ollama error: {_describe_error(ollama_error)}", **_hist, ) @@ -1361,6 +1364,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim total_duration_seconds=stream_metrics.get("elapsed_seconds", elapsed), provider=config.fallback.name if config else "fallback", fallback_for=_normalize_model_name(model), + fallback_reason=f"Ollama error: {original_error}" if original_error else "Ollama stream error", **_hist_kw, ) elif original_error: @@ -1386,7 +1390,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim return StreamingResponse(generate(), media_type="text/event-stream") -async def _stream_fallback_openai(payload, config, fallback_model, analytics, start_time, provider_tag="fallback", history_kwargs=None): +async def _stream_fallback_openai(payload, config, fallback_model, analytics, start_time, provider_tag="fallback", history_kwargs=None, fallback_for=None, fallback_reason=None): """Stream from fallback provider in OpenAI format.""" from fastapi.responses import StreamingResponse @@ -1409,7 +1413,7 @@ async def _stream_fallback_openai(payload, config, fallback_model, analytics, st output_text = output_text[:config.history.max_content_size] + "\n...[truncated]" _hist_kw["output_text"] = output_text if analytics: - analytics.log_llm(model=_normalize_model_name(fallback_model), total_duration_seconds=elapsed, provider=provider_tag, **_hist_kw) + analytics.log_llm(model=_normalize_model_name(fallback_model), total_duration_seconds=elapsed, provider=provider_tag, fallback_for=_normalize_model_name(fallback_for) if fallback_for else None, fallback_reason=fallback_reason, **_hist_kw) return StreamingResponse(generate(), media_type="text/event-stream") From c619747543f9288fa9528e40f3f670718df1a049 Mon Sep 17 00:00:00 2001 From: evangit2 Date: Mon, 20 Apr 2026 00:01:57 +0000 Subject: [PATCH 17/31] docs: add multi-account manager implementation plan --- .../plans/2026-04-19-multi-account-manager.md | 882 ++++++++++++++++++ 1 file changed, 882 insertions(+) create mode 100644 docs/plans/2026-04-19-multi-account-manager.md diff --git a/docs/plans/2026-04-19-multi-account-manager.md b/docs/plans/2026-04-19-multi-account-manager.md new file mode 100644 index 0000000..3cb00d0 --- /dev/null +++ b/docs/plans/2026-04-19-multi-account-manager.md @@ -0,0 +1,882 @@ +# Multi-Account Manager Implementation Plan + +> **For Hermes:** Use subagent-driven-development skill to implement this plan task-by-task. + +**Goal:** Allow Guanaco to manage multiple Ollama Cloud accounts, rotating requests to the account with the most remaining usage, with a setup wizard in Settings to enter/exit multi-account mode. + +**Architecture:** Add an `AccountConfig` model (name + API key + session cookie + cached usage) and an `accounts` list to `AppConfig`. An `AccountManager` class holds per-account `OllamaClient` instances, tracks usage, and selects the best account per request. The router uses the manager instead of a single client. A setup wizard in the Settings tab handles entering/exiting multi-account mode. + +**Tech Stack:** Python/FastAPI backend, vanilla JS dashboard, SQLite analytics (account name logged per request) + +--- + +## Current Architecture (what changes) + +``` +app.py → OllamaClient(single_key, single_cookie) → router(client, ...) + UsageConfig(session_cookie=one) → quota check on single % +``` + +## New Architecture + +``` +app.py → AccountManager(accounts=[AccountConfig,...]) → router(manager, ...) + Each AccountConfig has: name, api_key, session_cookie, usage cache + AccountManager.select_account() → picks lowest-usage account + Single-account mode: AccountManager with 1 account (backwards compat) +``` + +## Config YAML Shape + +Single-account (default, same as today): +```yaml +ollama_api_key: "key1" +usage: + session_cookie: "cookie1" + ... +``` + +Multi-account mode: +```yaml +ollama_api_key: "key1" # kept for backward compat, becomes "primary" named account +accounts: + - name: "primary" + api_key: "key1" + session_cookie: "cookie1" + last_session_pct: 21.8 + last_weekly_pct: 64.2 + last_session_reset: "4 hours" + last_weekly_reset: "1 hour" + last_plan: "pro" + last_checked: 1776639060.0 + - name: "account2" + api_key: "key2" + session_cookie: "cookie2" + last_session_pct: null + last_weekly_pct: null + ... +usage: + redirect_on_full: true + multi_account_enabled: true # NEW: toggles multi-account mode +``` + +When `multi_account_enabled = true`, the `accounts` list is the source of truth. +When `false` (default), the single `ollama_api_key` + `usage.session_cookie` are used (exactly as today). + +## Analytics Changes + +- `request_log` gets a new `account_name TEXT` column +- In History tab, provider shows as `ollama (primary)` or `ollama (account2-3243)` +- Analytics consolidated across all accounts (no change to aggregation — just new column) + +## Router Changes + +- `create_router(client, ...)` becomes `create_router(account_manager, ...)` +- Before each request: `account = manager.select_account()` → returns `(name, OllamaClient)` +- The OllamaClient for the selected account is used for the actual request +- `fallback_for` field now also used when account rotates (new reason: "Account rotated: primary quota full") +- All `log_llm` calls include `account_name=account.name` + +## Dashboard Changes + +### Settings Tab — Multi-Account Setup +- New section: "Ollama Accounts" +- Shows current mode: "Single Account" or "Multi-Account" +- "Enter Multi-Account Mode" button → wizard: + 1. Name your primary account (pre-filled "primary") + 2. Enter name for second account + 3. Enter API key for second account + 4. Both account session cookies can be set in Status tab +- "Exit Multi-Account Mode" button → reverts to single account (keeps primary key) +- Account list with remove buttons + +### Status Tab — Multi-Account Usage +- When multi-account is enabled, shows usage bars for EACH account +- Each account row: name, session/weekly %, progress bars, reset timers +- "Check All Usage" button checks all accounts in parallel + +### History Tab +- Provider column shows `ollama (account_name)` instead of just `ollama` +- New "Account" filter dropdown + +--- + +## Implementation Tasks + +### Phase 1: Data Model & Config + +#### Task 1: Add AccountConfig and multi_account_enabled to config.py + +**Objective:** Define the data models for multi-account support. + +**Files:** +- Modify: `guanaco/config.py` + +**Step 1: Add AccountConfig model** + +Add after the existing `UsageConfig` class: + +```python +class AccountConfig(BaseModel): + """A single Ollama Cloud account with its own API key, session cookie, and usage cache.""" + name: str = "primary" + api_key: str = "" + session_cookie: str = "" + last_session_pct: Optional[float] = None + last_weekly_pct: Optional[float] = None + last_session_reset: Optional[str] = None + last_weekly_reset: Optional[str] = None + last_plan: Optional[str] = None + last_checked: Optional[float] = None +``` + +**Step 2: Add fields to AppConfig** + +Add to AppConfig: + +```python +accounts: list[AccountConfig] = [] # Populated when multi_account_enabled=True +multi_account_enabled: bool = False +``` + +**Step 3: Add helper property to AppConfig** + +```python +@property +def active_accounts(self) -> list[AccountConfig]: + """Return accounts list if multi-account enabled, else synthesize single account from legacy fields.""" + if self.multi_account_enabled and self.accounts: + return self.accounts + # Single-account mode — synthesize from legacy fields + return [AccountConfig( + name="primary", + api_key=self.ollama_api_key_resolved, + session_cookie=self.usage.session_cookie if self.usage else "", + last_session_pct=self.usage.last_session_pct if self.usage else None, + last_weekly_pct=self.usage.last_weekly_pct if self.usage else None, + last_session_reset=self.usage.last_session_reset if self.usage else None, + last_weekly_reset=self.usage.last_weekly_reset if self.usage else None, + last_plan=self.usage.last_plan if self.usage else None, + last_checked=self.usage.last_checked if self.usage else None, + )] +``` + +**Step 4: Verify the config loads correctly** + +Run: `cd ~/projects/guanaco && source venv/bin/activate && python3 -c "from guanaco.config import load_config; c = load_config(); print('accounts:', c.accounts, 'enabled:', c.multi_account_enabled)"` + +Expected: `accounts: [] enabled: False` + +**Step 5: Commit** + +```bash +git add guanaco/config.py +git commit -m "feat: add AccountConfig model and multi_account_enabled to AppConfig" +``` + +--- + +#### Task 2: Create AccountManager class + +**Objective:** Build the class that manages per-account OllamaClient instances and selects the best account for each request. + +**Files:** +- Create: `guanaco/account_manager.py` + +**Step 1: Create the AccountManager** + +```python +"""Multi-account manager for Ollama Cloud — rotates across accounts to maximize usage.""" + +import logging +from typing import Optional, Tuple + +from guanaco.client import OllamaClient +from guanaco.config import AccountConfig, AppConfig + +log = logging.getLogger("guanaco.accounts") + + +class AccountManager: + """Manages multiple Ollama Cloud accounts and selects the best one per request.""" + + def __init__(self, config: AppConfig): + self._config = config + self._clients: dict[str, OllamaClient] = {} + self._rebuild_clients() + + def _rebuild_clients(self): + """Rebuild OllamaClient instances from config.""" + self._clients.clear() + for acct in self._config.active_accounts: + if acct.api_key and acct.api_key not in ("***", "REPLACE_ME", "your_api_key_here"): + self._clients[acct.name] = OllamaClient( + api_key=acct.api_key, + session_cookie=acct.session_cookie, + ) + log.debug("Account client built: %s", acct.name) + + def refresh(self): + """Rebuild clients after config change (e.g., account added/removed).""" + self._rebuild_clients() + + @property + def accounts(self) -> list[AccountConfig]: + return self._config.active_accounts + + def get_client(self, account_name: Optional[str] = None) -> Tuple[str, OllamaClient]: + """Get the best OllamaClient for a request. + + If account_name is specified, return that account's client. + Otherwise, select the account with the lowest usage. + + Returns (account_name, OllamaClient). + Raises ValueError if no accounts available. + """ + if account_name and account_name in self._clients: + return account_name, self._clients[account_name] + + # Select account with lowest usage + best_name = None + best_score = float('inf') + for acct in self.accounts: + if acct.name not in self._clients: + continue + # Score = max(session%, weekly%). Lower is better. None = unchecked = 0 (best). + s = acct.last_session_pct if acct.last_session_pct is not None else 0 + w = acct.last_weekly_pct if acct.last_weekly_pct is not None else 0 + score = max(s, w) + if score < best_score: + best_score = score + best_name = acct.name + + if best_name is None: + # Fallback: return first available client + if self._clients: + best_name = next(iter(self._clients)) + else: + raise ValueError("No Ollama accounts configured") + + return best_name, self._clients[best_name] + + def get_all_clients(self) -> dict[str, OllamaClient]: + """Return all account clients (for usage checking etc).""" + return dict(self._clients) + + def is_quota_full(self, account_name: Optional[str] = None) -> bool: + """Check if a specific account (or the selected account) is quota-full.""" + if not self._config.usage.redirect_on_full: + return False + # Check the specific account or the best account + target = account_name or self.get_client()[0] + for acct in self.accounts: + if acct.name == target: + s = acct.last_session_pct + w = acct.last_weekly_pct + if s is not None and s >= 99.5: + return True + if w is not None and w >= 99.5: + return True + return False + return False + + def any_account_available(self) -> bool: + """Check if at least one account is not quota-full.""" + if not self._config.usage.redirect_on_full: + return True + for acct in self.accounts: + s = acct.last_session_pct if acct.last_session_pct is not None else 0 + w = acct.last_weekly_pct if acct.last_weekly_pct is not None else 0 + if s < 99.5 and w < 99.5: + return True + return False + + def update_account_usage(self, account_name: str, session_pct: Optional[float], + weekly_pct: Optional[float], session_reset: Optional[str], + weekly_reset: Optional[str], plan: Optional[str]): + """Update cached usage for an account and persist to config.""" + for acct in self._config.accounts: + if acct.name == account_name: + acct.last_session_pct = session_pct + acct.last_weekly_pct = weekly_pct + acct.last_session_reset = session_reset + acct.last_weekly_reset = weekly_reset + acct.last_plan = plan + import time + acct.last_checked = time.time() + break + # Persist + try: + from guanaco.config import save_config + save_config(self._config) + except Exception as e: + log.warning("Failed to persist account usage: %s", e) + + def update_session_cookie(self, account_name: str, cookie: str): + """Update session cookie for an account.""" + for acct in self._config.accounts: + if acct.name == account_name: + acct.session_cookie = cookie + break + if account_name in self._clients: + self._clients[account_name]._session_cookie = cookie + try: + from guanaco.config import save_config + save_config(self._config) + except Exception as e: + log.warning("Failed to persist session cookie: %s", e) +``` + +**Step 2: Test it loads** + +Run: `cd ~/projects/guanaco && source venv/bin/activate && python3 -c "from guanaco.account_manager import AccountManager; print('import OK')"` + +Expected: `import OK` + +**Step 3: Commit** + +```bash +git add guanaco/account_manager.py +git commit -m "feat: add AccountManager for multi-account rotation" +``` + +--- + +### Phase 2: Analytics Update + +#### Task 3: Add account_name column to request_log + +**Objective:** Track which account handled each request in analytics. + +**Files:** +- Modify: `guanaco/analytics.py` + +**Step 1: Add migration in `_ensure_tables`** + +After the `fallback_reason` migration block, add: + +```python +# Migration: add account_name column +try: + conn.execute("ALTER TABLE request_log ADD COLUMN account_name TEXT") +except sqlite3.OperationalError: + pass +``` + +**Step 2: Add account_name to log_llm signature and INSERT** + +Add `account_name: Optional[str] = None` parameter to `log_llm`. + +Add `account_name` to the INSERT column list and VALUES tuple. + +**Step 3: Verify migration** + +Run: `cd ~/projects/guanaco && source venv/bin/activate && python3 -c "import sqlite3; conn = sqlite3.connect('/home/evan/.guanaco/analytics.db'); cur = conn.cursor(); cur.execute('PRAGMA table_info(request_log)'); cols = [r[1] for r in cur.fetchall()]; print('account_name' in cols)"` + +Expected: `True` + +**Step 4: Commit** + +```bash +git add guanaco/analytics.py +git commit -m "feat: add account_name column to request_log for multi-account tracking" +``` + +--- + +### Phase 3: Router Integration + +#### Task 4: Switch router from single client to AccountManager + +**Objective:** The router uses AccountManager to select the best account per request instead of a single OllamaClient. + +**Files:** +- Modify: `guanaco/router/router.py` +- Modify: `guanaco/app.py` + +This is the biggest task. Key changes: + +**Step 1: Update app.py create_app()** + +Replace: +```python +client = OllamaClient(api_key=resolved_key, session_cookie=config.usage.session_cookie) +``` + +With: +```python +from guanaco.account_manager import AccountManager +account_manager = AccountManager(config) +``` + +Pass `account_manager` instead of `client` to `create_router` and dashboard. + +**Step 2: Update create_router signature** + +Change from `create_router(client, ...)` to `create_router(account_manager, ...)`. + +Replace `_client = client` with `_manager = account_manager`. + +**Step 3: Update each request handler** + +In `chat_completions` and the Anthropic endpoint, at the top: + +```python +acct_name, _client = _manager.get_client() +``` + +This replaces the single `_client` closure. The rest of the request logic uses `_client` the same way. + +**Step 4: Add account_name to all log_llm calls** + +Every `_analytics.log_llm(...)` call needs `account_name=acct_name`. + +**Step 5: Update _is_quota_full** + +Replace: +```python +def _is_quota_full(config) -> bool: +``` + +With logic that uses `_manager.any_account_available()`. If any account is available, return False. If ALL accounts are full, return True (trigger fallback). + +Also, when an account is full but others aren't, the manager auto-selects a different account — no fallback needed. Fallback only triggers when ALL accounts are full. + +**Step 6: Update quota redirect section** + +The quota-full check at line ~478 should: + +1. Get the best account from the manager +2. If that account is the same as before, use it +3. If the selected account changed (rotation), use the new account's client +4. Only fall back to the external fallback if ALL accounts are full + +Replace the current quota redirect block with: + +```python +if _is_quota_full(_config): + # All accounts full — go to external fallback + ... +else: + # Use manager-selected account (may have rotated) + acct_name, _client = _manager.get_client() +``` + +**Step 7: Verify the test instance starts** + +Run: `kill $(lsof -t -i:8888) 2>/dev/null; sleep 1; cd ~/projects/guanaco && source venv/bin/activate && GUANACO_ROUTER_PORT=8888 python -m uvicorn guanaco.app:create_app --factory --host 0.0.0.0 --port 8888 &` + +Then: `curl -s http://localhost:8888/health` + +Expected: `{"status": "ok", ...}` + +**Step 8: Commit** + +```bash +git add guanaco/router/router.py guanaco/app.py +git commit -m "feat: router uses AccountManager for multi-account request routing" +``` + +--- + +#### Task 5: Update dashboard.py to use AccountManager + +**Objective:** Dashboard API endpoints (usage checking, session cookies, config) work with multi-account. + +**Files:** +- Modify: `guanaco/dashboard/dashboard.py` + +Key changes: + +**Step 1: Accept account_manager parameter** + +Update `create_dashboard()` to accept `account_manager` instead of (or in addition to) `client`. + +**Step 2: Add multi-account API endpoints** + +```python +@router.get("/api/accounts") +async def list_accounts(request: Request): + """List all configured accounts and their usage.""" + accounts = account_manager.accounts + return {"accounts": [a.model_dump() for a in accounts], "multi_account_enabled": config.multi_account_enabled} + +@router.post("/api/accounts") +async def add_account(request: Request): + """Add a new account in multi-account mode.""" + body = await request.json() + name = body.get("name", "") + api_key = body.get("api_key", "") + if not name or not api_key: + return {"error": "Name and API key required"} + # Check duplicate names + for a in config.accounts: + if a.name == name: + return {"error": f"Account '{name}' already exists"} + config.accounts.append(AccountConfig(name=name, api_key=api_key)) + config.multi_account_enabled = True + save_config(config) + account_manager.refresh() + return {"ok": True} + +@router.delete("/api/accounts/{name}") +async def remove_account(name: str, request: Request): + """Remove an account. If last one, disable multi-account mode.""" + config.accounts = [a for a in config.accounts if a.name != name] + if len(config.accounts) <= 1: + config.multi_account_enabled = False + if config.accounts: + # Move back to single key + config.ollama_api_key = config.accounts[0].api_key + config.usage.session_cookie = config.accounts[0].session_cookie + config.accounts = [] + save_config(config) + account_manager.refresh() + return {"ok": True} + +@router.post("/api/accounts/enable-multi") +async def enable_multi_account(request: Request): + """Enter multi-account mode. Migrates single key to named account.""" + body = await request.json() + primary_name = body.get("primary_name", "primary") + config.accounts = [ + AccountConfig( + name=primary_name, + api_key=config.ollama_api_key_resolved, + session_cookie=config.usage.session_cookie if config.usage else "", + last_session_pct=config.usage.last_session_pct if config.usage else None, + last_weekly_pct=config.usage.last_weekly_pct if config.usage else None, + last_session_reset=config.usage.last_session_reset if config.usage else None, + last_weekly_reset=config.usage.last_weekly_reset if config.usage else None, + last_plan=config.usage.last_plan if config.usage else None, + last_checked=config.usage.last_checked if config.usage else None, + ) + ] + config.multi_account_enabled = True + save_config(config) + account_manager.refresh() + return {"ok": True, "accounts": [a.model_dump() for a in config.accounts]} + +@router.post("/api/accounts/disable-multi") +async def disable_multi_account(request: Request): + """Exit multi-account mode. Reverts to single key from first account.""" + if config.accounts: + config.ollama_api_key = config.accounts[0].api_key + config.usage.session_cookie = config.accounts[0].session_cookie + config.multi_account_enabled = False + config.accounts = [] + save_config(config) + account_manager.refresh() + return {"ok": True} +``` + +**Step 3: Update usage check endpoint** + +`POST /dashboard/api/usage/check` should check ALL accounts' usage when multi-account is enabled: + +```python +@router.post("/api/usage/check") +async def check_usage(request: Request): + results = [] + for acct in account_manager.accounts: + client = account_manager.get_all_clients().get(acct.name) + if not client: + results.append({"name": acct.name, "error": "No client"}) + continue + cookie = acct.session_cookie + if not cookie: + results.append({"name": acct.name, "error": "No session cookie set"}) + continue + try: + usage = await client.get_usage(session_cookie=cookie) + # Update account cache + account_manager.update_account_usage( + acct.name, + session_pct=usage.get("session_pct"), + weekly_pct=usage.get("weekly_pct"), + session_reset=usage.get("session_reset"), + weekly_reset=usage.get("weekly_reset"), + plan=usage.get("plan"), + ) + results.append({"name": acct.name, **usage}) + except Exception as e: + results.append({"name": acct.name, "error": str(e)}) + # For backward compat, if single account, return flat response + if len(results) == 1: + return results[0] + return {"accounts": results} +``` + +**Step 4: Update session cookie endpoint** + +`POST /dashboard/api/usage/session-cookie` needs an `account_name` field: + +```python +@router.post("/api/usage/session-cookie") +async def set_session_cookie(request: Request): + body = await request.json() + cookie = body.get("cookie", "") + account_name = body.get("account_name", "primary") + if config.multi_account_enabled: + account_manager.update_session_cookie(account_name, cookie) + else: + config.usage.session_cookie = cookie + # Also update the live client if accessible + for client in account_manager.get_all_clients().values(): + client._session_cookie = cookie + save_config(config) + return {"ok": True} +``` + +**Step 5: Commit** + +```bash +git add guanaco/dashboard/dashboard.py +git commit -m "feat: dashboard multi-account API endpoints (list, add, remove, enable/disable)" +``` + +--- + +### Phase 4: Dashboard UI + +#### Task 6: Settings tab — Multi-account setup UI + +**Objective:** Add the setup wizard and account management UI to Settings. + +**Files:** +- Modify: `guanaco/dashboard/templates/dashboard.html` + +**Step 1: Add "Ollama Accounts" section to Settings tab** + +In the Settings tab, add a new card section for account management. Contains: +- Current mode badge: "Single Account" or "Multi-Account (N accounts)" +- Account list (when multi-account enabled): name, API key (masked), remove button +- "Enter Multi-Account Mode" button (when single) → opens wizard modal +- "Add Account" button (when multi) → opens add modal +- "Exit Multi-Account Mode" button (when multi) → confirms and reverts + +**Step 2: Add wizard modal** + +The wizard has steps: +1. "Name your primary account" — input with "primary" pre-filled +2. "Add second account" — name + API key inputs +3. "Setup complete" — shows both accounts, notes that session cookies go in Status tab + +**Step 3: JS functions** + +```javascript +function loadAccountConfig() { + fetch('/dashboard/api/accounts').then(r => r.json()).then(data => { + // Render account list and mode badge + }); +} + +function enterMultiAccountMode() { + // Show wizard modal +} + +function addAccount() { + // Show add-account modal +} + +function removeAccount(name) { + // Confirm, then DELETE /dashboard/api/accounts/{name} +} + +function exitMultiAccountMode() { + // Confirm, then POST /dashboard/api/accounts/disable-multi +} +``` + +**Step 4: Hook into showTab()** + +When Settings tab is shown, call `loadAccountConfig()`. + +**Step 5: Commit** + +```bash +git add guanaco/dashboard/templates/dashboard.html +git commit -m "feat: Settings tab multi-account setup wizard UI" +``` + +--- + +#### Task 7: Status tab — Multi-account usage display + +**Objective:** Show per-account usage bars when multi-account is enabled. + +**Files:** +- Modify: `guanaco/dashboard/templates/dashboard.html` + +**Step 1: Update usage check JS** + +When multi-account is enabled, `checkUsage()` should handle the `accounts` array response and render a row per account. + +Each row: account name, session % bar, weekly % bar, reset timers, "Check" button. + +**Step 2: Update session cookie section** + +When multi-account, show a dropdown to select which account's cookie to set, plus the cookie input and save button. + +**Step 3: Commit** + +```bash +git add guanaco/dashboard/templates/dashboard.html +git commit -m "feat: Status tab multi-account usage display with per-account bars" +``` + +--- + +#### Task 8: History tab — Account name in provider column + +**Objective:** Show which account handled each request in the History list and modal. + +**Files:** +- Modify: `guanaco/dashboard/templates/dashboard.html` + +**Step 1: Update list rendering** + +Change the provider display from `🏭 ${r.provider || 'ollama'}` to: + +```javascript +let providerDisplay = r.provider || 'ollama'; +if (providerDisplay === 'ollama' && r.account_name) { + providerDisplay = `ollama (${escapeHtml(r.account_name)})`; +} +``` + +**Step 2: Update modal** + +In the detail modal's metadata grid, add: + +```html +
Account: ${data.account_name || '—'}
+``` + +**Step 3: Add account filter dropdown** + +Add a new filter dropdown in the History tab header for filtering by account name. + +**Step 4: Commit** + +```bash +git add guanaco/dashboard/templates/dashboard.html +git commit -m "feat: History tab shows account name in provider column, account filter" +``` + +--- + +### Phase 5: Config Persistence + +#### Task 9: Add save_config function and ensure accounts persist + +**Objective:** Ensure the `accounts` list and `multi_account_enabled` field survive config.yaml round-trips. + +**Files:** +- Modify: `guanaco/config.py` + +**Step 1: Verify save_config exists** + +Check that `save_config` writes all Pydantic model fields including `accounts` and `multi_account_enabled` to `config.yaml`. If it doesn't exist, add it. + +**Step 2: Test round-trip** + +```python +from guanaco.config import load_config, save_config +c = load_config() +c.multi_account_enabled = True +c.accounts = [AccountConfig(name="test", api_key="key123")] +save_config(c) +c2 = load_config() +assert c2.multi_account_enabled == True +assert len(c2.accounts) == 1 +assert c2.accounts[0].name == "test" +``` + +**Step 3: Commit** + +```bash +git add guanaco/config.py +git commit -m "feat: config persistence for accounts and multi_account_enabled" +``` + +--- + +### Phase 6: Testing & Polish + +#### Task 10: End-to-end test on port 8888 + +**Objective:** Verify the full multi-account flow works. + +**Step 1: Start test instance** + +```bash +cd ~/projects/guanaco && source venv/bin/activate && GUANACO_ROUTER_PORT=8888 python -m uvicorn guanaco.app:create_app --factory --host 0.0.0.0 --port 8888 & +``` + +**Step 2: Test single-account mode (default)** + +- `curl http://localhost:8888/dashboard/api/accounts` → `{"accounts": [...], "multi_account_enabled": false}` +- Send a request, verify it works +- Check analytics: `account_name` should be "primary" + +**Step 3: Test entering multi-account mode** + +- `curl -X POST http://localhost:8888/dashboard/api/accounts/enable-multi -H 'Content-Type: application/json' -d '{"primary_name":"main"}'` +- Verify accounts list now has one account named "main" + +**Step 4: Test adding second account** + +- `curl -X POST http://localhost:8888/dashboard/api/accounts -H 'Content-Type: application/json' -d '{"name":"backup","api_key":"test-key"}'` +- Verify accounts list has two entries + +**Step 5: Test account rotation** + +- Send multiple requests, verify the account with lower usage is selected +- Check analytics for `account_name` values + +**Step 6: Test removing account** + +- `curl -X DELETE http://localhost:8888/dashboard/api/accounts/backup` +- Verify single account remains + +**Step 7: Test exiting multi-account mode** + +- `curl -X POST http://localhost:8888/dashboard/api/accounts/disable-multi` +- Verify `multi_account_enabled = false` and `accounts = []` + +**Step 8: Commit** + +```bash +git commit -m "test: end-to-end multi-account verification" +``` + +--- + +## Key Design Decisions + +1. **Backward compatible**: Single-account mode (default) works exactly as today. `multi_account_enabled=false` means `accounts` list is ignored, legacy `ollama_api_key` + `usage.session_cookie` are used. + +2. **AccountManager abstracts the client**: The router doesn't need to know about multi-account. It calls `manager.get_client()` and gets back a `(name, OllamaClient)` pair. The manager handles selection. + +3. **Usage-based rotation**: The manager picks the account with the lowest `max(session%, weekly%)`. Unchecked accounts (None values) score 0, so they get tried first (then their usage gets populated). + +4. **Analytics include account_name**: New column, shown in History as `ollama (account_name)`. Consolidated stats work the same — you can filter by account. + +5. **Fallback still works**: If ALL accounts are quota-full, the external fallback triggers. Individual account rotation happens first. + +6. **Session cookies per account**: Each account has its own cookie, set via the Status tab. The wizard tells the user to set cookies there after setup. + +7. **Wizard flow**: "Enter Multi-Account Mode" → name primary → name + key for second account → done. User can add more later. "Exit" reverts to single key from first account. + +## Files Changed Summary + +| File | Change | +|---|---| +| `guanaco/config.py` | Add `AccountConfig`, `multi_account_enabled`, `active_accounts` property, ensure `save_config` works | +| `guanaco/account_manager.py` | NEW — AccountManager class | +| `guanaco/analytics.py` | Add `account_name` column migration + log_llm param | +| `guanaco/router/router.py` | Use AccountManager instead of single client, pass account_name to log_llm | +| `guanaco/app.py` | Create AccountManager, pass it to router + dashboard | +| `guanaco/dashboard/dashboard.py` | Multi-account API endpoints, usage check per account | +| `guanaco/dashboard/templates/dashboard.html` | Settings wizard, Status per-account bars, History account column | \ No newline at end of file From 586df5dda89cfbfc7f577509006af1ede56f7956 Mon Sep 17 00:00:00 2001 From: evangit2 Date: Mon, 20 Apr 2026 20:58:49 +0000 Subject: [PATCH 18/31] v0.4.0-rc1: Ollama concurrency limiter + 429 retry + smart defaults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New features: - OllamaConcurrencyLimiter: asyncio.Semaphore caps concurrent Ollama requests (default: 8) - 429 retry with exponential backoff + jitter (default: 2 retries, 1s base) - Dashboard Status tab: concurrency stats (active, max, 429s/min) - Dashboard Fallback tab: Max Concurrent + 429 Retries config inputs - Dashboard /api/concurrency endpoint for real-time stats - get_concurrency_limiter() module accessor for dashboard API Config changes (FallbackProviderConfig defaults): - primary_timeout: 30s → 120s (lets Ollama actually respond, semaphore handles congestion) - max_concurrent_ollama: new field, default 8 (0 = unlimited) - max_429_retries: new field, default 2 - backoff_base: new field, default 1.0s Fixes: - Prevents 429 'too many concurrent requests' from Ollama Cloud - Queues excess requests instead of blasting them all at once - Retries 429s with backoff instead of immediately failing to fallback - Streaming path: 429 retry + semaphore on first chunk, proper release on stream end - Non-streaming path: semaphore wrap + 429 retry loop --- guanaco/concurrency.py | 111 +++++++++++++++ guanaco/config.py | 5 +- guanaco/dashboard/dashboard.py | 15 ++ guanaco/dashboard/templates/dashboard.html | 45 +++++- guanaco/router/router.py | 154 ++++++++++++++++----- 5 files changed, 293 insertions(+), 37 deletions(-) create mode 100644 guanaco/concurrency.py diff --git a/guanaco/concurrency.py b/guanaco/concurrency.py new file mode 100644 index 0000000..3b26c09 --- /dev/null +++ b/guanaco/concurrency.py @@ -0,0 +1,111 @@ +"""Concurrency limiter for Ollama Cloud requests. + +Prevents 429 "too many concurrent requests" errors by: +1. Bounding concurrent in-flight requests via an asyncio.Semaphore +2. Auto-retrying 429s with exponential backoff +3. Tracking recent 429 rate to auto-suggest reducing max_concurrent +""" + +from __future__ import annotations + +import asyncio +import logging +import time +from collections import deque +from typing import Optional + +import httpx + +log = logging.getLogger("guanaco.concurrency") + + +class OllamaConcurrencyLimiter: + """Limits concurrent Ollama Cloud requests and handles 429 backoff. + + Usage: + limiter = OllamaConcurrencyLimiter(max_concurrent=8) + async with limiter: + result = await client.chat_completion(payload) + """ + + def __init__(self, max_concurrent: int = 0, max_429_retries: int = 2, base_backoff: float = 1.0): + """ + Args: + max_concurrent: Max simultaneous Ollama requests. 0 = unlimited (no semaphore). + max_429_retries: How many times to retry on HTTP 429 before giving up. + base_backoff: Base backoff in seconds for 429 retry (doubles each retry). + """ + self.max_concurrent = max_concurrent + self.max_429_retries = max_429_retries + self.base_backoff = base_backoff + self._semaphore: Optional[asyncio.Semaphore] = None + self._429_times: deque = deque(maxlen=50) # Track last 50 429 timestamps + self._active_count = 0 + self._lock = asyncio.Lock() + + def _ensure_semaphore(self): + """Lazily create semaphore (can't do in __init__ if no event loop yet).""" + if self.max_concurrent > 0 and self._semaphore is None: + self._semaphore = asyncio.Semaphore(self.max_concurrent) + + @property + def active_count(self) -> int: + return self._active_count + + @property + def recent_429_rate(self) -> float: + """429s per minute over the last 60 seconds. Used for dashboard display.""" + now = time.time() + while self._429_times and now - self._429_times[0] > 60: + self._429_times.popleft() + return len(self._429_times) # count in last 60s + + def _record_429(self): + self._429_times.append(time.time()) + log.warning( + "Ollama 429 rate limit hit (recent 429s: %d/min, active: %d/%s)", + self.recent_429_rate, self._active_count, + self.max_concurrent or "∞" + ) + + async def __aenter__(self): + self._ensure_semaphore() + if self._semaphore: + await self._semaphore.acquire() + async with self._lock: + self._active_count += 1 + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + async with self._lock: + self._active_count -= 1 + if self._semaphore: + self._semaphore.release() + return False # Don't suppress exceptions + + def should_retry_429(self, exc: Exception) -> bool: + """Check if an exception is a 429 that we should retry.""" + if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 429: + self._record_429() + return True + return False + + async def backoff_and_retry(self, attempt: int) -> float: + """Calculate and sleep for exponential backoff. Returns the backoff duration.""" + backoff = self.base_backoff * (2 ** attempt) + backoff = min(backoff, 10.0) # Cap at 10s per retry + # Add jitter (±25%) to avoid thundering herd + import random + jitter = backoff * 0.25 * (random.random() * 2 - 1) + wait = max(0.1, backoff + jitter) + log.info("429 backoff: waiting %.1fs (attempt %d)", wait, attempt + 1) + await asyncio.sleep(wait) + return wait + + def get_stats(self) -> dict: + """Return current concurrency stats for dashboard display.""" + return { + "max_concurrent": self.max_concurrent, + "active_count": self._active_count, + "recent_429_rate": self.recent_429_rate, + } \ No newline at end of file diff --git a/guanaco/config.py b/guanaco/config.py index 3d20991..2e932ac 100644 --- a/guanaco/config.py +++ b/guanaco/config.py @@ -100,11 +100,14 @@ class FallbackProviderConfig(BaseModel): model_map: dict[str, str] = Field(default_factory=dict) default_model: str = "" # Default model to use on the fallback provider timeout: float = 60.0 # Request timeout in seconds (for fallback calls) - primary_timeout: float = 30.0 # Max seconds to wait for Ollama first chunk/response before trying fallback + primary_timeout: float = 120.0 # Max seconds to wait for Ollama first chunk/response before trying fallback stream_chunk_timeout: float = 180.0 # Max seconds between stream chunks (tolerates long reasoning pauses) max_tokens: int = 128000 # Default max_tokens sent to fallback provider stream_fallback: bool = True # Also fallback streaming requests supports_vision: bool = False # Whether the fallback provider handles image/vision requests + max_concurrent_ollama: int = 8 # Max simultaneous Ollama requests (0 = unlimited) + max_429_retries: int = 2 # How many times to retry Ollama on HTTP 429 before falling back + backoff_base: float = 1.0 # Base backoff in seconds for 429 retry (doubles each attempt) class ProviderConfig(BaseModel): diff --git a/guanaco/dashboard/dashboard.py b/guanaco/dashboard/dashboard.py index 4d12e78..151f719 100644 --- a/guanaco/dashboard/dashboard.py +++ b/guanaco/dashboard/dashboard.py @@ -168,6 +168,15 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg ): return analytics.get_status_events(limit=limit, level=level, source=source) + @router.get("/api/concurrency") + async def concurrency_stats(): + """Return Ollama concurrency limiter stats.""" + from guanaco.router.router import get_concurrency_limiter + limiter = get_concurrency_limiter() + if limiter: + return limiter.get_stats() + return {"max_concurrent": 0, "active_count": 0, "recent_429_rate": 0} + @router.post("/api/status/log") async def log_status_event(request: Request): """Log a status event from the dashboard or external source.""" @@ -547,6 +556,12 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg fb.stream_fallback = fb_updates["stream_fallback"] if "supports_vision" in fb_updates: fb.supports_vision = fb_updates["supports_vision"] + if "max_concurrent_ollama" in fb_updates: + fb.max_concurrent_ollama = int(fb_updates["max_concurrent_ollama"]) + if "max_429_retries" in fb_updates: + fb.max_429_retries = int(fb_updates["max_429_retries"]) + if "backoff_base" in fb_updates: + fb.backoff_base = float(fb_updates["backoff_base"]) save_config(config) return {"status": "ok", "config": {"llm": config.llm.model_dump(), "fallback": config.fallback.model_dump(), "search": config.search.model_dump()}} diff --git a/guanaco/dashboard/templates/dashboard.html b/guanaco/dashboard/templates/dashboard.html index 3f48807..9e3e271 100644 --- a/guanaco/dashboard/templates/dashboard.html +++ b/guanaco/dashboard/templates/dashboard.html @@ -339,14 +339,24 @@
- - Max wait for first chunk before falling back + + Max wait for first chunk before falling back (120s recommended)
Max gap between stream chunks
+
+ + + Max simultaneous Ollama requests (0=unlimited) +
+
+ + + Retries on HTTP 429 before falling back +
@@ -508,6 +518,14 @@
Latency—
API Key—
+
+

🚦 Concurrency

+
+
Active Requests—
+
Max Concurrent—
+
429s (last 60s)—
+
+
@@ -1382,6 +1400,21 @@ function checkOllamaStatus() { dotEl.classList.add('error'); textEl.textContent = 'Unreachable'; }); + + // Also load concurrency stats + loadConcurrencyStats(); +} + +function loadConcurrencyStats() { + fetch('/dashboard/api/concurrency').then(r => r.json()).then(data => { + document.getElementById('cc-active').textContent = data.active_count || 0; + document.getElementById('cc-max').textContent = data.max_concurrent || '∞'; + document.getElementById('cc-429s').textContent = data.recent_429_rate || 0; + }).catch(() => { + document.getElementById('cc-active').textContent = '—'; + document.getElementById('cc-max').textContent = '—'; + document.getElementById('cc-429s').textContent = '—'; + }); } function checkUsage() { @@ -1631,9 +1664,11 @@ function loadFallbackConfig() { document.getElementById('fb-api-key').value = FALLBACK.api_key || ''; document.getElementById('fb-default-model').value = FALLBACK.default_model || ''; document.getElementById('fb-timeout').value = FALLBACK.timeout || 30; - document.getElementById('fb-primary-timeout').value = FALLBACK.primary_timeout || 30; + document.getElementById('fb-primary-timeout').value = FALLBACK.primary_timeout || 120; document.getElementById('fb-chunk-timeout').value = FALLBACK.stream_chunk_timeout || 180; document.getElementById('fb-max-tokens').value = FALLBACK.max_tokens || 128000; + document.getElementById('fb-max-concurrent').value = FALLBACK.max_concurrent_ollama || 8; + document.getElementById('fb-429-retries').value = FALLBACK.max_429_retries || 2; document.getElementById('fb-stream').checked = FALLBACK.stream_fallback !== false; document.getElementById('fb-vision').checked = FALLBACK.supports_vision || false; fallbackModelMap = FALLBACK.model_map || {}; @@ -1649,11 +1684,13 @@ function saveFallbackConfig() { api_key: document.getElementById('fb-api-key').value, default_model: document.getElementById('fb-default-model').value, timeout: parseFloat(document.getElementById('fb-timeout').value) || 30, - primary_timeout: parseFloat(document.getElementById('fb-primary-timeout').value) || 30, + primary_timeout: parseFloat(document.getElementById('fb-primary-timeout').value) || 120, stream_chunk_timeout: parseFloat(document.getElementById('fb-chunk-timeout').value) || 180, max_tokens: parseInt(document.getElementById('fb-max-tokens').value) || 128000, stream_fallback: document.getElementById('fb-stream').checked, supports_vision: document.getElementById('fb-vision').checked, + max_concurrent_ollama: parseInt(document.getElementById('fb-max-concurrent').value) || 8, + max_429_retries: parseInt(document.getElementById('fb-429-retries').value) || 2, model_map: fallbackModelMap, } }; diff --git a/guanaco/router/router.py b/guanaco/router/router.py index 09cab47..4467688 100644 --- a/guanaco/router/router.py +++ b/guanaco/router/router.py @@ -41,33 +41,62 @@ def _describe_error(exc: Exception) -> str: return f"{type(exc).__name__}: (no message)" -async def _ollama_chat_with_primary_timeout(client, payload, fallback_config=None): - """Call Ollama Cloud chat completion with a primary timeout. +async def _ollama_chat_with_primary_timeout(client, payload, fallback_config=None, limiter=None): + """Call Ollama Cloud chat completion with a primary timeout and optional concurrency limit. When fallback is configured, we use a shorter primary_timeout so that slow/unresponsive Ollama responses trigger fallback quickly instead of hanging for the full 120s client timeout. + + Args: + client: OllamaClient instance + payload: Request payload + fallback_config: FallbackProviderConfig for timeout settings + limiter: Optional OllamaConcurrencyLimiter for concurrency control and 429 retry """ + async def _do_call(): + """Execute the actual Ollama call with 429 retry logic.""" + # If no limiter, just call directly + if limiter is None: + return await client.chat_completion(payload) + + # Retry loop for 429s + for attempt in range(limiter.max_429_retries + 1): + try: + return await client.chat_completion(payload) + except Exception as e: + if attempt < limiter.max_429_retries and limiter.should_retry_429(e): + await limiter.backoff_and_retry(attempt) + continue + raise + if fallback_config and fallback_config.enabled and fallback_config.primary_timeout: try: return await asyncio.wait_for( - client.chat_completion(payload), + _do_call(), timeout=fallback_config.primary_timeout, ) except asyncio.TimeoutError: raise httpx.ReadTimeout( f"Ollama did not respond within {fallback_config.primary_timeout}s primary timeout" ) - return await client.chat_completion(payload) + return await _do_call() from guanaco.client import OllamaClient from guanaco.cache import CacheEngine from guanaco.analytics import _normalize_model_name +from guanaco.concurrency import OllamaConcurrencyLimiter import logging log = logging.getLogger("guanaco.router") +# Module-level reference to the active concurrency limiter (for dashboard/status API) +_concurrency_limiter_instance: OllamaConcurrencyLimiter = None + +def get_concurrency_limiter() -> Optional[OllamaConcurrencyLimiter]: + return _concurrency_limiter_instance + # ── Empty Response Retry ── MAX_EMPTY_RETRIES = 1 # How many times to retry on empty responses @@ -330,6 +359,19 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute _config = config _cache = CacheEngine(config.cache) if config else None + # Concurrency limiter: prevents 429 "too many concurrent requests" from Ollama Cloud + _max_concurrent = getattr(config.fallback, 'max_concurrent_ollama', 0) if config and config.fallback else 0 + _max_429_retries = getattr(config.fallback, 'max_429_retries', 2) if config and config.fallback else 2 + _backoff_base = getattr(config.fallback, 'backoff_base', 1.0) if config and config.fallback else 1.0 + _concurrency_limiter = OllamaConcurrencyLimiter( + max_concurrent=_max_concurrent, + max_429_retries=_max_429_retries, + base_backoff=_backoff_base, + ) + # Expose for dashboard API (module-level reference) + global _concurrency_limiter_instance + _concurrency_limiter_instance = _concurrency_limiter + def _history_kwargs(request: Request, messages=None, output_text=None) -> dict: """Build history-related kwargs for analytics.log_llm() based on config. @@ -535,7 +577,8 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute try: # ── Retry on empty response ── for attempt in range(MAX_EMPTY_RETRIES + 1): - resp = await _ollama_chat_with_primary_timeout(client, p, _config.fallback if _config else None) + async with _concurrency_limiter: + resp = await _ollama_chat_with_primary_timeout(client, p, _config.fallback if _config else None, _concurrency_limiter) if not _is_empty_non_streaming_response(resp) or attempt == MAX_EMPTY_RETRIES: break log.warning("Empty cached-response from %s (attempt %d/%d), retrying...", resolved_model, attempt + 1, MAX_EMPTY_RETRIES + 1) @@ -638,11 +681,12 @@ def create_router(client: OllamaClient, analytics=None, config=None) -> APIRoute # Try Ollama Cloud first try: if body.stream: - return await _stream_completion_openai(client, payload, resolved_model, _analytics, start, _config, history_kwargs=_hist) + return await _stream_completion_openai(client, payload, resolved_model, _analytics, start, _config, history_kwargs=_hist, limiter=_concurrency_limiter) # ── Non-streaming: retry on empty response ── for attempt in range(MAX_EMPTY_RETRIES + 1): - resp = await _ollama_chat_with_primary_timeout(client, payload, _config.fallback if _config else None) + async with _concurrency_limiter: + resp = await _ollama_chat_with_primary_timeout(client, payload, _config.fallback if _config else None, _concurrency_limiter) if not _is_empty_non_streaming_response(resp) or attempt == MAX_EMPTY_RETRIES: break log.warning("Empty response from %s (attempt %d/%d), retrying...", resolved_model, attempt + 1, MAX_EMPTY_RETRIES + 1) @@ -1217,7 +1261,7 @@ def _accumulate_history_output(accumulated: list, chunk: str, history_kwargs: di accumulated.append(text) -async def _stream_completion_openai(client, payload, model, analytics, start_time, config=None, history_kwargs=None): +async def _stream_completion_openai(client, payload, model, analytics, start_time, config=None, history_kwargs=None, limiter=None): """Stream OpenAI-format SSE responses, with fallback and timeout support. Key design: When fallback is configured with primary_timeout, we apply @@ -1242,33 +1286,73 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim if use_timeouts: chunk_timeout = fb.stream_chunk_timeout if fb.stream_chunk_timeout else 180.0 # ── Timed streaming: fail fast on first chunk, tolerate gaps after ── - ollama_stream = client.chat_completion_stream(payload) + # Acquire semaphore for the duration of the Ollama stream + sem_ctx = limiter.__aenter__() if limiter else None + if sem_ctx: + await sem_ctx + ollama_stream = None stream_closed = False # Wait for the first chunk with a strict timeout (triggers fallback fast) - try: - first_chunk = await asyncio.wait_for( - ollama_stream.__anext__(), timeout=fb.primary_timeout - ) - except asyncio.TimeoutError: - # First chunk timeout — no data sent to client yet, can still fallback + # Also retry on 429 with backoff + first_chunk = None + max_429 = limiter.max_429_retries if limiter else 0 + for attempt in range(max_429 + 1): try: - await ollama_stream.aclose() - except RuntimeError: - pass # Generator already cleaned up by cancellation - stream_closed = True - raise httpx.ReadTimeout( - f"Ollama did not produce first stream chunk within {fb.primary_timeout}s" - ) - except StopAsyncIteration: - # Empty stream — treat as error so fallback can handle it - try: - await ollama_stream.aclose() - except RuntimeError: - pass - stream_closed = True - raise httpx.ReadTimeout( - f"Ollama stream ended before producing any chunks" - ) + if ollama_stream is None: + ollama_stream = client.chat_completion_stream(payload) + first_chunk = await asyncio.wait_for( + ollama_stream.__anext__(), timeout=fb.primary_timeout + ) + break # Got a chunk, exit retry loop + except httpx.HTTPStatusError as e: + if attempt < max_429 and limiter and limiter.should_retry_429(e): + try: + await ollama_stream.aclose() + except (RuntimeError, Exception): + pass + ollama_stream = None + await limiter.backoff_and_retry(attempt) + continue + # Not a 429 or out of retries — release semaphore and re-raise + if sem_ctx: + try: + await limiter.__aexit__(None, None, None) + except Exception: + pass + sem_ctx = None + raise + except asyncio.TimeoutError: + # First chunk timeout — no data sent to client yet, can still fallback + try: + await ollama_stream.aclose() + except RuntimeError: + pass + stream_closed = True + if sem_ctx: + try: + await limiter.__aexit__(None, None, None) + except Exception: + pass + sem_ctx = None + raise httpx.ReadTimeout( + f"Ollama did not produce first stream chunk within {fb.primary_timeout}s" + ) + except StopAsyncIteration: + # Empty stream — treat as error so fallback can handle it + try: + await ollama_stream.aclose() + except RuntimeError: + pass + stream_closed = True + if sem_ctx: + try: + await limiter.__aexit__(None, None, None) + except Exception: + pass + sem_ctx = None + raise httpx.ReadTimeout( + f"Ollama stream ended before producing any chunks" + ) # Got first chunk — process it (metrics chunks are internal, not yield) if first_chunk.startswith("__oct_metrics__:"): @@ -1310,6 +1394,12 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim await ollama_stream.aclose() except RuntimeError: pass # Already closed + # Release concurrency semaphore + if sem_ctx and limiter: + try: + await limiter.__aexit__(None, None, None) + except Exception: + pass else: # ── No timeout wrapping: original buffered behavior ── for attempt in range(MAX_EMPTY_RETRIES + 1): From 3bc114204dc311fda7e4d122c060a37243ac4957 Mon Sep 17 00:00:00 2001 From: evangit2 Date: Mon, 20 Apr 2026 21:01:43 +0000 Subject: [PATCH 19/31] Bump version to 0.4.0-rc.1 --- guanaco/app.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/guanaco/app.py b/guanaco/app.py index 702cc20..a9c2b63 100644 --- a/guanaco/app.py +++ b/guanaco/app.py @@ -13,7 +13,7 @@ from fastapi.middleware.cors import CORSMiddleware from guanaco.config import load_config, get_config, AppConfig, get_base_url, get_tailscale_ip from guanaco.client import OllamaClient -__version__ = "0.3.9" +__version__ = "0.4.0-rc.1" from guanaco.router.router import create_router as create_llm_router from guanaco.search.providers import ALL_PROVIDERS from guanaco.dashboard import create_dashboard_router diff --git a/pyproject.toml b/pyproject.toml index fdfb210..5e4c13c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "guanaco" -version = "0.3.9" +version = "0.4.0rc1" description = "OpenAI-compatible LLM proxy that maximizes Ollama Cloud subscriptions — search/scrape API emulation, usage tracking, fallback provider support, and a web dashboard" readme = "README.md" license = {text = "MIT"} From 20eee78effacaa2f094b49183b0b875a94e8f55e Mon Sep 17 00:00:00 2001 From: evangit2 Date: Mon, 20 Apr 2026 21:30:05 +0000 Subject: [PATCH 20/31] fix: prerelease update check discovers newer pre-releases over stable --- guanaco/dashboard/dashboard.py | 36 ++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/guanaco/dashboard/dashboard.py b/guanaco/dashboard/dashboard.py index 151f719..3927d0b 100644 --- a/guanaco/dashboard/dashboard.py +++ b/guanaco/dashboard/dashboard.py @@ -787,14 +787,12 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg result = {"current_version": current_version, "latest_version": None, "update_available": False, "error": None} try: - # Get release info from GitHub API - # Default: only check stable releases (/releases/latest) - # If allow_prerelease is set in config, also check prereleases config = get_config() allow_prerelease = getattr(config.router, "allow_prerelease", False) import httpx async with httpx.AsyncClient(timeout=10) as hc: release_data = None + # Always try stable release first resp = await hc.get( "https://api.github.com/repos/evangit2/guanaco/releases/latest", @@ -802,17 +800,32 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg ) if resp.status_code == 200: release_data = resp.json() - elif allow_prerelease: - # No stable release found — check all releases including prereleases + + # If allow_prerelease, also check all releases and pick the newest version + if allow_prerelease: resp = await hc.get( "https://api.github.com/repos/evangit2/guanaco/releases", headers={"Accept": "application/vnd.github+json"} ) if resp.status_code == 200 and resp.json(): - release_data = resp.json()[0] # GitHub sorts newest-first + def _ver_tuple(release): + """Parse tag_name into comparable tuple, stripping '-rc.X' suffixes.""" + tag = release.get("tag_name", "").lstrip("v") + # Handle pre-release suffixes like "0.4.0-rc.1" + base = tag.split("-")[0] # "0.4.0" + try: + return tuple(int(x) for x in base.split(".")) + except (ValueError, TypeError): + return (0,) + all_releases = resp.json() + # Pick the release with the highest version (regardless of prerelease flag) + newest = max(all_releases, key=_ver_tuple) + # Compare: use prerelease if it's newer than the stable one + if release_data is None or _ver_tuple(newest) > _ver_tuple(release_data): + release_data = newest + if release_data: tag = release_data.get("tag_name", "") - # Strip leading 'v' if present result["latest_version"] = tag.lstrip("v") result["release_notes"] = release_data.get("body", "")[:500] result["release_url"] = release_data.get("html_url", "") @@ -823,10 +836,13 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg result["error"] = str(e) if result["latest_version"]: - # Compare versions (simple semver comparison) + # Compare versions (strip pre-release suffixes like -rc.1 for comparison) try: - current_parts = [int(x) for x in current_version.split(".")] - latest_parts = [int(x) for x in result["latest_version"].split(".")] + def _parse_ver(v): + base = v.split("-")[0] # "0.4.0-rc.1" -> "0.4.0" + return [int(x) for x in base.split(".")] + current_parts = _parse_ver(current_version) + latest_parts = _parse_ver(result["latest_version"]) # Pad to same length while len(current_parts) < len(latest_parts): current_parts.append(0) From ff99d56e9b0d11e250a41a8737c5c4e97e70446e Mon Sep 17 00:00:00 2001 From: evangit2 Date: Mon, 20 Apr 2026 21:32:54 +0000 Subject: [PATCH 21/31] release: v0.4.0 --- guanaco/app.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/guanaco/app.py b/guanaco/app.py index a9c2b63..9e6b475 100644 --- a/guanaco/app.py +++ b/guanaco/app.py @@ -13,7 +13,7 @@ from fastapi.middleware.cors import CORSMiddleware from guanaco.config import load_config, get_config, AppConfig, get_base_url, get_tailscale_ip from guanaco.client import OllamaClient -__version__ = "0.4.0-rc.1" +__version__ = "0.4.0" from guanaco.router.router import create_router as create_llm_router from guanaco.search.providers import ALL_PROVIDERS from guanaco.dashboard import create_dashboard_router diff --git a/pyproject.toml b/pyproject.toml index 5e4c13c..550bcfc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "guanaco" -version = "0.4.0rc1" +version = "0.4.0" description = "OpenAI-compatible LLM proxy that maximizes Ollama Cloud subscriptions — search/scrape API emulation, usage tracking, fallback provider support, and a web dashboard" readme = "README.md" license = {text = "MIT"} From 44f132b4a76a7dcbdde98a614265110f820e12bf Mon Sep 17 00:00:00 2001 From: evangit2 Date: Mon, 20 Apr 2026 22:07:53 +0000 Subject: [PATCH 22/31] v0.4.1: fix update banner after applying, fix history toggle reverting, fix version reporting - checkForUpdate() now runs on page load (not just System tab click), so stale 'update available' banners clear after refreshing - checkForUpdate() retries on network errors (service may be restarting) - saveHistoryConfig() now syncs UI state from server POST response, preventing toggle drift on tab switch - __init__.py version parsing uses strict semver regex, preventing RC/pre-release versions like '0.4.0rc1' from overriding the hardcoded fallback. Only clean X.Y.Z versions >= baseline are accepted. - Bumped version to 0.4.1 across __init__.py, app.py, pyproject.toml --- guanaco/__init__.py | 9 ++- guanaco/app.py | 2 +- guanaco/dashboard/templates/dashboard.html | 93 ++++++++++++++-------- pyproject.toml | 2 +- 4 files changed, 69 insertions(+), 37 deletions(-) diff --git a/guanaco/__init__.py b/guanaco/__init__.py index 949cf43..ed3c9d8 100644 --- a/guanaco/__init__.py +++ b/guanaco/__init__.py @@ -3,13 +3,16 @@ # Single source of truth for version. # importlib.metadata can return stale values after git-pull without re-pip-install, # so we always use the hardcoded fallback and only override if metadata matches. -__version__ = "0.3.9" +__version__ = "0.4.1" try: from importlib.metadata import version as _version _pkg_ver = _version("guanaco") - # Only use pkg version if it's >= our hardcoded version (prevents stale 0.3.0 overrides) - if _pkg_ver and tuple(int(x) for x in _pkg_ver.split(".") if x.isdigit()) >= (0, 3, 9): + # Only use pkg version if it parses as a clean semver >= our hardcoded baseline. + # This prevents stale/RC versions like "0.4.0rc1" from overriding the hardcoded value. + import re + _m = re.match(r"^(\d+)\.(\d+)\.(\d+)$", _pkg_ver or "") + if _m and tuple(int(x) for x in _m.groups()) >= (0, 4, 1): __version__ = _pkg_ver except Exception: pass \ No newline at end of file diff --git a/guanaco/app.py b/guanaco/app.py index 9e6b475..924fcf0 100644 --- a/guanaco/app.py +++ b/guanaco/app.py @@ -13,7 +13,7 @@ from fastapi.middleware.cors import CORSMiddleware from guanaco.config import load_config, get_config, AppConfig, get_base_url, get_tailscale_ip from guanaco.client import OllamaClient -__version__ = "0.4.0" +__version__ = "0.4.1" from guanaco.router.router import create_router as create_llm_router from guanaco.search.providers import ALL_PROVIDERS from guanaco.dashboard import create_dashboard_router diff --git a/guanaco/dashboard/templates/dashboard.html b/guanaco/dashboard/templates/dashboard.html index 9e3e271..8ed8dbf 100644 --- a/guanaco/dashboard/templates/dashboard.html +++ b/guanaco/dashboard/templates/dashboard.html @@ -1168,7 +1168,18 @@ function saveHistoryConfig() { method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify(data) - }).then(r => r.json()).then(() => { + }).then(r => r.json()).then(resp => { + // Sync UI from server response to prevent drift + if (resp.history) { + document.getElementById('history-enabled').checked = resp.history.enabled || false; + document.getElementById('history-save-input').checked = resp.history.save_input !== false; + document.getElementById('history-save-output').checked = resp.history.save_output !== false; + document.getElementById('history-log-to-files').checked = resp.history.log_to_files || false; + document.getElementById('history-log-dir').value = resp.history.log_dir || ''; + document.getElementById('history-retention').value = resp.history.retention_days || 30; + document.getElementById('history-logdir-row').style.display = resp.history.log_to_files ? 'flex' : 'none'; + document.getElementById('history-config-panel').style.display = resp.history.enabled ? 'block' : 'none'; + } // Reload stats and log files loadHistoryStats(); loadHistoryLogs(); @@ -1894,6 +1905,9 @@ document.addEventListener('DOMContentLoaded', () => { loadHistory(); loadHistoryLogs(); + // Check for updates (auto-check on page load) + checkForUpdate(); + // Check connection checkOllamaStatus(); }); @@ -2067,42 +2081,57 @@ async function checkForUpdate() { btn.textContent = 'Checking...'; statusEl.textContent = ''; - try { - const resp = await fetch('/dashboard/api/update/check'); - const data = await resp.json(); + // Helper to actually perform the check + async function doCheck(attempt = 0) { + try { + const resp = await fetch('/dashboard/api/update/check'); + const data = await resp.json(); - document.getElementById('auto-update-toggle').checked = data.auto_update || false; + document.getElementById('auto-update-toggle').checked = data.auto_update || false; - if (data.error) { - statusEl.textContent = 'Error: ' + data.error; + if (data.error) { + // If service not ready yet, retry a couple times + if (attempt < 2 && data.error.includes('503')) { + await new Promise(r => setTimeout(r, 1500)); + return doCheck(attempt + 1); + } + statusEl.textContent = 'Error: ' + data.error; + statusEl.style.color = '#f87171'; + box.classList.add('hidden'); + return; + } + + statusEl.textContent = `v${data.current_version} installed`; + statusEl.style.color = '#94a3b8'; + + if (data.update_available) { + document.getElementById('update-title').textContent = `Update available: v${data.current_version} → v${data.latest_version}`; + document.getElementById('update-title').style.color = '#4ade80'; + document.getElementById('update-notes').textContent = data.release_notes ? data.release_notes.substring(0, 200) : 'No release notes available.'; + document.getElementById('update-link').innerHTML = data.release_url ? `View on GitHub →` : ''; + box.classList.remove('hidden'); + statusEl.textContent = `v${data.latest_version} available`; + statusEl.style.color = '#4ade80'; + } else { + box.classList.add('hidden'); + statusEl.textContent = `Up to date (v${data.current_version})`; + statusEl.style.color = '#4ade80'; + } + } catch (e) { + // Network error - service might be restarting + if (attempt < 2) { + await new Promise(r => setTimeout(r, 1500)); + return doCheck(attempt + 1); + } + statusEl.textContent = 'Failed to check: ' + e.message; statusEl.style.color = '#f87171'; - box.classList.add('hidden'); - return; } - - statusEl.textContent = `v${data.current_version} installed`; - statusEl.style.color = '#94a3b8'; - - if (data.update_available) { - document.getElementById('update-title').textContent = `Update available: v${data.current_version} → v${data.latest_version}`; - document.getElementById('update-title').style.color = '#4ade80'; - document.getElementById('update-notes').textContent = data.release_notes ? data.release_notes.substring(0, 200) : 'No release notes available.'; - document.getElementById('update-link').innerHTML = data.release_url ? `View on GitHub →` : ''; - box.classList.remove('hidden'); - statusEl.textContent = `v${data.latest_version} available`; - statusEl.style.color = '#4ade80'; - } else { - box.classList.add('hidden'); - statusEl.textContent = `Up to date (v${data.current_version})`; - statusEl.style.color = '#4ade80'; - } - } catch (e) { - statusEl.textContent = 'Failed to check: ' + e.message; - statusEl.style.color = '#f87171'; - } finally { - btn.disabled = false; - btn.textContent = 'Check for Update'; } + + await doCheck(); + + btn.disabled = false; + btn.textContent = 'Check for Update'; } async function applyUpdate() { diff --git a/pyproject.toml b/pyproject.toml index 550bcfc..8ee68f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "guanaco" -version = "0.4.0" +version = "0.4.1" description = "OpenAI-compatible LLM proxy that maximizes Ollama Cloud subscriptions — search/scrape API emulation, usage tracking, fallback provider support, and a web dashboard" readme = "README.md" license = {text = "MIT"} From c9f30161e2c2518b5033c2ee4e3c587c278cda5c Mon Sep 17 00:00:00 2001 From: evangit2 Date: Mon, 20 Apr 2026 23:49:57 +0000 Subject: [PATCH 23/31] feat: add OllamaAccount model and ollama_accounts list to AppConfig --- guanaco/config.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/guanaco/config.py b/guanaco/config.py index 2e932ac..257e19c 100644 --- a/guanaco/config.py +++ b/guanaco/config.py @@ -153,8 +153,23 @@ class UsageConfig(BaseModel): redirect_on_full: bool = False # Route all requests to fallback when quota is near limit +class OllamaAccount(BaseModel): + """A single Ollama Cloud account with its own API key and usage tracking.""" + name: str # Display name (unique, "ollama" is reserved for primary) + api_key: str = "" # API key for this account + session_cookie: str = "" # __Secure-session cookie for usage scraping + # Usage tracking (updated by background check) + last_session_pct: Optional[float] = None + last_weekly_pct: Optional[float] = None + last_plan: Optional[str] = None + last_session_reset: Optional[str] = None + last_weekly_reset: Optional[str] = None + last_checked: Optional[float] = None + + class AppConfig(BaseModel): ollama_api_key: str = "" + ollama_accounts: list[OllamaAccount] = Field(default_factory=list) router: RouterConfig = Field(default_factory=RouterConfig) llm: LLMConfig = Field(default_factory=LLMConfig) fallback: FallbackProviderConfig = Field(default_factory=FallbackProviderConfig) @@ -169,6 +184,20 @@ class AppConfig(BaseModel): """Resolve API key from config or environment.""" return self.ollama_api_key or os.environ.get("OLLAMA_API_KEY", "") + @property + def primary_account(self) -> "OllamaAccount": + """Get or create the primary 'ollama' account.""" + for acc in self.ollama_accounts: + if acc.name == "ollama": + return acc + # Auto-create from legacy single-key config + return OllamaAccount(name="ollama", api_key=self.ollama_api_key) + + @property + def active_accounts(self) -> list["OllamaAccount"]: + """All accounts that have a non-empty API key.""" + return [a for a in self.ollama_accounts if a.api_key and a.api_key not in ("***", "REPLACE_ME")] + _config: Optional[AppConfig] = None From cb163c0a990e7d862ea85f954efd138af3e4e910 Mon Sep 17 00:00:00 2001 From: evangit2 Date: Mon, 20 Apr 2026 23:50:44 +0000 Subject: [PATCH 24/31] feat: add AccountPool with quota-aware selection and round-robin fallback --- guanaco/accounts.py | 111 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 guanaco/accounts.py diff --git a/guanaco/accounts.py b/guanaco/accounts.py new file mode 100644 index 0000000..2b39136 --- /dev/null +++ b/guanaco/accounts.py @@ -0,0 +1,111 @@ +"""Multi-account Ollama key rotation with quota-aware selection.""" + +import logging +import time +from typing import Optional + +from guanaco.config import OllamaAccount + +logger = logging.getLogger(__name__) + + +class AccountPool: + """Manages a pool of Ollama accounts and selects the best one for each request. + + Selection strategy: + 1. Among accounts with usage data, pick the one with the lowest session_pct + 2. Among accounts without usage data, round-robin + 3. If only one account, always use it + """ + + def __init__(self, accounts: list[OllamaAccount]): + self._accounts: list[OllamaAccount] = accounts + self._rr_index: int = 0 + + @property + def accounts(self) -> list[OllamaAccount]: + return self._accounts + + def update_accounts(self, accounts: list[OllamaAccount]) -> None: + """Replace the account list (e.g., after config save).""" + self._accounts = accounts + + def get_account(self, preferred: Optional[str] = None) -> OllamaAccount: + """Select the best account for the next request. + + Args: + preferred: If set, try this account name first. + + Returns: + The selected OllamaAccount. + """ + active = [a for a in self._accounts if a.api_key and a.api_key not in ("***", "REPLACE_ME")] + if not active: + return self._accounts[0] if self._accounts else OllamaAccount(name="ollama") + + if len(active) == 1: + return active[0] + + if preferred: + for a in active: + if a.name == preferred: + return a + + with_usage = [a for a in active if a.last_session_pct is not None] + without_usage = [a for a in active if a.last_session_pct is None] + + if with_usage: + best = min(with_usage, key=lambda a: a.last_session_pct or 100) + logger.debug(f"Selected account '{best.name}' (session: {best.last_session_pct}%)") + return best + + if without_usage: + idx = self._rr_index % len(without_usage) + self._rr_index += 1 + return without_usage[idx] + + return active[0] + + def update_usage(self, account_name: str, session_pct: Optional[float], + weekly_pct: Optional[float], plan: Optional[str] = None, + session_reset: Optional[str] = None, weekly_reset: Optional[str] = None) -> None: + """Update usage data for a specific account.""" + for a in self._accounts: + if a.name == account_name: + a.last_session_pct = session_pct + a.last_weekly_pct = weekly_pct + a.last_plan = plan + a.last_session_reset = session_reset + a.last_weekly_reset = weekly_reset + a.last_checked = time.time() + break + + def mark_429(self, account_name: str) -> None: + """Mark that an account hit a 429. Temporarily deprioritize it.""" + for a in self._accounts: + if a.name == account_name: + if a.last_session_pct is not None: + a.last_session_pct = min(a.last_session_pct + 25, 100) + else: + a.last_session_pct = 75 + logger.info(f"Account '{account_name}' hit 429, deprioritizing (session est: {a.last_session_pct}%)") + break + + def account_names(self) -> list[str]: + """List all account names.""" + return [a.name for a in self._accounts] + + def get_by_name(self, name: str) -> Optional[OllamaAccount]: + """Find account by name.""" + for a in self._accounts: + if a.name == name: + return a + return None + + def name_taken(self, name: str) -> bool: + """Check if an account name is already used.""" + return any(a.name == name for a in self._accounts) + + def is_reserved_name(self, name: str) -> bool: + """Check if a name is reserved (case-insensitive).""" + return name.lower() in ("ollama", "primary", "default") \ No newline at end of file From c7f71f9414fecb12ad2ef7bdf2516c2f0d12a3e0 Mon Sep 17 00:00:00 2001 From: evangit2 Date: Mon, 20 Apr 2026 23:58:40 +0000 Subject: [PATCH 25/31] feat: OllamaClient supports per-request api_key override for multi-account --- guanaco/client.py | 244 ++++++++++++++++++++++++++++------------------ 1 file changed, 150 insertions(+), 94 deletions(-) diff --git a/guanaco/client.py b/guanaco/client.py index d12169a..4f3660f 100644 --- a/guanaco/client.py +++ b/guanaco/client.py @@ -68,10 +68,26 @@ class OllamaClient: self._models_cache_time: float = 0 self._models_cache_ttl: float = 300.0 # 5 minutes - async def _get_client(self) -> httpx.AsyncClient: + async def _get_client(self, api_key_override: Optional[str] = None) -> httpx.AsyncClient: + """Get or create the httpx client, optionally with a different API key. + + When api_key_override is provided and differs from the default key, + creates a temporary client that must be closed by the caller. + Returns (client, is_temp) tuple so callers know whether to close it. + """ + key = api_key_override or self.api_key + use_temp = api_key_override is not None and api_key_override != self.api_key + + if use_temp: + # Per-request key override — create a temporary client + headers = {"Content-Type": "application/json"} + if key and key not in ("***", "REPLACE_ME", "your_api_key_here"): + headers["Authorization"] = f"Bearer {key}" + return httpx.AsyncClient(timeout=self.timeout, headers=headers) + + # Default behavior — cached singleton client if self._client is None or self._client.is_closed: headers = {"Content-Type": "application/json"} - # Only send Authorization if we have a real API key (not empty, placeholder, or masked) if self.api_key and self.api_key not in ("***", "REPLACE_ME", "your_api_key_here"): headers["Authorization"] = f"Bearer {self.api_key}" self._client = httpx.AsyncClient( @@ -82,35 +98,46 @@ class OllamaClient: # ── Search & Fetch ── - async def search(self, query: str, max_results: int = 10) -> dict: + async def search(self, query: str, max_results: int = 10, api_key: Optional[str] = None) -> dict: """Search the web using Ollama's web_search API.""" - client = await self._get_client() - payload = {"query": query, "max_results": max(min(max_results, 10), 1)} - resp = await client.post(OLLAMA_SEARCH_URL, json=payload) - resp.raise_for_status() - return resp.json() + client = await self._get_client(api_key_override=api_key) + is_temp = api_key is not None and api_key != self.api_key + try: + payload = {"query": query, "max_results": max(min(max_results, 10), 1)} + resp = await client.post(OLLAMA_SEARCH_URL, json=payload) + resp.raise_for_status() + return resp.json() + finally: + if is_temp and not client.is_closed: + await client.aclose() - async def fetch(self, url: str) -> dict: + async def fetch(self, url: str, api_key: Optional[str] = None) -> dict: """Fetch/scrape a URL using Ollama's web_fetch API.""" - client = await self._get_client() - payload = {"url": url} - resp = await client.post(OLLAMA_FETCH_URL, json=payload) - resp.raise_for_status() - return resp.json() + client = await self._get_client(api_key_override=api_key) + is_temp = api_key is not None and api_key != self.api_key + try: + payload = {"url": url} + resp = await client.post(OLLAMA_FETCH_URL, json=payload) + resp.raise_for_status() + return resp.json() + finally: + if is_temp and not client.is_closed: + await client.aclose() # ── Models ── - async def list_models(self, force_refresh: bool = False) -> list[dict]: + async def list_models(self, force_refresh: bool = False, api_key: Optional[str] = None) -> list[dict]: """List available Ollama Cloud models, with caching. Uses the OpenAI-compatible /v1/models endpoint which returns model IDs in standard format (e.g. 'gemma4:31b', 'qwen3.5:397b'). """ now = time.time() - if not force_refresh and self._models_cache and (now - self._models_cache_time) < self._models_cache_ttl: + if not force_refresh and not api_key and self._models_cache and (now - self._models_cache_time) < self._models_cache_ttl: return self._models_cache - client = await self._get_client() + client = await self._get_client(api_key_override=api_key) + is_temp = api_key is not None and api_key != self.api_key try: resp = await client.get(OLLAMA_MODELS_URL) if resp.status_code == 401: @@ -143,6 +170,9 @@ class OllamaClient: except Exception as e: logger.error(f"Error fetching models: {e}") raise + finally: + if is_temp and not client.is_closed: + await client.aclose() async def check_model_available(self, model_name: str) -> bool: """Check if a specific model is available on Ollama Cloud.""" @@ -322,11 +352,16 @@ class OllamaClient: # ── Chat Completions ── - async def chat_completion(self, payload: dict) -> dict: + async def chat_completion(self, payload: dict, api_key: Optional[str] = None) -> dict: """Send a chat completion to Ollama Cloud (OpenAI-compatible format).""" - client = await self._get_client() + client = await self._get_client(api_key_override=api_key) + is_temp = api_key is not None and api_key != self.api_key start = time.time() - resp = await client.post(OLLAMA_CHAT_URL, json=payload) + try: + resp = await client.post(OLLAMA_CHAT_URL, json=payload) + finally: + if is_temp and not client.is_closed: + await client.aclose() elapsed = time.time() - start resp.raise_for_status() data = resp.json() @@ -371,9 +406,10 @@ class OllamaClient: data["_oct_metrics"] = metrics return data - async def chat_completion_stream(self, payload: dict): + async def chat_completion_stream(self, payload: dict, api_key: Optional[str] = None): """Stream chat completion responses from Ollama Cloud, yielding metrics via _oct_stream_metrics.""" - client = await self._get_client() + client = await self._get_client(api_key_override=api_key) + is_temp = api_key is not None and api_key != self.api_key payload_copy = dict(payload) payload_copy["stream"] = True @@ -384,79 +420,99 @@ class OllamaClient: completion_tokens = 0 # from usage data if available start = time.time() - async with client.stream("POST", OLLAMA_CHAT_URL, json=payload_copy) as resp: - resp.raise_for_status() - async for line in resp.aiter_lines(): - if line.startswith("data: "): - data = line[6:] - if data.strip() == "[DONE]": - # Estimate tokens from character count (4 chars ≈ 1 token) - estimated_content_tokens = max(1, content_chars // 4) if content_chars else 0 - estimated_reasoning_tokens = max(1, reasoning_chars // 4) if reasoning_chars else 0 - # Use API-provided completion_tokens if available, otherwise estimated content tokens - final_tokens = completion_tokens or estimated_content_tokens - elapsed = time.time() - start - ttft = (first_token_time - start) if first_token_time else None - generation_time = (elapsed - ttft) if ttft and elapsed > ttft else elapsed + try: + async with client.stream("POST", OLLAMA_CHAT_URL, json=payload_copy) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + if line.startswith("data: "): + data = line[6:] + if data.strip() == "[DONE]": + # Estimate tokens from character count (4 chars ≈ 1 token) + estimated_content_tokens = max(1, content_chars // 4) if content_chars else 0 + estimated_reasoning_tokens = max(1, reasoning_chars // 4) if reasoning_chars else 0 + # Use API-provided completion_tokens if available, otherwise estimated content tokens + final_tokens = completion_tokens or estimated_content_tokens + elapsed = time.time() - start + ttft = (first_token_time - start) if first_token_time else None + generation_time = (elapsed - ttft) if ttft and elapsed > ttft else elapsed - metrics = { - "eval_count": final_tokens, - "prompt_eval_count": prompt_tokens, - "reasoning_tokens": estimated_reasoning_tokens, - "elapsed_seconds": round(elapsed, 3), - "ttft_seconds": round(ttft, 3) if ttft else None, - } - if final_tokens and generation_time > 0: - metrics["tps"] = round(final_tokens / generation_time, 2) - if prompt_tokens: - metrics["prompt_eval_count"] = prompt_tokens - yield f"data: [DONE]\n\n" - # Store metrics on the response for analytics - yield f"__oct_metrics__:{json.dumps(metrics)}\n\n" - return # Exit generator, don't yield another [DONE] - try: - chunk_data = json.loads(data) - # Accumulate character counts for token estimation - for choice in chunk_data.get("choices", []): - delta = choice.get("delta", {}) - content = delta.get("content", "") - reasoning = delta.get("reasoning", "") or delta.get("reasoning_content", "") - if content or reasoning: - if first_token_time is None: - first_token_time = time.time() - content_chars += len(content) - reasoning_chars += len(reasoning) - # Capture usage data from final streaming chunk (Ollama/OpenAI format) - usage = chunk_data.get("usage") - if usage: - if usage.get("prompt_tokens"): - prompt_tokens = usage["prompt_tokens"] - if usage.get("completion_tokens"): - completion_tokens = usage["completion_tokens"] - except (json.JSONDecodeError, KeyError): - pass - yield f"data: {data}\n\n" - elif line.strip(): - yield f"data: {line}\n\n" - # If we get here without seeing [DONE], the stream ended unexpectedly - # Estimate tokens and yield [DONE] + metrics anyway - estimated_content_tokens = max(1, content_chars // 4) if content_chars else 0 - estimated_reasoning_tokens = max(1, reasoning_chars // 4) if reasoning_chars else 0 - final_tokens = completion_tokens or estimated_content_tokens - elapsed = time.time() - start - ttft = (first_token_time - start) if first_token_time else None - generation_time = (elapsed - ttft) if ttft and elapsed > ttft else elapsed - metrics = { - "eval_count": final_tokens, - "prompt_eval_count": prompt_tokens, - "reasoning_tokens": estimated_reasoning_tokens, - "elapsed_seconds": round(elapsed, 3), - "ttft_seconds": round(ttft, 3) if ttft else None, - } - if final_tokens and generation_time > 0: - metrics["tps"] = round(final_tokens / generation_time, 2) - yield "data: [DONE]\n\n" - yield f"__oct_metrics__:{json.dumps(metrics)}\n\n" + metrics = { + "eval_count": final_tokens, + "prompt_eval_count": prompt_tokens, + "reasoning_tokens": estimated_reasoning_tokens, + "elapsed_seconds": round(elapsed, 3), + "ttft_seconds": round(ttft, 3) if ttft else None, + } + if final_tokens and generation_time > 0: + metrics["tps"] = round(final_tokens / generation_time, 2) + if prompt_tokens: + metrics["prompt_eval_count"] = prompt_tokens + yield f"data: [DONE]\n\n" + # Store metrics on the response for analytics + yield f"__oct_metrics__:{json.dumps(metrics)}\n\n" + return # Exit generator, don't yield another [DONE] + try: + chunk_data = json.loads(data) + # Accumulate character counts for token estimation + for choice in chunk_data.get("choices", []): + delta = choice.get("delta", {}) + content = delta.get("content", "") + reasoning = delta.get("reasoning", "") or delta.get("reasoning_content", "") + if content or reasoning: + if first_token_time is None: + first_token_time = time.time() + content_chars += len(content) + reasoning_chars += len(reasoning) + # Capture usage data from final streaming chunk (Ollama/OpenAI format) + usage = chunk_data.get("usage") + if usage: + if usage.get("prompt_tokens"): + prompt_tokens = usage["prompt_tokens"] + if usage.get("completion_tokens"): + completion_tokens = usage["completion_tokens"] + except (json.JSONDecodeError, KeyError): + pass + yield f"data: {data}\n\n" + elif line.strip(): + yield f"data: {line}\n\n" + # If we get here without seeing [DONE], the stream ended unexpectedly + # Estimate tokens and yield [DONE] + metrics anyway + estimated_content_tokens = max(1, content_chars // 4) if content_chars else 0 + estimated_reasoning_tokens = max(1, reasoning_chars // 4) if reasoning_chars else 0 + final_tokens = completion_tokens or estimated_content_tokens + elapsed = time.time() - start + ttft = (first_token_time - start) if first_token_time else None + generation_time = (elapsed - ttft) if ttft and elapsed > ttft else elapsed + metrics = { + "eval_count": final_tokens, + "prompt_eval_count": prompt_tokens, + "reasoning_tokens": estimated_reasoning_tokens, + "elapsed_seconds": round(elapsed, 3), + "ttft_seconds": round(ttft, 3) if ttft else None, + } + if final_tokens and generation_time > 0: + metrics["tps"] = round(final_tokens / generation_time, 2) + yield "data: [DONE]\n\n" + yield f"__oct_metrics__:{json.dumps(metrics)}\n\n" + finally: + if is_temp and not client.is_closed: + await client.aclose() + + async def test_key(self, test_api_key: str) -> dict: + """Test if an Ollama API key works by listing models. Returns {ok: bool, error: str|None}.""" + client = await self._get_client(api_key_override=test_api_key) + try: + resp = await client.get(OLLAMA_MODELS_URL, timeout=10) + if resp.status_code == 200: + data = resp.json() + model_count = len(data.get("data", data.get("models", []))) + return {"ok": True, "error": None, "model_count": model_count} + return {"ok": False, "error": f"HTTP {resp.status_code}"} + except Exception as e: + return {"ok": False, "error": str(e)} + finally: + if not client.is_closed: + await client.aclose() async def close(self): if self._client and not self._client.is_closed: From 95da493c54d5edf856d0cd68193660f9c7b7d910 Mon Sep 17 00:00:00 2001 From: evangit2 Date: Tue, 21 Apr 2026 00:09:06 +0000 Subject: [PATCH 26/31] feat: wire AccountPool into app.py, router.py, and dashboard.py signatures --- guanaco/app.py | 15 +++++++++++++-- guanaco/dashboard/dashboard.py | 5 +++-- guanaco/router/router.py | 3 ++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/guanaco/app.py b/guanaco/app.py index 924fcf0..2dbeff0 100644 --- a/guanaco/app.py +++ b/guanaco/app.py @@ -13,6 +13,7 @@ from fastapi.middleware.cors import CORSMiddleware from guanaco.config import load_config, get_config, AppConfig, get_base_url, get_tailscale_ip from guanaco.client import OllamaClient +from guanaco.accounts import AccountPool __version__ = "0.4.1" from guanaco.router.router import create_router as create_llm_router from guanaco.search.providers import ALL_PROVIDERS @@ -32,6 +33,13 @@ def create_app(config: AppConfig | None = None) -> FastAPI: client = OllamaClient(api_key=resolved_key, session_cookie=config.usage.session_cookie) + # Ensure primary account exists in the accounts list + if not config.ollama_accounts: + config.ollama_accounts = [config.primary_account] + elif not any(a.name == "ollama" for a in config.ollama_accounts): + config.ollama_accounts.insert(0, config.primary_account) + account_pool = AccountPool(config.ollama_accounts) + from guanaco.config import get_default_config_dir key_manager = ApiKeyManager(get_default_config_dir()) analytics = AnalyticsLogger() @@ -129,7 +137,7 @@ def create_app(config: AppConfig | None = None) -> FastAPI: return {"status": "ok", "version": __version__} # ── LLM Router ── - app.include_router(create_llm_router(client, analytics=analytics, config=config)) + app.include_router(create_llm_router(client, analytics=analytics, config=config, account_pool=account_pool)) # ── Search Providers ── for provider_cls in ALL_PROVIDERS: @@ -259,7 +267,10 @@ def create_app(config: AppConfig | None = None) -> FastAPI: print(f" [WARN] Firecrawl SDK compat routes not loaded: {e}") # ── Dashboard ── - app.include_router(create_dashboard_router(key_manager, analytics, client), prefix="/dashboard") + app.include_router(create_dashboard_router(key_manager, analytics, client, account_pool=account_pool), prefix="/dashboard") + + # Store account_pool on app state for dashboard access + app.state.account_pool = account_pool # ── Ollama status & models (top-level API) ── diff --git a/guanaco/dashboard/dashboard.py b/guanaco/dashboard/dashboard.py index 3927d0b..51ea5bc 100644 --- a/guanaco/dashboard/dashboard.py +++ b/guanaco/dashboard/dashboard.py @@ -11,7 +11,7 @@ from fastapi import APIRouter, Request, BackgroundTasks from fastapi.responses import HTMLResponse, FileResponse import httpx -from guanaco.config import get_config, get_base_url, get_tailscale_ip, save_config, load_config +from guanaco.config import get_config, get_base_url, get_tailscale_ip, save_config, load_config, OllamaAccount from guanaco.utils.api_keys import ApiKeyManager from guanaco.analytics import AnalyticsLogger from guanaco.client import OllamaClient @@ -60,8 +60,9 @@ WantedBy=multi-user.target """ -def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogger, client=None) -> APIRouter: +def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogger, client=None, account_pool=None) -> APIRouter: router = APIRouter(tags=["Dashboard"]) + _account_pool = account_pool @router.get("/logo.png") async def logo(): diff --git a/guanaco/router/router.py b/guanaco/router/router.py index 4467688..a0072b4 100644 --- a/guanaco/router/router.py +++ b/guanaco/router/router.py @@ -353,10 +353,11 @@ async def _call_fallback_provider(payload: dict, fallback_config, stream: bool = # ── Provider creation ── -def create_router(client: OllamaClient, analytics=None, config=None) -> APIRouter: +def create_router(client: OllamaClient, analytics=None, config=None, account_pool=None) -> APIRouter: router = APIRouter(tags=["LLM Router"]) _analytics = analytics _config = config + _account_pool = account_pool _cache = CacheEngine(config.cache) if config else None # Concurrency limiter: prevents 429 "too many concurrent requests" from Ollama Cloud From c6741f92f326624876d3939f592e33d3d537a725 Mon Sep 17 00:00:00 2001 From: evangit2 Date: Tue, 21 Apr 2026 02:05:56 +0000 Subject: [PATCH 27/31] v0.4.2: Multi-account Ollama rotation with quota-aware load balancing Features: - Multi-account rotation: Add multiple Ollama Cloud accounts with API keys and session cookies via Dashboard Accounts tab or API - Quota-aware selection: AccountPool picker prefers least-loaded accounts; new/untested accounts get priority so they're validated immediately - Premium model routing: kimi-k2.6 and glm-5.1 restricted to Pro/Max accounts only; free-tier accounts show warning badge in dashboard - Account-level usage tracking: Per-account session/weekly percentages, plan type, and reset timers displayed in Dashboard Accounts tab - Analytics account_name: New DB column logs which account handled each request; provider field also includes account suffix (ollama:ollama2) - Dashboard Accounts tab: Full CRUD (add/remove/update-key/set-cookie/ check-usage/test), usage progress bars, plan badges, premium warnings - Config backward compatibility: Old configs without ollama_accounts auto-generate the primary account from legacy ollama_api_key + usage fields; no data loss on save/restart cycles APIs: - GET /api/accounts - List all accounts with usage data - POST /api/accounts/add - Add new account - POST /api/accounts/remove - Remove account by name - POST /api/accounts/test - Test account connectivity - POST /api/accounts/update-key - Update API key - POST /api/accounts/session-cookie - Set session cookie - POST /api/accounts/check-usage - Refresh usage from Ollama settings DB Migration: - account_name TEXT column added to request_log (auto-migrated on startup) All existing rows get NULL for account_name (no data loss) Files changed: - guanaco/accounts.py: AccountPool with model-aware filtering, PREMIUM_MODELS - guanaco/analytics.py: account_name param in log_llm(), migration - guanaco/app.py: AccountPool wiring into create_app() - guanaco/config.py: ollama_accounts list, legacy sync, primary_account property - guanaco/dashboard/dashboard.py: Account management API endpoints - guanaco/dashboard/templates/dashboard.html: Accounts tab UI - guanaco/router/router.py: _select_account(), provider=logging with account - pyproject.toml: version 0.4.2 --- guanaco/__init__.py | 2 +- guanaco/accounts.py | 53 ++++- guanaco/analytics.py | 12 +- guanaco/app.py | 2 +- guanaco/config.py | 29 ++- guanaco/dashboard/dashboard.py | 264 +++++++++++++++++++++ guanaco/dashboard/templates/dashboard.html | 247 ++++++++++++++++++- guanaco/router/router.py | 52 +++- pyproject.toml | 2 +- 9 files changed, 631 insertions(+), 32 deletions(-) diff --git a/guanaco/__init__.py b/guanaco/__init__.py index ed3c9d8..ee3f884 100644 --- a/guanaco/__init__.py +++ b/guanaco/__init__.py @@ -3,7 +3,7 @@ # Single source of truth for version. # importlib.metadata can return stale values after git-pull without re-pip-install, # so we always use the hardcoded fallback and only override if metadata matches. -__version__ = "0.4.1" +__version__ = "0.4.2-dev" try: from importlib.metadata import version as _version diff --git a/guanaco/accounts.py b/guanaco/accounts.py index 2b39136..f909ab1 100644 --- a/guanaco/accounts.py +++ b/guanaco/accounts.py @@ -8,6 +8,22 @@ from guanaco.config import OllamaAccount logger = logging.getLogger(__name__) +# Models that require a paid Ollama plan (not available on free tier). +PREMIUM_MODELS = {"kimi-k2.6", "glm-5.1"} + + +def model_requires_premium(model: str) -> bool: + """Check if a model requires a paid Ollama plan (pro/max). + + Matches case-insensitively against model name substrings. + E.g. 'kimi-k2.6-0915' matches 'k2.6'. + """ + model_lower = model.lower().strip() + for pm in PREMIUM_MODELS: + if pm in model_lower: + return True + return False + class AccountPool: """Manages a pool of Ollama accounts and selects the best one for each request. @@ -30,11 +46,18 @@ class AccountPool: """Replace the account list (e.g., after config save).""" self._accounts = accounts - def get_account(self, preferred: Optional[str] = None) -> OllamaAccount: + def get_account(self, preferred: Optional[str] = None, model: Optional[str] = None) -> OllamaAccount: """Select the best account for the next request. + Strategy: prefer accounts with the most available quota. + - If model is premium, skip free-tier accounts. + - Accounts with no usage data are assumed fresh (0%) — preferred. + - Among accounts with usage data, pick lowest session_pct. + - Tie-break with round-robin for equal-priority accounts. + Args: preferred: If set, try this account name first. + model: If set, check if model requires premium and filter accordingly. Returns: The selected OllamaAccount. @@ -43,6 +66,16 @@ class AccountPool: if not active: return self._accounts[0] if self._accounts else OllamaAccount(name="ollama") + # If model requires premium, filter out free-tier accounts + premium_needed = model and model_requires_premium(model) + if premium_needed: + eligible = [a for a in active if a.last_plan and a.last_plan.lower() != "free"] + if eligible: + active = eligible + logger.info(f"Model '{model}' requires premium plan, filtered to {len(eligible)} eligible accounts") + else: + logger.warning(f"Model '{model}' requires premium but no paid accounts available, trying all") + if len(active) == 1: return active[0] @@ -51,20 +84,22 @@ class AccountPool: if a.name == preferred: return a - with_usage = [a for a in active if a.last_session_pct is not None] + # Split by whether we have usage data without_usage = [a for a in active if a.last_session_pct is None] + with_usage = [a for a in active if a.last_session_pct is not None] - if with_usage: - best = min(with_usage, key=lambda a: a.last_session_pct or 100) - logger.debug(f"Selected account '{best.name}' (session: {best.last_session_pct}%)") - return best - + # Prefer accounts with no usage data (fresh/unknown quota) over known-heavy ones if without_usage: idx = self._rr_index % len(without_usage) self._rr_index += 1 - return without_usage[idx] + account = without_usage[idx] + logger.debug(f"Selected account '{account.name}' (no usage data, round-robin)") + return account - return active[0] + # All have usage data — pick the one with lowest usage + best = min(with_usage, key=lambda a: a.last_session_pct or 100) + logger.debug(f"Selected account '{best.name}' (session: {best.last_session_pct}%)") + return best def update_usage(self, account_name: str, session_pct: Optional[float], weekly_pct: Optional[float], plan: Optional[str] = None, diff --git a/guanaco/analytics.py b/guanaco/analytics.py index 2160ae1..8fb6212 100644 --- a/guanaco/analytics.py +++ b/guanaco/analytics.py @@ -83,6 +83,11 @@ class AnalyticsLogger: conn.execute("ALTER TABLE request_log ADD COLUMN fallback_reason TEXT") except sqlite3.OperationalError: pass + # Migration: add account_name column for multi-account rotation + try: + conn.execute("ALTER TABLE request_log ADD COLUMN account_name TEXT") + except sqlite3.OperationalError: + pass conn.execute(""" CREATE TABLE IF NOT EXISTS status_events ( id TEXT PRIMARY KEY, @@ -148,6 +153,7 @@ class AnalyticsLogger: input_text: Optional[str] = None, output_text: Optional[str] = None, fallback_reason: Optional[str] = None, + account_name: Optional[str] = None, ) -> str: """Log an LLM request. Returns the log entry ID.""" # Normalize model name so glm-5.1:cloud and glm-5.1 are grouped together @@ -160,12 +166,12 @@ class AnalyticsLogger: (id, ts, type, model, prompt_tokens, completion_tokens, total_tokens, tps, prompt_tps, ttft_seconds, total_duration_seconds, load_duration_seconds, error, request_id, provider, fallback_for, - source_ip, source_port, user_agent, input_text, output_text, fallback_reason) - VALUES (?, ?, 'llm', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + source_ip, source_port, user_agent, input_text, output_text, fallback_reason, account_name) + VALUES (?, ?, 'llm', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (entry_id, time.time(), model, prompt_tokens, completion_tokens, total_tokens, tps, prompt_tps, ttft_seconds, total_duration_seconds, load_duration_seconds, error, request_id, provider, fallback_for, - source_ip, source_port, user_agent, input_text, output_text, fallback_reason), + source_ip, source_port, user_agent, input_text, output_text, fallback_reason, account_name), ) # Write plaintext log file if configured diff --git a/guanaco/app.py b/guanaco/app.py index 2dbeff0..117876f 100644 --- a/guanaco/app.py +++ b/guanaco/app.py @@ -14,7 +14,7 @@ from fastapi.middleware.cors import CORSMiddleware from guanaco.config import load_config, get_config, AppConfig, get_base_url, get_tailscale_ip from guanaco.client import OllamaClient from guanaco.accounts import AccountPool -__version__ = "0.4.1" +__version__ = "0.4.2" from guanaco.router.router import create_router as create_llm_router from guanaco.search.providers import ALL_PROVIDERS from guanaco.dashboard import create_dashboard_router diff --git a/guanaco/config.py b/guanaco/config.py index 257e19c..c395af4 100644 --- a/guanaco/config.py +++ b/guanaco/config.py @@ -190,8 +190,18 @@ class AppConfig(BaseModel): for acc in self.ollama_accounts: if acc.name == "ollama": return acc - # Auto-create from legacy single-key config - return OllamaAccount(name="ollama", api_key=self.ollama_api_key) + # Auto-create from legacy single-key config, merging usage cookie/data + return OllamaAccount( + name="ollama", + api_key=self.ollama_api_key, + session_cookie=self.usage.session_cookie if hasattr(self, 'usage') else "", + last_session_pct=self.usage.last_session_pct if hasattr(self, 'usage') else None, + last_weekly_pct=self.usage.last_weekly_pct if hasattr(self, 'usage') else None, + last_plan=self.usage.last_plan if hasattr(self, 'usage') else None, + last_session_reset=self.usage.last_session_reset if hasattr(self, 'usage') else None, + last_weekly_reset=self.usage.last_weekly_reset if hasattr(self, 'usage') else None, + last_checked=self.usage.last_checked if hasattr(self, 'usage') else None, + ) @property def active_accounts(self) -> list["OllamaAccount"]: @@ -214,6 +224,21 @@ def load_config(path: Optional[Path] = None) -> AppConfig: else: _config = AppConfig() + # Ensure the primary "ollama" account is always in the accounts list + if not any(a.name == "ollama" for a in _config.ollama_accounts): + # Create primary from the legacy single-key config + usage data + _config.ollama_accounts.insert(0, OllamaAccount( + name="ollama", + api_key=_config.ollama_api_key, + session_cookie=_config.usage.session_cookie if hasattr(_config, 'usage') else "", + last_session_pct=_config.usage.last_session_pct if hasattr(_config, 'usage') else None, + last_weekly_pct=_config.usage.last_weekly_pct if hasattr(_config, 'usage') else None, + last_plan=_config.usage.last_plan if hasattr(_config, 'usage') else None, + last_session_reset=_config.usage.last_session_reset if hasattr(_config, 'usage') else None, + last_weekly_reset=_config.usage.last_weekly_reset if hasattr(_config, 'usage') else None, + last_checked=_config.usage.last_checked if hasattr(_config, 'usage') else None, + )) + return _config diff --git a/guanaco/dashboard/dashboard.py b/guanaco/dashboard/dashboard.py index 51ea5bc..f00c177 100644 --- a/guanaco/dashboard/dashboard.py +++ b/guanaco/dashboard/dashboard.py @@ -772,7 +772,19 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg config.usage.last_session_reset = usage_data.get("session_reset") config.usage.last_weekly_reset = usage_data.get("weekly_reset") config.usage.last_checked = time.time() + # Also sync to primary account in ollama_accounts if present + for acc in config.ollama_accounts: + if acc.name == "ollama": + acc.last_session_pct = usage_data.get("session_pct") + acc.last_weekly_pct = usage_data.get("weekly_pct") + acc.last_plan = usage_data.get("plan") + acc.last_session_reset = usage_data.get("session_reset") + acc.last_weekly_reset = usage_data.get("weekly_reset") + acc.last_checked = time.time() + break save_config(config) + if _account_pool: + _account_pool.update_accounts(config.ollama_accounts) return usage_data except Exception as e: return {"source": "error", "error": str(e)} @@ -964,4 +976,256 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg save_config(config) return {"status": "ok", "auto_update": config.router.auto_update} + # ── Accounts ── + + @router.get("/api/accounts") + async def list_accounts(request: Request): + """List all Ollama accounts with usage data (API keys masked). Always includes primary 'ollama' account.""" + config = get_config() + # Always ensure primary account is in the list + primary = config.primary_account + accounts_in_config = config.ollama_accounts + has_primary = any(a.name == "ollama" for a in accounts_in_config) + + if has_primary: + all_accounts = accounts_in_config + else: + all_accounts = [primary] + accounts_in_config + + # Merge legacy usage data into primary if it's missing + usage = config.usage + for acc in all_accounts: + if acc.name == "ollama": + if not acc.session_cookie and usage.session_cookie: + acc.session_cookie = usage.session_cookie + if acc.last_session_pct is None and usage.last_session_pct is not None: + acc.last_session_pct = usage.last_session_pct + if acc.last_weekly_pct is None and usage.last_weekly_pct is not None: + acc.last_weekly_pct = usage.last_weekly_pct + if acc.last_plan is None and usage.last_plan is not None: + acc.last_plan = usage.last_plan + if acc.last_session_reset is None and usage.last_session_reset is not None: + acc.last_session_reset = usage.last_session_reset + if acc.last_weekly_reset is None and usage.last_weekly_reset is not None: + acc.last_weekly_reset = usage.last_weekly_reset + if acc.last_checked is None and usage.last_checked is not None: + acc.last_checked = usage.last_checked + + accounts = [] + for acc in all_accounts: + accounts.append({ + "name": acc.name, + "api_key_masked": acc.api_key[:8] + "..." + acc.api_key[-4:] if len(acc.api_key) > 12 else ("***" if acc.api_key else ""), + "has_session_cookie": bool(acc.session_cookie), + "last_session_pct": acc.last_session_pct, + "last_weekly_pct": acc.last_weekly_pct, + "last_plan": acc.last_plan, + "last_session_reset": acc.last_session_reset, + "last_weekly_reset": acc.last_weekly_reset, + "last_checked": acc.last_checked, + }) + return {"accounts": accounts} + + @router.post("/api/accounts/add") + async def add_account(request: Request): + """Add a new Ollama account.""" + body = await request.json() + name = body.get("name", "").strip() + api_key = body.get("api_key", "").strip() + + if not name: + return {"status": "error", "message": "Account name is required"} + if not api_key: + return {"status": "error", "message": "API key is required"} + + config = get_config() + + # Reserved names + if name.lower() in ("ollama", "primary", "default"): + return {"status": "error", "message": f"'{name}' is a reserved account name"} + + # Check duplicate + if any(a.name == name for a in config.ollama_accounts): + return {"status": "error", "message": f"Account '{name}' already exists"} + + # Max 10 accounts + if len(config.ollama_accounts) >= 10: + return {"status": "error", "message": "Maximum of 10 accounts reached"} + + # Test the key first + if client: + result = await client.test_key(api_key) + if not result["ok"]: + return {"status": "error", "message": f"API key test failed: {result['error']}"} + + # Add the account + new_account = OllamaAccount(name=name, api_key=api_key) + config.ollama_accounts.append(new_account) + save_config(config) + + # Update the account pool if available + if _account_pool: + _account_pool.update_accounts(config.ollama_accounts) + + return {"status": "ok", "message": f"Account '{name}' added successfully"} + + @router.post("/api/accounts/remove") + async def remove_account(request: Request): + """Remove an Ollama account (cannot remove 'ollama' primary).""" + body = await request.json() + name = body.get("name", "").strip() + + if not name: + return {"status": "error", "message": "Account name is required"} + if name.lower() == "ollama": + return {"status": "error", "message": "Cannot remove the primary account"} + + config = get_config() + before = len(config.ollama_accounts) + config.ollama_accounts = [a for a in config.ollama_accounts if a.name != name] + + if len(config.ollama_accounts) == before: + return {"status": "error", "message": f"Account '{name}' not found"} + + save_config(config) + + if _account_pool: + _account_pool.update_accounts(config.ollama_accounts) + + return {"status": "ok", "message": f"Account '{name}' removed"} + + @router.post("/api/accounts/test") + async def test_account(request: Request): + """Test an API key (existing account by name, or a raw key).""" + body = await request.json() + name = body.get("name", "") + api_key = body.get("api_key", "") + + # If name given, look up key from config + if name and not api_key: + config = get_config() + acc = next((a for a in config.ollama_accounts if a.name == name), None) + if not acc: + return {"ok": False, "error": f"Account '{name}' not found"} + api_key = acc.api_key + + if not api_key: + return {"ok": False, "error": "No API key to test"} + + if client: + result = await client.test_key(api_key) + return result + return {"ok": False, "error": "No OllamaClient available"} + + @router.post("/api/accounts/session-cookie") + async def set_session_cookie(request: Request): + """Set the session cookie for an account (for usage scraping).""" + body = await request.json() + name = body.get("name", "ollama") + cookie = body.get("cookie", "").strip() + + config = get_config() + + # For primary account, also update the legacy usage config + if name == "ollama": + config.usage.session_cookie = cookie + + # Update in ollama_accounts list (may need to create entry for primary) + acc = next((a for a in config.ollama_accounts if a.name == name), None) + if acc: + acc.session_cookie = cookie + elif name == "ollama": + # Primary not in accounts list — set via usage config (already done above) + pass + else: + return {"status": "error", "message": f"Account '{name}' not found"} + + save_config(config) + + if _account_pool: + _account_pool.update_accounts(config.ollama_accounts) + + return {"status": "ok", "message": f"Session cookie set for '{name}'"} + + @router.post("/api/accounts/update-key") + async def update_account_key(request: Request): + """Update the API key for an existing account (including primary).""" + body = await request.json() + name = body.get("name", "").strip() + api_key = body.get("api_key", "").strip() + + if not name or not api_key: + return {"status": "error", "message": "Name and API key are required"} + + config = get_config() + + if name == "ollama": + # Primary account — update the main config key + config.ollama_api_key = api_key + save_config(config) + # Also update the running client if available + if client and hasattr(client, '_api_key'): + client._api_key = api_key + return {"status": "ok", "message": "Primary API key updated"} + + # Secondary account + acc = next((a for a in config.ollama_accounts if a.name == name), None) + if not acc: + return {"status": "error", "message": f"Account '{name}' not found"} + acc.api_key = api_key + save_config(config) + + if _account_pool: + _account_pool.update_accounts(config.ollama_accounts) + + return {"status": "ok", "message": f"Key updated for '{name}'"} + + @router.post("/api/accounts/check-usage") + async def check_account_usage(request: Request): + """Check usage/quota for a specific account using its session cookie.""" + body = await request.json() + name = body.get("name", "") + if not name: + return {"source": "unavailable", "error": "Account name required"} + + config = get_config() + + # Find the account — check ollama_accounts first, then legacy config for primary + acc = next((a for a in config.ollama_accounts if a.name == name), None) + if acc: + cookie = acc.session_cookie + elif name == "ollama": + cookie = config.usage.session_cookie + else: + return {"source": "unavailable", "error": f"Account '{name}' not found"} + + if not cookie: + return {"source": "unavailable", "error": f"No session cookie set for '{name}'. Set it in the Accounts tab."} + + try: + usage_data = await client.get_usage(session_cookie=cookie) + if usage_data.get("source") != "unavailable": + # Update the account in config + if acc: + acc.last_session_pct = usage_data.get("session_pct") + acc.last_weekly_pct = usage_data.get("weekly_pct") + acc.last_plan = usage_data.get("plan") + acc.last_session_reset = usage_data.get("session_reset") + acc.last_weekly_reset = usage_data.get("weekly_reset") + acc.last_checked = time.time() + elif name == "ollama": + # Sync to legacy usage config too + config.usage.last_session_pct = usage_data.get("session_pct") + config.usage.last_weekly_pct = usage_data.get("weekly_pct") + config.usage.last_plan = usage_data.get("plan") + config.usage.last_session_reset = usage_data.get("session_reset") + config.usage.last_weekly_reset = usage_data.get("weekly_reset") + config.usage.last_checked = time.time() + save_config(config) + if _account_pool: + _account_pool.update_accounts(config.ollama_accounts) + return usage_data + except Exception as e: + return {"source": "error", "error": str(e)} + return router \ No newline at end of file diff --git a/guanaco/dashboard/templates/dashboard.html b/guanaco/dashboard/templates/dashboard.html index 8ed8dbf..18e8a40 100644 --- a/guanaco/dashboard/templates/dashboard.html +++ b/guanaco/dashboard/templates/dashboard.html @@ -161,6 +161,7 @@
Status
Cache
API Keys
+
Accounts
System
@@ -699,6 +700,31 @@
+ + +
-
Model
Requests
Prompt Tok
Comp Tok
Avg TPS
Avg TTFT
+
Model
Reqs
Prompt
Comp
Total
Avg TPS
Avg TTFT
@@ -540,6 +545,7 @@
+
Weekly Usage @@ -548,6 +554,7 @@
+
@@ -581,6 +588,100 @@ + + +

📋 Event Log

@@ -810,7 +911,7 @@ function showTab(tab) { if (tab === 'cache') loadCacheStats(); if (tab === 'analytics') loadAnalytics(); if (tab === 'history') { loadHistoryConfig(); loadHistory(); loadHistoryLogs(); } - if (tab === 'status') { checkOllamaStatus(); loadUsageConfig(); checkUsage(); loadStatusLog(); } + if (tab === 'status') { checkOllamaStatus(); loadUsageConfig(); checkUsage(); loadStatusLog(); loadROIConfig(); } if (tab === 'accounts') { loadAccounts(); checkAllAccountUsage(); } if (tab === 'system') { loadAutostart(); checkForUpdate(); } } @@ -1208,7 +1309,7 @@ function loadModels() { // Fallback to config model list when API is down const cfg = CONFIG.llm || {}; const names = cfg.available_models || []; - const fallbackModels = names.map(n => ({name: n, model: n, id: n, capabilities: _getCapabilities(n)})); + const fallbackModels = names.map(n => ({name: n, model: n, id: n, capabilities: _getCapabilities(n), usage_multiplier: _getMultiplier(n)})); availableModels = fallbackModels; document.getElementById('models-loading').style.display = 'none'; document.getElementById('models-loading').textContent = ''; @@ -1237,6 +1338,9 @@ function renderModels(models) { const paramSize = details.parameter_size || ''; const quant = details.quantization_level || ''; const caps = m.capabilities || ['cloud']; + const mult = typeof m.usage_multiplier === 'number' ? m.usage_multiplier : (m.usage_multiplier || 0.75); + const filled = Math.round(mult * 4); // 4 slots, 0.25 = 1 filled, 1.00 = 4 filled + const slots = [0,1,2,3].map(i => ``).join(''); const capHtml = caps.map(c => `${c}`).join(''); return `
${display}
@@ -1246,6 +1350,7 @@ function renderModels(models) { ${quant ? `⚡ ${quant}` : ''}
${capHtml}
+
${slots}${mult.toFixed(2)}x
`; }).join(''); } @@ -1332,6 +1437,7 @@ function loadAnalytics() {
${m.requests}
${(m.prompt_tokens || 0).toLocaleString()}
${(m.completion_tokens || 0).toLocaleString()}
+
${((m.prompt_tokens || 0) + (m.completion_tokens || 0)).toLocaleString()}
${m.avg_tps || '—'}
${m.avg_ttft ? (m.avg_ttft * 1000).toFixed(0) + 'ms' : '—'}
@@ -1359,9 +1465,10 @@ function loadAnalytics() { `).join('') : '
No errors 🎉
'; - // Top stats + // Top stats — show raw tokens as primary count document.getElementById('stat-requests').textContent = (data.total_requests || 0).toLocaleString(); - document.getElementById('stat-tokens').textContent = ((data.prompt_tokens || 0) + (data.completion_tokens || 0)).toLocaleString(); + document.getElementById('stat-tokens').textContent = (data.total_tokens || 0).toLocaleString(); + document.getElementById('stat-tokens').nextElementSibling.textContent = 'Tokens'; document.getElementById('stat-tps').textContent = data.avg_tps || 0; document.getElementById('stat-ttft').textContent = data.avg_ttft ? (data.avg_ttft * 1000).toFixed(0) + 'ms' : '—'; document.getElementById('stat-keys').textContent = KEYS.length; @@ -1679,6 +1786,8 @@ function checkUsage() { const weeklyBar = document.getElementById('usage-weekly-bar'); const sessionReset = document.getElementById('usage-session-reset'); const weeklyReset = document.getElementById('usage-weekly-reset'); + const sessionBreakdown = document.getElementById('usage-session-breakdown'); + const weeklyBreakdown = document.getElementById('usage-weekly-breakdown'); const lastCheckedRow = document.getElementById('usage-last-checked-row'); const lastCheckedEl = document.getElementById('usage-last-checked'); planEl.textContent = 'Loading...'; @@ -1693,6 +1802,8 @@ function checkUsage() { weeklyBar.style.width = '0%'; sessionReset.textContent = ''; weeklyReset.textContent = ''; + sessionBreakdown.innerHTML = ''; + weeklyBreakdown.innerHTML = ''; lastCheckedRow.style.display = 'none'; } else { const plan = data.plan ? data.plan.charAt(0).toUpperCase() + data.plan.slice(1) : '—'; @@ -1715,6 +1826,72 @@ function checkUsage() { // Reset timers sessionReset.textContent = data.session_reset ? '⏱ Resets in ' + data.session_reset : ''; weeklyReset.textContent = data.weekly_reset ? '⏱ Resets in ' + data.weekly_reset : ''; + // Per-model breakdowns + function renderBreakdown(breakdown, container, idSuffix) { + if (!breakdown || breakdown.length === 0) { + container.innerHTML = ''; + return; + } + // Sort by percentage descending + const sorted = [...breakdown].sort((a, b) => b.pct - a.pct); + + // Separate significant vs tiny + const significant = []; + const tiny = []; + for (const b of sorted) { + if (b.pct >= 1.0) { + significant.push(b); + } else { + tiny.push(b); + } + } + + // Always show top 3 at minimum, even if tiny + while (significant.length < 3 && tiny.length > 0) { + significant.push(tiny.shift()); + } + + const hasMore = tiny.length > 0; + const all = [...significant, ...tiny]; + + function makeRow(b, hidden) { + const pctStr = b.pct >= 0.1 ? b.pct.toFixed(1) + '%' : '< 0.1%'; + return ` +
+ ${b.model} + ${b.requests.toLocaleString()} req · ${pctStr} +
+ `; + } + + const topHtml = significant.map(b => makeRow(b, false)).join(''); + const restHtml = tiny.map(b => makeRow(b, true)).join(''); + + const expandId = 'ub-expand-' + idSuffix; + const collapseId = 'ub-collapse-' + idSuffix; + + container.innerHTML = ` + +
+ ${topHtml} +
+ +
+ +
+ `; + } + renderBreakdown(data.session_breakdown, sessionBreakdown, 'session'); + renderBreakdown(data.weekly_breakdown, weeklyBreakdown, 'weekly'); // Last checked if (data.last_checked) { lastCheckedRow.style.display = ''; @@ -1772,6 +1949,8 @@ function clearSessionCookie() { document.getElementById('usage-weekly-bar').style.width = '0%'; document.getElementById('usage-session-reset').textContent = ''; document.getElementById('usage-weekly-reset').textContent = ''; + document.getElementById('usage-session-breakdown').innerHTML = ''; + document.getElementById('usage-weekly-breakdown').innerHTML = ''; } setTimeout(() => { statusEl.textContent = ''; statusEl.style.color = ''; }, 3000); }); @@ -2035,20 +2214,82 @@ function _getCapabilities(modelName) { 'gpt-oss': ['cloud','tools','thinking'], 'deepseek-v3.1': ['cloud','thinking'], 'deepseek-v3.2': ['cloud','thinking'], + 'deepseek-v4-pro': ['cloud','tools','thinking','vision'], + 'deepseek-v4-flash': ['cloud','tools','thinking','vision'], + 'gemini-3-flash-preview': ['cloud','tools','thinking','vision'], 'glm-5.1': ['cloud','tools','thinking'], 'glm-5': ['cloud','tools','thinking'], 'minimax-m2.7': ['cloud','tools','thinking'], 'minimax-m2.5': ['cloud','tools','thinking'], 'minimax-m2.1': ['cloud','tools','thinking'], + 'minimax-m2': ['cloud','tools','thinking'], + 'minimax-m3': ['cloud','tools','thinking','vision'], 'devstral-small-2': ['cloud','tools'], 'devstral-2': ['cloud','tools'], 'nemotron-3-super': ['cloud','tools','thinking'], + 'nemotron-3-nano': ['cloud','tools','thinking'], + 'nemotron-3-ultra': ['cloud','tools','thinking'], 'mistral-large-3': ['cloud','tools','thinking'], 'ministral-3': ['cloud','tools'], + 'kimi-k2.6': ['cloud','tools','thinking','vision'], 'kimi-k2.5': ['cloud','tools','thinking','vision'], 'cogito-2.1': ['cloud','thinking'], }; - return known[base] || ['cloud']; + if (known[base]) return known[base]; + // ── Inference for unknown models ── + const lc = base.toLowerCase(); + const caps = ['cloud']; + if (lc.includes('vl') || lc.includes('vision') || lc.includes('gemma') || lc.includes('gemini') || lc.includes('deepseek') || lc.startsWith('kimi-')) + caps.push('vision'); + if (lc.includes('coder') || lc.includes('minimax') || lc.startsWith('glm-') || lc.includes('mistral') || + lc.includes('ministral') || lc.includes('gpt-oss') || lc.includes('devstral') || lc.includes('nemotron') || lc.includes('deepseek') || lc.includes('rnj-1')) + caps.push('tools'); + if (lc.includes('deepseek') || lc.includes('cogito') || lc.includes('reason') || lc.includes('-thinking') || lc.includes('think')) + caps.push('thinking'); + else if (lc.startsWith('kimi-k') && !(lc.includes('k2.5') || lc.includes('k2.6'))) + caps.push('thinking'); + // deduplicate + return [...new Set(caps)].sort(); +} + +function _getMultiplier(modelName) { + const base = modelName.split(':')[0]; + const mults = { + 'nemotron-3-nano': 0.25, 'ministral-3': 0.25, 'rnj-1': 0.25, + 'gemma4': 0.50, 'gemma3': 0.50, 'minimax-m2.7': 0.50, 'minimax-m2.5': 0.50, 'minimax-m2.1': 0.50, 'minimax-m2': 0.50, + 'gpt-oss': 0.50, 'devstral-small-2': 0.50, 'nemotron-3-super': 0.50, 'gemini-3-flash-preview': 0.50, + 'glm-4.7': 0.50, 'glm-4.6': 0.50, + 'qwen3-vl': 0.75, 'qwen3-coder': 0.75, 'qwen3-next': 0.75, 'devstral-2': 0.75, + 'glm-5.1': 0.75, 'glm-5': 0.75, + 'kimi-k2.6': 0.75, 'kimi-k2.5': 0.75, 'kimi-k2-thinking': 0.75, + 'qwen3.5': 1.00, 'deepseek-v3.1': 1.00, 'deepseek-v3.2': 1.00, 'deepseek-v4-pro': 1.00, + 'mistral-large-3': 1.00, 'cogito-2.1': 1.00, 'kimi-k2': 1.00, + 'deepseek-v4-flash': 0.50, 'gemini-3-flash-preview': 0.50, + 'minimax-m3': 0.75, + 'nemotron-3-ultra': 1.00, + }; + if (mults[base]) return mults[base]; + // ── Inference from size hint ── + const m = modelName.match(/(\d+)(b|t)/i); + if (m) { + const num = parseInt(m[1], 10); + const unit = m[2].toLowerCase(); + if (unit === 't') return 1.00; + if (num <= 20) return 0.25; + if (num <= 80) return 0.50; + if (num <= 400) return 0.75; + return 1.00; + } + // ── Fallback heuristics ── + const lc = base.toLowerCase(); + if (lc.includes('nano') || lc.includes('mini') || lc.includes('small') || lc.includes('rnj-1')) return 0.25; + if (lc.includes('flash') || lc.includes('gemma') || lc.includes('gpt-oss') || lc.includes('minimax') || + lc.includes('devstral-small') || lc.includes('glm-4.') || lc.includes('super')) return 0.50; + if (lc.includes('kimi-k') || lc.includes('qwen3-vl') || lc.includes('qwen3-coder') || lc.includes('qwen3-next') || + lc.includes('devstral-2') || lc.includes('glm-5')) return 0.75; + if (lc.includes('pro') || lc.includes('qwen3.5') || lc.includes('deepseek-v3') || lc.includes('mistral-large') || + lc.includes('cogito')) return 1.00; + return 0.75; } // ─── Init ─── @@ -2466,6 +2707,262 @@ async function toggleAutoUpdate() { toggle.checked = !enabled; } } + +// ─── ROI Calculator (Experimental) ─── + +let roiEnabled = false; +let roiPrice = 20.0; +let roiCacheHit = 0.0; + +function loadROIConfig() { + fetch('/dashboard/api/roi/config').then(r => r.json()).then(data => { + roiEnabled = data.enabled; + roiPrice = data.subscription_price || 20.0; + roiCacheHit = data.cache_hit_pct || 0.0; + document.getElementById('roi-enabled').checked = roiEnabled; + document.getElementById('roi-card').style.display = 'block'; + const planSel = document.getElementById('roi-plan'); + const customInput = document.getElementById('roi-custom-price'); + if (roiPrice == 20) planSel.value = '20'; + else if (roiPrice == 100) planSel.value = '100'; + else { + planSel.value = '0'; + customInput.value = roiPrice; + customInput.disabled = false; + } + + // Set cache hit slider + const cacheSlider = document.getElementById('cache-hit-slider'); + const cacheValue = document.getElementById('cache-hit-value'); + if (cacheSlider && cacheValue) { + cacheSlider.value = roiCacheHit; + cacheValue.textContent = roiCacheHit + '%'; + } + + if (roiEnabled) { + document.getElementById('roi-disabled-msg').style.display = 'none'; + calculateROI(); + } else { + document.getElementById('roi-results').style.display = 'none'; + } + }).catch(() => { + // ROI endpoints may not exist yet — hide the card + document.getElementById('roi-card').style.display = 'none'; + }); +} + +function toggleROI() { + const enabled = document.getElementById('roi-enabled').checked; + roiEnabled = enabled; + const price = parseFloat(document.getElementById('roi-plan').value) || parseFloat(document.getElementById('roi-custom-price').value) || 20.0; + const cacheHit = roiCacheHit; + fetch('/dashboard/api/roi/config', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ enabled, subscription_price: price, cache_hit_pct: cacheHit }) + }).then(r => r.json()).then(data => { + if (data.status === 'ok') { + if (enabled) { + document.getElementById('roi-disabled-msg').style.display = 'none'; + calculateROI(); + } else { + document.getElementById('roi-results').style.display = 'none'; + document.getElementById('roi-disabled-msg').style.display = 'block'; + } + } + }); +} + +function updateCacheHitDisplay() { + const val = document.getElementById('cache-hit-slider').value; + document.getElementById('cache-hit-value').textContent = val + '%'; +} + +function setCacheHit() { + const val = parseFloat(document.getElementById('cache-hit-slider').value) || 0; + roiCacheHit = val; + const price = parseFloat(document.getElementById('roi-plan').value) || parseFloat(document.getElementById('roi-custom-price').value) || 20.0; + fetch('/dashboard/api/roi/config', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ enabled: roiEnabled, subscription_price: price, cache_hit_pct: val }) + }).then(r => r.json()).then(data => { + if (data.status === 'ok' && roiEnabled) { + calculateROI(); + } + }); +} + +function setROIPlan() { + const val = document.getElementById('roi-plan').value; + const custom = document.getElementById('roi-custom-price'); + if (val === '0') { + custom.disabled = false; + custom.focus(); + } else { + custom.disabled = true; + custom.value = ''; + roiPrice = parseFloat(val); + } + if (roiEnabled) { + fetch('/dashboard/api/roi/config', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ enabled: true, subscription_price: roiPrice }) + }); + } +} + +async function calculateROI() { + if (!roiEnabled) return; + const loading = document.getElementById('roi-loading'); + const results = document.getElementById('roi-results'); + loading.style.display = 'block'; + results.style.display = 'none'; + try { + const resp = await fetch('/dashboard/api/roi/calculate'); + const data = await resp.json(); + loading.style.display = 'none'; + if (data.error) { + document.getElementById('roi-model-rows').innerHTML = '
' + data.error + '
'; + results.style.display = 'block'; + return; + } + renderROIResults(data); + loadROIComparison(); + results.style.display = 'block'; + } catch (e) { + loading.style.display = 'none'; + document.getElementById('roi-model-rows').innerHTML = '
Failed to calculate ROI
'; + results.style.display = 'block'; + } +} + +function renderROIResults(data) { + document.getElementById('roi-total-cost').textContent = '$' + (data.total_cost || 0).toLocaleString(undefined, {maximumFractionDigits: 2}); + document.getElementById('roi-weekly').textContent = '$' + (data.weekly_value || 0).toLocaleString(undefined, {maximumFractionDigits: 2}); + const monthly = data.monthly_value || 0; + document.getElementById('roi-monthly').textContent = '$' + monthly.toLocaleString(undefined, {maximumFractionDigits: 2}); + document.getElementById('roi-sub-cost').textContent = '$' + (data.subscription_monthly || 0).toLocaleString(undefined, {maximumFractionDigits: 0}); + document.getElementById('roi-est-value').textContent = '$' + monthly.toLocaleString(undefined, {maximumFractionDigits: 2}); + const savings = Math.max(0, monthly - (data.subscription_monthly || 0)); + document.getElementById('roi-savings').textContent = '$' + savings.toLocaleString(undefined, {maximumFractionDigits: 2}); + const mult = data.roi_multiplier || 0; + document.getElementById('roi-mult').textContent = mult.toFixed(1) + 'x'; + document.getElementById('roi-mult').style.color = mult > 1 ? 'var(--success)' : (mult > 0.5 ? '#facc15' : 'var(--danger)'); + // Use total_raw_tokens if available (new calc), otherwise sum from by_model (cached) + let rawTotal = data.total_raw_tokens || 0; + if (!rawTotal && data.by_model) { + rawTotal = data.by_model.reduce((s, m) => s + (m.prompt_tokens || 0) + (m.completion_tokens || 0), 0); + } + document.getElementById('roi-total-tokens').textContent = rawTotal.toLocaleString(); + + const rows = document.getElementById('roi-model-rows'); + if (data.by_model && data.by_model.length > 0) { + rows.innerHTML = data.by_model.map(m => { + const pt = m.prompt_tokens || 0; + const ct = m.completion_tokens || 0; + const totalToks = pt + ct; + return '
' + + '
' + m.model + '
' + + '
$' + (m.prompt_per_mt || 0).toFixed(4) + '
' + + '
$' + (m.completion_per_mt || 0).toFixed(4) + '
' + + '
$' + (m.cost || 0).toFixed(2) + '
' + + '
' + (m.pct_of_total || 0).toFixed(1) + '%
' + + '
' + totalToks.toLocaleString() + '
' + + '
'; + }).join(''); + } else { + rows.innerHTML = '
No usage data yet this week
'; + } + + // Weekly usage % + const weeklyPct = data.weekly_pct_used || 0; + document.getElementById('roi-weekly-pct').textContent = weeklyPct > 0 ? '· Based on ' + weeklyPct.toFixed(1) + '% of weekly quota used' : ''; + + // Unmatched models warning + const unmatchedEl = document.getElementById('roi-unmatched'); + if (data.unmatched_models && data.unmatched_models.length > 0) { + unmatchedEl.textContent = '⚠ Could not price: ' + data.unmatched_models.join(', '); + unmatchedEl.style.display = 'block'; + } else { + unmatchedEl.style.display = 'none'; + } + + // Stale prices warning + document.getElementById('roi-stale').style.display = data.prices_stale ? 'block' : 'none'; +} + +// ─── Model Value Comparison ─── + +async function loadROIComparison() { + if (!roiEnabled) return; + try { + const resp = await fetch('/dashboard/api/roi/comparison?period=weekly'); + const data = await resp.json(); + renderROIComparison(data); + } catch (e) { + document.getElementById('roi-comparison-rows').innerHTML = '
Failed to load comparison
'; + } +} + +function renderROIComparison(data) { + const rows = document.getElementById('roi-comparison-rows'); + const summary = document.getElementById('roi-comparison-summary'); + + if (data.error) { + rows.innerHTML = '
' + data.error + '
'; + summary.style.display = 'none'; + return; + } + + if (!data.models || data.models.length === 0) { + rows.innerHTML = '
No usage data for this period
'; + summary.style.display = 'none'; + return; + } + + rows.innerHTML = data.models.map(m => { + const score = m.score || 0; + const scoreColor = score > 0 ? 'var(--success)' : (score < 0 ? 'var(--danger)' : 'var(--text2)'); + const scoreSign = score > 0 ? '+' : ''; + return '
' + + '
' + m.model + '
' + + '
' + (m.requests || 0).toLocaleString() + '
' + + '
' + (m.total_tokens || 0).toLocaleString() + '
' + + '
$' + (m.actual_value || 0).toFixed(2) + '
' + + '
$' + (m.fair_share || 0).toFixed(2) + '
' + + '
' + scoreSign + score.toFixed(2) + '
' + + '
'; + }).join(''); + + // Summary bar + const net = data.net_score || 0; + const netColor = net > 0 ? 'var(--success)' : (net < 0 ? 'var(--danger)' : 'var(--text2)'); + const netSign = net > 0 ? '+' : ''; + summary.innerHTML = + '
' + + 'Period cost: $' + (data.period_sub_cost || 0).toFixed(2) + ' · Total value: $' + (data.total_actual_value || 0).toFixed(2) + '' + + 'Net: ' + netSign + net.toFixed(2) + '' + + '
'; + summary.style.display = 'block'; +} + +function loadLastROI() { + fetch('/dashboard/api/roi/last').then(r => r.json()).then(data => { + if (!data.error && (data.by_model || []).length > 0) { + renderROIResults(data); + document.getElementById('roi-results').style.display = 'block'; + } + }).catch(() => {}); +} + +function resetROIData() { + fetch('/dashboard/api/roi/reset', {method: 'POST'}).then(() => { + document.getElementById('roi-results').style.display = 'none'; + if (roiEnabled) calculateROI(); + }); +} \ No newline at end of file diff --git a/guanaco/roi.py b/guanaco/roi.py new file mode 100644 index 0000000..978b170 --- /dev/null +++ b/guanaco/roi.py @@ -0,0 +1,473 @@ +""" +OpenRouter price-based subscription value calculator. + +This module: +1. Fetches live model prices from OpenRouter's API +2. Maps Ollama Cloud model names to OpenRouter model IDs +3. Calculates "what would this usage have cost on OpenRouter?" +4. Compares against subscription price to show value multiplier + +Prices are cached for 1 hour to avoid rate limits. +""" +from __future__ import annotations + +import logging +import sqlite3 +import time +from pathlib import Path +from typing import Optional + +import httpx + +logger = logging.getLogger(__name__) + +# ── OpenRouter API ── +OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" +OPENROUTER_CACHE_TTL = 3600 # 1 hour + +# Model family mappings: ollama_name_fragment -> openrouter_id_fragment +# These are used when exact match fails +FAMILY_MAP = { + "gemma": ("google/gemma", "google/gemma"), + "gemma3": ("google/gemma", "google/gemma"), + "gemma4": ("google/gemma", "google/gemma"), + "qwen": ("qwen/qwen", "qwen/qwen"), + "qwen3": ("qwen/qwen3", "qwen/qwen"), + "qwen3.5": ("qwen/qwen3.5", "qwen/qwen"), + "qwen3-vl": ("qwen/qwen3-vl", "qwen/qwen3-vl"), + "qwen3-coder": ("qwen/qwen3-coder", "qwen/qwen"), + "qwen3-next": ("qwen/qwen3-next", "qwen/qwen"), + "deepseek": ("deepseek/deepseek", "deepseek/deepseek"), + "deepseek-v3": ("deepseek/deepseek", "deepseek/deepseek-v3"), + "deepseek-v4": ("deepseek/deepseek", "deepseek/deepseek-v4"), + "gpt-oss": ("openai/gpt-oss", "openai/gpt"), + "minimax": ("minimax/minimax", "minimax/minimax"), + "glm": ("zhipu/glm", "zhipu/glm"), + "glm-5": ("zhipu/glm-5", "zhipu/glm"), + "kimi": ("moonshot/kimi", "moonshot/kimi"), + "kimi-k2": ("moonshot/kimi", "moonshot/kimi"), + "devstral": ("mistral/devstral", "mistral/devstral"), + "mistral": ("mistral/mistral", "mistral/mistral"), + "ministral": ("mistral/ministral", "mistral/ministral"), + "nemotron": ("nvidia/nemotron", "nvidia/nemotron"), + "cogito": ("cogito/cogito", "cogito/"), + "gemini": ("google/gemini", "google/gemini"), + "rnj": ("", ""), +} + + +def _normalized(name: str) -> str: + """Strip provider prefix, ~leaderboard prefix, :cloud/:local suffixes, and lower-case.""" + base = name.split(":")[0].lower() + # Strip ~ prefix (leaderboard indicator on OpenRouter) + if base.startswith("~"): + base = base[1:] + # Strip provider/ prefix (e.g. moonshotai/kimi-k2.6 → kimi-k2.6) + if "/" in base: + base = base.split("/", 1)[1] + if base.endswith("-cloud"): + base = base[:-6] + return base + + +def _model_size(name: str) -> int: + """Extract parameter size in billions from model name, 0 if unknown.""" + import re + m = re.search(r"(\d+)(b|t)", name, re.I) + if not m: + return 0 + n = int(m.group(1)) + unit = m.group(2).lower() + return n * 1000 if unit == "t" else n + + +class PriceCache: + """Holds cached OpenRouter prices in memory with TTL.""" + def __init__(self): + self.prices: dict[str, dict] = {} + self.fetched_at: float = 0 + + def is_fresh(self) -> bool: + return self.prices and (time.time() - self.fetched_at) < OPENROUTER_CACHE_TTL + + def fetch(self) -> dict[str, dict]: + if self.is_fresh(): + logger.debug("Using cached OpenRouter prices") + return self.prices + + prices = {} + try: + logger.info("Fetching OpenRouter model prices...") + r = httpx.get(OPENROUTER_MODELS_URL, timeout=30) + r.raise_for_status() + data = r.json() + for model in data.get("data", []): + model_id = model.get("id", "") + pricing = model.get("pricing", {}) + prompt = float(pricing.get("prompt", 0) or 0) + completion = float(pricing.get("completion", 0) or 0) + cache_read = float(pricing.get("input_cache_read", 0) or 0) + if prompt > 0 or completion > 0: + # Convert from per-token ($/token) to per-million-tokens ($/Mt) + entry = { + "prompt": prompt * 1_000_000, + "completion": completion * 1_000_000, + } + if cache_read > 0: + entry["input_cache_read"] = cache_read * 1_000_000 + prices[model_id] = entry + self.prices = prices + self.fetched_at = time.time() + logger.info(f"Fetched {len(prices)} OpenRouter price entries") + except Exception as e: + logger.warning(f"Failed to fetch OpenRouter prices: {e}") + return self.prices + + +# Singleton cache +_price_cache = PriceCache() + + +def _find_best_price(prices: dict, ollama_name: str) -> dict: + """ + Given OpenRouter prices dict {model_id: {prompt, completion}} and an + Ollama model name, return best matching price dict. + """ + norm = _normalized(ollama_name) + size = _model_size(ollama_name) + + # 1. Exact normalized match (handles provider-prefixed OR IDs like moonshotai/kimi-k2.6) + for orouter_id, price_info in prices.items(): + if _normalized(orouter_id) == norm: + return price_info + + # 2. Family prefix match — use raw orouter_id so provider/ prefix matches + best_family_price = None + best_family_score = -9999 + for orouter_id, price_info in prices.items(): + for frag, (family_exact, family_prefix) in FAMILY_MAP.items(): + if frag in norm and family_prefix and family_prefix in orouter_id: + # Score by size closeness + o_size = _model_size(orouter_id) + score = -(abs(o_size - size)) # higher = closer size + if score > best_family_score: + best_family_score = score + best_family_price = price_info + if best_family_price: + return best_family_price + + # 3. Same parameter size match + if size > 0: + for orouter_id, price_info in prices.items(): + if _model_size(orouter_id) == size: + return price_info + + # 4. Size-window fallback + candidates = [] + for orouter_id, price_info in prices.items(): + o_size = _model_size(orouter_id) + if o_size == 0: + continue + window = max(20, size * 0.5) + if abs(o_size - size) <= window: + candidates.append(price_info) + if candidates: + candidates.sort(key=lambda p: p["completion"] + p["prompt"]) + return candidates[len(candidates) // 2] + + # 5. Global average + all_prices = [p for p in prices.values() if p["prompt"] > 0 or p["completion"] > 0] + if all_prices: + avg_prompt = sum(p["prompt"] for p in all_prices) / len(all_prices) + avg_comp = sum(p["completion"] for p in all_prices) / len(all_prices) + return {"prompt": avg_prompt, "completion": avg_comp} + + return {"prompt": 0.0, "completion": 0.0} + + +def _map_usage_to_prices(usage_by_model: dict, prices: dict, cache_hit_pct: float = 0.0) -> dict: + """ + Map usage to prices, optionally applying prompt-cache hit discount. + + For models with input_cache_read pricing (e.g. Claude Fable, Qwen, Minimax): + - uncached_prompt = prompt_tokens * (1 - cache_hit_pct) + - cached_prompt = prompt_tokens * cache_hit_pct + - prompt cost = uncached_prompt * prompt_price + cached_prompt * cache_read_price + """ + result = {} + cache_rate = max(0.0, min(100.0, cache_hit_pct)) / 100.0 + for model, usage in usage_by_model.items(): + price = _find_best_price(prices, model) + pt = usage.get("prompt_tokens", 0) + ct = usage.get("completion_tokens", 0) + + # Apply cache discount if model supports it + if "input_cache_read" in price and cache_rate > 0: + uncached_pt = pt * (1 - cache_rate) + cached_pt = pt * cache_rate + prompt_cost = (uncached_pt / 1_000_000 * price["prompt"]) + (cached_pt / 1_000_000 * price["input_cache_read"]) + # Store effective prompt price for display + effective_prompt = prompt_cost / (pt / 1_000_000) if pt > 0 else price["prompt"] + else: + prompt_cost = (pt / 1_000_000) * price["prompt"] + effective_prompt = price["prompt"] + + comp_cost = (ct / 1_000_000) * price["completion"] + cost = prompt_cost + comp_cost + + result[model] = { + "prompt_tokens": pt, + "completion_tokens": ct, + "prompt_per_mt": effective_prompt, + "completion_per_mt": price["completion"], + "cost": cost, + "matched_price_model": _find_best_price.__module__, + "cache_applied": "input_cache_read" in price and cache_rate > 0, + "cache_read_per_mt": price.get("input_cache_read"), + } + return result + + +def get_usage_from_analytics(db_path: Path | str, since: float = 0) -> tuple[dict, float]: + usage_by_model = {} + total_weighted = 0.0 + try: + conn = sqlite3.connect(str(db_path)) + rows = conn.execute( + """SELECT model, + IFNULL(SUM(prompt_tokens),0), + IFNULL(SUM(completion_tokens),0), + IFNULL(SUM(prompt_tokens * IFNULL(usage_multiplier,1.0)),0), + IFNULL(SUM(completion_tokens * IFNULL(usage_multiplier,1.0)),0), + COUNT(*) + FROM request_log WHERE type='llm' AND ts > ? GROUP BY model""", + (since,), + ).fetchall() + for row in rows: + model, pt, ct, w_pt, w_ct, req_count = row + usage_by_model[model] = { + "prompt_tokens": pt, + "completion_tokens": ct, + "weighted_prompt": w_pt, + "weighted_completion": w_ct, + "requests": req_count, + } + total_weighted += (w_pt + w_ct) + conn.close() + except Exception as e: + logger.warning(f"Failed to read analytics DB: {e}") + return usage_by_model, total_weighted + + +def _get_ollama_week_start() -> float: + """Return Unix timestamp of the most recent Sunday at 20:00 UTC. + + Ollama resets its weekly quota every Sunday at 8 PM UTC. + """ + from datetime import datetime, timedelta, timezone + now = datetime.now(timezone.utc) + days_since_sunday = now.weekday() + 1 if now.weekday() != 6 else 0 + sunday = now - timedelta(days=days_since_sunday) + reset = sunday.replace(hour=20, minute=0, second=0, microsecond=0) + if now < reset: + reset -= timedelta(days=7) + return reset.timestamp() + + +def calculate_roi( + db_path: Path | str, + subscription_monthly: float = 20.0, + weekly_pct_used: float = 0.0, + cache_hit_pct: float = 0.0, +) -> dict: + """ + Calculate subscription value vs OpenRouter pay-as-you-go. + + Args: + db_path: path to analytics SQLite DB + subscription_monthly: monthly sub cost (20 Pro, 100 Max) + weekly_pct_used: % of weekly quota consumed (from usage check) + cache_hit_pct: estimated % of prompt tokens hitting cache (0-100). + Used for models with input_cache_read pricing on OpenRouter. + + Returns dict with: + total_cost, total_prompt_tokens, total_completion_tokens, + total_weighted_tokens, weekly_value, monthly_value, + subscription_monthly, plan ("pro"|"max"), roi_multiplier, + weekly_pct_used, cache_hit_pct, prices_stale, + by_model[] with prompt_tokens, completion_tokens, prompt_per_mt, completion_per_mt, cost, + unmatched_models[] names with no price match + """ + # 1. Fetch prices + prices = _price_cache.fetch() + prices_stale = not prices or len(prices) < 10 + plan = "pro" if subscription_monthly <= 25 else "max" + + # 2. Get usage + since = _get_ollama_week_start() + usage_by_model, total_weighted = get_usage_from_analytics(db_path, since) + + # 3. Map usage to prices (with cache hit estimation) + priced = _map_usage_to_prices(usage_by_model, prices, cache_hit_pct) + + total_cost = sum(m["cost"] for m in priced.values()) + total_prompt = sum(m["prompt_tokens"] for m in priced.values()) + total_comp = sum(m["completion_tokens"] for m in priced.values()) + + # 4. Extrapolate to 100% weekly + if weekly_pct_used > 0: + weekly_value = total_cost / (weekly_pct_used / 100.0) + else: + weekly_value = total_cost + + monthly_value = weekly_value * 4 + roi_multiplier = (monthly_value / subscription_monthly) if subscription_monthly > 0 else 0 + + unmatched = [m for m in usage_by_model if priced.get(m, {}).get("cost", 0) == 0] + + # Per-model breakdown + by_model = [] + for model, detail in priced.items(): + pt = detail["prompt_tokens"] + ct = detail["completion_tokens"] + by_model.append({ + "model": model, + "prompt_tokens": pt, + "completion_tokens": ct, + "prompt_per_mt": round(detail["prompt_per_mt"], 6), + "completion_per_mt": round(detail["completion_per_mt"], 6), + "cost": round(detail["cost"], 4), + "pct_of_total": round((detail["cost"] / total_cost * 100), 2) if total_cost > 0 else 0, + "cache_applied": detail.get("cache_applied", False), + "cache_read_per_mt": round(detail.get("cache_read_per_mt"), 6) if detail.get("cache_read_per_mt") else None, + }) + by_model.sort(key=lambda x: x["cost"], reverse=True) + + return { + "total_cost": round(total_cost, 2), + "total_prompt_tokens": int(total_prompt), + "total_completion_tokens": int(total_comp), + "total_raw_tokens": int(total_prompt + total_comp), + "total_weighted_tokens": int(total_weighted), + "weekly_value": round(weekly_value, 2), + "monthly_value": round(monthly_value, 2), + "subscription_monthly": subscription_monthly, + "plan": plan, + "roi_multiplier": round(roi_multiplier, 2), + "weekly_pct_used": weekly_pct_used, + "cache_hit_pct": cache_hit_pct, + "prices_stale": prices_stale, + "by_model": by_model, + "unmatched_models": unmatched, + "price_models_available": len(prices), + } + + +def calculate_model_value_comparison( + db_path: Path | str, + subscription_monthly: float = 20.0, + weekly_pct_used: float = 0.0, + session_pct_used: float = 0.0, + period: str = "weekly", # "weekly" or "session" +) -> dict: + """ + Score each model: positive = gave more value than its fair share of sub. + + For each model actually used: + - actual_value = what those tokens would cost on OpenRouter + - fair_share = (model's weighted tokens / total weighted tokens) * subscription_cost_for_period + - score = actual_value - fair_share + positive = model punches above its weight (good deal) + negative = model is expensive for its token share (bad deal) + """ + prices = _price_cache.fetch() + if not prices: + return {"error": "No OpenRouter prices available", "models": []} + + # Determine time window and usage % + now = time.time() + if period == "session": + since = now - (5 * 3600) # 5-hour session window + pct_used = session_pct_used + else: + since = now - (7 * 24 * 3600) # 7-day weekly window + pct_used = weekly_pct_used + + usage_by_model, total_weighted = get_usage_from_analytics(db_path, since) + if not usage_by_model: + return {"models": [], "summary": {}} + + # Period subscription cost + weekly_sub_cost = subscription_monthly / 4.0 + if pct_used > 0: + period_sub_cost = weekly_sub_cost * (pct_used / 100.0) + else: + period_sub_cost = weekly_sub_cost + + # Map to prices + priced = _map_usage_to_prices(usage_by_model, prices) + + # Per-model scoring + models = [] + total_actual_value = 0.0 + + for model, usage in usage_by_model.items(): + detail = priced.get(model, {}) + actual_value = detail.get("cost", 0.0) + pt = usage.get("prompt_tokens", 0) + ct = usage.get("completion_tokens", 0) + model_raw = pt + ct + w_pt = usage.get("weighted_prompt", 0) + w_ct = usage.get("weighted_completion", 0) + model_weighted = w_pt + w_ct + + # Fair share of subscription based on WEIGHTED token proportion + # (subscription quota is weighted; actual value uses raw tokens at OpenRouter prices) + fair_share = (model_weighted / total_weighted * period_sub_cost) if total_weighted > 0 else 0 + score = actual_value - fair_share + score_pct = (score / fair_share * 100) if fair_share > 0 else 0 + + total_actual_value += actual_value + + models.append({ + "model": model, + "requests": usage.get("requests", 0), + "prompt_tokens": pt, + "completion_tokens": ct, + "total_tokens": int(model_raw), + "weighted_tokens": int(model_weighted), + "pct_of_total_tokens": round((model_weighted / total_weighted * 100), 2) if total_weighted > 0 else 0, + "actual_value": round(actual_value, 2), + "fair_share": round(fair_share, 2), + "score": round(score, 2), + "score_pct": round(score_pct, 1), + "prompt_per_mt": round(detail.get("prompt_per_mt", 0), 6), + "completion_per_mt": round(detail.get("completion_per_mt", 0), 6), + }) + + # Sort by score descending (best value first) + models.sort(key=lambda x: x["score"], reverse=True) + + # Summary + net_score = total_actual_value - period_sub_cost + + # Compute total raw tokens across all models for the summary + total_raw = sum(m.get("prompt_tokens", 0) + m.get("completion_tokens", 0) for m in models) + + return { + "period": period, + "subscription_monthly": subscription_monthly, + "period_sub_cost": round(period_sub_cost, 2), + "total_actual_value": round(total_actual_value, 2), + "total_raw_tokens": int(total_raw), + "total_weighted_tokens": int(total_weighted), + "net_score": round(net_score, 2), + "models": models, + "prices_stale": len(prices) < 10, + "price_models_available": len(prices), + } + + +def get_cached_roi() -> dict: + """Return last calculated ROI, or minimal default.""" + return _price_cache.prices # placeholder; real caching is in config diff --git a/guanaco/router/router.py b/guanaco/router/router.py index 9282f20..4494349 100644 --- a/guanaco/router/router.py +++ b/guanaco/router/router.py @@ -469,11 +469,17 @@ def create_router(client: OllamaClient, analytics=None, config=None, account_poo """List available models by querying Ollama Cloud dynamically.""" try: models = await client.list_models() + # Fetch real usage levels from ollama.com library pages + model_names = [m.get("name", m.get("model", "")) for m in models] + usage_levels = await client.fetch_usage_levels(model_names) + data = [] for m in models: name = m.get("name", m.get("model", "")) display_name = name.replace("-cloud", "") if name.endswith("-cloud") else name details = m.get("details", {}) + level = usage_levels.get(name, 0) + multiplier = level * 0.25 if level else client._get_model_multiplier(name) data.append({ "id": display_name, "object": "model", @@ -483,6 +489,8 @@ def create_router(client: OllamaClient, analytics=None, config=None, account_poo "root": display_name, "parent": None, "capabilities": client._get_model_capabilities(name), + "usage_multiplier": multiplier, + "usage_level": level, # 1-4, 0 = unknown "details": { "parameter_size": details.get("parameter_size", ""), "quantization": details.get("quantization_level", ""), @@ -552,10 +560,45 @@ def create_router(client: OllamaClient, analytics=None, config=None, account_poo payload_fb["model"] = fallback_model if body.stream: if _config.fallback.stream_fallback: - fallback_payload = dict(payload) - fallback_payload["model"] = fallback_model + fallback_payload2 = dict(payload) + fallback_payload2["model"] = fallback_model + + async def _quota_fallback_stream(): + acc_content = [] + fb_start = time.time() + fb_first = None + try: + async for fb_chunk in await _call_fallback_provider(fallback_payload2, _config.fallback, stream=True): + yield fb_chunk + txt = _extract_sse_content(fb_chunk) + if txt: + if fb_first is None: + fb_first = time.time() + acc_content.append(txt) + finally: + if _analytics: + _hist_kw2 = dict(_hist) + if acc_content and _config and _config.history.enabled and _config.history.save_output: + out_text = "".join(acc_content) + if len(out_text) > _config.history.max_content_size: + out_text = out_text[:_config.history.max_content_size] + "\n...[truncated]" + _hist_kw2["output_text"] = out_text + fb_chars = len("".join(acc_content)) + fb_tokens = max(1, fb_chars // 4) if fb_chars else 0 + fb_elapsed = time.time() - fb_start + _analytics.log_llm( + model=_normalize_model_name(fallback_model), + prompt_tokens=0, + completion_tokens=fb_tokens, + total_duration_seconds=fb_elapsed, + provider=_config.fallback.name, + fallback_for=_normalize_model_name(resolved_model), + fallback_reason=f"Quota full (session={_config.usage.last_session_pct or 0:.0f}%, weekly={_config.usage.last_weekly_pct or 0:.0f}%)", + **_hist_kw2, + ) + return StreamingResponse( - await _call_fallback_provider(fallback_payload, _config.fallback, stream=True), + _quota_fallback_stream(), media_type="text/event-stream", ) # Can't stream from fallback — do non-streaming @@ -578,9 +621,16 @@ def create_router(client: OllamaClient, analytics=None, config=None, account_poo fallback_resp["_oct_fallback_provider"] = _config.fallback.name fallback_resp["_oct_original_model"] = _normalize_model_name(resolved_model) fallback_resp["_oct_quota_redirect"] = True + # Extract usage from fallback response when available + fb_usage = fallback_resp.get("usage", {}) + fb_prompt = fb_usage.get("prompt_tokens", 0) + fb_completion = fb_usage.get("completion_tokens", 0) if _analytics: _analytics.log_llm( model=_normalize_model_name(fallback_model), + prompt_tokens=fb_prompt, + completion_tokens=fb_completion, + total_tokens=fb_usage.get("total_tokens", fb_prompt + fb_completion), total_duration_seconds=time.time() - start, provider=_config.fallback.name, fallback_for=_normalize_model_name(resolved_model), @@ -755,9 +805,16 @@ def create_router(client: OllamaClient, analytics=None, config=None, account_poo fallback_resp = await _call_fallback_provider(fallback_payload, _config.fallback) elapsed = time.time() - start + # Extract usage from fallback response when available + fb_usage2 = fallback_resp.get("usage", {}) + fb_prompt2 = fb_usage2.get("prompt_tokens", 0) + fb_completion2 = fb_usage2.get("completion_tokens", 0) if _analytics: _analytics.log_llm( model=fallback_model, + prompt_tokens=fb_prompt2, + completion_tokens=fb_completion2, + total_tokens=fb_usage2.get("total_tokens", fb_prompt2 + fb_completion2), total_duration_seconds=elapsed, provider=_config.fallback.name, fallback_for=resolved_model, fallback_reason=f"Ollama error: {_describe_error(ollama_error)}", @@ -1472,13 +1529,20 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim _hist_kw["output_text"] = output_text if analytics: if used_fallback and fallback_model: + # Fallback providers don't emit __oct_metrics__ — estimate from accumulated content + fb_stream_metrics = dict(stream_metrics) + if not fb_stream_metrics.get("eval_count") and accumulated_content: + fb_chars = len("".join(accumulated_content)) + fb_tokens = max(1, fb_chars // 4) if fb_chars else 0 + fb_stream_metrics["eval_count"] = fb_tokens + fb_stream_metrics.setdefault("elapsed_seconds", elapsed) analytics.log_llm( model=_normalize_model_name(fallback_model), - prompt_tokens=stream_metrics.get("prompt_eval_count", 0), - completion_tokens=stream_metrics.get("eval_count"), - tps=stream_metrics.get("tps"), - ttft_seconds=stream_metrics.get("ttft_seconds"), - total_duration_seconds=stream_metrics.get("elapsed_seconds", elapsed), + prompt_tokens=fb_stream_metrics.get("prompt_eval_count", 0), + completion_tokens=fb_stream_metrics.get("eval_count"), + tps=fb_stream_metrics.get("tps"), + ttft_seconds=fb_stream_metrics.get("ttft_seconds"), + total_duration_seconds=fb_stream_metrics.get("elapsed_seconds", elapsed), provider=config.fallback.name if config else "fallback", fallback_for=_normalize_model_name(model), fallback_reason=f"Ollama error: {original_error}" if original_error else "Ollama stream error", @@ -1530,9 +1594,18 @@ async def _stream_fallback_openai(payload, config, fallback_model, analytics, st output_text = output_text[:config.history.max_content_size] + "\n...[truncated]" _hist_kw["output_text"] = output_text if analytics: - analytics.log_llm(model=_normalize_model_name(fallback_model), total_duration_seconds=elapsed, provider=provider_tag, fallback_for=_normalize_model_name(fallback_for) if fallback_for else None, fallback_reason=fallback_reason, **_hist_kw) - - return StreamingResponse(generate(), media_type="text/event-stream") + # Estimate tokens from accumulated content (fallbacks don't emit __oct_metrics__) + fb_chars = len("".join(accumulated_content)) + fb_tokens = max(1, fb_chars // 4) if fb_chars else 0 + analytics.log_llm( + model=_normalize_model_name(fallback_model), + completion_tokens=fb_tokens, + total_duration_seconds=elapsed, + provider=provider_tag, + fallback_for=_normalize_model_name(fallback_for) if fallback_for else None, + fallback_reason=fallback_reason, + **_hist_kw, + ) async def _stream_completion_anthropic(client, payload, model, max_tokens, analytics, start_time, history_kwargs=None, config=None): diff --git a/install.sh b/install.sh index 650ce40..2a0af80 100755 --- a/install.sh +++ b/install.sh @@ -188,6 +188,26 @@ echo "" prompt OLLAMA_API_KEY "Enter your Ollama API key" "" +if [ -n "$OLLAMA_API_KEY" ]; then + info "Validating API key with Ollama Cloud..." + VALIDATE_RESPONSE=$(curl -s -w "\n%{http_code}" "https://api.ollama.com/v1/models" \ + -H "Authorization: Bearer ${OLLAMA_API_KEY}" \ + -H "Accept: application/json" \ + --max-time 10 2>/dev/null || echo "timeout") + HTTP_CODE=$(echo "$VALIDATE_RESPONSE" | tail -1) + if [ "$HTTP_CODE" = "200" ]; then + success "API key validated (models endpoint responds OK)" + elif [ "$HTTP_CODE" = "401" ]; then + warn "API key returned 401 Unauthorized from Ollama Cloud" + warn "Key may be invalid or expired. Guanaco will still install but inference will fail." + warn "Fix with: guanaco setup (after install)" + elif [ "$HTTP_CODE" = "timeout" ]; then + warn "Could not reach Ollama Cloud (timeout). Skipping validation." + else + warn "API key validation returned HTTP $HTTP_CODE — key may not work" + fi +fi + if [ -z "$OLLAMA_API_KEY" ]; then echo "" warn "No API key provided. You can set it later with: guanaco setup" @@ -418,6 +438,8 @@ WorkingDirectory=${INSTALL_DIR} ExecStart=${VENV_DIR}/bin/python -m uvicorn guanaco.app:create_app --factory --host ${BIND_HOST} --port ${PORT} --log-level info Restart=on-failure RestartSec=5 +Environment=GUANACO_CONFIG_DIR=${CONFIG_DIR} +WorkingDirectory=${INSTALL_DIR}/repo [Install] WantedBy=multi-user.target diff --git a/pyproject.toml b/pyproject.toml index d88528c..b0812ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,8 +3,8 @@ requires = ["setuptools>=68.0", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "guanaco" -version = "0.4.2" +name = "guanaco-llm-proxy" +version = "0.4.3" description = "OpenAI-compatible LLM proxy that maximizes Ollama Cloud subscriptions — search/scrape API emulation, usage tracking, fallback provider support, and a web dashboard" readme = "README.md" license = {text = "MIT"} From aadb45d026f6d31cdae809a72a868e1f002f87d3 Mon Sep 17 00:00:00 2001 From: Evan Rice Date: Wed, 10 Jun 2026 01:16:50 +0000 Subject: [PATCH 29/31] hotfix(v0.5.0): correct version strings to 0.5.0 The squash-merge from develop included all v0.5.0 feature commits but excluded the version-bump commit (b4ed4fa), which was local-only and never pushed to origin/develop before the PR merge. This left main reporting v0.4.3 despite having all v0.5.0 code. Fixes: __init__.py, app.py, pyproject.toml all now report 0.5.0. Update-safe: only version string changes, zero functional diff. --- guanaco/__init__.py | 2 +- guanaco/app.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/guanaco/__init__.py b/guanaco/__init__.py index 5bf932f..640a122 100644 --- a/guanaco/__init__.py +++ b/guanaco/__init__.py @@ -3,7 +3,7 @@ # Single source of truth for version. # importlib.metadata can return stale values after git-pull without re-pip-install, # so we always use the hardcoded fallback and only override if metadata matches. -__version__ = "0.4.3" +__version__ = "0.5.0" try: from importlib.metadata import version as _version diff --git a/guanaco/app.py b/guanaco/app.py index 0efa761..3b7fddf 100644 --- a/guanaco/app.py +++ b/guanaco/app.py @@ -14,7 +14,7 @@ from fastapi.middleware.cors import CORSMiddleware from guanaco.config import load_config, get_config, AppConfig, get_base_url, get_tailscale_ip from guanaco.client import OllamaClient from guanaco.accounts import AccountPool -__version__ = "0.4.3" +__version__ = "0.5.0" from guanaco.router.router import create_router as create_llm_router from guanaco.search.providers import ALL_PROVIDERS from guanaco.dashboard import create_dashboard_router diff --git a/pyproject.toml b/pyproject.toml index b0812ed..bcefec3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "guanaco-llm-proxy" -version = "0.4.3" +version = "0.5.0" description = "OpenAI-compatible LLM proxy that maximizes Ollama Cloud subscriptions — search/scrape API emulation, usage tracking, fallback provider support, and a web dashboard" readme = "README.md" license = {text = "MIT"} From c0ec3546bd816daa01a5a63359143270473b92a2 Mon Sep 17 00:00:00 2001 From: Evan Rice Date: Wed, 10 Jun 2026 01:24:20 +0000 Subject: [PATCH 30/31] fix(version): use correct package name guanaco-llm-proxy and only override if metadata >= hardcoded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The importlib.metadata fallback was calling _version('guanaco') which matched the stale guanaco-0.4.2.dist-info package left in the venv. Since 0.4.2 >= 0.4.1, it always clobbered the hardcoded __version__. Changes: - Query 'guanaco-llm-proxy' (current package name) instead of 'guanaco' - Compare metadata version against the *hardcoded* value, not a fixed baseline — stale metadata can't downgrade the version anymore Fixes dashboard showing v0.4.3 after git-pull to v0.5.0 without re-pip-install. --- guanaco/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/guanaco/__init__.py b/guanaco/__init__.py index 640a122..07fbdf9 100644 --- a/guanaco/__init__.py +++ b/guanaco/__init__.py @@ -7,12 +7,15 @@ __version__ = "0.5.0" try: from importlib.metadata import version as _version - _pkg_ver = _version("guanaco") - # Only use pkg version if it parses as a clean semver >= our hardcoded baseline. - # This prevents stale/RC versions like "0.4.0rc1" from overriding the hardcoded value. + _pkg_ver = _version("guanaco-llm-proxy") + # Only override hardcoded if installed metadata is *newer or same* — + # prevents stale metadata from git-pull without re-pip-install from + # reverting the version to an old value. import re _m = re.match(r"^(\d+)\.(\d+)\.(\d+)$", _pkg_ver or "") - if _m and tuple(int(x) for x in _m.groups()) >= (0, 4, 1): - __version__ = _pkg_ver + if _m: + _hardcoded = tuple(int(x) for x in __version__.split(".")) + if tuple(int(x) for x in _m.groups()) >= _hardcoded: + __version__ = _pkg_ver except Exception: pass \ No newline at end of file From 87ff832ea38483bf6f3238699d68192705ff4907 Mon Sep 17 00:00:00 2001 From: Evan Rice Date: Wed, 10 Jun 2026 01:38:02 +0000 Subject: [PATCH 31/31] fix(updater): stash + hard-reset instead of merge-pull; release 0.5.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Apply Update endpoint used 'git pull origin {branch}', which fails with a merge conflict if the working tree has any local modifications. This happened to every install that had leftover version-string edits from a prior partial update, leaving the old code running silently. Fix: unconditionally stash (including untracked files) and reset to the exact remote commit before reinstalling. The update now succeeds even if the repo is dirty. Bumps all version strings to 0.5.1 so existing 0.4.2 → 0.5.x installs immediately receive the new updater logic. --- CHANGELOG.md | 177 +++++++++++++++++++++++++++++++++ guanaco/__init__.py | 2 +- guanaco/app.py | 2 +- guanaco/dashboard/dashboard.py | 19 +++- pyproject.toml | 2 +- 5 files changed, 194 insertions(+), 8 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..09eb137 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,177 @@ +# Changelog + +All notable changes to Guanaco will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +--- + +## [0.5.1] - 2026-06-10 + +### Fixed +- **Updater now stashes + hard-resets instead of merge-pulling.** Previously if the working tree had any local modifications (e.g. leftover version-string edits from a prior partial update), `git pull` would abort with a merge conflict and the update silently failed, leaving the old code running. The updater now unconditionally stashes any local changes (including untracked files) and resets to `origin/{branch}` before reinstalling. This makes the **Apply Update** button work reliably on every install, even if the repo is dirty. + +### Changed +- Bumped version to 0.5.1 to ensure existing 0.4.2 → 0.5.x update paths hit the new updater logic immediately. + +--- + +## [0.5.0] - 2026-06-10 + +### Major Release — Usage Tracking, ROI Dashboard, and Multi-Account Infrastructure + +This release represents a significant milestone: Guanaco now tracks every token accurately, displays real cost analytics via a web dashboard, rotates multiple Ollama Cloud accounts, and scrapes live usage tiers from ollama.com instead of guessing. + +--- + +### New Features + +#### Token Estimation & Accurate Usage Tracking +- **skimtoken estimation fallback** (`analytics.py`): When the upstream API (OpenRouter, Ollama Cloud) omits `usage` data in the response, Guanaco now falls back to `skimtoken` for token estimation instead of logging zero tokens. Estimates are ~15% accurate — dramatically better than silently losing usage data. +- **Proper total_tokens calculation**: Fixed `total_tokens = prompt_tokens + completion_tokens` in analytics logging. Previously some code paths double-counted or omitted totals. +- **`fallback_reason` audit column**: Added to `request_log` schema. When token estimation is used instead of API-reported usage, the reason is recorded (e.g. `"api_omitted_usage"`, `"stream_missing_usage"`) for later audit. +- **Input cache-read pricing** (`roi.py`): Tracks `input_cache_read` tokens separately from `input_cache_write`, applying the correct Ollama Cloud discount rate (typically 0.25× of input price for cache hits). + +#### ROI Dashboard & Per-Model Analytics +- **Web dashboard** (`dashboard/`): New FastAPI-mounted dashboard at `/dashboard/` showing: + - Total tokens consumed (last 24h, 7d, 30d, all-time) + - Per-model token distribution with visual bars + - Cost estimates in USD using live OpenRouter pricing + - **ROI configuration panel**: Slider for `cache_hit_pct` (default 70%), editable price multipliers per model + - **Per-model value scoring**: Each model gets a "value score" based on (capability / cost) ratio, helping users pick the cheapest model for a given task +- **OpenRouter price fetcher** (`roi.py`): Scrapes current model pricing from `https://openrouter.ai/api/v1/models` with 24h caching. Falls back to hardcoded prices if fetch fails. +- **Cache-hit discount logic**: ROI calculations apply the user-configured `cache_hit_pct` to reduce effective input costs, reflecting real-world Ollama Cloud behavior where repeated prompts are cached. + +#### Real Usage-Level Scraping from ollama.com +- **`_fetch_usage_level_sync()`** (`client.py`): New synchronous HTML scraper that parses `ollama.com/library/{model}` pages to determine actual GPU usage tiers: + - Handles **top-level model badges** (`x-test-model-cost-slot-active`) for unified-tier models + - Handles **per-tag listings** (`x-test-model-tag-cost` + `x-test-model-tag-usage-slot-active`) for models with multiple size variants + - Returns usage level 1-4, which maps to multiplier 0.25×, 0.50×, 0.75×, 1.00× +- **`fetch_usage_levels()`**: Async parallel fetcher with **1-hour global cache**. Fetches all library pages concurrently using `asyncio.gather()` with thread-pool execution for the blocking HTTP requests. +- **Wired into API responses**: Both `/v1/models` (OpenAI-compatible) and `/api/ollama/models` (internal) now return `usage_multiplier` and `usage_level` fields based on scraped live data. +- **Fixes major heuristic errors**: + - `gemma3:4b` and `gemma3:12b`: was 1.00×, now correctly **0.25×** (1 slot) + - `gemma3:27b`: was 1.00×, now correctly **0.50×** (2 slots) + - `ministral-3`: was 0.75×, now correctly **0.25×** (1 slot) + - `qwen3-vl`: stays **0.75×** (3 slots) — heuristic accidentally got this one right + - `deepseek-v4-pro`: stays **1.00×** (4 slots) + +#### Multi-Account Ollama Cloud Rotation +- **`accounts.py`**: New module managing multiple Ollama Cloud accounts: + - Each account has its own API key + session cookie + - Load-balanced request routing based on usage and subscription tier + - Quota-aware selection: least-loaded accounts preferred; new/untested accounts get priority for immediate validation +- **Premium model routing**: Models `kimi-k2.6` and `glm-5.1` restricted to Pro/Max accounts only. Free-tier accounts are skipped for these models. +- **Per-account usage tracking**: Analytics DB records which account handled each request, enabling per-account cost breakdowns. + +#### Model Catalog Expansion +- Added `minimax-m3` (new MiniMax flagship) +- Added `nemotron-3-ultra` (NVIDIA enterprise model) +- Added `kimi-k2.6` with 200k context window support + +#### Web Search / Scrape Emulation +- **Search provider plugins** (`search/providers/`): Modular search backend support: + - Brave Search API + - Cohere RAG API + - Exa (formerly Metaphor) + - Firecrawl (web scraping) + - Jina AI (neural search) + - SearXNG (self-hosted meta-search) + - Serper (Google Search API) + - Tavily (AI-native search) +- **Search router** (`search/base.py`): Unified interface — Guanaco presents a single `/search` endpoint regardless of which provider is configured. + +### Fixes + +#### Config & Install Robustness +- **Missing `UsageConfig` fields**: Added `last_plan`, `redirect_on_full`, `last_session_reset`, `last_weekly_reset`, `last_checked` to prevent `AttributeError` crashes on configs from v0.4.2 and earlier. +- **Config migration layer**: `load_config()` now auto-migrates v0.4.2 configs to v0.4.3+ schema on first load. No manual intervention needed. +- **Package rename**: Renamed PyPI package from `guanaco` → `guanaco-llm-proxy` to avoid collision with an existing `guanaco` package on PyPI. +- **Install script fixes** (`install.sh`): + - Ollama API key validation now uses the correct env var name + - Fixed `.env` file write pattern (was writing malformed key=value pairs) + - Fixed `grep` pattern for detecting existing config +- **Startup version sanity check**: Detects repo/venv version mismatch on boot and logs a warning. Prevents confusing "why is `/health` showing the old version?" issues. +- **systemd service**: Fixed `WorkingDirectory` to point at the actual repo checkout. Added `GUANACO_CONFIG_DIR` env var to service file. + +#### Dashboard & Analytics Fixes +- **Removed broken `usage_multiplier` column**: The analytics DB no longer tracks `usage_multiplier` per request (it was always wrong due to heuristic mismatch). Model-level multipliers are now fetched live from ollama.com. +- **Backward compat for `SearchConfig`**: Older installs missing search configuration no longer crash on startup. + +### Infrastructure + +#### CI/CD +- **GitHub Actions CI** (`.github/workflows/ci.yml`): Runs on every push — lint, type-check, unit tests. +- **GitHub Actions Release** (`.github/workflows/release.yml`): Automated PyPI publish on tag push. + +#### Docker +- **`Dockerfile.test`**: Containerized smoke-test environment for CI. +- **`test-local.sh`**: One-command local smoke test — builds Docker image, starts server, hits `/health`, validates version string. + +#### Project Hygiene +- Added `CODE_OF_CONDUCT.md`, `CONTRIBUTING.md`, `LICENSE` (MIT) +- Added `.gitignore` with Python/venv patterns +- Added macOS launch agent plist (`com.guanaco.start.plist`) +- Added systemd service templates (`guanaco.service`, `oct.service`) + +### API Changes + +#### Added Fields +- `/v1/models` response now includes: + - `usage_multiplier` (float): cost multiplier 0.25-1.00 + - `usage_level` (int): raw level 1-4, 0 = unknown +- `/api/ollama/models` response now includes: + - `usage_multiplier` (float) + - `usage_level` (int) + +#### Schema Changes +- `request_log` table: added `fallback_reason TEXT` column +- `request_log` table: removed `usage_multiplier` column (was unreliable) +- New `roi_config` table: stores `cache_hit_pct`, `price_multiplier`, per-model overrides + +### Performance +- **Parallel library scraping**: All ollama.com library pages are fetched concurrently. For a catalog of ~50 models, total scrape time is ~3-5 seconds vs. ~60 seconds sequential. +- **1-hour cache**: Scraped usage levels are cached globally, so the 3-5 second penalty only hits once per hour. +- **ROI price cache**: OpenRouter prices cached for 24 hours. Dashboard loads instantly after first visit. + +### Deprecated / Removed +- **Heuristic `_get_model_multiplier()`**: Still exists as fallback when ollama.com scraping fails, but is no longer the primary source. Returns `0.25` for ≤8B, `0.50` for ≤70B, `0.75` for ≤200B, `1.00` for larger. +- **`usage_multiplier` column in analytics DB**: Dropped. Use `/v1/models` or `/api/ollama/models` to get live multipliers. + +### Known Issues +- **Dev server restart unreliable on isolated instance**: The `uvicorn` process sometimes starts without producing logs. Production (`systemctl restart guanaco.service`) is unaffected. +- **Library scraper depends on ollama.com DOM**: If ollama.com changes their HTML test attributes (`x-test-model-*`), the scraper will fall back to heuristic. Monitor `/api/ollama/models` for sudden multiplier shifts. + +--- + +## [0.4.2] - 2026-05-15 + +### New Features +- Multi-account Ollama Cloud rotation with quota-aware selection +- Premium model routing (`kimi-k2.6`, `glm-5.1` → Pro/Max only) +- Per-account usage tracking + +--- + +## [0.4.1] - 2026-05-01 + +### Fixes +- Rate-limit retry logic for Ollama Cloud 429 responses +- SSE streaming stability improvements + +--- + +## [0.4.0] - 2026-04-20 + +### New Features +- Initial Ollama Cloud proxy support +- OpenAI-compatible `/v1/chat/completions` endpoint +- Token usage tracking with SQLite analytics DB +- Basic web dashboard + +--- + +## [0.3.9] and earlier + +See [GitHub releases](https://github.com/evangit2/guanaco/releases) for earlier versions. diff --git a/guanaco/__init__.py b/guanaco/__init__.py index 07fbdf9..c7e78f9 100644 --- a/guanaco/__init__.py +++ b/guanaco/__init__.py @@ -3,7 +3,7 @@ # Single source of truth for version. # importlib.metadata can return stale values after git-pull without re-pip-install, # so we always use the hardcoded fallback and only override if metadata matches. -__version__ = "0.5.0" +__version__ = "0.5.1" try: from importlib.metadata import version as _version diff --git a/guanaco/app.py b/guanaco/app.py index 3b7fddf..ea554b7 100644 --- a/guanaco/app.py +++ b/guanaco/app.py @@ -14,7 +14,7 @@ from fastapi.middleware.cors import CORSMiddleware from guanaco.config import load_config, get_config, AppConfig, get_base_url, get_tailscale_ip from guanaco.client import OllamaClient from guanaco.accounts import AccountPool -__version__ = "0.5.0" +__version__ = "0.5.1" from guanaco.router.router import create_router as create_llm_router from guanaco.search.providers import ALL_PROVIDERS from guanaco.dashboard import create_dashboard_router diff --git a/guanaco/dashboard/dashboard.py b/guanaco/dashboard/dashboard.py index c5fdaf9..4a4d048 100644 --- a/guanaco/dashboard/dashboard.py +++ b/guanaco/dashboard/dashboard.py @@ -895,7 +895,16 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg ) current_branch = branch_result.stdout.strip() or "main" - # Step 2: Git fetch + pull + # Step 2: Git fetch + hard reset — never fail because of local changes. + # We stash any local edits, reset to the exact remote commit, then pull. + # This guarantees the update always succeeds even if the user (or a prior + # partial update) left uncommitted files in the repo. + stash_result = subprocess.run( + ["git", "stash", "push", "-m", "pre-update-stash", "--include-untracked"], + cwd=project_dir, capture_output=True, text=True, timeout=15 + ) + # stash exit 0 = stashed something, exit 1 = nothing to stash — both OK + fetch_result = subprocess.run( ["git", "fetch", "origin", current_branch], cwd=project_dir, capture_output=True, text=True, timeout=30 @@ -903,12 +912,12 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg if fetch_result.returncode != 0: return {"status": "error", "step": "fetch", "message": fetch_result.stderr[:200]} - pull_result = subprocess.run( - ["git", "pull", "origin", current_branch], + reset_result = subprocess.run( + ["git", "reset", "--hard", f"origin/{current_branch}"], cwd=project_dir, capture_output=True, text=True, timeout=30 ) - if pull_result.returncode != 0: - return {"status": "error", "step": "pull", "message": pull_result.stderr[:200]} + if reset_result.returncode != 0: + return {"status": "error", "step": "reset", "message": reset_result.stderr[:200]} # Step 3: Reinstall into venv install_dir = Path.home() / ".guanaco" diff --git a/pyproject.toml b/pyproject.toml index bcefec3..02f60a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "guanaco-llm-proxy" -version = "0.5.0" +version = "0.5.1" description = "OpenAI-compatible LLM proxy that maximizes Ollama Cloud subscriptions — search/scrape API emulation, usage tracking, fallback provider support, and a web dashboard" readme = "README.md" license = {text = "MIT"}