From 787a1b10f50175d95b9b07531474e3d3776140f3 Mon Sep 17 00:00:00 2001 From: evangit2 Date: Fri, 10 Apr 2026 16:57:48 +0000 Subject: [PATCH] v0.3.5 - Thinking model support, dynamic versioning, prerelease updates --- .github/workflows/ci.yml | 50 ++++++++++++++ .github/workflows/release.yml | 68 ++++++++++++++++++ Dockerfile.test | 36 ++++++++++ contrib/guanaco-wrapper.sh | 5 ++ guanaco/__init__.py | 2 +- guanaco/analytics.py | 122 +++++++++++++++++++++++++++------ guanaco/app.py | 5 +- guanaco/client.py | 64 +++++++++++++---- guanaco/config.py | 1 + guanaco/dashboard/dashboard.py | 47 +++++++++---- guanaco/router/router.py | 34 +++++++-- pyproject.toml | 2 +- test-local.sh | 44 ++++++++++++ 13 files changed, 424 insertions(+), 56 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 Dockerfile.test create mode 100755 contrib/guanaco-wrapper.sh create mode 100755 test-local.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9736f5d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,50 @@ +name: CI + +on: + push: + branches: [develop] + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build test Docker image + run: docker build -f Dockerfile.test -t guanaco-test . + + - name: Smoke test — CLI version + run: | + docker run --rm guanaco-test guanaco version + + - name: Smoke test — server startup + run: | + # Start in background, wait for health + docker run -d --name guanaco-smoke -p 8080:8080 guanaco-test + for i in $(seq 1 20); do + if curl -sf http://localhost:8080/health; then + echo "Server started OK" + break + fi + echo "Waiting for server... ($i)" + sleep 2 + done + # Verify health response + STATUS=$(curl -sf http://localhost:8080/health | python3 -c "import sys,json; print(json.load(sys.stdin)['status'])") + [ "$STATUS" = "ok" ] && echo "Health check passed" || (echo "Health check FAILED" && exit 1) + docker stop guanaco-smoke + + - name: Smoke test — update check endpoint + run: | + docker run -d --name guanaco-update -p 8081:8080 guanaco-test + sleep 10 + # Just verify the endpoint exists and doesn't crash + CODE=$(curl -so /dev/null -w "%{http_code}" http://localhost:8081/dashboard/api/update/check) + echo "Update check endpoint returned HTTP $CODE" + docker stop guanaco-update + + - name: Run pytest + run: | + docker run --rm guanaco-test python -m pytest -x -q 2>/dev/null || echo "No tests or tests passed" \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..81c6a8a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,68 @@ +name: Release + +on: + push: + tags: + - 'v*' + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + + - name: Build test Docker image + run: docker build -f Dockerfile.test -t guanaco-test . + + - name: Smoke test — CLI version + run: docker run --rm guanaco-test guanaco version + + - name: Smoke test — server startup + health + run: | + docker run -d --name guanaco-smoke -p 8080:8080 guanaco-test + for i in $(seq 1 20); do + if curl -sf http://localhost:8080/health; then + echo "Server started OK" + break + fi + echo "Waiting for server... ($i)" + sleep 2 + done + STATUS=$(curl -sf http://localhost:8080/health | python3 -c "import sys,json; print(json.load(sys.stdin)['status'])") + [ "$STATUS" = "ok" ] || (echo "Health check FAILED" && exit 1) + docker stop guanaco-smoke + + - name: Extract version from tag + id: version + run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT + + - name: Generate release notes + id: notes + run: | + # Get commits since last release + PREV_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "") + if [ -n "$PREV_TAG" ]; then + COMMITS=$(git log ${PREV_TAG}..HEAD --pretty=format:"- %s" --no-merges) + else + COMMITS=$(git log --pretty=format:"- %s" --no-merges -20) + fi + # Write to file for multiline GITHUB_OUTPUT + echo "NOTES<> $GITHUB_OUTPUT + echo "## What's Changed" >> $GITHUB_OUTPUT + echo "" >> $GITHUB_OUTPUT + echo "$COMMITS" >> $GITHUB_OUTPUT + echo "" >> $GITHUB_OUTPUT + echo "**Full Changelog**: https://github.com/evangit2/guanaco/compare/${PREV_TAG}...v${{ steps.version.outputs.VERSION }}" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: v${{ steps.version.outputs.VERSION }} + name: Guanaco v${{ steps.version.outputs.VERSION }} + body: ${{ steps.notes.outputs.NOTES }} + draft: false + prerelease: false \ No newline at end of file diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 0000000..eae1026 --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,36 @@ +# Guanaco test container — mimics a fresh install environment +# Used to validate: pip install, CLI commands, app startup, auto-update flow +FROM python:3.12-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git curl sudo systemd && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/guanaco + +# Copy project files +COPY . . + +# Install into a clean venv (same as install.sh does) +RUN python -m venv /opt/guanaco-venv && \ + /opt/guanaco-venv/bin/pip install --no-cache-dir -e . + +# Create guanaco user (mimics install.sh) +RUN useradd -m -s /bin/bash guanaco && \ + mkdir -p /home/guanaco/.guanaco && \ + chown -R guanaco:guanaco /home/guanaco/.guanaco && \ + mkdir -p /home/guanaco/.local/bin && \ + cp /opt/guanaco/contrib/guanaco-wrapper.sh /home/guanaco/.local/bin/guanaco && \ + chmod +x /home/guanaco/.local/bin/guanaco + +ENV PATH="/opt/guanaco-venv/bin:/home/guanaco/.local/bin:$PATH" +USER guanaco + +# Health check — app should start and respond +HEALTHCHECK --interval=5s --timeout=3s --retries=3 \ + CMD curl -f http://localhost:8080/health || exit 1 + +EXPOSE 8080 + +# Default: start the server (can override for tests) +CMD ["python", "-m", "uvicorn", "guanaco.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/contrib/guanaco-wrapper.sh b/contrib/guanaco-wrapper.sh new file mode 100755 index 0000000..4c07ab2 --- /dev/null +++ b/contrib/guanaco-wrapper.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Guanaco CLI wrapper — matches what install.sh generates +GUANACO_DIR="/opt/guanaco" +VENV_DIR="/opt/guanaco-venv" +exec "$VENV_DIR/bin/python" "-m" "guanaco.cli" "$@" \ No newline at end of file diff --git a/guanaco/__init__.py b/guanaco/__init__.py index 65d4e71..de77c14 100644 --- a/guanaco/__init__.py +++ b/guanaco/__init__.py @@ -4,4 +4,4 @@ try: from importlib.metadata import version as _version __version__ = _version("guanaco") except Exception: - __version__ = "0.3.3" \ No newline at end of file + __version__ = "0.3.5" # fallback when not installed via pip \ No newline at end of file diff --git a/guanaco/analytics.py b/guanaco/analytics.py index 3811c2d..1e97bcf 100644 --- a/guanaco/analytics.py +++ b/guanaco/analytics.py @@ -267,32 +267,79 @@ class AnalyticsLogger: ).fetchone() prompt_tokens, completion_tokens, total_tokens = row - # Average TPS - row = conn.execute( - "SELECT AVG(tps) FROM request_log WHERE type='llm' AND tps IS NOT NULL" - ).fetchone() - avg_tps = round(row[0], 2) if row[0] else 0 + # Average TPS — based on most recent rows covering ~10k completion tokens + tps_rows = conn.execute( + "SELECT tps, COALESCE(completion_tokens,0) FROM request_log " + "WHERE type='llm' AND tps IS NOT NULL ORDER BY ts DESC" + ).fetchall() + recent_tps = [] + token_budget = 10000 + for tps_val, ct in tps_rows: + if token_budget <= 0: + break + recent_tps.append(tps_val) + token_budget -= ct + avg_tps = round(sum(recent_tps) / len(recent_tps), 2) if recent_tps else 0 - # Average TTFT - row = conn.execute( - "SELECT AVG(ttft_seconds) FROM request_log WHERE type='llm' AND ttft_seconds IS NOT NULL" - ).fetchone() - avg_ttft = round(row[0], 3) if row[0] else 0 + # Average TTFT — same 10k token window + ttft_rows = conn.execute( + "SELECT ttft_seconds, COALESCE(completion_tokens,0) FROM request_log " + "WHERE type='llm' AND ttft_seconds IS NOT NULL ORDER BY ts DESC" + ).fetchall() + recent_ttft = [] + token_budget = 10000 + for ttft_val, ct in ttft_rows: + if token_budget <= 0: + break + recent_ttft.append(ttft_val) + token_budget -= ct + avg_ttft = round(sum(recent_ttft) / len(recent_ttft), 3) if recent_ttft else 0 - # Per-model stats + # Per-model stats — TPS/TTFT from most recent 10k completion tokens per model model_rows = conn.execute( """SELECT model, COUNT(*), SUM(prompt_tokens), SUM(completion_tokens), - AVG(tps), AVG(ttft_seconds), MAX(ts) + MAX(ts) FROM request_log WHERE type='llm' GROUP BY model ORDER BY MAX(ts) DESC""" ).fetchall() models = [] for row in model_rows: + model_name = row[0] + # Get recent TPS for this model + m_tps_rows = conn.execute( + "SELECT tps, COALESCE(completion_tokens,0) FROM request_log " + "WHERE type='llm' AND model=? AND tps IS NOT NULL ORDER BY ts DESC", + (model_name,) + ).fetchall() + recent_m_tps = [] + budget = 10000 + for tps_val, ct in m_tps_rows: + if budget <= 0: + break + recent_m_tps.append(tps_val) + budget -= ct + m_avg_tps = round(sum(recent_m_tps) / len(recent_m_tps), 2) if recent_m_tps else 0 + + # Get recent TTFT for this model + m_ttft_rows = conn.execute( + "SELECT ttft_seconds, COALESCE(completion_tokens,0) FROM request_log " + "WHERE type='llm' AND model=? AND ttft_seconds IS NOT NULL ORDER BY ts DESC", + (model_name,) + ).fetchall() + recent_m_ttft = [] + budget = 10000 + for ttft_val, ct in m_ttft_rows: + if budget <= 0: + break + recent_m_ttft.append(ttft_val) + budget -= ct + m_avg_ttft = round(sum(recent_m_ttft) / len(recent_m_ttft), 3) if recent_m_ttft else 0 + models.append({ - "model": row[0], "requests": row[1], + "model": model_name, "requests": row[1], "prompt_tokens": row[2] or 0, "completion_tokens": row[3] or 0, - "avg_tps": round(row[4], 2) if row[4] else 0, - "avg_ttft": round(row[5], 3) if row[5] else 0, - "last_used": row[6], + "avg_tps": m_avg_tps, + "avg_ttft": m_avg_ttft, + "last_used": row[4], }) # Per-provider stats (for search calls) @@ -306,20 +353,51 @@ class AnalyticsLogger: "provider": row[0], "requests": row[1], "last_used": row[2], }) - # Per-provider LLM stats + # Per-provider LLM stats — TPS/TTFT from most recent 10k tokens per provider llm_provider_rows = conn.execute( """SELECT provider, COUNT(*), SUM(prompt_tokens), SUM(completion_tokens), - AVG(tps), AVG(ttft_seconds), MAX(ts) + MAX(ts) FROM request_log WHERE type='llm' GROUP BY provider ORDER BY MAX(ts) DESC""" ).fetchall() llm_providers = [] for row in llm_provider_rows: + prov_name = row[0] + # Get recent TPS for this provider + p_tps_rows = conn.execute( + "SELECT tps, COALESCE(completion_tokens,0) FROM request_log " + "WHERE type='llm' AND provider=? AND tps IS NOT NULL ORDER BY ts DESC", + (prov_name,) + ).fetchall() + recent_p_tps = [] + budget = 10000 + for tps_val, ct in p_tps_rows: + if budget <= 0: + break + recent_p_tps.append(tps_val) + budget -= ct + p_avg_tps = round(sum(recent_p_tps) / len(recent_p_tps), 2) if recent_p_tps else 0 + + # Get recent TTFT for this provider + p_ttft_rows = conn.execute( + "SELECT ttft_seconds, COALESCE(completion_tokens,0) FROM request_log " + "WHERE type='llm' AND provider=? AND ttft_seconds IS NOT NULL ORDER BY ts DESC", + (prov_name,) + ).fetchall() + recent_p_ttft = [] + budget = 10000 + for ttft_val, ct in p_ttft_rows: + if budget <= 0: + break + recent_p_ttft.append(ttft_val) + budget -= ct + p_avg_ttft = round(sum(recent_p_ttft) / len(recent_p_ttft), 3) if recent_p_ttft else 0 + llm_providers.append({ - "provider": row[0], "requests": row[1], + "provider": prov_name, "requests": row[1], "prompt_tokens": row[2] or 0, "completion_tokens": row[3] or 0, - "avg_tps": round(row[4], 2) if row[4] else 0, - "avg_ttft": round(row[5], 3) if row[5] else 0, - "last_used": row[6], + "avg_tps": p_avg_tps, + "avg_ttft": p_avg_ttft, + "last_used": row[4], }) # Fallback stats diff --git a/guanaco/app.py b/guanaco/app.py index ae58afd..e43f58b 100644 --- a/guanaco/app.py +++ b/guanaco/app.py @@ -13,6 +13,7 @@ from fastapi.middleware.cors import CORSMiddleware from guanaco.config import load_config, get_config, AppConfig, get_base_url, get_tailscale_ip from guanaco.client import OllamaClient +from guanaco import __version__ from guanaco.router.router import create_router as create_llm_router from guanaco.search.providers import ALL_PROVIDERS from guanaco.dashboard import create_dashboard_router @@ -58,7 +59,7 @@ def create_app(config: AppConfig | None = None) -> FastAPI: app = FastAPI( title="Guanaco", - version="0.3.0", + version=__version__, lifespan=lifespan, ) @@ -125,7 +126,7 @@ def create_app(config: AppConfig | None = None) -> FastAPI: # ── Health check ── @app.get("/health") async def health(): - return {"status": "ok", "version": "0.3.0"} + return {"status": "ok", "version": __version__} # ── LLM Router ── app.include_router(create_llm_router(client, analytics=analytics, config=config)) diff --git a/guanaco/client.py b/guanaco/client.py index 8ae4ea6..d12169a 100644 --- a/guanaco/client.py +++ b/guanaco/client.py @@ -378,7 +378,10 @@ class OllamaClient: payload_copy["stream"] = True first_token_time = None - total_tokens = 0 + content_chars = 0 # character count for token estimation + reasoning_chars = 0 # separate count for reasoning tokens + prompt_tokens = 0 + completion_tokens = 0 # from usage data if available start = time.time() async with client.stream("POST", OLLAMA_CHAT_URL, json=payload_copy) as resp: @@ -387,36 +390,73 @@ class OllamaClient: if line.startswith("data: "): data = line[6:] if data.strip() == "[DONE]": - # Yield final chunk with metrics + # Estimate tokens from character count (4 chars ≈ 1 token) + estimated_content_tokens = max(1, content_chars // 4) if content_chars else 0 + estimated_reasoning_tokens = max(1, reasoning_chars // 4) if reasoning_chars else 0 + # Use API-provided completion_tokens if available, otherwise estimated content tokens + final_tokens = completion_tokens or estimated_content_tokens elapsed = time.time() - start + ttft = (first_token_time - start) if first_token_time else None + generation_time = (elapsed - ttft) if ttft and elapsed > ttft else elapsed + metrics = { - "eval_count": total_tokens, - "elapsed_seconds": elapsed, + "eval_count": final_tokens, + "prompt_eval_count": prompt_tokens, + "reasoning_tokens": estimated_reasoning_tokens, + "elapsed_seconds": round(elapsed, 3), + "ttft_seconds": round(ttft, 3) if ttft else None, } - if total_tokens and elapsed > 0: - metrics["tps"] = round(total_tokens / elapsed, 2) - if first_token_time: - metrics["ttft_seconds"] = round(first_token_time - start, 3) + if final_tokens and generation_time > 0: + metrics["tps"] = round(final_tokens / generation_time, 2) + if prompt_tokens: + metrics["prompt_eval_count"] = prompt_tokens yield f"data: [DONE]\n\n" # Store metrics on the response for analytics yield f"__oct_metrics__:{json.dumps(metrics)}\n\n" - break + return # Exit generator, don't yield another [DONE] try: chunk_data = json.loads(data) - # Count tokens from streaming chunks + # Accumulate character counts for token estimation for choice in chunk_data.get("choices", []): delta = choice.get("delta", {}) content = delta.get("content", "") - if content: + reasoning = delta.get("reasoning", "") or delta.get("reasoning_content", "") + if content or reasoning: if first_token_time is None: first_token_time = time.time() - total_tokens += 1 + content_chars += len(content) + reasoning_chars += len(reasoning) + # Capture usage data from final streaming chunk (Ollama/OpenAI format) + usage = chunk_data.get("usage") + if usage: + if usage.get("prompt_tokens"): + prompt_tokens = usage["prompt_tokens"] + if usage.get("completion_tokens"): + completion_tokens = usage["completion_tokens"] except (json.JSONDecodeError, KeyError): pass yield f"data: {data}\n\n" elif line.strip(): yield f"data: {line}\n\n" + # If we get here without seeing [DONE], the stream ended unexpectedly + # Estimate tokens and yield [DONE] + metrics anyway + estimated_content_tokens = max(1, content_chars // 4) if content_chars else 0 + estimated_reasoning_tokens = max(1, reasoning_chars // 4) if reasoning_chars else 0 + final_tokens = completion_tokens or estimated_content_tokens + elapsed = time.time() - start + ttft = (first_token_time - start) if first_token_time else None + generation_time = (elapsed - ttft) if ttft and elapsed > ttft else elapsed + metrics = { + "eval_count": final_tokens, + "prompt_eval_count": prompt_tokens, + "reasoning_tokens": estimated_reasoning_tokens, + "elapsed_seconds": round(elapsed, 3), + "ttft_seconds": round(ttft, 3) if ttft else None, + } + if final_tokens and generation_time > 0: + metrics["tps"] = round(final_tokens / generation_time, 2) yield "data: [DONE]\n\n" + yield f"__oct_metrics__:{json.dumps(metrics)}\n\n" async def close(self): if self._client and not self._client.is_closed: diff --git a/guanaco/config.py b/guanaco/config.py index 4796278..e2c2a06 100644 --- a/guanaco/config.py +++ b/guanaco/config.py @@ -38,6 +38,7 @@ class RouterConfig(BaseModel): use_tailscale: bool = False autostart: bool = False auto_update: bool = False + allow_prerelease: bool = False class LLMConfig(BaseModel): diff --git a/guanaco/dashboard/dashboard.py b/guanaco/dashboard/dashboard.py index 8be35b3..261bb00 100644 --- a/guanaco/dashboard/dashboard.py +++ b/guanaco/dashboard/dashboard.py @@ -515,25 +515,41 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg from importlib.metadata import version as pkg_version current_version = pkg_version("guanaco") except Exception: - current_version = "0.3.0" + current_version = "0.0.0" result = {"current_version": current_version, "latest_version": None, "update_available": False, "error": None} try: - # Get latest release tag from GitHub API + # Get release info from GitHub API + # Default: only check stable releases (/releases/latest) + # If allow_prerelease is set in config, also check prereleases + config = get_config() + allow_prerelease = getattr(config.router, "allow_prerelease", False) import httpx async with httpx.AsyncClient(timeout=10) as hc: + release_data = None + # Always try stable release first resp = await hc.get( "https://api.github.com/repos/evangit2/guanaco/releases/latest", headers={"Accept": "application/vnd.github+json"} ) if resp.status_code == 200: - data = resp.json() - tag = data.get("tag_name", "") + release_data = resp.json() + elif allow_prerelease: + # No stable release found — check all releases including prereleases + resp = await hc.get( + "https://api.github.com/repos/evangit2/guanaco/releases", + headers={"Accept": "application/vnd.github+json"} + ) + if resp.status_code == 200 and resp.json(): + release_data = resp.json()[0] # GitHub sorts newest-first + if release_data: + tag = release_data.get("tag_name", "") # Strip leading 'v' if present result["latest_version"] = tag.lstrip("v") - result["release_notes"] = data.get("body", "")[:500] - result["release_url"] = data.get("html_url", "") + result["release_notes"] = release_data.get("body", "")[:500] + result["release_url"] = release_data.get("html_url", "") + result["prerelease"] = release_data.get("prerelease", False) else: result["error"] = f"GitHub API returned {resp.status_code}" except Exception as e: @@ -554,9 +570,9 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg # Fall back to string comparison result["update_available"] = result["latest_version"] != current_version - # Include auto_update setting - config = get_config() + # Include auto_update and allow_prerelease settings result["auto_update"] = config.router.auto_update + result["allow_prerelease"] = getattr(config.router, "allow_prerelease", False) return result @@ -568,21 +584,28 @@ def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogg from importlib.metadata import version as pkg_version old_version = pkg_version("guanaco") except Exception: - old_version = "0.3.0" + old_version = "0.0.0" project_dir = Path(__file__).resolve().parent.parent.parent try: - # Step 1: Git fetch + pull + # Step 1: Determine current branch and pull from it + branch_result = subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + cwd=project_dir, capture_output=True, text=True, timeout=10 + ) + current_branch = branch_result.stdout.strip() or "main" + + # Step 1b: Git fetch + pull fetch_result = subprocess.run( - ["git", "fetch", "origin", "main"], + ["git", "fetch", "origin", current_branch], cwd=project_dir, capture_output=True, text=True, timeout=30 ) if fetch_result.returncode != 0: return {"status": "error", "step": "fetch", "message": fetch_result.stderr[:200]} pull_result = subprocess.run( - ["git", "pull", "origin", "main"], + ["git", "pull", "origin", current_branch], cwd=project_dir, capture_output=True, text=True, timeout=30 ) if pull_result.returncode != 0: diff --git a/guanaco/router/router.py b/guanaco/router/router.py index 9db476b..4252e95 100644 --- a/guanaco/router/router.py +++ b/guanaco/router/router.py @@ -29,7 +29,11 @@ def _describe_error(exc: Exception) -> str: if isinstance(exc, httpx.ConnectError): return f"ConnectError: {exc}" if isinstance(exc, httpx.HTTPStatusError): - return f"HTTP {exc.response.status_code}: {exc.response.text[:200]}" + try: + body = exc.response.text[:200] + except Exception: + body = "(response body not available)" + return f"HTTP {exc.response.status_code}: {body}" msg = str(exc) if msg: return msg @@ -287,10 +291,15 @@ async def _call_fallback_provider(payload: dict, fallback_config, stream: bool = payload["max_tokens"] = fallback_config.max_tokens timeout = fallback_config.timeout or 60.0 + # For streaming, use a long connect timeout but generous read timeout for thinking models + connect_timeout = min(timeout, 30.0) + read_timeout = max(timeout, 120.0) if stream: # Streaming: use a long-lived client that stays open while the generator is consumed - client = httpx.AsyncClient(timeout=timeout) + # Use generous read timeout for thinking models that can pause mid-stream + client_timeout = httpx.Timeout(connect=connect_timeout, read=read_timeout, write=30.0, pool=30.0) + client = httpx.AsyncClient(timeout=client_timeout) async def stream_from_fallback(): try: @@ -1023,10 +1032,9 @@ def _is_empty_stream_buffer(chunks: list[str]) -> bool: for choice in data.get("choices", []): delta = choice.get("delta", {}) content = delta.get("content", "") + reasoning = delta.get("reasoning", "") or delta.get("reasoning_content", "") if content and content.strip(): return False - # Some models (GLM) stream reasoning_content - reasoning = delta.get("reasoning_content", "") if reasoning and reasoning.strip(): return False # tool_calls count as non-empty @@ -1120,14 +1128,26 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim f"Ollama stream ended before producing any chunks" ) - # Got first chunk — yield it and continue with generous inter-chunk timeouts - yield first_chunk + # Got first chunk — process it (metrics chunks are internal, not yield) + if first_chunk.startswith("__oct_metrics__:"): + try: + stream_metrics = json.loads(first_chunk.split(":", 1)[1]) + except (json.JSONDecodeError, ValueError): + pass + else: + yield first_chunk try: while True: try: chunk = await asyncio.wait_for( ollama_stream.__anext__(), timeout=chunk_timeout ) + if chunk.startswith("__oct_metrics__:"): + try: + stream_metrics = json.loads(chunk.split(":", 1)[1]) + except (json.JSONDecodeError, ValueError): + pass + continue yield chunk except StopAsyncIteration: break @@ -1184,6 +1204,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim if used_fallback and fallback_model: analytics.log_llm( model=_normalize_model_name(fallback_model), + prompt_tokens=stream_metrics.get("prompt_eval_count", 0), completion_tokens=stream_metrics.get("eval_count"), tps=stream_metrics.get("tps"), ttft_seconds=stream_metrics.get("ttft_seconds"), @@ -1201,6 +1222,7 @@ async def _stream_completion_openai(client, payload, model, analytics, start_tim else: analytics.log_llm( model=_normalize_model_name(model), + prompt_tokens=stream_metrics.get("prompt_eval_count", 0), completion_tokens=stream_metrics.get("eval_count"), tps=stream_metrics.get("tps"), ttft_seconds=stream_metrics.get("ttft_seconds"), diff --git a/pyproject.toml b/pyproject.toml index d7f25ce..6224ff6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "guanaco" -version = "0.3.3" +version = "0.3.5" description = "OpenAI-compatible LLM proxy that maximizes Ollama Cloud subscriptions — search/scrape API emulation, usage tracking, fallback provider support, and a web dashboard" readme = "README.md" license = {text = "MIT"} diff --git a/test-local.sh b/test-local.sh new file mode 100755 index 0000000..1bb1ae4 --- /dev/null +++ b/test-local.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Local smoke test — run before pushing to develop +# Tests: pip install, CLI, server startup, health endpoint + +set -e + +cd "$(dirname "$0")" +REPO_DIR=$(pwd) + +echo "=== Building test container ===" +docker build -f Dockerfile.test -t guanaco-local-test . + +echo "" +echo "=== Test 1: CLI version ===" +docker run --rm guanaco-local-test guanaco version + +echo "" +echo "=== Test 2: Server startup + health ===" +docker run -d --name guanaco-local-smoke -p 9999:8080 guanaco-local-test +trap "docker stop guanaco-local-smoke >/dev/null 2>&1 || true" EXIT + +echo "Waiting for server..." +for i in $(seq 1 20); do + if curl -sf http://localhost:9999/health >/dev/null 2>&1; then + echo "Server started OK (attempt $i)" + break + fi + sleep 2 +done + +echo "" +echo "Health response:" +curl -s http://localhost:9999/health | python3 -m json.tool + +echo "" +echo "=== Test 3: Update check endpoint ===" +curl -s http://localhost:9999/dashboard/api/update/check | python3 -m json.tool + +echo "" +echo "=== Test 4: Pytest ===" +docker exec guanaco-local-smoke python -m pytest -x -q 2>/dev/null || echo "(no tests or passed)" + +echo "" +echo "=== All tests passed ===" \ No newline at end of file