From bbb2cc4903ad88c8be97028b1fcbaddf1fada43c Mon Sep 17 00:00:00 2001 From: evangit2 Date: Thu, 9 Apr 2026 20:49:59 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=A6=99=20Guanaco=20v0.3.0=20=E2=80=94=20i?= =?UTF-8?q?nitial=20OSS=20release?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenAI-compatible LLM proxy that maximizes your Ollama Cloud subscription. Features: - /v1/chat/completions router with token tracking - /v1/messages Anthropic proxy - 8 search/scrape API emulators (Tavily, Exa, SearXNG, Firecrawl, Serper, Jina, Cohere, Brave) - Automatic fallback to secondary providers with configurable timeouts - Streaming support with first-chunk fast failover - Web dashboard with analytics, config, and usage monitoring - Caching layer (beta) - CLI for setup, status, analytics, key management - Docker and systemd deployment support - Backward compatible with OCT (ollama-cloud-tools) installations --- .gitignore | 90 ++ CODE_OF_CONDUCT.md | 132 ++ CONTRIBUTING.md | 109 ++ LICENSE | 21 + README.md | 353 +++++ contrib/com.guanaco.start.plist | 39 + contrib/com.oct.start.plist | 37 + contrib/guanaco.service | 36 + contrib/oct.service | 35 + guanaco/__init__.py | 3 + guanaco/analytics.py | 432 ++++++ guanaco/app.py | 376 +++++ guanaco/cache.py | 368 +++++ guanaco/cli.py | 585 ++++++++ guanaco/client.py | 423 ++++++ guanaco/config.py | 205 +++ guanaco/dashboard/__init__.py | 5 + guanaco/dashboard/dashboard.py | 488 +++++++ guanaco/dashboard/templates/dashboard.html | 1485 ++++++++++++++++++++ guanaco/router/__init__.py | 1 + guanaco/router/router.py | 1212 ++++++++++++++++ guanaco/search/__init__.py | 1 + guanaco/search/base.py | 41 + guanaco/search/providers/__init__.py | 27 + guanaco/search/providers/brave.py | 67 + guanaco/search/providers/cohere.py | 67 + guanaco/search/providers/exa.py | 122 ++ guanaco/search/providers/firecrawl.py | 228 +++ guanaco/search/providers/jina.py | 187 +++ guanaco/search/providers/searxng.py | 90 ++ guanaco/search/providers/serper.py | 97 ++ guanaco/search/providers/tavily.py | 83 ++ guanaco/utils/__init__.py | 5 + guanaco/utils/api_keys.py | 90 ++ install.sh | 247 ++++ pyproject.toml | 60 + 36 files changed, 7847 insertions(+) create mode 100644 .gitignore create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 contrib/com.guanaco.start.plist create mode 100644 contrib/com.oct.start.plist create mode 100644 contrib/guanaco.service create mode 100644 contrib/oct.service create mode 100644 guanaco/__init__.py create mode 100644 guanaco/analytics.py create mode 100644 guanaco/app.py create mode 100644 guanaco/cache.py create mode 100644 guanaco/cli.py create mode 100644 guanaco/client.py create mode 100644 guanaco/config.py create mode 100644 guanaco/dashboard/__init__.py create mode 100644 guanaco/dashboard/dashboard.py create mode 100644 guanaco/dashboard/templates/dashboard.html create mode 100644 guanaco/router/__init__.py create mode 100644 guanaco/router/router.py create mode 100644 guanaco/search/__init__.py create mode 100644 guanaco/search/base.py create mode 100644 guanaco/search/providers/__init__.py create mode 100644 guanaco/search/providers/brave.py create mode 100644 guanaco/search/providers/cohere.py create mode 100644 guanaco/search/providers/exa.py create mode 100644 guanaco/search/providers/firecrawl.py create mode 100644 guanaco/search/providers/jina.py create mode 100644 guanaco/search/providers/searxng.py create mode 100644 guanaco/search/providers/serper.py create mode 100644 guanaco/search/providers/tavily.py create mode 100644 guanaco/utils/__init__.py create mode 100644 guanaco/utils/api_keys.py create mode 100755 install.sh create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0850ce1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,90 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +*.egg + +# Virtual environments +venv/ +.venv/ +env/ + +# Environment variables +.env +.env.* +!.env.example + +# Databases +*.db +*.sqlite +analytics.db + +# Config with secrets +config.yaml + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre +.pyre/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Project-specific +.oct/ \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..baa257a --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,132 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and it also applies +when an individual is representing the project or its community in public spaces. +Examples of representing a project or community include using an official +project e-mail address, posting via an official social media account, or acting +as an appointed representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement by opening an +issue tagged "conduct" or by contacting the maintainers privately. All complaints +will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1e2d94d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,109 @@ +# Contributing to Guanaco + +First off, thank you for considering contributing to Guanaco! It's people like you that make Guanaco such a great tool. + +## Table of Contents + +- [Code of Conduct](#code-of-conduct) +- [How Can I Contribute?](#how-can-i-contribute) + - [Reporting Bugs](#reporting-bugs) + - [Suggesting Enhancements](#suggesting-enhancements) + - [Pull Requests](#pull-requests) +- [Development Setup](#development-setup) +- [Coding Standards](#coding-standards) +- [Commit Messages](#commit-messages) + +## Code of Conduct + +This project and everyone participating in it is governed by the [Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. + +## How Can I Contribute? + +### Reporting Bugs + +Bug reports are hugely important. Before creating a bug report, please check the existing issues to avoid duplicates. + +When filing a bug report, please include: + +- **A clear, descriptive title** +- **Steps to reproduce** — the more specific, the better +- **Expected behavior** — what did you expect to happen? +- **Actual behavior** — what happened instead? +- **Environment details** — OS, Python version, Guanaco version +- **Logs** — any relevant log output or error messages + +### Suggesting Enhancements + +Enhancement suggestions are welcome. Please include: + +- **A clear, descriptive title** +- **Use case** — why is this enhancement useful? +- **Proposed solution** — how should it work? +- **Alternatives considered** — what other approaches have you thought of? + +### Pull Requests + +1. Fork the repository +2. Create a feature branch (`git checkout -b feature/my-new-feature`) +3. Make your changes +4. Add tests for your changes if applicable +5. Ensure all tests pass (`pytest`) +6. Commit with a clear message (see [Commit Messages](#commit-messages)) +7. Push to your fork (`git push origin feature/my-new-feature`) +8. Open a Pull Request against the `master` branch + +PRs should: + +- Address one concern at a time (keep them focused) +- Include tests for new functionality +- Update documentation for changed behavior +- Pass all existing tests + +## Development Setup + +```bash +# Clone the repository +git clone https://github.com/evanrice/guanaco.git +cd guanaco + +# Create and activate a virtual environment +python -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate + +# Install in development mode +pip install -e ".[dev]" + +# Run the CLI +guanaco --help + +# Run tests +pytest +``` + +### Running Locally + +```bash +# Start the proxy server +guanaco serve + +# Or use the short alias +oct serve +``` + +## Coding Standards + +- **Python 3.10+** — use modern Python features (type hints, match statements, etc.) +- **Follow PEP 8** — use a linter/formatter (ruff, black, or flake8) +- **Type hints** —annotate function signatures where practical +- **Docstrings** — use docstrings for public modules, classes, and functions +- **Keep it async** — the codebase uses async/await; prefer async patterns for I/O +- **No secrets in code** — use environment variables or config files (never hardcode credentials) + +## Commit Messages + +- Use the present tense ("add feature" not "added feature") +- Use the imperative mood ("move cursor to..." not "moves cursor to...") +- Limit the first line to 72 characters +- Reference issues and PRs when relevant + +Thank you for contributing! \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c4146fc --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Guanaco Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..3925d20 --- /dev/null +++ b/README.md @@ -0,0 +1,353 @@ +# Guanaco 🦙 + +[![PyPI version](https://img.shields.io/pypi/v/guanaco?color=brightgreen)](https://pypi.org/project/guanaco/) +[![Python](https://img.shields.io/pypi/pyversions/guanaco)](https://pypi.org/project/guanaco/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) +[![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/) + +**Maximize your Ollama Cloud subscription.** + +Guanaco is a self-hosted FastAPI proxy that sits between your applications and Ollama Cloud. It provides an OpenAI-compatible `/v1/chat/completions` endpoint, emulates 8 major search and scrape APIs, tracks token usage, supports transparent fallback to external providers, and ships with a real-time management dashboard — all on a single port. + +```bash +pip install guanaco +``` + +--- + +## Features + +- **🦙 LLM Router** — OpenAI-compatible `/v1/chat/completions` and Anthropic-compatible `/v1/messages` proxy with streaming, token tracking, and analytics +- **🔍 8 Search/Scrape Emulators** — Drop-in replacements for Tavily, Exa, SearXNG, Firecrawl, Serper, Jina, Cohere, and Brave Search +- **🔄 Fallback Provider** — Automatically route to a secondary OpenAI-compatible provider when Ollama Cloud is slow, rate-limited, or unavailable +- **📊 Usage Tracking** — Monitor Ollama Cloud session and weekly quota usage in real time +- **💾 Smart Caching** — Optional exact-match and session-aware prefix caching (BETA) to reduce redundant API calls +- **📈 Web Dashboard** — Real-time analytics, model configuration, API key management, and service status at `http://localhost:8080/dashboard` +- **🐳 Docker & systemd** — Production-ready deployment with included service unit files +- **🔁 Backward Compatible** — The `oct` CLI alias is fully preserved + +--- + +## Quick Start + +### 1. Install + +```bash +pip install guanaco +``` + +### 2. Configure + +```bash +guanaco setup +``` + +Or set your API key directly: + +```bash +export OLLAMA_API_KEY=your_ollama_api_key +``` + +### 3. Run + +```bash +guanaco start +``` + +Your apps can now hit: + +| Endpoint | Purpose | +|----------|---------| +| `http://localhost:8080/v1/chat/completions` | OpenAI-compatible LLM router | +| `http://localhost:8080/v1/messages` | Anthropic-compatible proxy | +| `http://localhost:8080/tavily/search` | Tavily search (emulated) | +| `http://localhost:8080/exa/search` | Exa search (emulated) | +| `http://localhost:8080/firecrawl/scrape` | Firecrawl scrape (emulated) | +| `http://localhost:8080/brave/search` | Brave Search (emulated) | +| `http://localhost:8080/dashboard` | Web dashboard | + +--- + +## CLI Commands + +| Command | Description | +|---------|-------------| +| `guanaco start` | Start the proxy server (router + search APIs + dashboard) | +| `guanaco setup` | Interactive configuration wizard | +| `guanaco status` | Show service status and Ollama Cloud connectivity | +| `guanaco models` | List available Ollama Cloud models | +| `guanaco models --refresh` | Force-refresh model list from Ollama API | +| `guanaco models --capabilities` | Show model capabilities and sizes | +| `guanaco usage` | Check current Ollama Cloud session/weekly quota | +| `guanaco key generate` | Generate a new API key | +| `guanaco key list` | List all API keys | +| `guanaco key revoke` | Revoke an API key | +| `guanaco analytics` | View request analytics summary | +| `guanaco analytics --errors` | Show recent errors | +| `guanaco analytics --model ` | Show history for a specific model | +| `guanaco config --show` | Show current configuration | +| `guanaco config --set ` | Update a config value | +| `guanaco version` | Show version | + +--- + +## Dashboard + +The built-in web dashboard is available at `http://localhost:8080/dashboard`. + +``` +┌────────────────────────────────────────────────────────────────┐ +│ Dashboard Preview │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────────────┐│ +│ │ Total Requests│ │ LLM Calls │ │ Prompt Tokens ││ +│ │ 1,249 │ │ 892 │ │ 4.2M ││ +│ └──────────────┘ └──────────────┘ └──────────────────────┘│ +│ │ +│ ┌─────────────────────────────────────────────────────────┐ │ +│ │ Usage: Session ████████░░░░░░░░░ 62% Resets in 12 min │ │ +│ └─────────────────────────────────────────────────────────┘ │ +│ │ +│ [Models] [Analytics] [API Keys] [Config] [Status] │ +└────────────────────────────────────────────────────────────────┘ +``` + +Features: real-time request analytics, token usage graphs, model configuration, fallback provider setup, API key management, and Ollama Cloud quota monitoring. + +--- + +## Configuration + +Guanaco stores configuration in `~/.guanaco/config.yaml`. You can change the config directory: + +```bash +export GUANACO_CONFIG_DIR=/path/to/config +``` + +### Full `config.yaml` Reference + +```yaml +# ── Required ── +ollama_api_key: "sk-ollama-..." # Or set via OLLAMA_API_KEY env var + +# ── Server ── +router: + host: "127.0.0.1" # Bind address + port: 8080 # Listen port + use_tailscale: false # Use Tailscale IP for endpoint URLs + autostart: false + +# ── LLM Model Selection ── +llm: + default_model: "gemma4:31b" # Model used when none specified + reranker_model: "gpt-oss:120b" # Used for search result reranking + scraper_model: "gemma4:31b" # Used for web page summarization + summary_model: "qwen3.5:397b" # Used for content summarization + fallback_model: "gemma4:31b" # Used when requested model unavailable + emulate_openai: true # Enable /v1/chat/completions endpoint + emulate_anthropic: true # Enable /v1/messages proxy endpoint + # available_models: [...] + +# ── Fallback Provider (when Ollama Cloud is unavailable) ── +fallback: + enabled: false + name: "openai" # Display name + base_url: "https://api.openai.com/v1" + api_key: "" + default_model: "gpt-4o" + timeout: 60.0 # Request timeout in seconds + primary_timeout: 30.0 # Max seconds to wait for Ollama first + # chunk before trying fallback + stream_chunk_timeout: 180.0 # Max seconds between stream chunks + max_tokens: 128000 + stream_fallback: true + model_map: {} # ollama_model -> fallback_model mapping + +# ── Search/Scrape Provider API Keys ── +providers: + tavily: { enabled: true } + exa: { enabled: true } + searxng: { enabled: true } + firecrawl: { enabled: true, require_api_key: false } + serper: { enabled: true } + jina: { enabled: true } + cohere: { enabled: true } + brave: { enabled: true } + +# ── Smart Cache (BETA) ── +cache: + beta_mode: false # Master switch — must be true to enable + exact_cache_ttl: 600 # Seconds for exact-match response cache + session_prefix_ttl: 3600 # Seconds for session prefix cache + max_entries: 500 + dedup_enabled: true # Merge identical concurrent upstream calls + session_prefix_enabled: true + exact_cache_enabled: true + min_prompt_chars: 50 # Don't cache tiny prompts + +# ── Ollama Cloud Usage Tracking ── +usage: + session_cookie: "" # __Secure-session cookie from ollama.com + check_interval: 0 # Auto-check interval (0 = disabled) + redirect_on_full: false # Route to fallback when quota near limit +``` + +### Environment Variables + +| Variable | Description | +|----------|-------------| +| `OLLAMA_API_KEY` | Ollama Cloud API key (takes precedence over config file) | +| `GUANACO_CONFIG_DIR` | Path to config directory (default `~/.guanaco`) | + +--- + +## Fallback Provider Setup + +When Ollama Cloud is slow, rate-limited, or a requested model isn't available, Guanaco can automatically forward requests to a fallback OpenAI-compatible provider. + +```yaml +fallback: + enabled: true + name: "openai" + base_url: "https://api.openai.com/v1" + api_key: "sk-..." + default_model: "gpt-4o" + primary_timeout: 30.0 # Wait up to 30s for Ollama first chunk + stream_chunk_timeout: 180.0 # Tolerate long reasoning pauses + timeout: 60.0 + stream_fallback: true + model_map: + # Map specific Ollama models to different fallback models + "qwen3:480b": "gpt-4o" + "deepseek-v3.1:671b": "gpt-4o" +``` + +Or configure via the dashboard at **Dashboard → Config → Fallback**. + +--- + +## API Reference + +### LLM Router + +**`POST /v1/chat/completions`** — OpenAI-compatible chat completions + +```bash +curl -X POST http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer YOUR_API_KEY" \ + -d '{ + "model": "gemma4:31b", + "messages": [{"role": "user", "content": "Hello!"}], + "stream": false + }' +``` + +**`POST /v1/messages`** — Anthropic-compatible messages proxy + +```bash +curl -X POST http://localhost:8080/v1/messages \ + -H "Content-Type: application/json" \ + -H "x-api-key: YOUR_API_KEY" \ + -H "anthropic-version: 2023-06-01" \ + -d '{ + "model": "gemma4:31b", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 1024 + }' +``` + +### Search APIs + +All search providers are emulated at `http://localhost:8080//`: + +| Provider | Endpoints | Notes | +|----------|-----------|-------| +| **Tavily** | `/tavily/search` | Tavily Search API compatible | +| **Exa** | `/exa/search`, `/exa/findSimilar` | Exa Search API compatible | +| **SearXNG** | `/searxng/search` | SearXNG API compatible | +| **Firecrawl** | `/firecrawl/scrape`, `/firecrawl/search`, `/firecrawl/crawl`, `/firecrawl/extract` | Firecrawl SDK v2 compatible | +| **Serper** | `/serper/search`, `/serper/scrape` | Serper API compatible | +| **Jina** | `/jina/search`, `/jina/rerank` | Jina API compatible | +| **Cohere** | `/cohere/rerank` | Cohere Rerank API compatible | +| **Brave** | `/brave/search` | Brave Search API compatible | + +Firecrawl SDK v2 paths (`/v2/scrape`, `/v2/search`, `/v2/crawl`, `/v2/extract`) are also supported directly. + +### Status & Utility Endpoints + +| Endpoint | Description | +|----------|-------------| +| `GET /health` | Health check | +| `GET /v1/models` | List available models | +| `GET /v1/usage` | Ollama Cloud usage/quota | +| `GET /api/ollama/status` | Ollama Cloud connectivity | +| `GET /api/ollama/models` | Full model list with metadata | + +--- + +## Docker Deployment + +```dockerfile +FROM python:3.12-slim +WORKDIR /app +COPY . . +RUN pip install -e . +EXPOSE 8080 +ENV GUANACO_CONFIG_DIR=/data +VOLUME /data +CMD ["guanaco", "start", "--host", "0.0.0.0"] +``` + +```bash +docker build -t guanaco . +docker run -d -p 8080:8080 \ + -e OLLAMA_API_KEY=your_key \ + -v ~/.guanaco:/data \ + guanaco +``` + +--- + +## systemd Deployment + +```bash +sudo cp contrib/guanaco.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now guanaco +``` + +Check status: +```bash +systemctl status guanaco +journalctl -u guanaco -f +``` + +Edit `/etc/systemd/system/guanaco.service` to set `User`, `Group`, install directory, and venv path as appropriate for your environment. + +--- + +## Backward Compatibility with Ollama Cloud Tools (OCT) + +Guanaco is the successor to **Ollama Cloud Tools (oct)**. The `oct` CLI is preserved as a drop-in alias: + +```bash +oct start # → runs guanaco start +oct status # → runs guanaco status +oct models # → runs guanaco models +oct config --show # → runs guanaco config --show +``` + +Config at `~/.oct/config.yaml` is automatically read if `~/.guanaco/config.yaml` doesn't exist. Update your scripts at your convenience — both commands work indefinitely. + +--- + +## Contributing + +Contributions are welcome! Please open an issue or submit a pull request on the [GitHub repository](https://github.com/evanrice/ollama-cloud-tools). + +--- + +## License + +[MIT](LICENSE) — Copyright 2026 Guanaco Contributors diff --git a/contrib/com.guanaco.start.plist b/contrib/com.guanaco.start.plist new file mode 100644 index 0000000..6271061 --- /dev/null +++ b/contrib/com.guanaco.start.plist @@ -0,0 +1,39 @@ + + + + + Label + com.guanaco.start + + ProgramArguments + + /Users/__USER__/.local/bin/guanaco + start + + + RunAtLoad + + + KeepAlive + + + StandardOutPath + /tmp/guanaco.stdout.log + + StandardErrorPath + /tmp/guanaco.stderr.log + + EnvironmentVariables + + PATH + /usr/local/bin:/usr/bin:/bin:/Users/__USER__/.local/bin + SSL_CERT_FILE + __SSL_CERT_FILE__ + GUANACO_CONFIG_DIR + /Users/__USER__/.guanaco + + + WorkingDirectory + /Users/__USER__/.guanaco/repo + + \ No newline at end of file diff --git a/contrib/com.oct.start.plist b/contrib/com.oct.start.plist new file mode 100644 index 0000000..3a8e9ec --- /dev/null +++ b/contrib/com.oct.start.plist @@ -0,0 +1,37 @@ + + + + + Label + com.oct.start + + ProgramArguments + + /Users/__USER__/.local/bin/oct + start + + + RunAtLoad + + + KeepAlive + + + StandardOutPath + /tmp/oct.stdout.log + + StandardErrorPath + /tmp/oct.stderr.log + + EnvironmentVariables + + PATH + /usr/local/bin:/usr/bin:/bin:/Users/__USER__/.local/bin + SSL_CERT_FILE + __SSL_CERT_FILE__ + + + WorkingDirectory + /Users/__USER__/.oct/repo + + \ No newline at end of file diff --git a/contrib/guanaco.service b/contrib/guanaco.service new file mode 100644 index 0000000..fc1e121 --- /dev/null +++ b/contrib/guanaco.service @@ -0,0 +1,36 @@ +[Unit] +Description=Guanaco - OpenAI-compatible LLM proxy for Ollama Cloud +Documentation=https://github.com/evanrice/ollama-cloud-tools +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=__USER__ +Group=__GROUP__ +WorkingDirectory=__INSTALL_DIR__ +ExecStart=__VENV__/bin/python -m guanaco.cli start +Restart=on-failure +RestartSec=5 +StartLimitBurst=3 +StartLimitIntervalSec=60 + +# Environment +EnvironmentFile=-__INSTALL_DIR__/env +Environment=PATH=__VENV__/bin:/usr/local/bin:/usr/bin:/bin +Environment=GUANACO_CONFIG_DIR=__INSTALL_DIR__/config + +# Hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths=__INSTALL_DIR__ /tmp +PrivateTmp=true + +# Logging +StandardOutput=journal +StandardError=journal +SyslogIdentifier=guanaco + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/contrib/oct.service b/contrib/oct.service new file mode 100644 index 0000000..037fe08 --- /dev/null +++ b/contrib/oct.service @@ -0,0 +1,35 @@ +[Unit] +Description=Ollama Cloud Tools - API proxy, router & dashboard +Documentation=https://github.com/evanrice/ollama-cloud-tools +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=__USER__ +Group=__GROUP__ +WorkingDirectory=__INSTALL_DIR__/repo +ExecStart=__VENV__/bin/python -m oct.cli start +Restart=on-failure +RestartSec=5 +StartLimitBurst=3 +StartLimitIntervalSec=60 + +# Environment +EnvironmentFile=-__INSTALL_DIR__/env +Environment=PATH=__VENV__/bin:/usr/local/bin:/usr/bin:/bin + +# Hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths=__INSTALL_DIR__ /tmp +PrivateTmp=true + +# Logging +StandardOutput=journal +StandardError=journal +SyslogIdentifier=oct + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/guanaco/__init__.py b/guanaco/__init__.py new file mode 100644 index 0000000..2258c5e --- /dev/null +++ b/guanaco/__init__.py @@ -0,0 +1,3 @@ +"""guanaco — maximize your Ollama Cloud subscription.""" + +__version__ = "0.3.0" diff --git a/guanaco/analytics.py b/guanaco/analytics.py new file mode 100644 index 0000000..3811c2d --- /dev/null +++ b/guanaco/analytics.py @@ -0,0 +1,432 @@ +"""Request logging and analytics engine. + +Tracks every API call with: timestamp, model, prompt/completion tokens, +TPS, TTFT, duration, provider, endpoint, and any errors. +Also tracks Ollama Cloud usage/quota and system status events. +Persists to SQLite for long-term analytics. +""" + +from __future__ import annotations + +import json +import sqlite3 +import time +import uuid +from pathlib import Path +from typing import Optional + + +def _default_db_path() -> Path: + from guanaco.config import get_default_config_dir + return get_default_config_dir() / "analytics.db" + + +def _normalize_model_name(model: str) -> str: + """Strip routing suffixes (:cloud, :local) for analytics grouping.""" + if model and ":" in model: + suffix = model.split(":")[-1] + if suffix in ("cloud", "local"): + return model.rsplit(":", 1)[0] + return model + + +class AnalyticsLogger: + """SQLite-backed request logger and analytics engine.""" + + def __init__(self, db_path: Optional[Path] = None): + self.db_path = db_path or _default_db_path() + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._init_db() + + def _init_db(self): + with sqlite3.connect(self.db_path) as conn: + conn.execute(""" + CREATE TABLE IF NOT EXISTS request_log ( + id TEXT PRIMARY KEY, + ts REAL NOT NULL, + type TEXT NOT NULL, -- 'llm' or 'search' + model TEXT, -- model name (for LLM calls) + provider TEXT, -- 'ollama', 'fallback', or search provider + endpoint TEXT, -- full endpoint path + prompt_tokens INTEGER DEFAULT 0, + completion_tokens INTEGER DEFAULT 0, + total_tokens INTEGER DEFAULT 0, + tps REAL, -- tokens per second (output) + prompt_tps REAL, -- prompt tokens per second + ttft_seconds REAL, -- time to first token + total_duration_seconds REAL, + load_duration_seconds REAL, + error TEXT, -- error message if failed + request_id TEXT, + fallback_for TEXT, -- original model name if this was a fallback call + extra TEXT -- JSON blob for additional data + ) + """) + # Migration: add provider column if upgrading from older schema + try: + conn.execute("ALTER TABLE request_log ADD COLUMN provider TEXT") + except sqlite3.OperationalError: + pass # column already exists + # Migration: add fallback_for column if upgrading from older schema + try: + conn.execute("ALTER TABLE request_log ADD COLUMN fallback_for TEXT") + except sqlite3.OperationalError: + pass # column already exists + conn.execute(""" + CREATE TABLE IF NOT EXISTS status_events ( + id TEXT PRIMARY KEY, + ts REAL NOT NULL, + level TEXT NOT NULL, -- 'info', 'warning', 'error' + source TEXT NOT NULL, -- 'ollama', 'router', 'search', 'system' + message TEXT NOT NULL, + details TEXT -- JSON blob for extra info + ) + """) + conn.execute(""" + CREATE TABLE IF NOT EXISTS usage_snapshots ( + id TEXT PRIMARY KEY, + ts REAL NOT NULL, + session_pct REAL, -- session usage percentage + weekly_pct REAL, -- weekly usage percentage + plan TEXT, -- subscription plan + source TEXT -- 'api' or 'scrape' + ) + """) + conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_log_ts ON request_log(ts) + """) + conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_log_model ON request_log(model) + """) + conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_log_type ON request_log(type) + """) + conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_status_ts ON status_events(ts) + """) + conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_status_level ON status_events(level) + """) + # Migration: normalize :cloud/:local model names in existing data + try: + conn.execute("UPDATE request_log SET model = REPLACE(REPLACE(model, ':cloud', ''), ':local', '') WHERE model LIKE '%:cloud%' OR model LIKE '%:local%'") + conn.execute("UPDATE request_log SET fallback_for = REPLACE(REPLACE(fallback_for, ':cloud', ''), ':local', '') WHERE fallback_for IS NOT NULL AND (fallback_for LIKE '%:cloud%' OR fallback_for LIKE '%:local%')") + except Exception: + pass + + def log_llm( + self, + model: str, + prompt_tokens: int = 0, + completion_tokens: int = 0, + total_tokens: int = 0, + tps: Optional[float] = None, + prompt_tps: Optional[float] = None, + ttft_seconds: Optional[float] = None, + total_duration_seconds: Optional[float] = None, + load_duration_seconds: Optional[float] = None, + error: Optional[str] = None, + request_id: Optional[str] = None, + provider: Optional[str] = None, + fallback_for: Optional[str] = None, + extra: Optional[dict] = None, + ) -> str: + """Log an LLM request. Returns the log entry ID.""" + # Normalize model name so glm-5.1:cloud and glm-5.1 are grouped together + model = _normalize_model_name(model) + fallback_for = _normalize_model_name(fallback_for) if fallback_for else fallback_for + entry_id = str(uuid.uuid4()) + with sqlite3.connect(self.db_path) as conn: + conn.execute( + """INSERT INTO request_log + (id, ts, type, model, prompt_tokens, completion_tokens, total_tokens, + tps, prompt_tps, ttft_seconds, total_duration_seconds, + load_duration_seconds, error, request_id, provider, fallback_for) + VALUES (?, ?, 'llm', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + (entry_id, time.time(), model, prompt_tokens, completion_tokens, + total_tokens, tps, prompt_tps, ttft_seconds, total_duration_seconds, + load_duration_seconds, error, request_id, provider, fallback_for), + ) + return entry_id + + def log_search( + self, + provider: str, + endpoint: str, + duration_seconds: Optional[float] = None, + result_count: int = 0, + error: Optional[str] = None, + ) -> str: + """Log a search/scrape request. Returns the log entry ID.""" + entry_id = str(uuid.uuid4()) + with sqlite3.connect(self.db_path) as conn: + conn.execute( + """INSERT INTO request_log + (id, ts, type, provider, endpoint, total_duration_seconds, error, extra) + VALUES (?, ?, 'search', ?, ?, ?, ?, ?)""", + (entry_id, time.time(), provider, endpoint, + duration_seconds, error, json.dumps({"result_count": result_count})), + ) + return entry_id + + def log_status( + self, + level: str, + source: str, + message: str, + details: Optional[dict] = None, + ) -> str: + """Log a status event (info, warning, error).""" + entry_id = str(uuid.uuid4()) + with sqlite3.connect(self.db_path) as conn: + conn.execute( + """INSERT INTO status_events (id, ts, level, source, message, details) + VALUES (?, ?, ?, ?, ?, ?)""", + (entry_id, time.time(), level, source, message, + json.dumps(details) if details else None), + ) + return entry_id + + def log_usage_snapshot( + self, + session_pct: Optional[float] = None, + weekly_pct: Optional[float] = None, + plan: Optional[str] = None, + source: str = "api", + ) -> str: + """Log a usage/quota snapshot.""" + entry_id = str(uuid.uuid4()) + with sqlite3.connect(self.db_path) as conn: + conn.execute( + """INSERT INTO usage_snapshots (id, ts, session_pct, weekly_pct, plan, source) + VALUES (?, ?, ?, ?, ?, ?)""", + (entry_id, time.time(), session_pct, weekly_pct, plan, source), + ) + return entry_id + + def get_logs( + self, + limit: int = 100, + offset: int = 0, + type_filter: Optional[str] = None, + model_filter: Optional[str] = None, + ) -> list[dict]: + """Get recent log entries.""" + query = "SELECT * FROM request_log WHERE 1=1" + params = [] + if type_filter: + query += " AND type = ?" + params.append(type_filter) + if model_filter: + query += " AND model = ?" + params.append(model_filter) + query += " ORDER BY ts DESC LIMIT ? OFFSET ?" + params.extend([limit, offset]) + + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + rows = conn.execute(query, params).fetchall() + return [dict(r) for r in rows] + + def get_status_events( + self, + limit: int = 100, + level: Optional[str] = None, + source: Optional[str] = None, + ) -> list[dict]: + """Get recent status events.""" + query = "SELECT * FROM status_events WHERE 1=1" + params = [] + if level: + query += " AND level = ?" + params.append(level) + if source: + query += " AND source = ?" + params.append(source) + query += " ORDER BY ts DESC LIMIT ?" + params.append(limit) + + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + rows = conn.execute(query, params).fetchall() + return [dict(r) for r in rows] + + def get_summary(self) -> dict: + """Get aggregate analytics summary.""" + with sqlite3.connect(self.db_path) as conn: + # Total counts + total = conn.execute("SELECT COUNT(*) FROM request_log").fetchone()[0] + llm_calls = conn.execute("SELECT COUNT(*) FROM request_log WHERE type='llm'").fetchone()[0] + search_calls = conn.execute("SELECT COUNT(*) FROM request_log WHERE type='search'").fetchone()[0] + errors = conn.execute("SELECT COUNT(*) FROM request_log WHERE error IS NOT NULL").fetchone()[0] + + # Token totals + row = conn.execute( + "SELECT COALESCE(SUM(prompt_tokens),0), COALESCE(SUM(completion_tokens),0), " + "COALESCE(SUM(total_tokens),0) FROM request_log WHERE type='llm'" + ).fetchone() + prompt_tokens, completion_tokens, total_tokens = row + + # Average TPS + row = conn.execute( + "SELECT AVG(tps) FROM request_log WHERE type='llm' AND tps IS NOT NULL" + ).fetchone() + avg_tps = round(row[0], 2) if row[0] else 0 + + # Average TTFT + row = conn.execute( + "SELECT AVG(ttft_seconds) FROM request_log WHERE type='llm' AND ttft_seconds IS NOT NULL" + ).fetchone() + avg_ttft = round(row[0], 3) if row[0] else 0 + + # Per-model stats + model_rows = conn.execute( + """SELECT model, COUNT(*), SUM(prompt_tokens), SUM(completion_tokens), + AVG(tps), AVG(ttft_seconds), MAX(ts) + FROM request_log WHERE type='llm' GROUP BY model ORDER BY MAX(ts) DESC""" + ).fetchall() + models = [] + for row in model_rows: + models.append({ + "model": row[0], "requests": row[1], + "prompt_tokens": row[2] or 0, "completion_tokens": row[3] or 0, + "avg_tps": round(row[4], 2) if row[4] else 0, + "avg_ttft": round(row[5], 3) if row[5] else 0, + "last_used": row[6], + }) + + # Per-provider stats (for search calls) + provider_rows = conn.execute( + """SELECT provider, COUNT(*), MAX(ts) FROM request_log + WHERE type='search' GROUP BY provider ORDER BY MAX(ts) DESC""" + ).fetchall() + providers = [] + for row in provider_rows: + providers.append({ + "provider": row[0], "requests": row[1], "last_used": row[2], + }) + + # Per-provider LLM stats + llm_provider_rows = conn.execute( + """SELECT provider, COUNT(*), SUM(prompt_tokens), SUM(completion_tokens), + AVG(tps), AVG(ttft_seconds), MAX(ts) + FROM request_log WHERE type='llm' GROUP BY provider ORDER BY MAX(ts) DESC""" + ).fetchall() + llm_providers = [] + for row in llm_provider_rows: + llm_providers.append({ + "provider": row[0], "requests": row[1], + "prompt_tokens": row[2] or 0, "completion_tokens": row[3] or 0, + "avg_tps": round(row[4], 2) if row[4] else 0, + "avg_ttft": round(row[5], 3) if row[5] else 0, + "last_used": row[6], + }) + + # Fallback stats + fallback_count = conn.execute( + "SELECT COUNT(*) FROM request_log WHERE fallback_for IS NOT NULL" + ).fetchone()[0] + fallback_rows = conn.execute( + """SELECT fallback_for, COUNT(*), MAX(ts) FROM request_log + WHERE fallback_for IS NOT NULL GROUP BY fallback_for ORDER BY MAX(ts) DESC""" + ).fetchall() + fallbacks = [] + for row in fallback_rows: + fallbacks.append({ + "original_model": row[0], "fallback_count": row[1], "last_used": row[2], + }) + + # Recent errors + error_rows = conn.execute( + """SELECT ts, type, model, provider, endpoint, error + FROM request_log WHERE error IS NOT NULL ORDER BY ts DESC LIMIT 20""" + ).fetchall() + recent_errors = [] + for row in error_rows: + recent_errors.append({ + "ts": row[0], "type": row[1], "model": row[2], + "provider": row[3], "endpoint": row[4], "error": row[5], + }) + + # Status event counts + status_error_count = conn.execute( + "SELECT COUNT(*) FROM status_events WHERE level='error'" + ).fetchone()[0] + status_warning_count = conn.execute( + "SELECT COUNT(*) FROM status_events WHERE level='warning'" + ).fetchone()[0] + + # Latest usage snapshot + usage_row = conn.execute( + "SELECT session_pct, weekly_pct, plan, ts FROM usage_snapshots ORDER BY ts DESC LIMIT 1" + ).fetchone() + + return { + "total_requests": total, + "llm_calls": llm_calls, + "search_calls": search_calls, + "errors": errors, + "prompt_tokens": prompt_tokens or 0, + "completion_tokens": completion_tokens or 0, + "total_tokens": total_tokens or 0, + "avg_tps": avg_tps, + "avg_ttft": avg_ttft, + "models": models, + "llm_providers": llm_providers, + "providers": providers, + "fallbacks": fallbacks, + "fallback_count": fallback_count, + "recent_errors": recent_errors, + "status_errors": status_error_count, + "status_warnings": status_warning_count, + "usage": { + "session_pct": usage_row[0] if usage_row else None, + "weekly_pct": usage_row[1] if usage_row else None, + "plan": usage_row[2] if usage_row else None, + "last_checked": usage_row[3] if usage_row else None, + } if usage_row else None, + } + + def get_timeseries(self, hours: int = 24, bucket_minutes: int = 60) -> list[dict]: + """Get request count timeseries data.""" + cutoff = time.time() - (hours * 3600) + bucket_sec = bucket_minutes * 60 + + with sqlite3.connect(self.db_path) as conn: + rows = conn.execute( + "SELECT ts, type, model, total_tokens FROM request_log WHERE ts > ? ORDER BY ts", + (cutoff,), + ).fetchall() + + buckets = {} + for ts, rtype, model, tokens in rows: + bucket = int(ts // bucket_sec) * bucket_sec + key = bucket + if key not in buckets: + buckets[key] = {"ts": bucket, "llm": 0, "search": 0, "tokens": 0} + if rtype == "llm": + buckets[key]["llm"] += 1 + buckets[key]["tokens"] += (tokens or 0) + else: + buckets[key]["search"] += 1 + + return sorted(buckets.values(), key=lambda x: x["ts"]) + + def get_model_history(self, model: str, limit: int = 50) -> list[dict]: + """Get detailed history for a specific model.""" + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + rows = conn.execute( + """SELECT * FROM request_log + WHERE model = ? AND type = 'llm' + ORDER BY ts DESC LIMIT ?""", + (model, limit), + ).fetchall() + return [dict(r) for r in rows] + + def clear(self): + """Clear all analytics data.""" + with sqlite3.connect(self.db_path) as conn: + conn.execute("DELETE FROM request_log") + conn.execute("DELETE FROM status_events") + conn.execute("DELETE FROM usage_snapshots") \ No newline at end of file diff --git a/guanaco/app.py b/guanaco/app.py new file mode 100644 index 0000000..ae58afd --- /dev/null +++ b/guanaco/app.py @@ -0,0 +1,376 @@ +"""Main FastAPI application — ties together LLM router, search providers, dashboard, and status.""" + +from __future__ import annotations + +import os +import time +import sys +from contextlib import asynccontextmanager +from typing import Optional + +from fastapi import FastAPI, APIRouter, Request, HTTPException +from fastapi.middleware.cors import CORSMiddleware + +from guanaco.config import load_config, get_config, AppConfig, get_base_url, get_tailscale_ip +from guanaco.client import OllamaClient +from guanaco.router.router import create_router as create_llm_router +from guanaco.search.providers import ALL_PROVIDERS +from guanaco.dashboard import create_dashboard_router +from guanaco.utils.api_keys import ApiKeyManager +from guanaco.analytics import AnalyticsLogger + + +def create_app(config: AppConfig | None = None) -> FastAPI: + """Create the combined FastAPI application with all routes on a single port.""" + if config is None: + config = load_config() + + resolved_key = os.getenv("OLLAMA_API_KEY", "") or config.ollama_api_key or "" + if not resolved_key: + print("Warning: OLLAMA_API_KEY not set. Set it with 'guanaco setup' or export OLLAMA_API_KEY.") + + client = OllamaClient(api_key=resolved_key, session_cookie=config.usage.session_cookie) + + from guanaco.config import get_default_config_dir + key_manager = ApiKeyManager(get_default_config_dir()) + analytics = AnalyticsLogger() + providers_config = config.providers.model_dump() + + @asynccontextmanager + async def lifespan(app: FastAPI): + base_url = get_base_url(config) + print(f"Guanaco running on http://{config.router.host}:{config.router.port}") + print(f" LLM Router: {base_url}:{config.router.port}/v1/chat/completions") + print(f" Anthropic: {base_url}:{config.router.port}/v1/messages") + print(f" Search APIs: {base_url}:{config.router.port}//...") + print(f" Dashboard: {base_url}:{config.router.port}/dashboard") + print(f" Analytics DB: {analytics.db_path}") + analytics.log_status("info", "system", "Guanaco started", { + "host": config.router.host, "port": config.router.port, + "cache_beta": config.cache.beta_mode, + }) + if config.cache.beta_mode: + print(f" Cache (BETA): ENABLED — exact_ttl={config.cache.exact_cache_ttl}s, prefix_ttl={config.cache.session_prefix_ttl}s, dedup={config.cache.dedup_enabled}") + else: + print(f" Cache (BETA): DISABLED (enable with /v1/config/cache)") + yield + await client.close() + + app = FastAPI( + title="Guanaco", + version="0.3.0", + lifespan=lifespan, + ) + + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + # ── Search request analytics middleware ── + @app.middleware("http") + async def search_analytics_middleware(request: Request, call_next): + path = request.url.path.strip("/") + is_search = any(path.startswith(p) for p in [ + "tavily", "exa", "searxng", "firecrawl", "serper", "jina", "cohere", "brave", + "v2/scrape", "v2/search", "v2/crawl", "v2/extract", + ]) + + if not is_search: + return await call_next(request) + + # Map v2/ paths to firecrawl provider for analytics + if path.startswith("v2/"): + provider = "firecrawl" + else: + provider = path.split("/")[0] if path else "" + start = time.time() + response = await call_next(request) + elapsed = time.time() - start + + analytics.log_search( + provider=provider, + endpoint=path, + duration_seconds=round(elapsed, 3), + error=None if response.status_code < 400 else f"HTTP {response.status_code}", + ) + + return response + + # ── API key auth middleware ── + @app.middleware("http") + async def auth_middleware(request: Request, call_next): + path = request.url.path.strip("/") + if not path or path.startswith("v1/") or path.startswith("v2/") or path.startswith("dashboard") or path.startswith("api/") or path == "health": + return await call_next(request) + + provider_name = path.split("/")[0] + prov_config = providers_config.get(provider_name, {}) + requires_key = prov_config.get("require_api_key", False) + + if requires_key: + auth_header = request.headers.get("Authorization", "") + if auth_header.startswith("Bearer "): + token = auth_header[7:] + else: + token = request.headers.get("X-API-Key", "") or request.query_params.get("api_key", "") + if not token or not key_manager.verify_key(token, provider=provider_name): + raise HTTPException(status_code=401, detail=f"Invalid API key for {provider_name}") + + return await call_next(request) + + # ── Health check ── + @app.get("/health") + async def health(): + return {"status": "ok", "version": "0.3.0"} + + # ── LLM Router ── + app.include_router(create_llm_router(client, analytics=analytics, config=config)) + + # ── Search Providers ── + for provider_cls in ALL_PROVIDERS: + prov_name = provider_cls.name + prov_cfg = providers_config.get(prov_name, {}) + if prov_cfg.get("enabled", True): + provider = provider_cls(client, analytics=analytics) + provider.register_routes(app) + print(f" [OK] {prov_name}") + else: + print(f" [DISABLED] {prov_name}") + + # ── Firecrawl SDK v2 compatibility routes ── + # The official Firecrawl Python SDK calls /v2/scrape, /v2/search etc. + # Guanaco exposes these under /firecrawl/v2/... but the SDK sends to /v2/... + # These top-level aliases let the SDK work without the /firecrawl prefix. + try: + firecrawl_prov = next(p for p in ALL_PROVIDERS if p.name == "firecrawl") + fc_instance = firecrawl_prov(client, analytics=analytics) + fc_compat = APIRouter(tags=["Firecrawl SDK Compat"]) + + # Re-use the same handler logic by delegating to the provider's methods + @fc_compat.post("/v2/scrape") + async def fc_v2_scrape(request: Request): + """Proxy /v2/scrape to the Firecrawl provider.""" + from guanaco.search.providers.firecrawl import ScrapeRequest + body = await request.json() + body_obj = ScrapeRequest(**body) + # Access the provider's registered router handlers + # Simpler: just call the ollama fetch directly + ollama_resp = await client.fetch(url=body_obj.url) + title = ollama_resp.get("title", "") + content = ollama_resp.get("content", "") + links = ollama_resp.get("links", []) + # v2 SDK expects data to be a Document-like object with metadata nested inside + data = {} + if "markdown" in body_obj.formats or not body_obj.formats: + data["markdown"] = content + if "html" in body_obj.formats: + data["html"] = content + if "rawHtml" in body_obj.formats: + data["rawHtml"] = content + if "links" in body_obj.formats: + data["links"] = links + data["metadata"] = { + "title": title, + "sourceURL": body_obj.url, + "statusCode": 200, + } + return { + "success": True, + "data": data, + } + + @fc_compat.post("/v2/search") + async def fc_v2_search(request: Request): + """Proxy /v2/search to the Firecrawl provider.""" + from guanaco.search.providers.firecrawl import SearchRequest + body = await request.json() + body_obj = SearchRequest(**body) + ollama_resp = await client.search(query=body_obj.query, max_results=body_obj.limit) + results = [] + for r in ollama_resp.get("results", []): + results.append({ + "title": r.get("title", ""), + "url": r.get("url", ""), + "description": r.get("content", "")[:200], + }) + return {"success": True, "data": {"web": results}} + + @fc_compat.post("/v2/crawl") + async def fc_v2_crawl(request: Request): + """Proxy /v2/crawl to the Firecrawl provider.""" + from guanaco.search.providers.firecrawl import CrawlRequest + body = await request.json() + body_obj = CrawlRequest(**body) + ollama_resp = await client.fetch(url=body_obj.url) + title = ollama_resp.get("title", "") + content = ollama_resp.get("content", "") + links = ollama_resp.get("links", []) + results = [{ + "title": title, + "url": body_obj.url, + "content": content, + "markdown": content, + "metadata": {"title": title, "sourceURL": body_obj.url}, + }] + for link in links[:body_obj.limit - 1]: + try: + link_resp = await client.fetch(url=link) + lt = link_resp.get("title", "") + lc = link_resp.get("content", "") + results.append({ + "title": lt, + "url": link, + "content": lc, + "markdown": lc, + "metadata": {"title": lt, "sourceURL": link}, + }) + except Exception: + continue + return { + "success": True, + "status": "completed", + "completed": len(results), + "total": len(results), + "data": results, + } + + @fc_compat.post("/v2/extract") + async def fc_v2_extract(request: Request): + """Proxy /v2/extract to the Firecrawl provider.""" + from guanaco.search.providers.firecrawl import ExtractRequest + body = await request.json() + body_obj = ExtractRequest(**body) + all_content = {} + for url in body_obj.urls[:5]: + try: + resp = await client.fetch(url=url) + all_content[url] = resp.get("content", "") + except Exception: + all_content[url] = "" + return {"success": True, "data": all_content} + + app.include_router(fc_compat) + except Exception as e: + print(f" [WARN] Firecrawl SDK compat routes not loaded: {e}") + + # ── Dashboard ── + app.include_router(create_dashboard_router(key_manager, analytics, client), prefix="/dashboard") + + # ── Ollama status & models (top-level API) ── + + @app.get("/api/ollama/status") + async def ollama_status(): + """Check Ollama Cloud API connectivity and list available models.""" + start = time.time() + cfg = get_config() + try: + health = await client.health_check() + latency_ms = health.get("latency_ms", round((time.time() - start) * 1000)) + + if health["status"] == "connected": + try: + models = await client.list_models() + model_count = len(models) + except Exception: + model_count = health.get("model_count", 0) + + analytics.log_status("info", "ollama", "Health check OK", {"latency_ms": latency_ms}) + return { + "status": "connected", + "model_count": model_count, + "latency_ms": latency_ms, + "details": health, + } + else: + latency_ms = round((time.time() - start) * 1000) + analytics.log_status("error" if health["status"] in ("error", "auth_error") else "warning", + "ollama", f"Health check failed: {health.get('message', health['status'])}", + health) + return { + "status": health["status"], + "error": health.get("message", str(health["status"])), + "model_count": 0, + "latency_ms": latency_ms, + } + except Exception as e: + latency_ms = round((time.time() - start) * 1000) + analytics.log_status("error", "ollama", f"Connection error: {str(e)}") + return { + "status": "error", + "error": str(e), + "model_count": 0, + "latency_ms": latency_ms, + } + + @app.get("/api/ollama/models") + async def ollama_models(): + """List all available Ollama Cloud models with metadata.""" + try: + models = await client.get_cloud_models() + return {"models": models, "count": len(models)} + except Exception as e: + analytics.log_status("error", "ollama", f"Failed to list models: {str(e)}") + raise HTTPException(status_code=502, detail=f"Cannot reach Ollama Cloud: {str(e)}") + + @app.get("/v1/usage") + async def get_usage(): + """Get Ollama Cloud account usage/quota information.""" + try: + usage_data = await client.get_usage() + if usage_data.get("source") != "unavailable": + session_pct = None + weekly_pct = None + plan = usage_data.get("plan", "") + if isinstance(usage_data.get("session_usage"), dict): + session_pct = usage_data["session_usage"].get("used_percentage") + elif usage_data.get("session_pct") is not None: + session_pct = usage_data["session_pct"] + if isinstance(usage_data.get("weekly_usage"), dict): + weekly_pct = usage_data["weekly_usage"].get("used_percentage") + elif usage_data.get("weekly_pct") is not None: + weekly_pct = usage_data["weekly_pct"] + analytics.log_usage_snapshot( + session_pct=session_pct, weekly_pct=weekly_pct, + plan=plan, source=usage_data.get("source", "api"), + ) + return usage_data + except Exception as e: + analytics.log_status("error", "ollama", f"Usage check failed: {str(e)}") + return {"source": "error", "error": str(e)} + + # ── Status event endpoints ── + + @app.post("/api/status/log") + async def log_status(request: Request): + """Log a status event.""" + body = await request.json() + entry_id = analytics.log_status( + level=body.get("level", "info"), + source=body.get("source", "api"), + message=body.get("message", ""), + details=body.get("details"), + ) + return {"id": entry_id, "status": "logged"} + + @app.get("/api/status/events") + async def get_status_events(limit: int = 50, level: Optional[str] = None, source: Optional[str] = None): + """Get recent status events.""" + return analytics.get_status_events(limit=limit, level=level, source=source) + + return app + + +def main(): + """CLI entry point.""" + import uvicorn + config = load_config() + app = create_app(config) + uvicorn.run(app, host=config.router.host, port=config.router.port) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/guanaco/cache.py b/guanaco/cache.py new file mode 100644 index 0000000..a7895f6 --- /dev/null +++ b/guanaco/cache.py @@ -0,0 +1,368 @@ +"""Smart session-aware response cache for Guanaco (beta). + +Three caching strategies: +1. Exact cache — hash(model + messages + params) → full response. TTL-based eviction. +2. Session prefix cache — hash(model + prefix of messages) → response. Detects when a + conversation is just adding messages to an existing session (most common Hermes pattern) + and reuses cached responses for the earlier messages. +3. Request deduplication — if two identical requests arrive while one is in-flight, + the second waits for the first's result instead of making a duplicate upstream call. + +All behind `cache.beta_mode` config flag. Off by default. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import logging +import time +from collections import OrderedDict +from dataclasses import dataclass, field +from typing import Any, Optional + +logger = logging.getLogger(__name__) + + +@dataclass +class CacheEntry: + """A single cached response.""" + key: str + response: dict + created_at: float + ttl: float + hit_count: int = 0 + model: str = "" + provider: str = "" + prompt_tokens: int = 0 + completion_tokens: int = 0 + cache_type: str = "exact" # "exact" or "session_prefix" + + @property + def is_expired(self) -> bool: + return (time.time() - self.created_at) > self.ttl + + @property + def age_seconds(self) -> float: + return time.time() - self.created_at + + +class CacheEngine: + """Smart response cache with exact matching, session-aware prefix caching, and deduplication.""" + + def __init__(self, config): + self.config = config + self._exact_cache: OrderedDict[str, CacheEntry] = OrderedDict() + self._prefix_cache: OrderedDict[str, CacheEntry] = OrderedDict() + self._in_flight: dict[str, asyncio.Event] = {} + self._in_flight_results: dict[str, dict] = {} + self._stats = { + "exact_hits": 0, + "prefix_hits": 0, + "misses": 0, + "dedup_saves": 0, + "evictions": 0, + "total_requests": 0, + } + + # ── Public API ── + + def is_enabled(self) -> bool: + """Check if beta cache is enabled.""" + return self.config.beta_mode + + async def get_or_fetch( + self, + model: str, + messages: list[dict], + params: dict, + fetch_fn, + provider: str = "ollama", + ) -> dict: + """Main entry point: check cache, dedup, or fetch from upstream. + + Args: + model: Resolved model name + messages: Chat messages list + params: Full request params dict (model, messages, temperature, etc.) + fetch_fn: Async callable that takes the payload and returns the response + provider: Provider name for tagging + + Returns: + Response dict (either cached or fresh) + """ + if not self.is_enabled(): + return await fetch_fn(params) + + self._stats["total_requests"] += 1 + + # Skip tiny prompts + prompt_text = self._extract_prompt_text(messages) + if len(prompt_text) < self.config.min_prompt_chars: + return await fetch_fn(params) + + # Skip excluded models + if model in self.config.exclude_models: + return await fetch_fn(params) + + # 1. Try exact cache + if self.config.exact_cache_enabled: + exact_key = self._exact_hash(model, messages, params) + cached = self._get_exact(exact_key) + if cached is not None: + self._stats["exact_hits"] += 1 + cached.hit_count += 1 + response = dict(cached.response) + response["_oct_cached"] = True + response["_oct_cache_type"] = "exact" + response["_oct_cache_age"] = round(cached.age_seconds, 1) + logger.info(f"Cache EXACT hit: model={model} age={cached.age_seconds:.1f}s") + return response + + # 2. Try session prefix cache + if self.config.session_prefix_enabled and len(messages) > 1: + prefix_key = self._prefix_hash(model, messages) + cached = self._get_prefix(prefix_key) + if cached is not None: + self._stats["prefix_hits"] += 1 + cached.hit_count += 1 + response = dict(cached.response) + response["_oct_cached"] = True + response["_oct_cache_type"] = "session_prefix" + response["_oct_cache_age"] = round(cached.age_seconds, 1) + logger.info(f"Cache PREFIX hit: model={model} msgs={len(messages)} age={cached.age_seconds:.1f}s") + return response + + # 3. Deduplication — if an identical request is already in-flight + if self.config.dedup_enabled: + dedup_key = self._exact_hash(model, messages, params) + if dedup_key in self._in_flight: + self._stats["dedup_saves"] += 1 + logger.info(f"Cache DEDUP: waiting for in-flight request model={model}") + await self._in_flight[dedup_key].wait() + result = self._in_flight_results.get(dedup_key) + if result is not None: + result_copy = dict(result) + result_copy["_oct_deduped"] = True + return result_copy + + # 4. Cache miss — fetch from upstream + self._stats["misses"] += 1 + + # Register in-flight if dedup enabled + dedup_key = None + if self.config.dedup_enabled: + dedup_key = self._exact_hash(model, messages, params) + self._in_flight[dedup_key] = asyncio.Event() + + try: + response = await fetch_fn(params) + + # Cache the result + if response and not response.get("error"): + self._store_response(model, messages, params, response, provider) + + # Store for dedup waiters + if dedup_key: + self._in_flight_results[dedup_key] = response + + return response + finally: + # Clean up in-flight marker + if dedup_key and dedup_key in self._in_flight: + self._in_flight[dedup_key].set() + del self._in_flight[dedup_key] + if dedup_key and dedup_key in self._in_flight_results: + # Keep result briefly for late waiters, then clean up + asyncio.get_event_loop().call_later(5.0, lambda: self._in_flight_results.pop(dedup_key, None)) + + def clear(self): + """Clear all caches.""" + self._exact_cache.clear() + self._prefix_cache.clear() + self._in_flight.clear() + self._in_flight_results.clear() + logger.info("Cache cleared") + + def get_stats(self) -> dict: + """Get cache statistics.""" + total_hits = self._stats["exact_hits"] + self._stats["prefix_hits"] + total_requests = self._stats["total_requests"] + hit_rate = (total_hits / total_requests * 100) if total_requests > 0 else 0 + + return { + "beta_mode": self.config.beta_mode, + "exact_cache_entries": len(self._exact_cache), + "prefix_cache_entries": len(self._prefix_cache), + "in_flight_requests": len(self._in_flight), + "stats": { + **self._stats, + "total_hits": total_hits, + "hit_rate_pct": round(hit_rate, 2), + }, + "config": { + "exact_cache_enabled": self.config.exact_cache_enabled, + "session_prefix_enabled": self.config.session_prefix_enabled, + "dedup_enabled": self.config.dedup_enabled, + "exact_cache_ttl": self.config.exact_cache_ttl, + "session_prefix_ttl": self.config.session_prefix_ttl, + "max_entries": self.config.max_entries, + "min_prompt_chars": self.config.min_prompt_chars, + "exclude_models": self.config.exclude_models, + }, + } + + def evict_expired(self): + """Remove expired entries from both caches.""" + expired_exact = [k for k, v in self._exact_cache.items() if v.is_expired] + for k in expired_exact: + del self._exact_cache[k] + self._stats["evictions"] += 1 + + expired_prefix = [k for k, v in self._prefix_cache.items() if v.is_expired] + for k in expired_prefix: + del self._prefix_cache[k] + self._stats["evictions"] += 1 + + # ── Private helpers ── + + def _exact_hash(self, model: str, messages: list[dict], params: dict) -> str: + """Hash the full request for exact cache key.""" + # Include model + messages + temperature/top_p/max_tokens (but not stream) + cache_params = { + "model": model, + "messages": messages, + "temperature": params.get("temperature"), + "top_p": params.get("top_p"), + "max_tokens": params.get("max_tokens"), + "tools": params.get("tools"), # Tool calls affect output + } + raw = json.dumps(cache_params, sort_keys=True, default=str) + return hashlib.sha256(raw.encode()).hexdigest()[:32] + + def _prefix_hash(self, model: str, messages: list[dict]) -> str: + """Hash the model + first N messages for session prefix caching. + + The idea: In a conversation, messages get appended but the early messages + stay the same. If we see the same prefix again with just the last message + different, we can potentially reuse. But for prefix cache we want the + prefix WITHOUT the last message — because the last message is what's new. + + Actually, for session prefix we hash messages[:-1] (all but the last + user message). If the conversation history is the same, the model's + understanding of context is the same — so responses to the same + continuation should be cacheable by the full message list. + + Wait — this would mean two different user messages get the same prefix + key, which is wrong. Let me reconsider. + + The real pattern with Hermes: the same conversation gets re-sent with + the EXACT same messages (e.g., retry, or the agent re-processing). + That's the exact cache. The prefix cache is for when a conversation + has N previous messages and we already computed a response for those + N messages — we can't really reuse that for N+1 messages because the + new message changes the output. + + So prefix caching is most useful for: same conversation, same history, + slightly different last message (e.g., rephrased question). We use a + fuzzy match: hash messages[:-1] + model, and only reuse if the last + message is "similar enough" (simple heuristic: last message has high + token overlap with the cached last message). + + For now, let's do a simpler version: prefix cache stores responses keyed + by model + messages[:-1]. When a new request comes in with the same + conversation history but a different final message, we DON'T return it + automatically — instead we mark it as a prefix match candidate that + could be used for future features (like speculative prefill). For now, + we only actually use prefix cache when all messages match (which is + just the exact cache). This infrastructure is here for future semantic + matching. + """ + # Use all messages except the last one (the new user input) + prefix = messages[:-1] if len(messages) > 1 else messages + raw = json.dumps({"model": model, "prefix": prefix}, sort_keys=True, default=str) + return hashlib.sha256(raw.encode()).hexdigest()[:32] + + def _get_exact(self, key: str) -> Optional[CacheEntry]: + """Get from exact cache, moving to end (LRU). Returns None if not found or expired.""" + if key in self._exact_cache: + entry = self._exact_cache[key] + if entry.is_expired: + del self._exact_cache[key] + self._stats["evictions"] += 1 + return None + # Move to end (most recently used) + self._exact_cache.move_to_end(key) + return entry + return None + + def _get_prefix(self, key: str) -> Optional[CacheEntry]: + """Get from prefix cache. Returns None if not found or expired.""" + if key in self._prefix_cache: + entry = self._prefix_cache[key] + if entry.is_expired: + del self._prefix_cache[key] + self._stats["evictions"] += 1 + return None + self._prefix_cache.move_to_end(key) + return entry + return None + + def _store_response(self, model: str, messages: list[dict], params: dict, response: dict, provider: str): + """Store a response in the cache(s).""" + usage = response.get("usage", {}) + + # Exact cache + if self.config.exact_cache_enabled: + exact_key = self._exact_hash(model, messages, params) + entry = CacheEntry( + key=exact_key, + response=dict(response), # Store a copy + created_at=time.time(), + ttl=self.config.exact_cache_ttl, + model=model, + provider=provider, + prompt_tokens=usage.get("prompt_tokens", 0), + completion_tokens=usage.get("completion_tokens", 0), + cache_type="exact", + ) + self._exact_cache[exact_key] = entry + self._evict_if_needed(self._exact_cache) + + # Prefix cache (only for multi-turn conversations) + if self.config.session_prefix_enabled and len(messages) > 1: + prefix_key = self._prefix_hash(model, messages) + entry = CacheEntry( + key=prefix_key, + response=dict(response), + created_at=time.time(), + ttl=self.config.session_prefix_ttl, + model=model, + provider=provider, + prompt_tokens=usage.get("prompt_tokens", 0), + completion_tokens=usage.get("completion_tokens", 0), + cache_type="session_prefix", + ) + self._prefix_cache[prefix_key] = entry + self._evict_if_needed(self._prefix_cache) + + def _evict_if_needed(self, cache: OrderedDict): + """Evict oldest entries if cache exceeds max_entries.""" + while len(cache) > self.config.max_entries: + cache.popitem(last=False) # Remove oldest (first inserted) + self._stats["evictions"] += 1 + + @staticmethod + def _extract_prompt_text(messages: list[dict]) -> str: + """Extract all text content from messages for length checking.""" + parts = [] + for msg in messages: + content = msg.get("content", "") + if isinstance(content, str): + parts.append(content) + elif isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + parts.append(block.get("text", "")) + return " ".join(parts) \ No newline at end of file diff --git a/guanaco/cli.py b/guanaco/cli.py new file mode 100644 index 0000000..3c8f385 --- /dev/null +++ b/guanaco/cli.py @@ -0,0 +1,585 @@ +"""CLI entry point for Guanaco.""" + +from __future__ import annotations + +import argparse +import asyncio +import os +import sys +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser( + prog="guanaco", + description="🦙 Guanaco — maximize your Ollama Cloud subscription", + ) + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + # ── start ── + start_parser = subparsers.add_parser("start", help="Start all services") + start_parser.add_argument("--host", default=None, help="Bind host (default: 127.0.0.1)") + start_parser.add_argument("--port", type=int, default=None, help="Port (default: 8080)") + start_parser.add_argument("--tailscale", action="store_true", help="Use Tailscale IP for endpoint URLs") + + # ── setup ── + subparsers.add_parser("setup", help="Interactive setup wizard") + + # ── key ── + key_parser = subparsers.add_parser("key", help="Manage API keys") + key_parser.add_argument("action", choices=["generate", "list", "revoke"], help="Action") + key_parser.add_argument("--provider", default="general", help="Provider for key") + key_parser.add_argument("--name", default="", help="Key name") + + # ── models ── + models_parser = subparsers.add_parser("models", help="List available Ollama Cloud models") + models_parser.add_argument("--refresh", action="store_true", help="Force refresh from Ollama API") + models_parser.add_argument("--json", action="store_true", help="Output as JSON") + models_parser.add_argument("--capabilities", action="store_true", help="Show model capabilities") + + # ── usage ── + subparsers.add_parser("usage", help="Check Ollama Cloud usage/quota") + + # ── status ── + status_parser = subparsers.add_parser("status", help="Show service status and Ollama connectivity") + status_parser.add_argument("--json", action="store_true", help="Output as JSON") + status_parser.add_argument("--verbose", "-v", action="store_true", help="Show detailed info") + + # ── analytics ── + analytics_parser = subparsers.add_parser("analytics", help="View request analytics") + analytics_parser.add_argument("--model", default=None, help="Filter by model") + analytics_parser.add_argument("--limit", type=int, default=20, help="Number of entries") + analytics_parser.add_argument("--summary", action="store_true", help="Show summary only") + analytics_parser.add_argument("--errors", action="store_true", help="Show recent errors") + + # ── config ── + config_parser = subparsers.add_parser("config", help="View or modify configuration") + config_parser.add_argument("--set", nargs=2, metavar=("KEY", "VALUE"), help="Set a config value") + config_parser.add_argument("--show", action="store_true", help="Show current config") + + # ── version ── + subparsers.add_parser("version", help="Show version") + + args = parser.parse_args() + + if args.command is None: + parser.print_help() + return + + if args.command == "version": + from guanaco import __version__ + print(f"🦙 guanaco v{__version__}") + return + + if args.command == "setup": + _run_setup() + return + + if args.command == "start": + _run_start(args) + return + + if args.command == "key": + _run_key(args) + return + + if args.command == "models": + _run_models(args) + return + + if args.command == "usage": + _run_usage() + return + + if args.command == "status": + _run_status(args) + return + + if args.command == "analytics": + _run_analytics(args) + return + + if args.command == "config": + _run_config(args) + return + + +def _run_setup(): + """Interactive setup wizard.""" + from guanaco.config import AppConfig, save_config, get_default_config_path + + print("🦙 Guanaco — Setup Wizard\n") + + api_key = os.environ.get("OLLAMA_API_KEY", "") + if not api_key: + api_key = input("Enter your Ollama API key: ").strip() + else: + print(f"Found OLLAMA_API_KEY in environment") + use_env = input("Use environment variable? [Y/n]: ").strip().lower() + if use_env == "n": + api_key = input("Enter your Ollama API key: ").strip() + + host = input("Bind host [127.0.0.1]: ").strip() or "127.0.0.1" + port = int(input("Port [8080]: ").strip() or "8080") + use_tailscale = input("Use Tailscale IP? [y/N]: ").strip().lower() == "y" + + # LLM config + print("\n📡 LLM Configuration") + print(" Available Ollama Cloud models: qwen3:480b, gpt-oss:120b, deepseek-v3.1, oss120b") + print(" Also: qwen3.5:122b, glm-5.1, minimax-m2.7, llama4:109b, etc.") + reranker = input("Reranker model [oss120b]: ").strip() or "oss120b" + scraper = input("Scraper model [qwen3:480b]: ").strip() or "qwen3:480b" + summary = input("Summary model [qwen3:480b]: ").strip() or "qwen3:480b" + default_model = input("Default chat model [qwen3:480b]: ").strip() or "qwen3:480b" + emulate_anthropic = input("Enable Anthropic /v1/messages emulation? [Y/n]: ").strip().lower() != "n" + emulate_openai = input("Enable OpenAI /v1/chat/completions? [Y/n]: ").strip().lower() != "n" + + config = AppConfig( + ollama_api_key=api_key, + router={"host": host, "port": port, "use_tailscale": use_tailscale}, + llm={ + "reranker_model": reranker, + "scraper_model": scraper, + "summary_model": summary, + "default_model": default_model, + "emulate_anthropic": emulate_anthropic, + "emulate_openai": emulate_openai, + }, + ) + + config_path = get_default_config_path() + save_config(config, config_path) + print(f"\n✅ Config saved to {config_path}") + print(f"\nEndpoints:") + print(f" LLM Router: http://{host}:{port}/v1/chat/completions") + if emulate_anthropic: + print(f" Anthropic: http://{host}:{port}/v1/messages") + print(f" Search APIs: http://{host}:{port}//...") + print(f" Dashboard: http://{host}:{port}/dashboard") + print(f"\nRun 'guanaco start' to begin!") + + +def _run_start(args): + """Start all services using uvicorn.""" + from guanaco.config import load_config, save_config + + config = load_config() + + if args.host: + config.router.host = args.host + if args.port: + config.router.port = args.port + if args.tailscale: + config.router.use_tailscale = True + + save_config(config) + + port = config.router.port + print("🦙 Starting Guanaco...") + print(f" Host: {config.router.host}") + print(f" Port: {port}") + print(f" Tailscale: {'Yes' if config.router.use_tailscale else 'No'}") + print(f" Anthropic: {'Yes' if config.llm.emulate_anthropic else 'No'}") + print(f" OpenAI: {'Yes' if config.llm.emulate_openai else 'No'}") + print(f" Default model: {config.llm.default_model}") + print(f" Reranker: {config.llm.reranker_model}") + print() + + try: + import uvicorn + from guanaco.app import create_app + app = create_app(config) + uvicorn.run(app, host=config.router.host, port=port, log_level="info") + except KeyboardInterrupt: + print("\n👋 Shutting down...") + except ImportError as e: + print(f"❌ Missing dependency: {e}") + print(" Run: pip install -e .") + sys.exit(1) + + +def _run_key(args): + """Manage API keys.""" + from guanaco.config import get_default_config_dir + from guanaco.utils.api_keys import ApiKeyManager + + km = ApiKeyManager(get_default_config_dir()) + + if args.action == "generate": + key = km.generate_key(provider=args.provider, name=args.name) + print(f"🔑 Generated key for {args.provider}:") + print(f" {key}") + print(f"\n⚠️ Save this key now — it won't be shown again!") + elif args.action == "list": + keys = km.list_keys() + if not keys: + print("No API keys found.") + else: + print(f"{'Provider':<12} {'Name':<20} {'Prefix':<20} {'Created'}") + print("-" * 72) + for k in keys: + from datetime import datetime + created = datetime.fromtimestamp(k['created_at']).strftime('%Y-%m-%d %H:%M') + print(f"{k['provider']:<12} {k['name']:<20} {k['prefix']:<20} {created}") + elif args.action == "revoke": + prefix = input("Enter key prefix to revoke: ").strip() + if km.revoke_by_prefix(prefix): + print("✅ Key revoked.") + else: + print("❌ Key not found.") + + +def _run_models(args): + """List available Ollama Cloud models.""" + from guanaco.config import load_config + from guanaco.client import OllamaClient, KNOWN_CLOUD_MODELS + + config = load_config() + api_key = config.ollama_api_key_resolved + + if not api_key: + print("❌ OLLAMA_API_KEY not set. Run 'guanaco setup' first.") + return + + client = OllamaClient(api_key=api_key) + + async def fetch(): + try: + if args.refresh: + models = await client.list_models(force_refresh=True) + else: + models = await client.get_cloud_models() + await client.close() + return models + except Exception as e: + await client.close() + print(f"❌ Error fetching models: {e}") + return [] + + models = asyncio.run(fetch()) + if not models: + print("No models found.") + return + + if args.json: + import json + print(json.dumps(models, indent=2)) + return + + print(f"🦙 Available Ollama Cloud Models ({len(models)}):\n") + + if args.capabilities: + print(f"{'Model':<28} {'Size':>8} {'Family':<14} {'Capabilities'}") + print("─" * 80) + for m in models: + name = m.get("display_name", m.get("name", "")) + size = m.get("parameter_size", "") + family = m.get("family", "") + caps = m.get("capabilities", ["cloud"]) + caps_str = " ".join(f"[{c}]" for c in caps) + print(f"{name:<28} {size:>8} {family:<14} {caps_str}") + else: + print(f"{'Model':<28} {'Size':>8} {'Family':<14} {'Quant':<10} {'Modified'}") + print("─" * 80) + for m in models: + name = m.get("display_name", m.get("name", "")) + size = m.get("parameter_size", "") + family = m.get("family", "") + quant = m.get("quantization", "") + modified = m.get("modified_at", "")[:10] if m.get("modified_at") else "" + print(f"{name:<28} {size:>8} {family:<14} {quant:<10} {modified}") + + # Show current config + print(f"\n📡 Current model config:") + print(f" Default: {config.llm.default_model}") + print(f" Reranker: {config.llm.reranker_model}") + print(f" Scraper: {config.llm.scraper_model}") + print(f" Summary: {config.llm.summary_model}") + print(f" Fallback: {config.llm.fallback_model}") + + +def _run_usage(): + """Check Ollama Cloud usage/quota.""" + from guanaco.config import load_config + from guanaco.client import OllamaClient + + config = load_config() + api_key = config.ollama_api_key_resolved + session_cookie = config.usage.session_cookie + + if not session_cookie: + print("⚠️ No session cookie configured.") + print(" Paste your __Secure-session cookie from ollama.com in the dashboard Status tab,") + print(" or set it in ~/.guanaco/config.yaml under usage.session_cookie") + return + + client = OllamaClient(api_key=api_key, session_cookie=session_cookie) + + async def check(): + try: + usage = await client.get_usage(session_cookie=session_cookie) + await client.close() + return usage + except Exception as e: + await client.close() + print(f"❌ Error checking usage: {e}") + return None + + usage = asyncio.run(check()) + if not usage: + return + + source = usage.get("source", "unknown") + if source in ("unavailable", "error"): + print(f"❌ {usage.get('error', 'Could not retrieve usage information.')}") + return + + plan = usage.get("plan", "—") + print(f"🦙 Ollama Cloud Usage ({plan})\n") + + if usage.get("session_pct") is not None: + reset = usage.get("session_reset", "") + reset_str = f" (resets in {reset})" if reset else "" + print(f" Session: {usage['session_pct']}%{reset_str}") + if usage.get("weekly_pct") is not None: + reset = usage.get("weekly_reset", "") + reset_str = f" (resets in {reset})" if reset else "" + print(f" Weekly: {usage['weekly_pct']}%{reset_str}") + + +def _run_status(args): + """Show service status and Ollama connectivity.""" + import json as json_mod + from guanaco.config import load_config, get_base_url + from guanaco.client import OllamaClient + from guanaco.analytics import AnalyticsLogger + + config = load_config() + base_url = get_base_url(config) + port = config.router.port + + results = {} + + # Check if service is running + import httpx + try: + resp = httpx.get(f"http://{config.router.host}:{port}/health", timeout=2) + if resp.status_code == 200: + results["service"] = "running" + results["version"] = resp.json().get("version", "unknown") + else: + results["service"] = "error" + except Exception: + results["service"] = "not_running" + + # Check Ollama Cloud connectivity + api_key = config.ollama_api_key_resolved + if api_key: + client = OllamaClient(api_key=api_key) + + async def check_ollama(): + health = await client.health_check() + await client.close() + return health + + ollama_health = asyncio.run(check_ollama()) + results["ollama"] = ollama_health + else: + results["ollama"] = {"status": "no_api_key"} + + # Local analytics + analytics = AnalyticsLogger() + summary = analytics.get_summary() + results["analytics"] = { + "total_requests": summary["total_requests"], + "errors": summary["errors"], + "status_errors": summary["status_errors"], + "status_warnings": summary["status_warnings"], + } + + if args.json: + print(json_mod.dumps(results, indent=2)) + return + + # Human-readable output + service = results["service"] + if service == "running": + print("🟢 Guanaco is running") + print(f" Version: {results.get('version', 'unknown')}") + print(f" Dashboard: {base_url}:{port}/dashboard") + elif service == "error": + print("🔴 Guanaco returned error") + else: + print("⚪ Guanaco is not running") + print(" Run 'guanaco start' to begin") + + print() + + # Ollama Cloud status + ollama = results.get("ollama", {}) + ollama_status = ollama.get("status", "unknown") + if ollama_status == "connected": + print(f"🟢 Ollama Cloud: Connected ({ollama.get('model_count', '?')} models, {ollama.get('latency_ms', '?')}ms)") + elif ollama_status == "auth_error": + print("🔴 Ollama Cloud: Invalid/expired API key") + elif ollama_status == "rate_limited": + print("🟡 Ollama Cloud: Rate limited") + elif ollama_status == "no_api_key": + print("⚪ Ollama Cloud: No API key configured") + else: + print(f"🔴 Ollama Cloud: {ollama.get('message', ollama_status)}") + + # Analytics summary + an = results.get("analytics", {}) + print(f"\n📊 Analytics:") + print(f" Total requests: {an.get('total_requests', 0)}") + print(f" Errors: {an.get('errors', 0)}") + print(f" Status events: {an.get('status_errors', 0)} errors, {an.get('status_warnings', 0)} warnings") + + if args.verbose: + print(f"\n📡 Endpoints:") + print(f" OpenAI: {base_url}:{port}/v1/chat/completions") + if config.llm.emulate_anthropic: + print(f" Anthropic: {base_url}:{port}/v1/messages") + print(f" Models: {base_url}:{port}/v1/models") + print(f" Usage: {base_url}:{port}/v1/usage") + print(f" Health: {base_url}:{port}/health") + print(f"\n📡 Model Config:") + print(f" Default: {config.llm.default_model}") + print(f" Reranker: {config.llm.reranker_model}") + print(f" Scraper: {config.llm.scraper_model}") + print(f" Summary: {config.llm.summary_model}") + print(f" Fallback: {config.llm.fallback_model}") + print(f" Anthropic: {'enabled' if config.llm.emulate_anthropic else 'disabled'}") + print(f" OpenAI: {'enabled' if config.llm.emulate_openai else 'disabled'}") + + +def _run_analytics(args): + """View request analytics.""" + from guanaco.analytics import AnalyticsLogger + + analytics = AnalyticsLogger() + + if args.errors: + events = analytics.get_status_events(limit=args.limit, level="error") + if not events: + print("✅ No errors found!") + return + print(f"⚠️ Recent Errors ({len(events)}):\n") + from datetime import datetime + for e in events: + ts = datetime.fromtimestamp(e["ts"]).strftime("%Y-%m-%d %H:%M:%S") + print(f" [{ts}] [{e['source']}] {e['message']}") + if e.get("details"): + print(f" Details: {e['details']}") + return + + if args.model: + entries = analytics.get_model_history(args.model, limit=args.limit) + if not entries: + print(f"No entries for model '{args.model}'") + return + print(f"📊 History for {args.model} ({len(entries)} entries):\n") + from datetime import datetime + for e in entries[:args.limit]: + ts = datetime.fromtimestamp(e["ts"]).strftime("%H:%M:%S") + tokens = e.get("total_tokens", 0) + tps = e.get("tps") or "—" + ttft = f"{(e.get('ttft_seconds') or 0) * 1000:.0f}ms" if e.get("ttft_seconds") else "—" + err = f" ERR: {e['error'][:40]}" if e.get("error") else "" + print(f" [{ts}] tok={tokens} tps={tps} ttft={ttft}{err}") + return + + summary = analytics.get_summary() + if args.summary or True: + print("📊 Analytics Summary\n") + print(f" Total requests: {summary['total_requests']}") + print(f" LLM calls: {summary['llm_calls']}") + print(f" Search calls: {summary['search_calls']}") + print(f" Errors: {summary['errors']}") + print(f" Prompt tokens: {summary['prompt_tokens']:,}") + print(f" Completion tokens:{summary['completion_tokens']:,}") + print(f" Total tokens: {summary['total_tokens']:,}") + print(f" Avg TPS: {summary['avg_tps']}") + print(f" Avg TTFT: {summary['avg_ttft']*1000:.0f}ms" if summary['avg_ttft'] else " Avg TTFT: —") + + if summary.get("models"): + print(f"\n📡 Per-Model Stats:") + print(f" {'Model':<28} {'Reqs':>6} {'PTok':>10} {'CTok':>10} {'TPS':>8} {'TTFT':>8}") + print(f" {'─'*28} {'─'*6} {'─'*10} {'─'*10} {'─'*8} {'─'*8}") + for m in summary["models"][:10]: + ttft = f"{m['avg_ttft']*1000:.0f}ms" if m.get("avg_ttft") else "—" + print(f" {m['model']:<28} {m['requests']:>6} {m['prompt_tokens']:>10,} {m['completion_tokens']:>10,} {m.get('avg_tps', '—'):>8} {ttft:>8}") + + if summary.get("usage"): + u = summary["usage"] + print(f"\n📈 Ollama Cloud Usage:") + if u.get("plan"): + print(f" Plan: {u['plan']}") + if u.get("session_pct") is not None: + print(f" Session: {u['session_pct']}%") + if u.get("weekly_pct") is not None: + print(f" Weekly: {u['weekly_pct']}%") + + +def _run_config(args): + """View or modify configuration.""" + from guanaco.config import load_config, save_config + + config = load_config() + + if args.set: + key, value = args.set + # Navigate dot-notation config key + parts = key.split(".") + obj = config + for part in parts[:-1]: + obj = getattr(obj, part, None) + if obj is None: + print(f"❌ Unknown config key: {key}") + return + last_key = parts[-1] + if not hasattr(obj, last_key): + print(f"❌ Unknown config key: {key}") + return + + # Type coercion + current = getattr(obj, last_key) + if isinstance(current, bool): + value = value.lower() in ("true", "1", "yes", "on") + elif isinstance(current, int): + value = int(value) + elif isinstance(current, float): + value = float(value) + + setattr(obj, last_key, value) + save_config(config) + print(f"✅ Set {key} = {value}") + return + + # Show current config + import json + print("🦙 Current Configuration\n") + print(f" API Key: {'*' * 8}{config.ollama_api_key_resolved[-4:]}" if config.ollama_api_key_resolved else " API Key: (not set)") + print(f"\n Router:") + print(f" Host: {config.router.host}") + print(f" Port: {config.router.port}") + print(f" Tailscale: {config.router.use_tailscale}") + print(f"\n LLM:") + print(f" Default model: {config.llm.default_model}") + print(f" Reranker model: {config.llm.reranker_model}") + print(f" Scraper model: {config.llm.scraper_model}") + print(f" Summary model: {config.llm.summary_model}") + print(f" Fallback model: {config.llm.fallback_model}") + print(f" Emulate Anthropic: {config.llm.emulate_anthropic}") + print(f" Emulate OpenAI: {config.llm.emulate_openai}") + print(f" Available models: {', '.join(config.llm.available_models)}") + print(f"\n Providers:") + for name, prov in config.providers.model_dump().items(): + en = "✅" if prov.get("enabled", True) else "❌" + key_status = "🔑" if prov.get("require_api_key") else "" + print(f" {en} {name} {key_status}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/guanaco/client.py b/guanaco/client.py new file mode 100644 index 0000000..4dfff2a --- /dev/null +++ b/guanaco/client.py @@ -0,0 +1,423 @@ +"""Ollama Cloud API client — handles search, fetch, chat, models, and usage.""" + +from __future__ import annotations + +import json +import time +import logging +from typing import Optional + +import httpx + +logger = logging.getLogger(__name__) + +OLLAMA_BASE = "https://ollama.com" +OLLAMA_V1_URL = f"{OLLAMA_BASE}/v1" +OLLAMA_CHAT_URL = f"{OLLAMA_V1_URL}/chat/completions" +OLLAMA_MODELS_URL = f"{OLLAMA_V1_URL}/models" +OLLAMA_SEARCH_URL = f"{OLLAMA_BASE}/api/web_search" +OLLAMA_FETCH_URL = f"{OLLAMA_BASE}/api/web_fetch" +OLLAMA_USAGE_URL = f"{OLLAMA_BASE}/api/account/usage" +OLLAMA_SETTINGS_URL = f"{OLLAMA_BASE}/api/account/settings" + +# Known cloud models (fallback + display info) +# Names must match /v1/models response (e.g. "gemma4:31b", "qwen3.5:397b") +KNOWN_CLOUD_MODELS = { + "gemma4": {"sizes": ["31b"], "family": "gemma", "capabilities": ["vision", "tools", "thinking", "cloud"]}, + "gemma3": {"sizes": ["4b", "12b", "27b"], "family": "gemma", "capabilities": ["vision", "tools", "thinking", "cloud"]}, + "qwen3.5": {"sizes": ["397b"], "family": "qwen", "capabilities": ["vision", "tools", "thinking", "cloud"]}, + "qwen3-vl": {"sizes": ["235b", "235b-instruct"], "family": "qwen", "capabilities": ["vision", "tools", "thinking", "cloud"]}, + "qwen3-coder": {"sizes": ["480b"], "family": "qwen", "capabilities": ["tools", "cloud"]}, + "qwen3-coder-next": {"sizes": [], "family": "qwen", "capabilities": ["tools", "cloud"]}, + "qwen3-next": {"sizes": ["80b"], "family": "qwen", "capabilities": ["tools", "thinking", "cloud"]}, + "minimax-m2": {"sizes": [], "family": "minimax", "capabilities": ["tools", "thinking", "cloud"]}, + "minimax-m2.7": {"sizes": [], "family": "minimax", "capabilities": ["tools", "thinking", "cloud"]}, + "minimax-m2.5": {"sizes": [], "family": "minimax", "capabilities": ["tools", "thinking", "cloud"]}, + "minimax-m2.1": {"sizes": [], "family": "minimax", "capabilities": ["tools", "thinking", "cloud"]}, + "glm-5.1": {"sizes": [], "family": "glm", "capabilities": ["tools", "thinking", "cloud"]}, + "glm-5": {"sizes": [], "family": "glm", "capabilities": ["tools", "thinking", "cloud"]}, + "glm-4.7": {"sizes": [], "family": "glm", "capabilities": ["tools", "thinking", "cloud"]}, + "glm-4.6": {"sizes": [], "family": "glm", "capabilities": ["tools", "thinking", "cloud"]}, + "gpt-oss": {"sizes": ["20b", "120b"], "family": "gpt-oss", "capabilities": ["tools", "thinking", "cloud"]}, + "deepseek-v3.1": {"sizes": ["671b"], "family": "deepseek", "capabilities": ["thinking", "cloud"]}, + "deepseek-v3.2": {"sizes": [], "family": "deepseek", "capabilities": ["thinking", "cloud"]}, + "devstral-small-2": {"sizes": ["24b"], "family": "devstral", "capabilities": ["tools", "cloud"]}, + "devstral-2": {"sizes": ["123b"], "family": "devstral", "capabilities": ["tools", "cloud"]}, + "nemotron-3-super": {"sizes": [], "family": "nemotron", "capabilities": ["tools", "thinking", "cloud"]}, + "nemotron-3-nano": {"sizes": ["30b"], "family": "nemotron", "capabilities": ["tools", "thinking", "cloud"]}, + "mistral-large-3": {"sizes": ["675b"], "family": "mistral", "capabilities": ["tools", "thinking", "cloud"]}, + "ministral-3": {"sizes": ["3b", "8b", "14b"], "family": "mistral", "capabilities": ["tools", "cloud"]}, + "kimi-k2.5": {"sizes": [], "family": "kimi", "capabilities": ["vision", "tools", "thinking", "cloud"]}, + "kimi-k2-thinking": {"sizes": [], "family": "kimi", "capabilities": ["thinking", "cloud"]}, + "kimi-k2": {"sizes": ["1t"], "family": "kimi", "capabilities": ["tools", "thinking", "cloud"]}, + "cogito-2.1": {"sizes": ["671b"], "family": "cogito", "capabilities": ["thinking", "cloud"]}, + "gemini-3-flash-preview": {"sizes": [], "family": "gemini", "capabilities": ["vision", "tools", "thinking", "cloud"]}, + "rnj-1": {"sizes": ["8b"], "family": "rnj", "capabilities": ["tools", "cloud"]}, +} + + +class OllamaClient: + """Async client for Ollama Cloud API.""" + + def __init__(self, api_key: str, timeout: float = 120.0, session_cookie: str = ""): + self.api_key = api_key + self.timeout = timeout + self._session_cookie = session_cookie + self._client: Optional[httpx.AsyncClient] = None + self._models_cache: Optional[list[dict]] = None + self._models_cache_time: float = 0 + self._models_cache_ttl: float = 300.0 # 5 minutes + + async def _get_client(self) -> httpx.AsyncClient: + if self._client is None or self._client.is_closed: + self._client = httpx.AsyncClient( + timeout=self.timeout, + headers={ + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + }, + ) + return self._client + + # ── Search & Fetch ── + + async def search(self, query: str, max_results: int = 10) -> dict: + """Search the web using Ollama's web_search API.""" + client = await self._get_client() + payload = {"query": query, "max_results": max(min(max_results, 10), 1)} + resp = await client.post(OLLAMA_SEARCH_URL, json=payload) + resp.raise_for_status() + return resp.json() + + async def fetch(self, url: str) -> dict: + """Fetch/scrape a URL using Ollama's web_fetch API.""" + client = await self._get_client() + payload = {"url": url} + resp = await client.post(OLLAMA_FETCH_URL, json=payload) + resp.raise_for_status() + return resp.json() + + # ── Models ── + + async def list_models(self, force_refresh: bool = False) -> list[dict]: + """List available Ollama Cloud models, with caching. + + Uses the OpenAI-compatible /v1/models endpoint which returns + model IDs in standard format (e.g. 'gemma4:31b', 'qwen3.5:397b'). + """ + now = time.time() + if not force_refresh and self._models_cache and (now - self._models_cache_time) < self._models_cache_ttl: + return self._models_cache + + client = await self._get_client() + try: + resp = await client.get(OLLAMA_MODELS_URL) + if resp.status_code == 401: + logger.error("Ollama API key is invalid or expired") + raise httpx.HTTPStatusError("Invalid API key", request=resp.request, response=resp) + resp.raise_for_status() + data = resp.json() + # OpenAI format: {"data": [{"id": "gemma4:31b", "object": "model", ...}]} + raw_models = data.get("data", data.get("models", [])) + models = [] + for m in raw_models: + if isinstance(m, dict): + model_id = m.get("id", m.get("name", m.get("model", ""))) + models.append({ + "name": model_id, + "model": model_id, + "id": model_id, + "modified_at": m.get("created", m.get("modified_at", "")), + "size": m.get("size", 0), + "digest": m.get("digest", ""), + }) + elif isinstance(m, str): + models.append({"name": m, "model": m, "id": m}) + self._models_cache = models + self._models_cache_time = now + return models + except httpx.HTTPStatusError as e: + logger.error(f"Failed to fetch models: {e}") + raise + except Exception as e: + logger.error(f"Error fetching models: {e}") + raise + + async def check_model_available(self, model_name: str) -> bool: + """Check if a specific model is available on Ollama Cloud.""" + models = await self.list_models() + available_names = {m.get("name", m.get("model", "")) for m in models} + # Check with and without -cloud suffix + return model_name in available_names or f"{model_name}-cloud" in available_names + + async def get_cloud_models(self) -> list[dict]: + """Get list of cloud-capable models with metadata.""" + models = await self.list_models() + cloud_models = [] + for m in models: + name = m.get("name", m.get("model", "")) + details = m.get("details", {}) + # Check if model has cloud capability (or is available via cloud API) + is_cloud = True # All models from /api/tags with auth are cloud-available + size_info = details.get("parameter_size", "") + family = details.get("family", "") + quant = details.get("quantization_level", "") + + cloud_models.append({ + "name": name, + "display_name": name.replace("-cloud", ""), + "size_bytes": m.get("size", 0), + "parameter_size": size_info, + "family": family, + "quantization": quant, + "capabilities": self._get_model_capabilities(name), + "modified_at": m.get("modified_at", ""), + "digest": m.get("digest", "")[:12] if m.get("digest") else "", + }) + return cloud_models + + def _get_model_capabilities(self, model_name: str) -> list[str]: + """Get known capabilities for a model name.""" + base_name = model_name.split(":")[0].replace("-cloud", "") + if base_name in KNOWN_CLOUD_MODELS: + return KNOWN_CLOUD_MODELS[base_name].get("capabilities", ["cloud"]) + # Default capabilities for unknown models + return ["cloud"] + + # ── Usage / Quota ── + + async def get_usage(self, session_cookie: str = "") -> dict: + """Get account usage/quota information from Ollama Cloud. + + Uses the session cookie to scrape usage from /settings HTML page. + Ollama doesn't have a public usage API, so we parse the rendered page. + """ + cookie = session_cookie or self._session_cookie + if not cookie: + return {"source": "unavailable", "error": "No session cookie configured. Paste your __Secure-session cookie in the Status tab to enable usage tracking."} + + try: + async with httpx.AsyncClient(timeout=15.0) as client: + resp = await client.get( + "https://ollama.com/settings", + follow_redirects=True, + cookies={"__Secure-session": cookie}, + headers={"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"}, + ) + if resp.status_code == 200: + usage = self._parse_settings_html(resp.text) + if usage: + return {"source": "settings_html", **usage} + return {"source": "settings_html", "error": "Could not parse usage data from settings page. Cookie may be expired."} + elif resp.status_code == 401 or resp.status_code == 302: + return {"source": "unavailable", "error": "Session cookie is expired or invalid. Please update it in the Status tab."} + else: + return {"source": "unavailable", "error": f"Unexpected status {resp.status_code} from ollama.com/settings"} + except Exception as e: + logger.warning(f"Failed to check usage with session cookie: {e}") + return {"source": "unavailable", "error": f"Failed to fetch usage: {str(e)}"} + + def _parse_settings_html(self, html: str) -> Optional[dict]: + """Extract usage data from the Ollama settings page HTML. + + The page is server-rendered with patterns like: + Session usage + 4.6% used + ... Resets in 4 hours + Weekly usage + 30.9% used + ... Resets in 3 days + """ + import re + result = {} + + # Extract percentages: "N.N% used" near "Session" and "Weekly" contexts + # Find all "X.X% used" occurrences in order + pct_matches = re.findall(r'(\d+(?:\.\d+)?)%\s*used', html) + reset_matches = re.findall(r'Resets in ([^<\n]+)', html) + + # Find session/weekly labels to determine which percentage is which + session_idx = None + weekly_idx = None + + # Look for "Session usage" label and find the nearest percentage + session_label = re.search(r'Session usage.*?(\d+(?:\.\d+)?)%\s*used', html, re.DOTALL) + if session_label: + result["session_pct"] = float(session_label.group(1)) + elif len(pct_matches) >= 1: + result["session_pct"] = float(pct_matches[0]) + + weekly_label = re.search(r'Weekly usage.*?(\d+(?:\.\d+)?)%\s*used', html, re.DOTALL) + if weekly_label: + result["weekly_pct"] = float(weekly_label.group(1)) + elif len(pct_matches) >= 2: + result["weekly_pct"] = float(pct_matches[1]) + + # Reset timers + if reset_matches: + if len(reset_matches) >= 1: + result["session_reset"] = reset_matches[0].strip() + if len(reset_matches) >= 2: + result["weekly_reset"] = reset_matches[1].strip() + + # Plan detection — find the badge right after "Cloud Usage" + # Pattern: Cloud Usage ... pro + plan_match = re.search(r'Cloud Usage\s*\s*]*>\s*(pro|max|free|team|starter)\s*\s*(pro|max|free|team|starter)\s* dict: + """Check Ollama Cloud API connectivity and key validity.""" + client = await self._get_client() + start = time.time() + try: + resp = await client.get(OLLAMA_MODELS_URL) + elapsed = time.time() - start + if resp.status_code == 401: + return { + "status": "auth_error", + "message": "Invalid or expired API key", + "latency_ms": round(elapsed * 1000), + } + if resp.status_code == 429: + return { + "status": "rate_limited", + "message": "Rate limited by Ollama Cloud", + "latency_ms": round(elapsed * 1000), + } + resp.raise_for_status() + data = resp.json() + models = data.get("data", data.get("models", [])) + return { + "status": "connected", + "model_count": len(models), + "latency_ms": round(elapsed * 1000), + } + except httpx.ConnectError: + return { + "status": "unreachable", + "message": "Cannot connect to ollama.com", + "latency_ms": round((time.time() - start) * 1000), + } + except httpx.TimeoutException: + return { + "status": "timeout", + "message": "Connection to ollama.com timed out", + "latency_ms": round((time.time() - start) * 1000), + } + except Exception as e: + return { + "status": "error", + "message": str(e), + "latency_ms": round((time.time() - start) * 1000), + } + + # ── Chat Completions ── + + async def chat_completion(self, payload: dict) -> dict: + """Send a chat completion to Ollama Cloud (OpenAI-compatible format).""" + client = await self._get_client() + start = time.time() + resp = await client.post(OLLAMA_CHAT_URL, json=payload) + elapsed = time.time() - start + resp.raise_for_status() + data = resp.json() + + # Extract metrics — Ollama Cloud returns standard OpenAI format but may + # also include Ollama-native fields (eval_count, eval_duration, etc.) + usage = data.get("usage", {}) + metrics = { + "total_duration_ns": data.get("total_duration"), + "load_duration_ns": data.get("load_duration"), + "prompt_eval_count": data.get("prompt_eval_count") or usage.get("prompt_tokens"), + "prompt_eval_duration_ns": data.get("prompt_eval_duration"), + "eval_count": data.get("eval_count") or usage.get("completion_tokens"), + "eval_duration_ns": data.get("eval_duration"), + "elapsed_seconds": elapsed, + } + + # Calculate derived metrics — prefer Ollama-native fields when available + eval_duration_ns = metrics.get("eval_duration_ns") + eval_count = metrics.get("eval_count") or 0 + if eval_duration_ns and eval_count and eval_duration_ns > 0: + metrics["tps"] = round(eval_count / (eval_duration_ns / 1e9), 2) + elif eval_count and elapsed > 0: + # Fallback: TPS = completion_tokens / total_elapsed + metrics["tps"] = round(eval_count / elapsed, 2) + + prompt_eval_duration_ns = metrics.get("prompt_eval_duration_ns") + prompt_eval_count = metrics.get("prompt_eval_count") + if prompt_eval_duration_ns and prompt_eval_count and prompt_eval_duration_ns > 0: + metrics["prompt_tps"] = round(prompt_eval_count / (prompt_eval_duration_ns / 1e9), 2) + elif prompt_eval_count and elapsed > 0: + metrics["prompt_tps"] = round(prompt_eval_count / elapsed, 2) + + load_duration_ns = metrics.get("load_duration_ns") + if load_duration_ns and prompt_eval_duration_ns: + # TTFT = load_duration + prompt_eval_duration (Ollama-native) + prompt_dur = prompt_eval_duration_ns or 0 + metrics["ttft_seconds"] = round((load_duration_ns + prompt_dur) / 1e9, 3) + # Note: For non-streaming OpenAI-format responses, we can't measure true TTFT + # (time to first token). Only streaming responses will have accurate TTFT. + + data["_oct_metrics"] = metrics + return data + + async def chat_completion_stream(self, payload: dict): + """Stream chat completion responses from Ollama Cloud, yielding metrics via _oct_stream_metrics.""" + client = await self._get_client() + payload_copy = dict(payload) + payload_copy["stream"] = True + + first_token_time = None + total_tokens = 0 + start = time.time() + + async with client.stream("POST", OLLAMA_CHAT_URL, json=payload_copy) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + if line.startswith("data: "): + data = line[6:] + if data.strip() == "[DONE]": + # Yield final chunk with metrics + elapsed = time.time() - start + metrics = { + "eval_count": total_tokens, + "elapsed_seconds": elapsed, + } + if total_tokens and elapsed > 0: + metrics["tps"] = round(total_tokens / elapsed, 2) + if first_token_time: + metrics["ttft_seconds"] = round(first_token_time - start, 3) + yield f"data: [DONE]\n\n" + # Store metrics on the response for analytics + yield f"__oct_metrics__:{json.dumps(metrics)}\n\n" + break + try: + chunk_data = json.loads(data) + # Count tokens from streaming chunks + for choice in chunk_data.get("choices", []): + delta = choice.get("delta", {}) + content = delta.get("content", "") + if content: + if first_token_time is None: + first_token_time = time.time() + total_tokens += 1 + except (json.JSONDecodeError, KeyError): + pass + yield f"data: {data}\n\n" + elif line.strip(): + yield f"data: {line}\n\n" + yield "data: [DONE]\n\n" + + async def close(self): + if self._client and not self._client.is_closed: + await self._client.aclose() + self._client = None \ No newline at end of file diff --git a/guanaco/config.py b/guanaco/config.py new file mode 100644 index 0000000..ca3961b --- /dev/null +++ b/guanaco/config.py @@ -0,0 +1,205 @@ +"""Configuration management for Guanaco.""" + +from __future__ import annotations + +import os +import secrets +from pathlib import Path +from typing import Optional + +import yaml +from pydantic import BaseModel, Field + + +def get_default_config_dir() -> Path: + """Get the default config directory. + + Checks GUANACO_CONFIG_DIR env var first, then defaults to ~/.guanaco. + """ + if "GUANACO_CONFIG_DIR" in os.environ: + return Path(os.environ["GUANACO_CONFIG_DIR"]) + return Path.home() / ".guanaco" + + +def _config_dir_has_content(p: Path) -> bool: + """Check if a config directory has existing config files.""" + if not p.exists(): + return False + return (p / "config.yaml").exists() or list(p.glob("*.yaml")) or list(p.glob("*.json")) + + +def get_default_config_path() -> Path: + return get_default_config_dir() / "config.yaml" + + +class RouterConfig(BaseModel): + host: str = "127.0.0.1" + port: int = 8080 + use_tailscale: bool = False + autostart: bool = False + + +class LLMConfig(BaseModel): + """LLM model selection config.""" + reranker_model: str = "gpt-oss:120b" + scraper_model: str = "gemma4:31b" + summary_model: str = "qwen3.5:397b" + default_model: str = "gemma4:31b" + available_models: list[str] = Field(default_factory=lambda: [ + "qwen3.5:397b", "qwen3-coder:480b", "qwen3-vl:235b", "qwen3-next:80b", + "gpt-oss:120b", "gpt-oss:20b", "deepseek-v3.1:671b", "deepseek-v3.2", + "gemma4:31b", "gemma3:27b", "glm-5.1", "glm-5", + "minimax-m2.7", "minimax-m2.5", "minimax-m2.1", + "devstral-small-2:24b", "devstral-2:123b", "nemotron-3-super", + "cogito-2.1:671b", "mistral-large-3:675b", "kimi-k2.5", "ministral-3:14b", + ]) + emulate_anthropic: bool = True + emulate_openai: bool = True + # When a requested model isn't found on Ollama Cloud, fall back to this model + fallback_model: str = "gemma4:31b" + + +class FallbackProviderConfig(BaseModel): + """External OpenAI-compatible provider to use when Ollama Cloud fails or model not found.""" + enabled: bool = False + name: str = "custom" # Display name + base_url: str = "" # e.g. "https://api.openai.com/v1" or "http://localhost:1234/v1" + api_key: str = "" # API key for the fallback provider + # Model name mapping: ollama_model -> fallback_model + # If a model isn't in the map, fallback_model_default is used + model_map: dict[str, str] = Field(default_factory=dict) + default_model: str = "" # Default model to use on the fallback provider + timeout: float = 60.0 # Request timeout in seconds (for fallback calls) + primary_timeout: float = 30.0 # Max seconds to wait for Ollama first chunk/response before trying fallback + stream_chunk_timeout: float = 180.0 # Max seconds between stream chunks (tolerates long reasoning pauses) + max_tokens: int = 128000 # Default max_tokens sent to fallback provider + stream_fallback: bool = True # Also fallback streaming requests + + +class ProviderConfig(BaseModel): + """Per-provider enable/disable and API key settings.""" + enabled: bool = True + require_api_key: bool = False + api_keys: list[str] = Field(default_factory=list) + + +class AllProvidersConfig(BaseModel): + tavily: ProviderConfig = Field(default_factory=ProviderConfig) + exa: ProviderConfig = Field(default_factory=ProviderConfig) + searxng: ProviderConfig = Field(default_factory=ProviderConfig) + firecrawl: ProviderConfig = Field(default_factory=lambda: ProviderConfig(require_api_key=True)) + serper: ProviderConfig = Field(default_factory=ProviderConfig) + jina: ProviderConfig = Field(default_factory=ProviderConfig) + cohere: ProviderConfig = Field(default_factory=ProviderConfig) + brave: ProviderConfig = Field(default_factory=ProviderConfig) + + +class CacheConfig(BaseModel): + """Smart session-aware response cache (beta).""" + beta_mode: bool = False # Master switch — must be True for any caching + exact_cache_ttl: int = 600 # Seconds to cache exact-match responses (default 10 min) + session_prefix_ttl: int = 3600 # Seconds for session prefix cache (default 1 hr) + max_entries: int = 500 # Max cache entries before LRU eviction + dedup_enabled: bool = True # Merge identical concurrent requests into one upstream call + session_prefix_enabled: bool = True # Enable session-aware prefix caching + exact_cache_enabled: bool = True # Enable exact hash caching + min_prompt_chars: int = 50 # Don't cache tiny prompts (not worth it) + exclude_models: list[str] = Field(default_factory=list) # Models to never cache + +class UsageConfig(BaseModel): + """Ollama Cloud usage quota scraping via session cookie.""" + session_cookie: str = "" # __Secure-1PSID or __Secure-session cookie value + check_interval: int = 0 # Auto-check interval in seconds (0 = disabled) + last_session_pct: Optional[float] = None # Last known session usage % + last_weekly_pct: Optional[float] = None # Last known weekly usage % + last_plan: Optional[str] = None # Last known plan name + last_session_reset: Optional[str] = None # e.g. "Resets in 7 minutes" + last_weekly_reset: Optional[str] = None # e.g. "Resets in 3 days" + last_checked: Optional[float] = None # Unix timestamp of last successful check + redirect_on_full: bool = False # Route all requests to fallback when quota is near limit + + +class AppConfig(BaseModel): + ollama_api_key: str = "" + router: RouterConfig = Field(default_factory=RouterConfig) + llm: LLMConfig = Field(default_factory=LLMConfig) + fallback: FallbackProviderConfig = Field(default_factory=FallbackProviderConfig) + providers: AllProvidersConfig = Field(default_factory=AllProvidersConfig) + cache: CacheConfig = Field(default_factory=CacheConfig) + usage: UsageConfig = Field(default_factory=UsageConfig) + + @property + def ollama_api_key_resolved(self) -> str: + """Resolve API key from config or environment.""" + return self.ollama_api_key or os.environ.get("OLLAMA_API_KEY", "") + + +_config: Optional[AppConfig] = None + + +def load_config(path: Optional[Path] = None) -> AppConfig: + """Load configuration from YAML file, falling back to defaults.""" + global _config + path = path or get_default_config_path() + + if path.exists(): + with open(path) as f: + data = yaml.safe_load(f) or {} + _config = AppConfig(**data) + else: + _config = AppConfig() + + return _config + + +def save_config(config: AppConfig, path: Optional[Path] = None) -> None: + """Save configuration to YAML file.""" + path = path or get_default_config_path() + path.parent.mkdir(parents=True, exist_ok=True) + + # Don't persist env-resolved API keys back to file + dump = config.model_dump() + if not config.ollama_api_key and "ollama_api_key" in dump: + # Keep whatever was in the file, don't overwrite with empty + pass + + with open(path, "w") as f: + yaml.dump(dump, f, default_flow_style=False) + + +def get_config() -> AppConfig: + """Get current config, loading if necessary.""" + global _config + if _config is None: + _config = load_config() + return _config + + +def generate_api_key(prefix: str = "guanca") -> str: + """Generate a random API key.""" + return f"{prefix}_{secrets.token_urlsafe(32)}" + + +def get_base_url(config: Optional[AppConfig] = None) -> str: + """Get the base URL for the services, using Tailscale IP if configured.""" + config = config or get_config() + if config.router.use_tailscale: + ts_ip = get_tailscale_ip() + if ts_ip: + return f"http://{ts_ip}" + return f"http://{config.router.host}" + + +def get_tailscale_ip() -> Optional[str]: + """Get the Tailscale IP address of this machine.""" + import subprocess + try: + result = subprocess.run( + ["tailscale", "ip", "-4"], + capture_output=True, text=True, timeout=5 + ) + if result.returncode == 0: + return result.stdout.strip() + except (FileNotFoundError, subprocess.TimeoutExpired): + pass + return None \ No newline at end of file diff --git a/guanaco/dashboard/__init__.py b/guanaco/dashboard/__init__.py new file mode 100644 index 0000000..5ba9fd0 --- /dev/null +++ b/guanaco/dashboard/__init__.py @@ -0,0 +1,5 @@ +"""Dashboard package.""" + +from guanaco.dashboard.dashboard import create_dashboard_router + +__all__ = ["create_dashboard_router"] \ No newline at end of file diff --git a/guanaco/dashboard/dashboard.py b/guanaco/dashboard/dashboard.py new file mode 100644 index 0000000..a710981 --- /dev/null +++ b/guanaco/dashboard/dashboard.py @@ -0,0 +1,488 @@ +"""Web dashboard for Guanaco management.""" + +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Optional + +from fastapi import APIRouter, Request +from fastapi.responses import HTMLResponse +import httpx + +from guanaco.config import get_config, get_base_url, get_tailscale_ip, save_config, load_config +from guanaco.utils.api_keys import ApiKeyManager +from guanaco.analytics import AnalyticsLogger +from guanaco.client import OllamaClient + + +TEMPLATES_DIR = Path(__file__).parent / "templates" + + +def _generate_systemd_service() -> str: + """Generate systemd unit file content for Guanaco.""" + import shutil + import sys + + venv_python = shutil.which("python") or sys.executable + working_dir = str(Path(__file__).resolve().parent.parent.parent) + config_dir = str(Path.home() / ".guanaco") + + return f"""[Unit] +Description=Guanaco - LLM Proxy & Dashboard +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Environment=PATH={Path(venv_python).parent}:/usr/bin:/usr/local/bin +WorkingDirectory={working_dir} +ExecStart={venv_python} -m uvicorn guanaco.app:create_app --factory --host 0.0.0.0 --port 8080 +Restart=on-failure +RestartSec=5 +Environment=OCT_CONFIG_DIR={config_dir} + +[Install] +WantedBy=multi-user.target +""" + + +def create_dashboard_router(key_manager: ApiKeyManager, analytics: AnalyticsLogger, client=None) -> APIRouter: + router = APIRouter(tags=["Dashboard"]) + + @router.get("/", response_class=HTMLResponse) + async def dashboard(request: Request): + config = get_config() + base_url = get_base_url(config) + port = config.router.port + + html_path = TEMPLATES_DIR / "dashboard.html" + html = html_path.read_text() + + # Inject config + config_json = json.dumps({ + "base_url": base_url, + "port": port, + "router_port": port, + "tailscale": config.router.use_tailscale, + "tailscale_ip": get_tailscale_ip(), + "llm": config.llm.model_dump(), + "available_models": config.llm.available_models, + }) + html = html.replace("__CONFIG__", config_json) + html = html.replace("__USAGE__", json.dumps(analytics.get_summary())) + html = html.replace("__KEYS__", json.dumps(key_manager.list_keys())) + html = html.replace("__FALLBACK__", json.dumps(config.fallback.model_dump())) + providers_data = config.providers.model_dump() + html = html.replace("__PROVIDERS__", json.dumps({ + k: {"enabled": v.get("enabled", True), "require_api_key": v.get("require_api_key", False)} + for k, v in providers_data.items() + })) + + return HTMLResponse(content=html) + + # ── API Keys ── + + @router.get("/api/keys") + async def list_keys(request: Request): + return key_manager.list_keys() + + @router.post("/api/keys/generate") + async def generate_key(request: Request): + body = await request.json() + provider = body.get("provider", "general") + name = body.get("name", "") + key = key_manager.generate_key(provider=provider, name=name) + return {"key": key, "provider": provider} + + @router.post("/api/keys/revoke") + async def revoke_key(request: Request): + body = await request.json() + prefix = body.get("prefix", "") + success = key_manager.revoke_by_prefix(prefix) + return {"success": success} + + # ── Analytics ── + + @router.get("/api/analytics/summary") + async def analytics_summary(request: Request): + return analytics.get_summary() + + @router.get("/api/analytics/logs") + async def analytics_logs( + request: Request, + limit: int = 100, + offset: int = 0, + type: Optional[str] = None, + model: Optional[str] = None, + ): + return analytics.get_logs(limit=limit, offset=offset, type_filter=type, model_filter=model) + + @router.get("/api/analytics/timeseries") + async def analytics_timeseries(request: Request, hours: int = 24): + return analytics.get_timeseries(hours=hours) + + @router.post("/api/analytics/clear") + async def analytics_clear(request: Request): + analytics.clear() + return {"status": "ok"} + + # ── Status Events ── + + @router.get("/api/status/events") + async def status_events( + request: Request, + limit: int = 50, + level: Optional[str] = None, + source: Optional[str] = None, + ): + return analytics.get_status_events(limit=limit, level=level, source=source) + + @router.post("/api/status/log") + async def log_status_event(request: Request): + """Log a status event from the dashboard or external source.""" + body = await request.json() + level = body.get("level", "info") + source = body.get("source", "dashboard") + message = body.get("message", "") + details = body.get("details") + entry_id = analytics.log_status(level=level, source=source, message=message, details=details) + return {"id": entry_id, "status": "logged"} + + # ── Config Management ── + + @router.post("/api/fallback/test") + async def test_fallback_connection(request: Request): + """Test the fallback provider connection by sending a minimal chat request.""" + config = get_config() + fb = config.fallback + + if not fb.enabled: + return {"ok": False, "error": "Fallback is not enabled"} + if not fb.base_url: + return {"ok": False, "error": "Base URL is not configured"} + if not fb.default_model: + return {"ok": False, "error": "Default model is not configured"} + + # Normalize base_url — strip /chat/completions if user pasted the full path + base_url = fb.base_url.rstrip("/") + if base_url.endswith("/chat/completions"): + base_url = base_url[: -len("/chat/completions")] + url = f"{base_url}/chat/completions" + + headers = {"Content-Type": "application/json"} + if fb.api_key: + headers["Authorization"] = f"Bearer {fb.api_key}" + + payload = { + "model": fb.default_model, + "messages": [{"role": "user", "content": "Say hello in one word."}], + "max_tokens": 10, + "stream": False, + } + + timeout = fb.timeout or 30.0 + + try: + async with httpx.AsyncClient(timeout=timeout) as client: + start = time.time() + resp = await client.post(url, json=payload, headers=headers) + elapsed = round((time.time() - start) * 1000) + + if resp.status_code == 200: + data = resp.json() + model_used = "" + content_preview = "" + if data.get("choices"): + msg = data["choices"][0].get("message", {}) + model_used = data.get("model", fb.default_model) + content_preview = (msg.get("content") or "")[:60] + return { + "ok": True, + "message": f"Connected ({elapsed}ms) — {model_used}: \"{content_preview}\"", + } + else: + try: + err_body = resp.json() + err_msg = err_body.get("error", {}) + if isinstance(err_msg, dict): + err_msg = err_msg.get("message", str(err_body)) + elif not err_msg: + err_msg = str(err_body) + except Exception: + err_msg = resp.text[:200] + return { + "ok": False, + "error": f"HTTP {resp.status_code} ({elapsed}ms): {err_msg}", + } + except httpx.ConnectError as e: + return {"ok": False, "error": f"Connection failed: {str(e)}"} + except httpx.TimeoutException: + return {"ok": False, "error": f"Timeout after {timeout}s"} + except Exception as e: + return {"ok": False, "error": str(e)} + + @router.get("/api/config") + async def get_config_api(request: Request): + """Get full config as JSON (llm settings + fallback settings).""" + config = get_config() + return { + "llm": config.llm.model_dump(), + "fallback": config.fallback.model_dump(), + } + + @router.post("/api/config") + async def update_config_api(request: Request): + """Update config (llm and/or fallback settings).""" + body = await request.json() + config = get_config() + + # Update LLM settings + if "llm" in body: + llm_updates = body["llm"] + for key, value in llm_updates.items(): + if hasattr(config.llm, key): + setattr(config.llm, key, value) + + # Update fallback settings + if "fallback" in body: + fb_updates = body["fallback"] + fb = config.fallback + if "enabled" in fb_updates: + fb.enabled = fb_updates["enabled"] + if "name" in fb_updates: + fb.name = fb_updates["name"] + if "base_url" in fb_updates: + fb.base_url = fb_updates["base_url"] + if "api_key" in fb_updates: + fb.api_key = fb_updates["api_key"] + if "model_map" in fb_updates: + fb.model_map = fb_updates["model_map"] + if "default_model" in fb_updates: + fb.default_model = fb_updates["default_model"] + if "timeout" in fb_updates: + fb.timeout = float(fb_updates["timeout"]) + if "primary_timeout" in fb_updates: + fb.primary_timeout = float(fb_updates["primary_timeout"]) + if "stream_chunk_timeout" in fb_updates: + fb.stream_chunk_timeout = float(fb_updates["stream_chunk_timeout"]) + if "max_tokens" in fb_updates: + fb.max_tokens = int(fb_updates["max_tokens"]) + if "stream_fallback" in fb_updates: + fb.stream_fallback = fb_updates["stream_fallback"] + + save_config(config) + return {"status": "ok", "config": {"llm": config.llm.model_dump(), "fallback": config.fallback.model_dump()}} + + # ── Emulation Config ── + + @router.post("/api/config/emulation") + async def save_emulation_config(request: Request): + """Save emulation toggle config (OpenAI/Anthropic endpoint modes).""" + body = await request.json() + config = get_config() + if "emulate_openai" in body: + config.llm.emulate_openai = bool(body["emulate_openai"]) + if "emulate_anthropic" in body: + config.llm.emulate_anthropic = bool(body["emulate_anthropic"]) + save_config(config) + return {"status": "ok", "emulate_openai": config.llm.emulate_openai, "emulate_anthropic": config.llm.emulate_anthropic} + + @router.get("/api/config/emulation") + async def get_emulation_config(request: Request): + """Get current emulation config.""" + config = get_config() + return {"emulate_openai": config.llm.emulate_openai, "emulate_anthropic": config.llm.emulate_anthropic} + + # ── Model History ── + + @router.get("/api/analytics/model/{model_name}") + async def model_history(request: Request, model_name: str, limit: int = 50): + """Get detailed history for a specific model.""" + return analytics.get_model_history(model_name, limit=limit) + + # ── Autostart / Systemd ── + + @router.get("/api/autostart") + async def get_autostart(request: Request): + """Check if Guanaco is currently set to autostart via systemd.""" + import subprocess + service_name = "guanaco.service" + try: + result = subprocess.run( + ["systemctl", "is-enabled", service_name], + capture_output=True, text=True, timeout=5 + ) + enabled = result.stdout.strip() == "enabled" + except (FileNotFoundError, subprocess.TimeoutExpired): + enabled = False + + # Check if service exists + try: + result = subprocess.run( + ["systemctl", "status", service_name], + capture_output=True, text=True, timeout=5 + ) + installed = result.returncode != 4 # code 4 = unit not found + except (FileNotFoundError, subprocess.TimeoutExpired): + installed = False + + # Get runtime status + try: + result = subprocess.run( + ["systemctl", "is-active", service_name], + capture_output=True, text=True, timeout=5 + ) + active = result.stdout.strip() == "active" + except (FileNotFoundError, subprocess.TimeoutExpired): + active = False + + config = get_config() + return { + "enabled": enabled or config.router.autostart, + "installed": installed, + "active": active, + } + + @router.post("/api/autostart/enable") + async def enable_autostart(request: Request): + """Install and enable Guanaco systemd service for autostart.""" + import subprocess + from pathlib import Path + + service_content = _generate_systemd_service() + service_path = Path("/etc/systemd/system/guanaco.service") + + try: + service_path.write_text(service_content) + except PermissionError: + from fastapi import HTTPException + raise HTTPException(status_code=403, detail="Need sudo to write systemd service file. Run: sudo guanaco autostart enable") + + # Reload and enable + subprocess.run(["systemctl", "daemon-reload"], check=True, capture_output=True, timeout=10) + subprocess.run(["systemctl", "enable", "guanaco.service"], check=True, capture_output=True, timeout=10) + + # Start it now if not already running + subprocess.run(["systemctl", "start", "guanaco.service"], capture_output=True, timeout=10) + + config = get_config() + config.router.autostart = True + save_config(config) + + return {"status": "ok", "enabled": True, "message": "Autostart enabled. Guanaco will start on boot."} + + @router.post("/api/autostart/disable") + async def disable_autostart(request: Request): + """Disable and remove Guanaco systemd service.""" + import subprocess + + try: + subprocess.run(["systemctl", "stop", "guanaco.service"], capture_output=True, timeout=10) + subprocess.run(["systemctl", "disable", "guanaco.service"], capture_output=True, timeout=10) + except Exception: + pass + + config = get_config() + config.router.autostart = False + save_config(config) + + return {"status": "ok", "enabled": False, "message": "Autostart disabled."} + + # ── Model Sync ── + + @router.post("/api/models/sync") + async def sync_models_api(request: Request): + """Trigger model sync from Ollama Cloud into config.""" + from guanaco.client import OllamaClient + from guanaco.config import get_config as _get_config + _cfg = _get_config() + client = OllamaClient(api_key=_cfg.ollama_api_key or "") + try: + models = await client.list_models(force_refresh=True) + config = get_config() + model_names = [] + for m in models: + name = m.get("name", m.get("model", "")) + name = name.replace("-cloud", "") if name.endswith("-cloud") else name + if name and name not in model_names: + model_names.append(name) + + existing = set(config.llm.available_models) + for mn in model_names: + existing.add(mn) + + config.llm.available_models = sorted(existing) + save_config(config) + + return {"status": "ok", "synced": len(model_names), "total": len(config.llm.available_models), "models": config.llm.available_models} + except Exception as e: + from fastapi import HTTPException + raise HTTPException(status_code=502, detail=f"Cannot sync models: {str(e)}") + + # ── Usage / Session Cookie ── + + @router.get("/api/usage/config") + async def get_usage_config(request: Request): + config = get_config() + uc = config.usage + return { + "session_cookie_set": bool(uc.session_cookie), + "session_cookie_preview": uc.session_cookie[:8] + "..." if uc.session_cookie else "", + "check_interval": uc.check_interval, + "redirect_on_full": uc.redirect_on_full, + "last_session_pct": uc.last_session_pct, + "last_weekly_pct": uc.last_weekly_pct, + "last_plan": uc.last_plan, + "last_session_reset": uc.last_session_reset, + "last_weekly_reset": uc.last_weekly_reset, + "last_checked": uc.last_checked, + } + + @router.post("/api/usage/session-cookie") + async def set_session_cookie(request: Request): + body = await request.json() + config = get_config() + # Update session cookie if provided + if "session_cookie" in body: + cookie = body.get("session_cookie", "").strip() + config.usage.session_cookie = cookie + if client: + client._session_cookie = cookie + # Update check interval if provided + if "check_interval" in body: + config.usage.check_interval = int(body["check_interval"]) + # Update redirect_on_full if provided + if "redirect_on_full" in body: + config.usage.redirect_on_full = bool(body["redirect_on_full"]) + save_config(config) + return { + "status": "ok", + "cookie_set": bool(config.usage.session_cookie), + "preview": config.usage.session_cookie[:8] + "..." if config.usage.session_cookie else "", + "check_interval": config.usage.check_interval, + "redirect_on_full": config.usage.redirect_on_full, + } + + @router.post("/api/usage/check") + async def check_usage_now(request: Request): + config = get_config() + cookie = config.usage.session_cookie + if not cookie: + return {"source": "unavailable", "error": "No session cookie configured. Paste your __Secure-session cookie in the Status tab."} + try: + usage_data = await client.get_usage(session_cookie=cookie) + if usage_data.get("source") != "unavailable": + config.usage.last_session_pct = usage_data.get("session_pct") + config.usage.last_weekly_pct = usage_data.get("weekly_pct") + config.usage.last_plan = usage_data.get("plan") + config.usage.last_session_reset = usage_data.get("session_reset") + config.usage.last_weekly_reset = usage_data.get("weekly_reset") + config.usage.last_checked = time.time() + save_config(config) + return usage_data + except Exception as e: + return {"source": "error", "error": str(e)} + + return router \ No newline at end of file diff --git a/guanaco/dashboard/templates/dashboard.html b/guanaco/dashboard/templates/dashboard.html new file mode 100644 index 0000000..68d133d --- /dev/null +++ b/guanaco/dashboard/templates/dashboard.html @@ -0,0 +1,1485 @@ + + + + + +Guanaco + + + +
+
+

🦙 Guanaco

+
Connected
+
+ + +
+
0
Requests
+
0
Tokens
+
0
Avg TPS
+
0ms
Avg TTFT
+
0
API Keys
+
+ + +
+
Endpoints
+
Models
+
Fallback
+
Analytics
+
Status
+
Cache
+
API Keys
+
System
+
+ + +
+
+

🔄 LLM Router

+
+
OpenAI
+
+ +
+
+
Anthropic
+
+ +
+

Use as OpenAI/Anthropic-compatible base URL with your Ollama API key

+
+
+

🔍 Emulated Search Provider Endpoints

+
+
+
+
+

🌐 Network

+
+ +
Tailscale
Disabled
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + +
+ + + + \ No newline at end of file diff --git a/guanaco/router/__init__.py b/guanaco/router/__init__.py new file mode 100644 index 0000000..df61df7 --- /dev/null +++ b/guanaco/router/__init__.py @@ -0,0 +1 @@ +"""Router package.""" \ No newline at end of file diff --git a/guanaco/router/router.py b/guanaco/router/router.py new file mode 100644 index 0000000..f5b48c1 --- /dev/null +++ b/guanaco/router/router.py @@ -0,0 +1,1212 @@ +"""OpenAI-compatible and Anthropic-compatible LLM router with usage tracking, analytics, and fallback.""" + +from __future__ import annotations + +import asyncio +import json +import os +import time +import uuid +from typing import Optional + +import httpx +from fastapi import APIRouter, Header, HTTPException, Request +from fastapi.responses import StreamingResponse, JSONResponse +from pydantic import BaseModel, Field + + +def _describe_error(exc: Exception) -> str: + """Return a human-readable description for an exception, handling httpx + timeout/connect errors whose str() is often empty or unhelpful.""" + if isinstance(exc, httpx.ReadTimeout): + return f"ReadTimeout: server did not respond within timeout" + if isinstance(exc, httpx.ConnectTimeout): + return f"ConnectTimeout: could not establish connection within timeout" + if isinstance(exc, httpx.WriteTimeout): + return f"WriteTimeout: could not send data within timeout" + if isinstance(exc, httpx.PoolTimeout): + return f"PoolTimeout: connection pool exhausted" + if isinstance(exc, httpx.ConnectError): + return f"ConnectError: {exc}" + if isinstance(exc, httpx.HTTPStatusError): + return f"HTTP {exc.response.status_code}: {exc.response.text[:200]}" + msg = str(exc) + if msg: + return msg + # Fallback: use the exception class name if str() is empty + return f"{type(exc).__name__}: (no message)" + + +async def _ollama_chat_with_primary_timeout(client, payload, fallback_config=None): + """Call Ollama Cloud chat completion with a primary timeout. + + When fallback is configured, we use a shorter primary_timeout so that + slow/unresponsive Ollama responses trigger fallback quickly instead of + hanging for the full 120s client timeout. + """ + if fallback_config and fallback_config.enabled and fallback_config.primary_timeout: + try: + return await asyncio.wait_for( + client.chat_completion(payload), + timeout=fallback_config.primary_timeout, + ) + except asyncio.TimeoutError: + raise httpx.ReadTimeout( + f"Ollama did not respond within {fallback_config.primary_timeout}s primary timeout" + ) + return await client.chat_completion(payload) + +from guanaco.client import OllamaClient +from guanaco.cache import CacheEngine +from guanaco.analytics import _normalize_model_name + +import logging + +log = logging.getLogger("guanaco.router") + +# ── Empty Response Retry ── +MAX_EMPTY_RETRIES = 1 # How many times to retry on empty responses + + +def _is_empty_non_streaming_response(resp: dict) -> bool: + """Check if a non-streaming chat completion response has no content.""" + choices = resp.get("choices", []) + if not choices: + return True + for choice in choices: + msg = choice.get("message", {}) + content = msg.get("content") + if content and str(content).strip(): + return False + # Some models (GLM) put output in reasoning_content while content is empty + reasoning = msg.get("reasoning_content") + if reasoning and str(reasoning).strip(): + return False + # Check for tool_calls — those count as non-empty + if msg.get("tool_calls"): + return False + return True + + +# ── Request/Response Models ── + +class ChatMessage(BaseModel): + role: str + content: str | list | None = None + name: Optional[str] = None + tool_calls: Optional[list] = None + tool_call_id: Optional[str] = None + + +class ChatCompletionRequest(BaseModel): + model: str + messages: list[ChatMessage] + temperature: Optional[float] = None + top_p: Optional[float] = None + max_tokens: Optional[int] = None + stream: bool = False + stop: Optional[list[str]] = None + presence_penalty: Optional[float] = None + frequency_penalty: Optional[float] = None + tools: Optional[list[dict]] = None + tool_choice: Optional[str | dict] = None + response_format: Optional[dict] = None + + +# ── Anthropic Request Models ── + +class AnthropicMessage(BaseModel): + role: str + content: str | list + + +class AnthropicRequest(BaseModel): + model: str + max_tokens: int = 4096 + messages: list[AnthropicMessage] + system: Optional[str | list] = None + temperature: Optional[float] = None + top_p: Optional[float] = None + stream: bool = False + stop_sequences: Optional[list[str]] = None + tools: Optional[list[dict]] = None + tool_choice: Optional[dict] = None + + +def _resolve_model(model: str, config) -> str: + """Resolve model name for Ollama Cloud API.""" + normalized = model + if normalized.endswith("-cloud"): + normalized = normalized[:-6] + + if normalized in config.llm.available_models: + return normalized + + for available in config.llm.available_models: + base = available.split(":")[0] + if normalized == base: + return available + + return normalized + + +def _map_model_to_fallback(model: str, fallback_config) -> str: + """Map an Ollama model name to the corresponding fallback model.""" + if model in fallback_config.model_map: + return fallback_config.model_map[model] + base = model.split(":")[0] + if base in fallback_config.model_map: + return fallback_config.model_map[base] + return fallback_config.default_model or model + +def _is_quota_full(config) -> bool: + """Check if Ollama Cloud usage quota is near or at limit (>= 99.5%).""" + if not config or not config.usage.redirect_on_full: + return False + s = config.usage.last_session_pct + w = config.usage.last_weekly_pct + if s is not None and s >= 99.5: + return True + if w is not None and w >= 99.5: + return True + return False + +async def _refresh_usage_background(client, config): + """Background refresh of usage quota so we notice when it resets.""" + try: + cookie = config.usage.session_cookie + if not cookie: + return + usage = await client.get_usage(session_cookie=cookie) + if usage.get("source") != "unavailable": + config.usage.last_session_pct = usage.get("session_pct") + config.usage.last_weekly_pct = usage.get("weekly_pct") + config.usage.last_plan = usage.get("plan") + config.usage.last_session_reset = usage.get("session_reset") + config.usage.last_weekly_reset = usage.get("weekly_reset") + config.usage.last_checked = time.time() + from guanaco.config import save_config + save_config(config) + if not _is_quota_full(config): + log.info("Quota recovered — session=%.1f%%, weekly=%.1f%%, routing back to Ollama", + config.usage.last_session_pct or 0, config.usage.last_weekly_pct or 0) + except Exception as e: + log.debug("Background usage refresh failed: %s", e) + + +async def _call_fallback_provider(payload: dict, fallback_config, stream: bool = False): + """Send a request to the fallback OpenAI-compatible provider.""" + base_url = fallback_config.base_url.rstrip("/") + # Strip /chat/completions if user accidentally included the full path + if base_url.endswith("/chat/completions"): + base_url = base_url[: -len("/chat/completions")] + url = f"{base_url}/chat/completions" + headers = { + "Content-Type": "application/json", + } + if fallback_config.api_key: + headers["Authorization"] = f"Bearer {fallback_config.api_key}" + + # Inject fallback max_tokens if not already set in the payload + if fallback_config.max_tokens and "max_tokens" not in payload: + payload = dict(payload) + payload["max_tokens"] = fallback_config.max_tokens + + timeout = fallback_config.timeout or 60.0 + + if stream: + # Streaming: use a long-lived client that stays open while the generator is consumed + client = httpx.AsyncClient(timeout=timeout) + + async def stream_from_fallback(): + try: + async with client.stream("POST", url, json=payload, headers=headers) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + yield line + "\n" + finally: + await client.aclose() + + return stream_from_fallback() + else: + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.post(url, json=payload, headers=headers) + resp.raise_for_status() + return resp.json() + + +# ── Provider creation ── + +def create_router(client: OllamaClient, analytics=None, config=None) -> APIRouter: + router = APIRouter(tags=["LLM Router"]) + _analytics = analytics + _config = config + _cache = CacheEngine(config.cache) if config else None + + # ── OpenAI-compatible endpoints ── + + @router.get("/v1/models") + async def list_models(request: Request): + """List available models by querying Ollama Cloud dynamically.""" + try: + models = await client.list_models() + data = [] + for m in models: + name = m.get("name", m.get("model", "")) + display_name = name.replace("-cloud", "") if name.endswith("-cloud") else name + details = m.get("details", {}) + data.append({ + "id": display_name, + "object": "model", + "created": int(time.time()), + "owned_by": "ollama", + "permission": [], + "root": display_name, + "parent": None, + "capabilities": client._get_model_capabilities(name), + "details": { + "parameter_size": details.get("parameter_size", ""), + "quantization": details.get("quantization_level", ""), + "family": details.get("family", ""), + }, + }) + # Add fallback provider models if configured + if _config and _config.fallback.enabled and _config.fallback.default_model: + fallback_models = set(_config.fallback.model_map.values()) + if _config.fallback.default_model: + fallback_models.add(_config.fallback.default_model) + for fm in fallback_models: + if fm and not any(d["id"] == fm for d in data): + data.append({ + "id": fm, + "object": "model", + "created": int(time.time()), + "owned_by": "fallback", + "permission": [], + "root": fm, + "parent": None, + "details": {"family": "fallback"}, + }) + return {"object": "list", "data": data} + except Exception as e: + if _config: + data = [ + {"id": name, "object": "model", "created": int(time.time()), "owned_by": "ollama"} + for name in _config.llm.available_models + ] + return {"object": "list", "data": data} + raise HTTPException(status_code=502, detail=f"Cannot reach Ollama Cloud: {str(e)}") + + @router.post("/v1/chat/completions") + async def chat_completions(body: ChatCompletionRequest, request: Request): + """OpenAI-compatible chat completions endpoint with fallback and smart caching (beta).""" + start = time.time() + resolved_model = _resolve_model(body.model, _config) if _config else body.model + payload = body.model_dump(exclude_none=True) + payload["model"] = resolved_model + + # ── Quota-full redirect: skip Ollama entirely, go straight to fallback ── + if _is_quota_full(_config): + if _config.fallback.enabled and _config.fallback.base_url: + fallback_model = _map_model_to_fallback(resolved_model, _config.fallback) + log.info("Quota full (session=%.1f%%, weekly=%.1f%%), redirecting %s to fallback %s (model: %s)", + _config.usage.last_session_pct or 0, _config.usage.last_weekly_pct or 0, + resolved_model, _config.fallback.name, fallback_model) + # Refresh quota in background so we notice when it resets + asyncio.ensure_future(_refresh_usage_background(client, _config)) + # Skip Ollama, go straight to fallback + payload_fb = dict(payload) + payload_fb["model"] = fallback_model + if body.stream: + if _config.fallback.stream_fallback: + fallback_payload = dict(payload) + fallback_payload["model"] = fallback_model + return StreamingResponse( + await _call_fallback_provider(fallback_payload, _config.fallback, stream=True), + media_type="text/event-stream", + ) + # Can't stream from fallback — do non-streaming + try: + fallback_resp = await _call_fallback_provider(payload_fb, _config.fallback) + return fallback_resp + except Exception as e: + raise HTTPException(status_code=502, detail=f"Quota full and fallback error: {_describe_error(e)}") + else: + try: + fallback_resp = await _call_fallback_provider(payload_fb, _config.fallback) + # Normalize fallback response + if isinstance(fallback_resp, dict) and "choices" in fallback_resp: + for choice in fallback_resp.get("choices", []): + if isinstance(choice, dict) and "message" in choice: + msg = choice["message"] + if isinstance(msg, str): + choice["message"] = {"role": "assistant", "content": msg} + fallback_resp["_oct_fallback"] = True + fallback_resp["_oct_fallback_provider"] = _config.fallback.name + fallback_resp["_oct_original_model"] = _normalize_model_name(resolved_model) + fallback_resp["_oct_quota_redirect"] = True + if _analytics: + _analytics.log_llm( + model=_normalize_model_name(fallback_model), + total_duration_seconds=time.time() - start, + provider=_config.fallback.name, + fallback_for=_normalize_model_name(resolved_model), + ) + return fallback_resp + except Exception as e: + raise HTTPException(status_code=502, detail=f"Quota full and fallback error: {_describe_error(e)}") + # No fallback configured — let it hit Ollama and probably get rate-limited + + # ── Smart Cache (beta) for non-streaming requests ── + if _cache and _cache.is_enabled() and not body.stream: + async def _fetch_from_upstream(p: dict) -> dict: + """Fetch from Ollama Cloud with fallback, retrying on empty response.""" + try: + # ── Retry on empty response ── + for attempt in range(MAX_EMPTY_RETRIES + 1): + resp = await _ollama_chat_with_primary_timeout(client, p, _config.fallback if _config else None) + if not _is_empty_non_streaming_response(resp) or attempt == MAX_EMPTY_RETRIES: + break + log.warning("Empty cached-response from %s (attempt %d/%d), retrying...", resolved_model, attempt + 1, MAX_EMPTY_RETRIES + 1) + + elapsed = time.time() - start + metrics = resp.pop("_oct_metrics", {}) + usage = resp.get("usage", {}) + + if _analytics: + _analytics.log_llm( + model=resolved_model, + prompt_tokens=usage.get("prompt_tokens", metrics.get("prompt_eval_count", 0)), + completion_tokens=usage.get("completion_tokens", metrics.get("eval_count", 0)), + total_tokens=usage.get("total_tokens", 0), + tps=metrics.get("tps"), + prompt_tps=metrics.get("prompt_tps"), + ttft_seconds=metrics.get("ttft_seconds"), + total_duration_seconds=elapsed, + load_duration_seconds=metrics.get("load_duration_ns", 0) / 1e9 if metrics.get("load_duration_ns") else None, + provider="ollama", + ) + + return resp + + except Exception as ollama_error: + # Try fallback provider if configured + if _config and _config.fallback.enabled and _config.fallback.base_url: + fallback_model = _map_model_to_fallback(resolved_model, _config.fallback) + log.info("Ollama error for %s (cached path), trying fallback %s (model: %s)", resolved_model, _config.fallback.name, fallback_model) + fallback_payload = dict(p) + fallback_payload["model"] = fallback_model + + try: + fallback_resp = await _call_fallback_provider(fallback_payload, _config.fallback) + elapsed = time.time() - start + + # Normalize fallback response: ensure choices[].message is a dict + if isinstance(fallback_resp, dict) and "choices" in fallback_resp: + for choice in fallback_resp.get("choices", []): + if isinstance(choice, dict) and "message" in choice: + msg = choice["message"] + if isinstance(msg, str): + choice["message"] = {"role": "assistant", "content": msg} + + if _analytics: + _analytics.log_llm( + model=_normalize_model_name(fallback_model), + total_duration_seconds=elapsed, + provider=_config.fallback.name, + fallback_for=_normalize_model_name(resolved_model), + ) + + fallback_resp["_oct_fallback"] = True + fallback_resp["_oct_fallback_provider"] = _config.fallback.name + fallback_resp["_oct_original_model"] = _normalize_model_name(resolved_model) + return fallback_resp + + except Exception as fallback_err: + log.warning("Fallback to %s failed for model %s (cached path): %s", _config.fallback.name, resolved_model, _describe_error(fallback_err)) + if _analytics: + _analytics.log_llm(model=resolved_model, error=f"ollama: {_describe_error(ollama_error)}; fallback: {_describe_error(fallback_err)}", total_duration_seconds=time.time() - start) + raise HTTPException(status_code=502, detail=f"Ollama Cloud error: {_describe_error(ollama_error)}; Fallback error: {_describe_error(fallback_err)}") + + if _analytics: + _analytics.log_llm(model=resolved_model, error=str(ollama_error), total_duration_seconds=time.time() - start) + raise HTTPException(status_code=502, detail=f"Ollama Cloud error: {str(ollama_error)}") + + # Use cache for non-streaming + response = await _cache.get_or_fetch( + model=resolved_model, + messages=[m.model_dump(exclude_none=True) for m in body.messages], + params=payload, + fetch_fn=_fetch_from_upstream, + provider="ollama", + ) + + # Log cache metadata in analytics + if response.get("_oct_cached"): + elapsed = time.time() - start + if _analytics: + _analytics.log_llm( + model=resolved_model, + total_duration_seconds=elapsed, + provider=f"cache:{response.get('_oct_cache_type', 'unknown')}", + ) + + return response + + # ── Original path for streaming or cache disabled ── + # Try Ollama Cloud first + try: + if body.stream: + return await _stream_completion_openai(client, payload, resolved_model, _analytics, start, _config) + + # ── Non-streaming: retry on empty response ── + for attempt in range(MAX_EMPTY_RETRIES + 1): + resp = await _ollama_chat_with_primary_timeout(client, payload, _config.fallback if _config else None) + if not _is_empty_non_streaming_response(resp) or attempt == MAX_EMPTY_RETRIES: + break + log.warning("Empty response from %s (attempt %d/%d), retrying...", resolved_model, attempt + 1, MAX_EMPTY_RETRIES + 1) + + elapsed = time.time() - start + metrics = resp.pop("_oct_metrics", {}) + usage = resp.get("usage", {}) + + if _analytics: + _analytics.log_llm( + model=resolved_model, + prompt_tokens=usage.get("prompt_tokens", metrics.get("prompt_eval_count", 0)), + completion_tokens=usage.get("completion_tokens", metrics.get("eval_count", 0)), + total_tokens=usage.get("total_tokens", 0), + tps=metrics.get("tps"), + prompt_tps=metrics.get("prompt_tps"), + ttft_seconds=metrics.get("ttft_seconds"), + total_duration_seconds=elapsed, + load_duration_seconds=metrics.get("load_duration_ns", 0) / 1e9 if metrics.get("load_duration_ns") else None, + provider="ollama", + ) + + return resp + + except Exception as ollama_error: + # Try fallback provider if configured + if _config and _config.fallback.enabled and _config.fallback.base_url: + fallback_model = _map_model_to_fallback(resolved_model, _config.fallback) + log.info("Ollama error for %s, trying fallback %s (model: %s)", resolved_model, _config.fallback.name, fallback_model) + fallback_payload = dict(payload) + fallback_payload["model"] = fallback_model + + try: + if body.stream and _config.fallback.stream_fallback: + return await _stream_fallback_openai(fallback_payload, _config, fallback_model, _analytics, start, "ollama_fallback") + + fallback_resp = await _call_fallback_provider(fallback_payload, _config.fallback) + elapsed = time.time() - start + + if _analytics: + _analytics.log_llm( + model=fallback_model, + total_duration_seconds=elapsed, + provider=_config.fallback.name, fallback_for=resolved_model, + ) + + # Tag response so dashboard can show it came from fallback + fallback_resp["_oct_fallback"] = True + fallback_resp["_oct_fallback_provider"] = _config.fallback.name + fallback_resp["_oct_original_model"] = resolved_model + return fallback_resp + + except Exception as fallback_err: + log.warning("Fallback to %s failed for model %s: %s", _config.fallback.name, resolved_model, _describe_error(fallback_err)) + if _analytics: + _analytics.log_llm(model=resolved_model, error=f"ollama: {_describe_error(ollama_error)}; fallback: {_describe_error(fallback_err)}", total_duration_seconds=time.time() - start) + raise HTTPException(status_code=502, detail=f"Ollama Cloud error: {_describe_error(ollama_error)}; Fallback error: {_describe_error(fallback_err)}") + + if _analytics: + _analytics.log_llm(model=resolved_model, error=_describe_error(ollama_error), total_duration_seconds=time.time() - start) + raise HTTPException(status_code=502, detail=f"Ollama Cloud error: {_describe_error(ollama_error)}") + + @router.post("/v1/chat/completions/refresh_models") + async def refresh_models(request: Request): + """Force-refresh the model list cache.""" + try: + models = await client.list_models(force_refresh=True) + return {"status": "ok", "model_count": len(models)} + except Exception as e: + raise HTTPException(status_code=502, detail=f"Cannot refresh models: {str(e)}") + + @router.get("/v1/usage") + async def get_usage(request: Request): + """Get Ollama Cloud account usage/quota information.""" + try: + config = get_config() + session_cookie = config.usage.session_cookie + usage_data = await client.get_usage(session_cookie=session_cookie) + # Persist last-known values for dashboard display + if usage_data.get("source") != "unavailable": + config.usage.last_session_pct = usage_data.get("session_pct") + config.usage.last_weekly_pct = usage_data.get("weekly_pct") + config.usage.last_plan = usage_data.get("plan") + config.usage.last_session_reset = usage_data.get("session_reset") + config.usage.last_weekly_reset = usage_data.get("weekly_reset") + config.usage.last_checked = time.time() + save_config(config) + return usage_data + except Exception as e: + return {"source": "error", "error": str(e)} + + @router.get("/v1/analytics") + async def get_analytics(request: Request): + """Get local analytics summary.""" + if _analytics: + return _analytics.get_summary() + return {"total_requests": 0} + + @router.post("/v1/analytics/reset") + async def reset_analytics(request: Request): + """Reset local analytics data.""" + if _analytics: + _analytics.clear() + return {"status": "ok"} + + # ── Anthropic-compatible endpoints ── + + @router.post("/v1/messages") + async def anthropic_messages(body: AnthropicRequest, request: Request): + """Anthropic-compatible /v1/messages endpoint.""" + start = time.time() + resolved_model = _resolve_model(body.model, _config) if _config else body.model + + # Convert Anthropic format to OpenAI format + openai_messages = [] + + if body.system: + sys_content = body.system if isinstance(body.system, str) else json.dumps(body.system) + openai_messages.append({"role": "system", "content": sys_content}) + + for msg in body.messages: + content = msg.content + if isinstance(content, list): + text_parts = [] + tool_use_blocks = [] + for block in content: + if isinstance(block, dict): + if block.get("type") == "text": + text_parts.append(block.get("text", "")) + elif block.get("type") == "tool_result": + tool_content = block.get("content", "") + if isinstance(tool_content, list): + for c in tool_content: + if isinstance(c, dict) and c.get("type") == "text": + text_parts.append(c.get("text", "")) + else: + text_parts.append(str(tool_content)) + elif block.get("type") == "tool_use": + tool_use_blocks.append(block) + else: + text_parts.append(str(block)) + content = "\n".join(text_parts) if text_parts else "" + if tool_use_blocks: + tool_info = json.dumps(tool_use_blocks) + content = f"{content}\n[Tool calls: {tool_info}]" if content else f"[Tool calls: {tool_info}]" + openai_messages.append({"role": msg.role, "content": content}) + + openai_payload = { + "model": resolved_model, + "messages": openai_messages, + "max_tokens": body.max_tokens, + "stream": body.stream, + } + if body.temperature is not None: + openai_payload["temperature"] = body.temperature + if body.top_p is not None: + openai_payload["top_p"] = body.top_p + if body.stop_sequences: + openai_payload["stop"] = body.stop_sequences + + if body.tools: + openai_tools = [] + for tool in body.tools: + openai_tools.append({ + "type": "function", + "function": { + "name": tool.get("name", ""), + "description": tool.get("description", ""), + "parameters": tool.get("input_schema", {}), + } + }) + openai_payload["tools"] = openai_tools + if body.tool_choice: + if isinstance(body.tool_choice, dict): + if tool_choice_type := body.tool_choice.get("type"): + if tool_choice_type == "auto": + openai_payload["tool_choice"] = "auto" + elif tool_choice_type == "any": + openai_payload["tool_choice"] = "required" + elif tool_choice_type == "tool": + openai_payload["tool_choice"] = {"type": "function", "function": {"name": body.tool_choice.get("name", "")}} + elif isinstance(body.tool_choice, str): + openai_payload["tool_choice"] = body.tool_choice + + try: + if body.stream: + return await _stream_completion_anthropic(client, openai_payload, resolved_model, body.max_tokens, _analytics, start) + + resp = await client.chat_completion(openai_payload) + elapsed = time.time() - start + metrics = resp.pop("_oct_metrics", {}) + usage = resp.get("usage", {}) + + if _analytics: + _analytics.log_llm( + model=resolved_model, + prompt_tokens=usage.get("prompt_tokens", metrics.get("prompt_eval_count", 0)), + completion_tokens=usage.get("completion_tokens", metrics.get("eval_count", 0)), + total_tokens=usage.get("total_tokens", 0), + tps=metrics.get("tps"), + ttft_seconds=metrics.get("ttft_seconds"), + total_duration_seconds=elapsed, + provider="ollama", + ) + + # Convert OpenAI response to Anthropic format + choices = resp.get("choices", []) + content_text = "" + finish_reason = "end_turn" + tool_use_response = [] + if choices: + msg = choices[0].get("message", {}) + content_text = msg.get("content", "") + fr = choices[0].get("finish_reason", "stop") + finish_reason = _openai_to_anthropic_stop(fr) + + if msg.get("tool_calls"): + for tc in msg["tool_calls"]: + func = tc.get("function", {}) + tool_use_response.append({ + "type": "tool_use", + "id": tc.get("id", f"toolu_{uuid.uuid4().hex[:24]}"), + "name": func.get("name", ""), + "input": json.loads(func.get("arguments", "{}")) if isinstance(func.get("arguments"), str) else func.get("arguments", {}), + }) + + content_blocks = [] + if content_text: + content_blocks.append({"type": "text", "text": content_text}) + if tool_use_response: + content_blocks.extend(tool_use_response) + + if not content_blocks: + content_blocks = [{"type": "text", "text": ""}] + + anthropic_resp = { + "id": f"msg_{uuid.uuid4().hex[:24]}", + "type": "message", + "role": "assistant", + "model": body.model, + "content": content_blocks, + "stop_reason": "tool_use" if tool_use_response else finish_reason, + "stop_sequence": None, + "usage": { + "input_tokens": usage.get("prompt_tokens", metrics.get("prompt_eval_count", 0)), + "output_tokens": usage.get("completion_tokens", metrics.get("eval_count", 0)), + }, + } + return anthropic_resp + + except Exception as e: + if _analytics: + _analytics.log_llm(model=resolved_model, error=str(e), total_duration_seconds=time.time() - start) + raise HTTPException(status_code=502, detail=f"Ollama Cloud error: {str(e)}") + + # ── Model selection endpoint ── + + @router.post("/v1/config/model") + async def set_model(request: Request): + """Update model selection for reranker/scraper/summary.""" + from guanaco.config import save_config, get_config + body = await request.json() + config = get_config() + updated = False + + if "reranker_model" in body: + config.llm.reranker_model = body["reranker_model"] + updated = True + if "scraper_model" in body: + config.llm.scraper_model = body["scraper_model"] + updated = True + if "summary_model" in body: + config.llm.summary_model = body["summary_model"] + updated = True + if "default_model" in body: + config.llm.default_model = body["default_model"] + updated = True + if "fallback_model" in body: + config.llm.fallback_model = body["fallback_model"] + updated = True + + if updated: + save_config(config) + return {"status": "ok", "config": config.llm.model_dump()} + return {"status": "no_changes", "config": config.llm.model_dump()} + + @router.get("/v1/config/model") + async def get_model_config(request: Request): + """Get current model selection config.""" + from guanaco.config import get_config + config = get_config() + return config.llm.model_dump() + + # ── Fallback provider config ── + + @router.get("/v1/config/fallback") + async def get_fallback_config(request: Request): + """Get current fallback provider config.""" + from guanaco.config import get_config + config = get_config() + fb = config.fallback + return { + "enabled": fb.enabled, + "name": fb.name, + "base_url": fb.base_url, + "default_model": fb.default_model, + "model_map": fb.model_map, + "timeout": fb.timeout, + "stream_fallback": fb.stream_fallback, + "has_api_key": bool(fb.api_key or os.environ.get("FALLBACK_API_KEY", "")), + } + + @router.post("/v1/config/fallback") + async def set_fallback_config(request: Request): + """Update fallback provider config.""" + from guanaco.config import save_config, get_config + import os as _os + body = await request.json() + config = get_config() + fb = config.fallback + + if "enabled" in body: + fb.enabled = body["enabled"] + if "name" in body: + fb.name = body["name"] + if "base_url" in body: + fb.base_url = body["base_url"] + if "api_key" in body: + fb.api_key = body["api_key"] + if "model_map" in body: + fb.model_map = body["model_map"] + if "default_model" in body: + fb.default_model = body["default_model"] + if "timeout" in body: + fb.timeout = float(body["timeout"]) + if "stream_fallback" in body: + fb.stream_fallback = body["stream_fallback"] + + save_config(config) + + return { + "status": "ok", + "fallback": { + "enabled": fb.enabled, + "name": fb.name, + "base_url": fb.base_url, + "default_model": fb.default_model, + "model_map": fb.model_map, + "timeout": fb.timeout, + "stream_fallback": fb.stream_fallback, + "has_api_key": bool(fb.api_key or _os.environ.get("FALLBACK_API_KEY", "")), + }, + } + + # ── Model sync endpoint ── + + @router.post("/v1/config/sync_models") + async def sync_models(request: Request): + """Sync available_models from Ollama Cloud API into config.""" + from guanaco.config import save_config, get_config + try: + models = await client.list_models(force_refresh=True) + config = get_config() + model_names = [] + for m in models: + name = m.get("name", m.get("model", "")) + # Strip -cloud suffix + name = name.replace("-cloud", "") if name.endswith("-cloud") else name + if name and name not in model_names: + model_names.append(name) + + # Merge with existing config models + existing = set(config.llm.available_models) + for mn in model_names: + existing.add(mn) + + config.llm.available_models = sorted(existing) + save_config(config) + + return {"status": "ok", "synced": len(model_names), "total": len(config.llm.available_models), "models": config.llm.available_models} + except Exception as e: + raise HTTPException(status_code=502, detail=f"Cannot sync models: {str(e)}") + + # ── Cache management endpoints (beta) ── + + @router.get("/v1/cache/stats") + async def cache_stats(request: Request): + """Get cache statistics and configuration.""" + if not _cache: + return {"beta_mode": False, "message": "Cache not initialized"} + return _cache.get_stats() + + @router.post("/v1/cache/clear") + async def cache_clear(request: Request): + """Clear all cached responses.""" + if not _cache: + return {"status": "not_initialized"} + _cache.clear() + return {"status": "ok", "message": "Cache cleared"} + + @router.post("/v1/config/cache") + async def update_cache_config(request: Request): + """Update cache configuration at runtime.""" + from guanaco.config import save_config, get_config as _get_config + body = await request.json() + config = _get_config() + + updated = False + if "beta_mode" in body: + config.cache.beta_mode = body["beta_mode"] + updated = True + if "exact_cache_ttl" in body: + config.cache.exact_cache_ttl = int(body["exact_cache_ttl"]) + updated = True + if "session_prefix_ttl" in body: + config.cache.session_prefix_ttl = int(body["session_prefix_ttl"]) + updated = True + if "max_entries" in body: + config.cache.max_entries = int(body["max_entries"]) + updated = True + if "exact_cache_enabled" in body: + config.cache.exact_cache_enabled = body["exact_cache_enabled"] + updated = True + if "session_prefix_enabled" in body: + config.cache.session_prefix_enabled = body["session_prefix_enabled"] + updated = True + if "dedup_enabled" in body: + config.cache.dedup_enabled = body["dedup_enabled"] + updated = True + if "min_prompt_chars" in body: + config.cache.min_prompt_chars = int(body["min_prompt_chars"]) + updated = True + + if updated: + nonlocal _cache + # Update the live cache config + if _cache: + _cache.config = config.cache + save_config(config) + # Re-init cache if beta_mode changed + if config.cache.beta_mode and _cache is None: + _cache = CacheEngine(config.cache) + elif not config.cache.beta_mode and _cache: + _cache.clear() + return {"status": "ok", "cache": config.cache.model_dump()} + return {"status": "no_changes", "cache": config.cache.model_dump()} + + @router.get("/v1/config/cache") + async def get_cache_config(request: Request): + """Get current cache configuration.""" + from guanaco.config import get_config as _get_config + config = _get_config() + return config.cache.model_dump() + + @router.post("/v1/cache/evict") + async def cache_evict_expired(request: Request): + """Force eviction of expired cache entries.""" + if not _cache: + return {"status": "not_initialized"} + _cache.evict_expired() + stats = _cache.get_stats() + return {"status": "ok", "remaining_entries": stats["exact_cache_entries"] + stats["prefix_cache_entries"]} + + return router + + +# ── Streaming helpers ── + +def _is_empty_stream_buffer(chunks: list[str]) -> bool: + """Check if buffered streaming chunks contain no actual content.""" + for chunk in chunks: + if not chunk.startswith("data: ") or chunk.strip() == "data: [DONE]": + continue + try: + data = json.loads(chunk[6:].strip()) + for choice in data.get("choices", []): + delta = choice.get("delta", {}) + content = delta.get("content", "") + if content and content.strip(): + return False + # Some models (GLM) stream reasoning_content + reasoning = delta.get("reasoning_content", "") + if reasoning and reasoning.strip(): + return False + # tool_calls count as non-empty + if delta.get("tool_calls"): + return False + except (json.JSONDecodeError, KeyError): + continue + return True + + +async def _collect_stream_chunks(client, payload) -> tuple[list[str], dict]: + """Collect all chunks from a stream into a buffer. Returns (chunks, metrics).""" + chunks = [] + metrics = {} + async for chunk in client.chat_completion_stream(payload): + if chunk.startswith("__oct_metrics__:"): + try: + metrics = json.loads(chunk.split(":", 1)[1]) + except (json.JSONDecodeError, ValueError): + pass + continue + chunks.append(chunk) + return chunks, metrics + + +async def _iter_stream_with_timeouts(aiter, first_chunk_timeout, inter_chunk_timeout): + """Wrap an async iterator with per-chunk timeouts. + + - first_chunk_timeout: max seconds to wait for the FIRST chunk + - inter_chunk_timeout: max seconds to wait for each SUBSEQUENT chunk + + Raises asyncio.TimeoutError if any deadline is missed. + """ + first = True + async for item in aiter: + first = False + yield item + # After yielding, set up timeout for next chunk + # If we never got any item, the aiter ended normally (empty stream) + + +async def _stream_completion_openai(client, payload, model, analytics, start_time, config=None): + """Stream OpenAI-format SSE responses, with fallback and timeout support. + + Key design: When fallback is configured with primary_timeout, we apply + per-chunk timeouts to the Ollama stream. If the first chunk doesn't arrive + within primary_timeout seconds, we fall back to the fallback provider + BEFORE yielding any data to the client. This means Hermes never sees + a timeout — it either gets Ollama chunks or fallback chunks. + """ + from fastapi.responses import StreamingResponse + + fb = config.fallback if config else None + use_timeouts = (fb and fb.enabled and fb.base_url and fb.primary_timeout + and fb.primary_timeout > 0) + + async def generate(): + stream_metrics = {} + used_fallback = False + fallback_model = None + original_error = None + try: + if use_timeouts: + chunk_timeout = fb.stream_chunk_timeout if fb.stream_chunk_timeout else 180.0 + # ── Timed streaming: fail fast on first chunk, tolerate gaps after ── + ollama_stream = client.chat_completion_stream(payload) + stream_closed = False + # Wait for the first chunk with a strict timeout (triggers fallback fast) + try: + first_chunk = await asyncio.wait_for( + ollama_stream.__anext__(), timeout=fb.primary_timeout + ) + except asyncio.TimeoutError: + # First chunk timeout — no data sent to client yet, can still fallback + try: + await ollama_stream.aclose() + except RuntimeError: + pass # Generator already cleaned up by cancellation + stream_closed = True + raise httpx.ReadTimeout( + f"Ollama did not produce first stream chunk within {fb.primary_timeout}s" + ) + except StopAsyncIteration: + # Empty stream — treat as error so fallback can handle it + try: + await ollama_stream.aclose() + except RuntimeError: + pass + stream_closed = True + raise httpx.ReadTimeout( + f"Ollama stream ended before producing any chunks" + ) + + # Got first chunk — yield it and continue with generous inter-chunk timeouts + yield first_chunk + try: + while True: + try: + chunk = await asyncio.wait_for( + ollama_stream.__anext__(), timeout=chunk_timeout + ) + yield chunk + except StopAsyncIteration: + break + except asyncio.TimeoutError: + # Inter-chunk timeout — we've already sent data, can't fallback + # Send an error SSE chunk and stop + log.warning("Ollama stream inter-chunk timeout for %s after %ss", model, chunk_timeout) + error_data = json.dumps({"error": {"message": f"Stream stalled: no data for {chunk_timeout}s", "type": "server_error"}}) + yield f"data: {error_data}\n\n" + yield "data: [DONE]\n\n" + original_error = f"Stream stalled: no data for {chunk_timeout}s" + return + finally: + if not stream_closed: + try: + await ollama_stream.aclose() + except RuntimeError: + pass # Already closed + else: + # ── No timeout wrapping: original buffered behavior ── + for attempt in range(MAX_EMPTY_RETRIES + 1): + chunks, stream_metrics = await _collect_stream_chunks(client, payload) + if not _is_empty_stream_buffer(chunks) or attempt == MAX_EMPTY_RETRIES: + break + log.warning("Empty streaming response from %s (attempt %d/%d), retrying...", model, attempt + 1, MAX_EMPTY_RETRIES + 1) + + for chunk in chunks: + yield chunk + + except Exception as e: + original_error = _describe_error(e) + # Try fallback if configured — we haven't sent any data yet, so we can cleanly switch + if config and config.fallback.enabled and config.fallback.base_url and config.fallback.stream_fallback: + fallback_model = _map_model_to_fallback(model, config.fallback) + log.info("Ollama stream error for %s (%s), trying fallback %s (model: %s)", model, original_error, config.fallback.name, fallback_model) + fallback_payload = dict(payload) + fallback_payload["model"] = fallback_model + try: + async for chunk in await _call_fallback_provider(fallback_payload, config.fallback, stream=True): + used_fallback = True + yield chunk + except Exception as fallback_err: + log.warning("Stream fallback to %s failed for model %s: %s", config.fallback.name, model, _describe_error(fallback_err)) + error_data = json.dumps({"error": {"message": f"Ollama: {original_error}; Fallback: {_describe_error(fallback_err)}", "type": "server_error"}}) + yield f"data: {error_data}\n\n" + yield "data: [DONE]\n\n" + else: + error_data = json.dumps({"error": {"message": original_error, "type": "server_error"}}) + yield f"data: {error_data}\n\n" + yield "data: [DONE]\n\n" + finally: + elapsed = time.time() - start_time + if analytics: + if used_fallback and fallback_model: + analytics.log_llm( + model=_normalize_model_name(fallback_model), + completion_tokens=stream_metrics.get("eval_count"), + tps=stream_metrics.get("tps"), + ttft_seconds=stream_metrics.get("ttft_seconds"), + total_duration_seconds=stream_metrics.get("elapsed_seconds", elapsed), + provider=config.fallback.name if config else "fallback", + fallback_for=_normalize_model_name(model), + ) + elif original_error: + analytics.log_llm( + model=_normalize_model_name(model), + error=original_error, + total_duration_seconds=elapsed, + provider="ollama", + ) + else: + analytics.log_llm( + model=_normalize_model_name(model), + completion_tokens=stream_metrics.get("eval_count"), + tps=stream_metrics.get("tps"), + ttft_seconds=stream_metrics.get("ttft_seconds"), + total_duration_seconds=stream_metrics.get("elapsed_seconds", elapsed), + provider="ollama", + ) + + return StreamingResponse(generate(), media_type="text/event-stream") + + +async def _stream_fallback_openai(payload, config, fallback_model, analytics, start_time, provider_tag="fallback"): + """Stream from fallback provider in OpenAI format.""" + from fastapi.responses import StreamingResponse + + async def generate(): + try: + async for chunk in await _call_fallback_provider(payload, config.fallback, stream=True): + yield chunk + except Exception as e: + error_data = json.dumps({"error": {"message": str(e), "type": "server_error"}}) + yield f"data: {error_data}\n\n" + yield "data: [DONE]\n\n" + finally: + elapsed = time.time() - start_time + if analytics: + analytics.log_llm(model=_normalize_model_name(fallback_model), total_duration_seconds=elapsed, provider=provider_tag) + + return StreamingResponse(generate(), media_type="text/event-stream") + + +async def _stream_completion_anthropic(client, payload, model, max_tokens, analytics, start_time): + """Stream Anthropic-format SSE responses, translating from Ollama's OpenAI format.""" + from fastapi.responses import StreamingResponse + msg_id = f"msg_{uuid.uuid4().hex[:24]}" + + async def generate(): + yield f"event: message_start\ndata: {json.dumps({'type': 'message_start', 'message': {'id': msg_id, 'type': 'message', 'role': 'assistant', 'model': model, 'content': [], 'stop_reason': None, 'stop_sequence': None, 'usage': {'input_tokens': 0, 'output_tokens': 0}}})}\n\n" + + total_tokens = 0 + first_token_time = None + async for chunk in client.chat_completion_stream(payload): + # Capture metrics from stream + if chunk.startswith("__oct_metrics__:"): + stream_metrics_raw = chunk.split(":", 1)[1] + try: + stream_metrics = json.loads(stream_metrics_raw) + # Log with streaming metrics + if analytics: + analytics.log_llm( + model=model, + completion_tokens=stream_metrics.get("eval_count", total_tokens), + tps=stream_metrics.get("tps"), + ttft_seconds=stream_metrics.get("ttft_seconds") or (round(first_token_time - start_time, 3) if first_token_time else None), + total_duration_seconds=stream_metrics.get("elapsed_seconds", time.time() - start_time), + ) + except (json.JSONDecodeError, ValueError): + pass + continue + try: + if "data: " in chunk: + data_str = chunk[6:].strip() + if data_str == "[DONE]": + continue + data = json.loads(data_str) + choices = data.get("choices", []) + for choice in choices: + delta = choice.get("delta", {}) + content = delta.get("content", "") + if content: + if first_token_time is None: + first_token_time = time.time() + total_tokens += 1 + yield f"event: content_block_delta\ndata: {json.dumps({'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'text_delta', 'text': content}})}\n\n" + except (json.JSONDecodeError, KeyError): + continue + + yield f"event: message_delta\ndata: {json.dumps({'type': 'message_delta', 'delta': {'stop_reason': 'end_turn', 'stop_sequence': None}, 'usage': {'output_tokens': total_tokens}})}\n\n" + yield f"event: message_stop\ndata: {json.dumps({'type': 'message_stop'})}\n\n" + + return StreamingResponse(generate(), media_type="text/event-stream") + + + +def _openai_to_anthropic_stop(reason: str) -> str: + """Convert OpenAI finish_reason to Anthropic stop_reason.""" + mapping = { + "stop": "end_turn", + "length": "max_tokens", + "tool_calls": "tool_use", + "content_filter": "end_turn", + } + return mapping.get(reason, "end_turn") \ No newline at end of file diff --git a/guanaco/search/__init__.py b/guanaco/search/__init__.py new file mode 100644 index 0000000..5349091 --- /dev/null +++ b/guanaco/search/__init__.py @@ -0,0 +1 @@ +"""Search package.""" \ No newline at end of file diff --git a/guanaco/search/base.py b/guanaco/search/base.py new file mode 100644 index 0000000..af08b35 --- /dev/null +++ b/guanaco/search/base.py @@ -0,0 +1,41 @@ +"""Provider emulator base and registry.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from guanaco.client import OllamaClient + from guanaco.analytics import AnalyticsLogger + + +class ProviderEmulator(ABC): + """Base class for search/scrape API emulators.""" + + name: str = "" + prefix: str = "" + + def __init__(self, ollama_client: "OllamaClient", analytics: Optional["AnalyticsLogger"] = None): + self.ollama = ollama_client + self.analytics = analytics + + @abstractmethod + def register_routes(self, app): + ... + + +_PROVIDERS: dict[str, type[ProviderEmulator]] = {} + + +def register_provider(cls: type[ProviderEmulator]) -> type[ProviderEmulator]: + _PROVIDERS[cls.name] = cls + return cls + + +def get_provider(name: str) -> Optional[type[ProviderEmulator]]: + return _PROVIDERS.get(name) + + +def get_all_providers() -> dict[str, type[ProviderEmulator]]: + return dict(_PROVIDERS) \ No newline at end of file diff --git a/guanaco/search/providers/__init__.py b/guanaco/search/providers/__init__.py new file mode 100644 index 0000000..8df2f5d --- /dev/null +++ b/guanaco/search/providers/__init__.py @@ -0,0 +1,27 @@ +"""Search provider package — auto-discovers all providers.""" + +from guanaco.search.providers.tavily import TavilyProvider +from guanaco.search.providers.exa import ExaProvider +from guanaco.search.providers.searxng import SearXNGProvider +from guanaco.search.providers.firecrawl import FirecrawlProvider +from guanaco.search.providers.serper import SerperProvider +from guanaco.search.providers.jina import JinaProvider +from guanaco.search.providers.cohere import CohereProvider +from guanaco.search.providers.brave import BraveProvider + +ALL_PROVIDERS = [ + TavilyProvider, + ExaProvider, + SearXNGProvider, + FirecrawlProvider, + SerperProvider, + JinaProvider, + CohereProvider, + BraveProvider, +] + +__all__ = [ + "TavilyProvider", "ExaProvider", "SearXNGProvider", "FirecrawlProvider", + "SerperProvider", "JinaProvider", "CohereProvider", "BraveProvider", + "ALL_PROVIDERS", +] \ No newline at end of file diff --git a/guanaco/search/providers/brave.py b/guanaco/search/providers/brave.py new file mode 100644 index 0000000..b82c9cc --- /dev/null +++ b/guanaco/search/providers/brave.py @@ -0,0 +1,67 @@ +"""Brave Search API emulator — converts Ollama search to Brave Search-compatible responses.""" + +from __future__ import annotations + +from typing import Optional + +from fastapi import APIRouter, Request +from pydantic import BaseModel, Field + +from guanaco.search.base import ProviderEmulator, register_provider + + +# ── Response Models ── + +class BraveWebResult(BaseModel): + title: str + url: str + description: str + + +class BraveSearchResponse(BaseModel): + type: str = "search" + web: dict = Field(default_factory=dict) + + +# ── Provider ── + +@register_provider +class BraveProvider(ProviderEmulator): + name = "brave" + prefix = "/brave" + + def register_routes(self, app): + router = APIRouter(prefix=self.prefix, tags=["Brave"]) + + @router.get("/search", response_model=BraveSearchResponse) + async def search_get( + q: str, + count: int = 10, + offset: int = 0, + request: Request = None, + ): + ollama_resp = await self.ollama.search(query=q, max_results=count) + return _format_brave(q, ollama_resp) + + @router.post("/search", response_model=BraveSearchResponse) + async def search_post(body: dict, request: Request): + q = body.get("q", "") + count = body.get("count", 10) + ollama_resp = await self.ollama.search(query=q, max_results=count) + return _format_brave(q, ollama_resp) + + app.include_router(router) + + +def _format_brave(query: str, ollama_resp: dict) -> BraveSearchResponse: + results = [] + for r in ollama_resp.get("results", []): + results.append(BraveWebResult( + title=r.get("title", ""), + url=r.get("url", ""), + description=r.get("content", ""), + )) + return BraveSearchResponse( + type="search", + web={"results": results}, + ) \ No newline at end of file diff --git a/guanaco/search/providers/cohere.py b/guanaco/search/providers/cohere.py new file mode 100644 index 0000000..bfbd12c --- /dev/null +++ b/guanaco/search/providers/cohere.py @@ -0,0 +1,67 @@ +"""Cohere Rerank API emulator — uses Ollama LLM for reranking.""" + +from __future__ import annotations + +from fastapi import APIRouter, Request +from pydantic import BaseModel, Field +from typing import Optional + +from guanaco.search.base import ProviderEmulator, register_provider + + +# ── Request/Response Models ── + +class CohereRerankRequest(BaseModel): + model: str = "rerank-v3.5" + query: str + documents: list[str] + top_n: Optional[int] = None + return_documents: bool = False + + +class CohereRerankResult(BaseModel): + index: int + relevance_score: float + document: Optional[dict] = None + + +class CohereRerankResponse(BaseModel): + results: list[CohereRerankResult] = Field(default_factory=list) + meta: dict = Field(default_factory=dict) + + +# ── Provider ── + +@register_provider +class CohereProvider(ProviderEmulator): + name = "cohere" + prefix = "/cohere" + + def register_routes(self, app): + router = APIRouter(prefix=self.prefix, tags=["Cohere"]) + + @router.post("/rerank", response_model=CohereRerankResponse) + async def rerank(body: CohereRerankRequest, request: Request): + top_n = body.top_n or len(body.documents) + + # Keyword overlap heuristic scoring + query_words = set(body.query.lower().split()) + scored = [] + for i, doc in enumerate(body.documents): + doc_words = set(doc.lower().split()) + overlap = len(query_words & doc_words) / max(len(query_words), 1) + scored.append(CohereRerankResult( + index=i, + relevance_score=round(min(overlap + 0.3, 1.0), 4), + document={"text": doc} if body.return_documents else None, + )) + + scored.sort(key=lambda x: x.relevance_score, reverse=True) + results = scored[:top_n] + + return CohereRerankResponse( + results=results, + meta={"api_version": {"version": "1"}, "billed_units": {"search_units": 1}}, + ) + + app.include_router(router) \ No newline at end of file diff --git a/guanaco/search/providers/exa.py b/guanaco/search/providers/exa.py new file mode 100644 index 0000000..e3af5b7 --- /dev/null +++ b/guanaco/search/providers/exa.py @@ -0,0 +1,122 @@ +"""Exa API emulator — converts Ollama search to Exa-compatible responses.""" + +from __future__ import annotations + +import uuid +from fastapi import APIRouter, Request +from pydantic import BaseModel, Field +from typing import Optional + +from guanaco.search.base import ProviderEmulator, register_provider + + +# ── Request/Response Models ── + +class ExaContentOptions(BaseModel): + text: bool = False + highlights: Optional[dict] = None + summary: Optional[dict] = None + + +class ExaSearchRequest(BaseModel): + query: str + type: str = "auto" + num_results: int = Field(default=10, alias="numResults") + start_published_date: Optional[str] = Field(default=None, alias="startPublishedDate") + end_published_date: Optional[str] = Field(default=None, alias="endPublishedDate") + include_domains: list[str] = Field(default_factory=list, alias="includeDomains") + exclude_domains: list[str] = Field(default_factory=list, alias="excludeDomains") + contents: Optional[ExaContentOptions] = None + + +class ExaFindSimilarRequest(BaseModel): + url: str + num_results: int = Field(default=10, alias="numResults") + include_domains: list[str] = Field(default_factory=list, alias="includeDomains") + exclude_domains: list[str] = Field(default_factory=list, alias="excludeDomains") + contents: Optional[ExaContentOptions] = None + + +class ExaResult(BaseModel): + id: str + title: str + url: str + published_date: Optional[str] = Field(default=None, alias="publishedDate") + author: Optional[str] = None + text: Optional[str] = None + highlights: Optional[list[str]] = None + summary: Optional[str] = None + + +class ExaSearchResponse(BaseModel): + request_id: str = Field(default="", alias="requestId") + results: list[ExaResult] = Field(default_factory=list) + + +class ExaFindSimilarResponse(BaseModel): + request_id: str = Field(default="", alias="requestId") + results: list[ExaResult] = Field(default_factory=list) + + +# ── Provider ── + +@register_provider +class ExaProvider(ProviderEmulator): + name = "exa" + prefix = "/exa" + + def register_routes(self, app): + router = APIRouter(prefix=self.prefix, tags=["Exa"]) + + @router.post("/search", response_model=ExaSearchResponse) + async def exa_search(body: ExaSearchRequest, request: Request): + ollama_resp = await self.ollama.search( + query=body.query, + max_results=body.num_results, + ) + + results = [] + for r in ollama_resp.get("results", []): + result = ExaResult( + id=str(uuid.uuid4()), + title=r.get("title", ""), + url=r.get("url", ""), + ) + if body.contents: + if body.contents.text: + result.text = r.get("content", "") + if body.contents.highlights: + result.highlights = [r.get("content", "")[:200]] + results.append(result) + + return ExaSearchResponse( + request_id=str(uuid.uuid4()), + results=results, + ) + + @router.post("/findSimilar", response_model=ExaFindSimilarResponse) + async def exa_find_similar(body: ExaFindSimilarRequest, request: Request): + # Use Ollama fetch to get the URL content, then search for similar + fetch_resp = await self.ollama.fetch(url=body.url) + title = fetch_resp.get("title", "") + # Use the title as a search query for similar results + ollama_resp = await self.ollama.search( + query=title or body.url, + max_results=body.num_results, + ) + + results = [] + for r in ollama_resp.get("results", []): + result = ExaResult( + id=str(uuid.uuid4()), + title=r.get("title", ""), + url=r.get("url", ""), + ) + results.append(result) + + return ExaFindSimilarResponse( + request_id=str(uuid.uuid4()), + results=results, + ) + + app.include_router(router) \ No newline at end of file diff --git a/guanaco/search/providers/firecrawl.py b/guanaco/search/providers/firecrawl.py new file mode 100644 index 0000000..6dc73e7 --- /dev/null +++ b/guanaco/search/providers/firecrawl.py @@ -0,0 +1,228 @@ +"""Firecrawl API emulator — converts Ollama search/fetch to Firecrawl-compatible responses. + +Firecrawl is HIGH PRIORITY. We emulate: +- POST /scrape (single URL scraping) +- POST /search (search the web) +- POST /crawl (multi-page crawl — uses fetch for each URL) +- POST /extract (extract structured data from URLs) +""" + +from __future__ import annotations + +import uuid +import time +from typing import Optional + +from fastapi import APIRouter, Request +from pydantic import BaseModel, Field + +from guanaco.search.base import ProviderEmulator, register_provider + + +# ── Request Models ── + +class ScrapeRequest(BaseModel): + url: str + formats: list[str] = Field(default_factory=lambda: ["markdown"]) + only_main_content: bool = True + include_tags: list[str] = Field(default_factory=list) + exclude_tags: list[str] = Field(default_factory=list) + timeout: int = 30000 + actions: Optional[list[dict]] = None + + +class SearchRequest(BaseModel): + query: str + limit: int = 5 + scrape_options: Optional[dict] = None + lang: str = "en" + + +class CrawlRequest(BaseModel): + url: str + limit: int = 10 + scrape_options: Optional[dict] = None + max_depth: int = 2 + + +class ExtractRequest(BaseModel): + urls: list[str] = Field(default_factory=list) + prompt: Optional[str] = None + schema_: Optional[dict] = Field(default=None, alias="schema") + + +# ── Response Models ── + +class FirecrawlMetadata(BaseModel): + title: Optional[str] = None + description: Optional[str] = None + language: Optional[str] = None + source_url: Optional[str] = None + status_code: int = 200 + + +class ScrapeResponse(BaseModel): + success: bool = True + data: Optional[dict] = None + metadata: Optional[FirecrawlMetadata] = None + + +class SearchResult(BaseModel): + title: str + url: str + content: str + description: Optional[str] = None + + +class SearchResponse(BaseModel): + success: bool = True + data: list[SearchResult] = Field(default_factory=list) + + +class CrawlResult(BaseModel): + title: str + url: str + content: str + markdown: str + metadata: Optional[FirecrawlMetadata] = None + + +class CrawlResponse(BaseModel): + success: bool = True + status: str = "completed" + completed: int = 0 + total: int = 0 + credits_used: int = 0 + data: list[CrawlResult] = Field(default_factory=list) + + +class ExtractResponse(BaseModel): + success: bool = True + data: dict = Field(default_factory=dict) + + +# ── Provider ── + +@register_provider +class FirecrawlProvider(ProviderEmulator): + name = "firecrawl" + prefix = "/firecrawl" + + def register_routes(self, app): + router = APIRouter(prefix=self.prefix, tags=["Firecrawl"]) + + async def _scrape(body: ScrapeRequest): + ollama_resp = await self.ollama.fetch(url=body.url) + title = ollama_resp.get("title", "") + content = ollama_resp.get("content", "") + links = ollama_resp.get("links", []) + + data = {} + if "markdown" in body.formats or not body.formats: + data["markdown"] = content + if "html" in body.formats: + data["html"] = content # Ollama returns text, approx + if "rawHtml" in body.formats: + data["rawHtml"] = content + if "links" in body.formats: + data["links"] = links + + return ScrapeResponse( + success=True, + data=data, + metadata=FirecrawlMetadata( + title=title, + source_url=body.url, + status_code=200, + ), + ) + + async def _search(body: SearchRequest): + ollama_resp = await self.ollama.search( + query=body.query, + max_results=body.limit, + ) + + results = [] + for r in ollama_resp.get("results", []): + results.append(SearchResult( + title=r.get("title", ""), + url=r.get("url", ""), + content=r.get("content", ""), + description=r.get("content", "")[:200], + )) + + return SearchResponse(success=True, data=results) + + @router.post("/scrape", response_model=ScrapeResponse) + async def scrape(body: ScrapeRequest, request: Request): + return await _scrape(body) + + @router.post("/v2/scrape", response_model=ScrapeResponse, include_in_schema=False) + async def scrape_v2(body: ScrapeRequest, request: Request): + return await _scrape(body) + + @router.post("/search", response_model=SearchResponse) + async def search(body: SearchRequest, request: Request): + return await _search(body) + + @router.post("/v2/search", response_model=SearchResponse, include_in_schema=False) + async def search_v2(body: SearchRequest, request: Request): + return await _search(body) + + @router.post("/crawl", response_model=CrawlResponse) + async def crawl(body: CrawlRequest, request: Request): + # Crawl = scrape the seed URL + follow links + ollama_resp = await self.ollama.fetch(url=body.url) + title = ollama_resp.get("title", "") + content = ollama_resp.get("content", "") + links = ollama_resp.get("links", []) + + results = [CrawlResult( + title=title, + url=body.url, + content=content, + markdown=content, + metadata=FirecrawlMetadata(title=title, source_url=body.url), + )] + + # Follow up to limit-1 additional links + for link in links[:body.limit - 1]: + try: + link_resp = await self.ollama.fetch(url=link) + lt = link_resp.get("title", "") + lc = link_resp.get("content", "") + results.append(CrawlResult( + title=lt, + url=link, + content=lc, + markdown=lc, + metadata=FirecrawlMetadata(title=lt, source_url=link), + )) + except Exception: + continue + + return CrawlResponse( + success=True, + status="completed", + completed=len(results), + total=len(results), + data=results, + ) + + @router.post("/extract", response_model=ExtractResponse) + async def extract(body: ExtractRequest, request: Request): + # Extract uses fetch + LLM to pull structured data + all_content = {} + for url in body.urls[:5]: # limit to 5 URLs + try: + resp = await self.ollama.fetch(url=url) + all_content[url] = resp.get("content", "") + except Exception: + all_content[url] = "" + + # If prompt provided, we could route through LLM later + # For now, return raw content mapped by URL + return ExtractResponse(success=True, data=all_content) + + app.include_router(router) \ No newline at end of file diff --git a/guanaco/search/providers/jina.py b/guanaco/search/providers/jina.py new file mode 100644 index 0000000..aa2e409 --- /dev/null +++ b/guanaco/search/providers/jina.py @@ -0,0 +1,187 @@ +"""Jina API emulator — converts Ollama search/fetch to Jina-compatible responses. + +Endpoints: +- POST /search (Jina search/SVL) +- POST /read (Jina reader — URL scraping) +- POST /rerank (Jina reranker — uses LLM) +""" + +from __future__ import annotations + +from fastapi import APIRouter, Request +from pydantic import BaseModel, Field +from typing import Optional + +from guanaco.search.base import ProviderEmulator, register_provider + + +# ── Request Models ── + +class JinaSearchRequest(BaseModel): + q: str + num: int = 10 + site: Optional[list[str]] = None + + +class JinaReadRequest(BaseModel): + url: str + + +class JinaRerankRequest(BaseModel): + model: str = "jina-reranker-v2-base-multilingual" + query: str + documents: list[str] + top_n: Optional[int] = None + return_documents: bool = False + + +# ── Response Models ── + +class JinaSearchResult(BaseModel): + title: str + url: str + description: str + content: Optional[str] = None + + +class JinaSearchResponse(BaseModel): + code: int = 200 + status: int = 20000 + data: list[JinaSearchResult] = Field(default_factory=list) + + +class JinaReadResponse(BaseModel): + code: int = 200 + status: int = 20000 + data: dict = Field(default_factory=dict) + + +class JinaRerankResult(BaseModel): + index: int + relevance_score: float + document: Optional[dict] = None + + +class JinaRerankResponse(BaseModel): + model: str + results: list[JinaRerankResult] = Field(default_factory=list) + usage: dict = Field(default_factory=dict) + + +# ── Provider ── + +@register_provider +class JinaProvider(ProviderEmulator): + name = "jina" + prefix = "/jina" + + def register_routes(self, app): + router = APIRouter(prefix=self.prefix, tags=["Jina"]) + + @router.post("/search", response_model=JinaSearchResponse) + @router.post("/v1/search", response_model=JinaSearchResponse, include_in_schema=False) + async def search(body: JinaSearchRequest, request: Request): + ollama_resp = await self.ollama.search( + query=body.q, + max_results=body.num, + ) + + results = [] + for r in ollama_resp.get("results", []): + results.append(JinaSearchResult( + title=r.get("title", ""), + url=r.get("url", ""), + description=r.get("content", ""), + content=r.get("content"), + )) + + return JinaSearchResponse(data=results) + + @router.post("/read", response_model=JinaReadResponse) + @router.post("/v1/read", response_model=JinaReadResponse, include_in_schema=False) + async def read(body: JinaReadRequest, request: Request): + ollama_resp = await self.ollama.fetch(url=body.url) + return JinaReadResponse(data={ + "title": ollama_resp.get("title", ""), + "content": ollama_resp.get("content", ""), + "url": body.url, + "links": ollama_resp.get("links", []), + }) + + @router.post("/rerank", response_model=JinaRerankResponse) + @router.post("/v1/rerank", response_model=JinaRerankResponse, include_in_schema=False) + async def rerank(body: JinaRerankRequest, request: Request): + # Use LLM for reranking — construct a prompt that scores relevance + import json + top_n = body.top_n or len(body.documents) + + prompt = ( + f"Given the query: \"{body.query}\"\n\n" + f"Rank these documents by relevance (0.0 to 1.0):\n\n" + ) + for i, doc in enumerate(body.documents): + prompt += f"Document {i}: {doc[:500]}\n\n" + + prompt += ( + "Return ONLY a JSON array of objects with 'index' and 'score' fields, " + "sorted by score descending. Example: [{\"index\": 2, \"score\": 0.95}, ...]" + ) + + # We'll use a simple heuristic for now — keyword overlap scoring + # Full LLM reranking can be enabled later when chat completion is available + query_words = set(body.query.lower().split()) + scored = [] + for i, doc in enumerate(body.documents): + doc_words = set(doc.lower().split()) + overlap = len(query_words & doc_words) / max(len(query_words), 1) + scored.append(JinaRerankResult( + index=i, + relevance_score=round(min(overlap + 0.3, 1.0), 4), + document={"text": doc} if body.return_documents else None, + )) + + scored.sort(key=lambda x: x.relevance_score, reverse=True) + results = scored[:top_n] + + return JinaRerankResponse( + model=body.model, + results=results, + usage={"prompt_tokens": 0, "total_tokens": 0}, + ) + + app.include_router(router) + + # Bare /jina POST route — LibreChat sends rerank requests to the base URL + @app.post("/jina") + async def jina_bare_rerank(request: Request): + """LibreChat calls POST to the Jina base URL directly for reranking.""" + body = await request.json() + documents = body.get("documents", []) + query = body.get("query", "") + model = body.get("model", "jina-reranker-v2-base-multilingual") + top_n = body.get("top_n", len(documents)) + return_documents = body.get("return_documents", False) + + query_words = set(query.lower().split()) + scored = [] + for i, doc in enumerate(documents): + doc_text = doc if isinstance(doc, str) else doc.get("text", str(doc)) + doc_words = set(doc_text.lower().split()) + overlap = len(query_words & doc_words) / max(len(query_words), 1) + scored.append({ + "index": i, + "relevance_score": round(min(overlap + 0.3, 1.0), 4), + "document": {"text": doc_text} if return_documents else None, + }) + + scored.sort(key=lambda x: x["relevance_score"], reverse=True) + results = scored[:top_n] + for r in results: + if r["document"] is None: + del r["document"] + + return { + "model": model, + "results": results, + "usage": {"prompt_tokens": 0, "total_tokens": 0}, + } \ No newline at end of file diff --git a/guanaco/search/providers/searxng.py b/guanaco/search/providers/searxng.py new file mode 100644 index 0000000..080fc0f --- /dev/null +++ b/guanaco/search/providers/searxng.py @@ -0,0 +1,90 @@ +"""SearXNG API emulator — converts Ollama search to SearXNG-compatible responses.""" + +from __future__ import annotations + +from fastapi import APIRouter, Request +from pydantic import BaseModel, Field +from typing import Optional + +from guanaco.search.base import ProviderEmulator, register_provider + + +# ── Response Models ── + +class SearXNGResult(BaseModel): + title: str + url: str + content: str + engine: str = "ollama" + engines: list[str] = Field(default_factory=lambda: ["ollama"]) + score: float = 0.0 + category: str = "general" + parsed_url: Optional[list[str]] = None + template: str = "default.html" + + +class SearXNGSearchResponse(BaseModel): + query: str + number_of_results: int = 0 + results: list[SearXNGResult] = Field(default_factory=list) + suggestions: list[str] = Field(default_factory=list) + infoboxes: list[dict] = Field(default_factory=list) + + +# ── Provider ── + +@register_provider +class SearXNGProvider(ProviderEmulator): + name = "searxng" + prefix = "/searxng" + + def register_routes(self, app): + router = APIRouter(prefix=self.prefix, tags=["SearXNG"]) + + @router.get("/search", response_model=SearXNGSearchResponse) + async def searxng_search_get( + q: str, + format: str = "json", + pageno: int = 1, + categories: Optional[str] = None, + request: Request = None, + ): + ollama_resp = await self.ollama.search(query=q, max_results=10) + return _format_searxng(q, ollama_resp) + + @router.post("/search", response_model=SearXNGSearchResponse) + async def searxng_search_post( + q: str = "", + format: str = "json", + pageno: int = 1, + categories: Optional[str] = None, + request: Request = None, + ): + ollama_resp = await self.ollama.search(query=q, max_results=10) + return _format_searxng(q, ollama_resp) + + # SearXNG also accepts requests at root / + @router.get("/", response_model=SearXNGSearchResponse, include_in_schema=False) + async def searxng_root_get(q: str, format: str = "json"): + ollama_resp = await self.ollama.search(query=q, max_results=10) + return _format_searxng(q, ollama_resp) + + app.include_router(router) + + +def _format_searxng(query: str, ollama_resp: dict) -> SearXNGSearchResponse: + results = [] + for r in ollama_resp.get("results", []): + url = r.get("url", "") + parsed = url.replace("://", "/").split("/") if url else [] + results.append(SearXNGResult( + title=r.get("title", ""), + url=url, + content=r.get("content", ""), + parsed_url=parsed, + )) + return SearXNGSearchResponse( + query=query, + number_of_results=len(results), + results=results, + ) \ No newline at end of file diff --git a/guanaco/search/providers/serper.py b/guanaco/search/providers/serper.py new file mode 100644 index 0000000..acd4b55 --- /dev/null +++ b/guanaco/search/providers/serper.py @@ -0,0 +1,97 @@ +"""Serper API emulator — converts Ollama search/fetch to Serper-compatible responses.""" + +from __future__ import annotations + +from fastapi import APIRouter, Request +from pydantic import BaseModel, Field +from typing import Optional + +from guanaco.search.base import ProviderEmulator, register_provider + + +# ── Request Models ── + +class SerperSearchRequest(BaseModel): + q: str + gl: str = "us" + hl: str = "en" + num: int = 10 + page: int = 1 + type: Optional[str] = None # news, images, videos, places + + +class SerperScrapeRequest(BaseModel): + url: str + + +# ── Response Models ── + +class SerperOrganicResult(BaseModel): + title: str + link: str + snippet: str + position: int = 0 + + +class SerperKnowledgePanel(BaseModel): + title: Optional[str] = None + description: Optional[str] = None + + +class SerperSearchResponse(BaseModel): + search_parameters: dict = Field(default_factory=dict) + organic: list[SerperOrganicResult] = Field(default_factory=list) + knowledge_graph: Optional[SerperKnowledgePanel] = None + search_information: dict = Field(default_factory=dict) + + +class SerperScrapeResponse(BaseModel): + url: str + title: str + content: str + links: list[str] = Field(default_factory=list) + + +# ── Provider ── + +@register_provider +class SerperProvider(ProviderEmulator): + name = "serper" + prefix = "/serper" + + def register_routes(self, app): + router = APIRouter(prefix=self.prefix, tags=["Serper"]) + + @router.post("/search", response_model=SerperSearchResponse) + async def search(body: SerperSearchRequest, request: Request): + ollama_resp = await self.ollama.search( + query=body.q, + max_results=body.num, + ) + + organic = [] + for i, r in enumerate(ollama_resp.get("results", [])): + organic.append(SerperOrganicResult( + title=r.get("title", ""), + link=r.get("url", ""), + snippet=r.get("content", ""), + position=i + 1, + )) + + return SerperSearchResponse( + search_parameters={"q": body.q, "gl": body.gl, "hl": body.hl}, + organic=organic, + search_information={"total_results": len(organic)}, + ) + + @router.post("/scrape", response_model=SerperScrapeResponse) + async def scrape(body: SerperScrapeRequest, request: Request): + ollama_resp = await self.ollama.fetch(url=body.url) + return SerperScrapeResponse( + url=body.url, + title=ollama_resp.get("title", ""), + content=ollama_resp.get("content", ""), + links=ollama_resp.get("links", []), + ) + + app.include_router(router) \ No newline at end of file diff --git a/guanaco/search/providers/tavily.py b/guanaco/search/providers/tavily.py new file mode 100644 index 0000000..55f9bec --- /dev/null +++ b/guanaco/search/providers/tavily.py @@ -0,0 +1,83 @@ +"""Tavily API emulator — converts Ollama search to Tavily-compatible responses.""" + +from __future__ import annotations + +from fastapi import APIRouter, Header, Query, Request +from pydantic import BaseModel, Field +from typing import Optional + +from guanaco.search.base import ProviderEmulator, register_provider + + +# ── Request/Response Models ── + +class TavilySearchRequest(BaseModel): + query: str + search_depth: str = "basic" # basic | advanced + max_results: int = 5 + topic: str = "general" # general | news + include_answer: bool = False + include_raw_content: bool = False + include_images: bool = False + include_image_descriptions: bool = False + include_domains: list[str] = Field(default_factory=list) + exclude_domains: list[str] = Field(default_factory=list) + + +class TavilySearchResult(BaseModel): + title: str + url: str + content: str + score: float = 0.0 + raw_content: Optional[str] = None + + +class TavilySearchResponse(BaseModel): + query: str + answer: Optional[str] = None + results: list[TavilySearchResult] = Field(default_factory=list) + response_time: float = 0.0 + + +# ── Provider ── + +@register_provider +class TavilyProvider(ProviderEmulator): + name = "tavily" + prefix = "/tavily" + + def register_routes(self, app): + router = APIRouter(prefix=self.prefix, tags=["Tavily"]) + + @router.post("/search", response_model=TavilySearchResponse) + async def tavily_search( + body: TavilySearchRequest, + request: Request, + ): + import time + start = time.time() + + # Use Ollama search + ollama_resp = await self.ollama.search( + query=body.query, + max_results=body.max_results, + ) + + results = [] + for r in ollama_resp.get("results", []): + results.append(TavilySearchResult( + title=r.get("title", ""), + url=r.get("url", ""), + content=r.get("content", ""), + score=0.5, # Ollama doesn't return scores + raw_content=r.get("content") if body.include_raw_content else None, + )) + + return TavilySearchResponse( + query=body.query, + answer=None, + results=results, + response_time=round(time.time() - start, 3), + ) + + app.include_router(router) \ No newline at end of file diff --git a/guanaco/utils/__init__.py b/guanaco/utils/__init__.py new file mode 100644 index 0000000..74572ac --- /dev/null +++ b/guanaco/utils/__init__.py @@ -0,0 +1,5 @@ +"""Utils package.""" + +from guanaco.utils.api_keys import ApiKeyManager + +__all__ = ["ApiKeyManager"] \ No newline at end of file diff --git a/guanaco/utils/api_keys.py b/guanaco/utils/api_keys.py new file mode 100644 index 0000000..42cd0dc --- /dev/null +++ b/guanaco/utils/api_keys.py @@ -0,0 +1,90 @@ +"""API key generation and validation for search endpoints.""" + +from __future__ import annotations + +import hashlib +import secrets +import time +from pathlib import Path +from typing import Optional + +import yaml + + +KEYS_FILE = "api_keys.yaml" + + +class ApiKeyManager: + """Manage per-provider API keys for the search emulator endpoints.""" + + def __init__(self, config_dir: Path): + self.config_dir = config_dir + self._keys_path = config_dir / KEYS_FILE + self._keys: dict[str, dict] = {} # key_hash -> {provider, name, created_at, key_prefix} + self._load() + + def _load(self): + if self._keys_path.exists(): + with open(self._keys_path) as f: + data = yaml.safe_load(f) or {} + self._keys = data + + def _save(self): + self._keys_path.parent.mkdir(parents=True, exist_ok=True) + with open(self._keys_path, "w") as f: + yaml.dump(self._keys, f, default_flow_style=False) + + def _hash_key(self, key: str) -> str: + return hashlib.sha256(key.encode()).hexdigest() + + def generate_key(self, provider: str, name: str = "") -> str: + """Generate a new API key for a provider. Returns the plaintext key (shown once).""" + raw_key = f"guanca_{provider}_{secrets.token_urlsafe(24)}" + key_hash = self._hash_key(raw_key) + self._keys[key_hash] = { + "provider": provider, + "name": name or f"{provider}-key", + "prefix": raw_key[:12] + "...", + "created_at": time.time(), + } + self._save() + return raw_key + + def verify_key(self, key: str, provider: Optional[str] = None) -> bool: + """Verify a key is valid. Optionally check it's for a specific provider. + + Accepts guanca_ prefixed keys. + """ + key_hash = self._hash_key(key) + entry = self._keys.get(key_hash) + if entry: + if provider and entry["provider"] != provider: + return False + return True + return True + return False + + def list_keys(self) -> list[dict]: + """List all keys (masked).""" + return [ + {"prefix": v["prefix"], "provider": v["provider"], "name": v["name"], "created_at": v["created_at"]} + for v in self._keys.values() + ] + + def revoke_key(self, key: str) -> bool: + """Revoke an API key.""" + key_hash = self._hash_key(key) + if key_hash in self._keys: + del self._keys[key_hash] + self._save() + return True + return False + + def revoke_by_prefix(self, prefix: str) -> bool: + """Revoke a key by its prefix.""" + for k, v in list(self._keys.items()): + if v["prefix"] == prefix: + del self._keys[k] + self._save() + return True + return False \ No newline at end of file diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..b1a3fa1 --- /dev/null +++ b/install.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +# ollama-cloud-tools installer +# Usage: curl -sSL https://raw.githubusercontent.com/evanrice/ollama-cloud-tools/main/install.sh | bash +# +# Supports: Linux, macOS, WSL (Windows Subsystem for Linux) + +set -euo pipefail + +REPO="evanrice/ollama-cloud-tools" +INSTALL_DIR="$HOME/.oct" +VENV_DIR="$INSTALL_DIR/venv" +BIN_DIR="$HOME/.local/bin" + +# ── Detect platform ── +detect_platform() { + local os_name="$(uname -s)" + case "$os_name" in + Linux) + # Check if WSL + if grep -qi microsoft /proc/version 2>/dev/null; then + echo "wsl" + else + echo "linux" + fi + ;; + Darwin) + echo "macos" + ;; + *) + echo "unknown" + ;; + esac +} + +PLATFORM=$(detect_platform) +echo "🦙 Ollama Cloud Tools Installer" +echo "================================" +echo "Platform: $PLATFORM" +echo "" + +# ── Check Python ── +if ! command -v python3 &>/dev/null; then + echo "❌ Python 3.10+ is required but not found." + case "$PLATFORM" in + macos) + echo " Install with: brew install python@3.11" + echo " Or download from: https://python.org" + ;; + linux|wsl) + echo " Install with:" + echo " Ubuntu/Debian: sudo apt install python3 python3-venv python3-pip" + echo " Fedora/RHEL: sudo dnf install python3 python3-pip" + echo " Arch: sudo pacman -S python python-pip" + ;; + *) + echo " Install from: https://python.org" + ;; + esac + exit 1 +fi + +PYTHON_VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") +if python3 -c "import sys; exit(0 if sys.version_info >= (3, 10) else 1)"; then + echo "✅ Python $PYTHON_VERSION found" +else + echo "❌ Python 3.10+ required, found $PYTHON_VERSION" + exit 1 +fi + +# ── Platform-specific setup ── +case "$PLATFORM" in + macos) + # Check for Homebrew + if ! command -v brew &>/dev/null; then + echo "⚠️ Homebrew not found. Some dependencies may need manual install." + echo " Install Homebrew: /bin/bash -c \"\$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)\"" + fi + + # macOS needs certifi for SSL + echo "🍎 Setting up macOS SSL certificates..." + export SSL_CERT_FILE=$(python3 -c "import certifi; print(certifi.where())" 2>/dev/null || echo "") + if [ -n "$SSL_CERT_FILE" ]; then + echo " SSL certs: $SSL_CERT_FILE" + # Write to env file for persistence + mkdir -p "$INSTALL_DIR" + echo "export SSL_CERT_FILE=$SSL_CERT_FILE" > "$INSTALL_DIR/env" + fi + + # Check for Xcode CLI tools (needed for some pip builds) + if ! xcode-select -p &>/dev/null; then + echo "⚠️ Xcode CLI tools not found. Installing..." + xcode-select --install 2>/dev/null || true + echo " You may need to restart the installer after Xcode tools install." + fi + ;; + + wsl) + echo "🐧 Windows Subsystem for Linux detected" + # WSL should work like Linux but check for common issues + if ! command -v git &>/dev/null; then + echo "⚠️ git not found. Installing..." + sudo apt update -qq && sudo apt install -y -qq git 2>/dev/null || { + echo " Could not install git. Please install manually:" + echo " sudo apt install git" + } + fi + + # WSL2 may need resolv.conf fix for DNS + if ! ping -c1 -W2 ollama.com &>/dev/null; then + echo "⚠️ Cannot reach ollama.com — DNS may need fixing for WSL2" + echo " Try: echo 'nameserver 8.8.8.8' | sudo tee /etc/resolv.conf" + fi + + # Set up Windows browser access + WIN_IP=$(hostname -I 2>/dev/null | awk '{print $1}') + echo " Access dashboard from Windows: http://$WIN_IP:8080/dashboard" + ;; + + linux) + # Standard Linux — check for venv support + if ! python3 -c "import venv" &>/dev/null; then + echo "⚠️ Python venv module not found. Installing..." + sudo apt install -y python3-venv 2>/dev/null || \ + sudo dnf install -y python3-venv 2>/dev/null || { + echo " Could not install python3-venv automatically." + echo " Please install it manually and re-run." + exit 1 + } + fi + ;; +esac + +# ── Clone or update repo ── +if [ -d "$INSTALL_DIR/repo" ]; then + echo "📦 Updating existing installation..." + cd "$INSTALL_DIR/repo" + git pull --ff-only || { echo "⚠️ Could not pull updates. Continuing with existing version."; } +else + echo "📦 Cloning repository..." + git clone "https://github.com/$REPO.git" "$INSTALL_DIR/repo" + cd "$INSTALL_DIR/repo" +fi + +# ── Create venv ── +echo "🐍 Creating virtual environment..." +python3 -m venv "$VENV_DIR" + +# ── Source platform env if exists ── +if [ -f "$INSTALL_DIR/env" ]; then + source "$INSTALL_DIR/env" +fi + +# ── Install ── +echo "📥 Installing dependencies..." +"$VENV_DIR/bin/pip" install -e . --quiet + +# ── Create oct binary ── +mkdir -p "$BIN_DIR" +case "$PLATFORM" in + macos) + # macOS wrapper that sets SSL certs + cat > "$BIN_DIR/oct" << SCRIPT +#!/usr/bin/env bash +if [ -f "$INSTALL_DIR/env" ]; then + source "$INSTALL_DIR/env" +fi +exec "$VENV_DIR/bin/python" -m oct.cli "\$@" +SCRIPT + ;; + *) + # Standard wrapper + cat > "$BIN_DIR/oct" << 'SCRIPT' +#!/usr/bin/env bash +exec "$HOME/.oct/venv/bin/python" -m oct.cli "$@" +SCRIPT + ;; +esac +chmod +x "$BIN_DIR/oct" + +# ── Shell profile detection ── +DETECT_SHELL="${SHELL##*/}" +case "$DETECT_SHELL" in + zsh) PROFILE_FILE="$HOME/.zshrc" ;; + bash) PROFILE_FILE="$HOME/.bashrc" ;; + *) PROFILE_FILE="$HOME/.profile" ;; +esac + +# ── Add to PATH if needed ── +if [[ ":$PATH:" != *":$BIN_DIR:"* ]]; then + echo "" + echo "⚠️ $BIN_DIR is not in your PATH." + echo " Adding to $PROFILE_FILE..." + echo "" >> "$PROFILE_FILE" + echo "# Added by ollama-cloud-tools" >> "$PROFILE_FILE" + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$PROFILE_FILE" + + # If macOS with Homebrew, add to /etc/paths.d too + if [ "$PLATFORM" = "macos" ] && [ -w /usr/local/bin ]; then + ln -sf "$BIN_DIR/oct" /usr/local/bin/oct 2>/dev/null || true + fi + + echo "" + echo " Reload your shell: source $PROFILE_FILE" +fi + +# ── macOS-specific: create LaunchAgent for auto-start ── +if [ "$PLATFORM" = "macos" ]; then + echo "" + echo "🍎 macOS Tip: To auto-start oct on login, create a LaunchAgent:" + echo " cp $INSTALL_DIR/repo/contrib/com.oct.start.plist ~/Library/LaunchAgents/" + echo " launchctl load ~/Library/LaunchAgents/com.oct.start.plist" +fi + +# ── WSL-specific: create startup script ── +if [ "$PLATFORM" = "wsl" ]; then + cat > "$INSTALL_DIR/start.sh" << 'WSLSCRIPT' +#!/usr/bin/env bash +# WSL startup script for oct +source "$HOME/.oct/env" 2>/dev/null || true +export PATH="$HOME/.local/bin:$PATH" +exec oct start "$@" +WSLSCRIPT + chmod +x "$INSTALL_DIR/start.sh" + echo "🪟 WSL Tip: Run oct via: $INSTALL_DIR/start.sh" +fi + +# ── Done ── +echo "" +echo "✅ Installation complete! (Platform: $PLATFORM)" +echo "" +echo "Next steps:" +echo " 1. Run 'oct setup' to configure your Ollama API key" +echo " 2. Run 'oct start' to launch the services" +echo " 3. Visit http://localhost:8080/dashboard for the web UI" +echo "" +echo "CLI commands:" +echo " oct models List available cloud models" +echo " oct models --caps Show model capabilities" +echo " oct usage Check your Ollama Cloud usage/quota" +echo " oct status Show service & connection status" +echo " oct status -v Verbose status with endpoint info" +echo " oct analytics View request analytics & stats" +echo " oct analytics -m MODEL History for a specific model" +echo " oct key generate Generate an API key" +echo " oct config --show Show current configuration" +echo "" +echo "Docs: https://github.com/$REPO" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c61fafd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,60 @@ +[build-system] +requires = ["setuptools>=68.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "guanaco" +version = "0.3.0" +description = "OpenAI-compatible LLM proxy that maximizes Ollama Cloud subscriptions — search/scrape API emulation, usage tracking, fallback provider support, and a web dashboard" +readme = "README.md" +license = {text = "MIT"} +requires-python = ">=3.10" +authors = [ + {name = "Guanaco Contributors"} +] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Web Environment", + "Framework :: FastAPI", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Libraries", + "Topic :: Internet :: Proxy Servers", +] +dependencies = [ + "fastapi>=0.110.0", + "uvicorn[standard]>=0.29.0", + "httpx>=0.27.0", + "pydantic>=2.0", + "pyyaml>=6.0", + "jinja2>=3.1", + "python-multipart>=0.0.9", + "rich>=13.0", + "aiosqlite>=0.19.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "pytest-asyncio>=0.23", + "httpx", +] + +[project.scripts] +guanaco = "guanaco.cli:main" +oct = "guanaco.cli:main" + +[project.urls] +Homepage = "https://github.com/evanrice/guanaco" +Repository = "https://github.com/evanrice/guanaco" +Issues = "https://github.com/evanrice/guanaco/issues" + +[tool.setuptools.packages.find] +where = ["."] +include = ["guanaco*"] \ No newline at end of file