mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-07-09 16:08:31 +00:00
* feat: add redis stream bridge * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * fix(gateway): address redis stream bridge review Redis was imported eagerly through deerflow.runtime and declared as a hard dependency, which made memory-only installs load redis.asyncio at startup and left the lazy factory import ineffective. Move redis behind an optional extra, remove the public eager re-export, and keep make_stream_bridge as the only runtime import path with an actionable install hint when the extra is missing. Because Docker deployments now default the stream bridge to Redis via DEER_FLOW_STREAM_BRIDGE_REDIS_URL, install the redis extra explicitly in Docker/dev container flows and teach the local uv-extra detector to infer redis from both stream_bridge.type and the Redis URL env var. This keeps Docker working while preserving slim non-Docker installs. Harden the Redis bridge by batching XREAD replay, replacing brittle ResponseError string matching with a single fallback to 0-0 for malformed Last-Event-ID values, documenting connection/retention/fail-hard behavior, and adding fake plus opt-in real Redis coverage for XADD/XREAD, replay, invalid IDs, and MAXLEN trimming. * fix(config): bump config version for stream bridge * fix redis stream bridge terminal handling * fix: repair uv.lock, format redis.py, and align Dockerfile extras test The uv.lock file was missing a closing bracket for the redis extras section, redis.py had a formatting issue caught by ruff, and the Dockerfile extras test did not account for the hardcoded --extra redis flag. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
181 lines
7.9 KiB
YAML
181 lines
7.9 KiB
YAML
# DeerFlow Production Environment
|
|
# Usage: make up
|
|
#
|
|
# Services:
|
|
# - nginx: Reverse proxy (port 2026, configurable via PORT env var)
|
|
# - frontend: Next.js production server
|
|
# - gateway: FastAPI Gateway API + agent runtime
|
|
# - redis: Redis Streams backend for cross-worker SSE stream bridge
|
|
# - provisioner: (optional) Sandbox provisioner for Kubernetes mode
|
|
#
|
|
# Key environment variables (set via environment/.env or scripts/deploy.sh):
|
|
# DEER_FLOW_PROJECT_ROOT — project root for relative runtime paths
|
|
# DEER_FLOW_HOME — runtime data dir, default .deer-flow under $DEER_FLOW_PROJECT_ROOT (or cwd)
|
|
# DEER_FLOW_CONFIG_PATH — path to config.yaml
|
|
# DEER_FLOW_EXTENSIONS_CONFIG_PATH — path to extensions_config.json
|
|
# DEER_FLOW_SKILLS_PATH — skills dir, default $DEER_FLOW_PROJECT_ROOT/skills
|
|
# DEER_FLOW_DOCKER_SOCKET — Docker socket path for aio/DooD mode, default /var/run/docker.sock (used only by the opt-in docker-compose.dood.yaml overlay)
|
|
# DEER_FLOW_REPO_ROOT — repo root (used for skills host path in DooD)
|
|
# BETTER_AUTH_SECRET — required for frontend auth/session security
|
|
# DEER_FLOW_INTERNAL_AUTH_TOKEN — shared internal Gateway auth token for multi-worker IM channels
|
|
#
|
|
# LangSmith tracing is disabled by default (LANGSMITH_TRACING=false).
|
|
# Set LANGSMITH_TRACING=true and LANGSMITH_API_KEY in .env to enable it.
|
|
#
|
|
# Access: http://localhost:${PORT:-2026}
|
|
|
|
services:
|
|
# ── Redis Stream Bridge ────────────────────────────────────────────────────
|
|
redis:
|
|
image: redis:7-alpine
|
|
container_name: deer-flow-redis
|
|
command: ["redis-server", "--appendonly", "yes"]
|
|
volumes:
|
|
- redis-data:/data
|
|
healthcheck:
|
|
test: ["CMD", "redis-cli", "ping"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 10
|
|
networks:
|
|
- deer-flow
|
|
restart: unless-stopped
|
|
|
|
# ── Reverse Proxy ──────────────────────────────────────────────────────────
|
|
nginx:
|
|
image: nginx:alpine
|
|
container_name: deer-flow-nginx
|
|
ports:
|
|
- "${PORT:-2026}:2026"
|
|
volumes:
|
|
- ./nginx/nginx.conf:/etc/nginx/nginx.conf.template:ro
|
|
command: >
|
|
sh -c "cp /etc/nginx/nginx.conf.template /etc/nginx/nginx.conf
|
|
&& nginx -g 'daemon off;'"
|
|
depends_on:
|
|
- frontend
|
|
- gateway
|
|
networks:
|
|
- deer-flow
|
|
restart: unless-stopped
|
|
|
|
# ── Frontend: Next.js Production ───────────────────────────────────────────
|
|
frontend:
|
|
build:
|
|
context: ../
|
|
dockerfile: frontend/Dockerfile
|
|
target: prod
|
|
args:
|
|
PNPM_STORE_PATH: ${PNPM_STORE_PATH:-/root/.local/share/pnpm/store}
|
|
NPM_REGISTRY: ${NPM_REGISTRY:-}
|
|
container_name: deer-flow-frontend
|
|
environment:
|
|
- BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET}
|
|
- DEER_FLOW_INTERNAL_GATEWAY_BASE_URL=http://gateway:8001
|
|
env_file:
|
|
- ../frontend/.env
|
|
networks:
|
|
- deer-flow
|
|
restart: unless-stopped
|
|
|
|
# ── Gateway API ────────────────────────────────────────────────────────────
|
|
gateway:
|
|
build:
|
|
context: ../
|
|
dockerfile: backend/Dockerfile
|
|
args:
|
|
APT_MIRROR: ${APT_MIRROR:-}
|
|
UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:0.7.20}
|
|
UV_INDEX_URL: ${UV_INDEX_URL:-https://pypi.org/simple}
|
|
UV_EXTRAS: ${UV_EXTRAS:-}
|
|
container_name: deer-flow-gateway
|
|
# Gateway hosts the agent runtime with in-process RunManager + StreamBridge
|
|
# singletons -- run state lives in this worker's memory. Default to a single
|
|
# worker: the Redis stream bridge solves cross-worker SSE delivery and
|
|
# reconnect, but run cancel, request dedup, and per-worker IM channel
|
|
# services remain worker-local. Override GATEWAY_WORKERS only when the
|
|
# Redis stream bridge is enabled and those limitations are acceptable.
|
|
command: sh -c "cd backend && PYTHONPATH=. uv run uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 --workers ${GATEWAY_WORKERS:-1}"
|
|
volumes:
|
|
- ${DEER_FLOW_CONFIG_PATH}:/app/backend/config.yaml:ro
|
|
- ${DEER_FLOW_EXTENSIONS_CONFIG_PATH}:/app/backend/extensions_config.json:ro
|
|
- ../skills:/app/skills:ro
|
|
- ${DEER_FLOW_HOME}:/app/backend/.deer-flow
|
|
# DooD: the host Docker socket is NOT mounted by default. It is added only
|
|
# for aio (pure-DooD) sandbox mode via the opt-in docker-compose.dood.yaml
|
|
# overlay (appended by scripts/deploy.sh). See backend/docs/CONFIGURATION.md
|
|
# Security Note section for details.
|
|
|
|
# CLI auth dirs (Claude Code / Codex) are NOT mounted by default: they
|
|
# expose the entire ~/.claude and ~/.codex (history, projects, global
|
|
# config, credentials) into the container. Mount them only when you use
|
|
# the Claude/Codex CLI login as a model provider or ACP agent, via the
|
|
# opt-in docker-compose.cli-auth.yaml overlay. Prefer an env token
|
|
# (CLAUDE_CODE_OAUTH_TOKEN, see .env.example / backend/docs/CONFIGURATION.md).
|
|
working_dir: /app
|
|
environment:
|
|
- CI=true
|
|
- DEER_FLOW_PROJECT_ROOT=/app
|
|
- DEER_FLOW_HOME=/app/backend/.deer-flow
|
|
- DEER_FLOW_CONFIG_PATH=/app/backend/config.yaml
|
|
- DEER_FLOW_EXTENSIONS_CONFIG_PATH=/app/backend/extensions_config.json
|
|
- DEER_FLOW_STREAM_BRIDGE_REDIS_URL=${DEER_FLOW_STREAM_BRIDGE_REDIS_URL:-redis://redis:6379/0}
|
|
- DEER_FLOW_CHANNELS_LANGGRAPH_URL=${DEER_FLOW_CHANNELS_LANGGRAPH_URL:-http://gateway:8001/api}
|
|
- DEER_FLOW_CHANNELS_GATEWAY_URL=${DEER_FLOW_CHANNELS_GATEWAY_URL:-http://gateway:8001}
|
|
- DEER_FLOW_INTERNAL_AUTH_TOKEN=${DEER_FLOW_INTERNAL_AUTH_TOKEN}
|
|
# DooD path/network translation
|
|
- DEER_FLOW_HOST_BASE_DIR=${DEER_FLOW_HOME}
|
|
- DEER_FLOW_HOST_SKILLS_PATH=${DEER_FLOW_REPO_ROOT}/skills
|
|
- DEER_FLOW_SANDBOX_HOST=host.docker.internal
|
|
# Proxy values (HTTP_PROXY/HTTPS_PROXY/ALL_PROXY) are inherited from ../.env via env_file.
|
|
# Only NO_PROXY is declared here so internal service hostnames are always exempt from the proxy.
|
|
- NO_PROXY=${NO_PROXY:-}${NO_PROXY:+,}localhost,127.0.0.1,::1,gateway,frontend,nginx,provisioner,host.docker.internal
|
|
- no_proxy=${no_proxy:-}${no_proxy:+,}localhost,127.0.0.1,::1,gateway,frontend,nginx,provisioner,host.docker.internal
|
|
env_file:
|
|
- ../.env
|
|
extra_hosts:
|
|
- "host.docker.internal:host-gateway"
|
|
depends_on:
|
|
redis:
|
|
condition: service_healthy
|
|
networks:
|
|
- deer-flow
|
|
restart: unless-stopped
|
|
|
|
# ── Sandbox Provisioner (optional, Kubernetes mode) ────────────────────────
|
|
provisioner:
|
|
build:
|
|
context: ./provisioner
|
|
dockerfile: Dockerfile
|
|
args:
|
|
APT_MIRROR: ${APT_MIRROR:-}
|
|
PIP_INDEX_URL: ${PIP_INDEX_URL:-}
|
|
container_name: deer-flow-provisioner
|
|
volumes:
|
|
- ~/.kube/config:/root/.kube/config:ro
|
|
environment:
|
|
- K8S_NAMESPACE=deer-flow
|
|
- SANDBOX_IMAGE=enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest
|
|
- SKILLS_HOST_PATH=${DEER_FLOW_REPO_ROOT}/skills
|
|
- THREADS_HOST_PATH=${DEER_FLOW_HOME}/threads
|
|
- KUBECONFIG_PATH=/root/.kube/config
|
|
- NODE_HOST=host.docker.internal
|
|
- K8S_API_SERVER=https://host.docker.internal:26443
|
|
env_file:
|
|
- ../.env
|
|
extra_hosts:
|
|
- "host.docker.internal:host-gateway"
|
|
networks:
|
|
- deer-flow
|
|
restart: unless-stopped
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8002/health"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 6
|
|
volumes:
|
|
redis-data:
|
|
|
|
networks:
|
|
deer-flow:
|
|
driver: bridge
|