mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-07-09 16:08:31 +00:00
72f033fbbe
* feat: add redis stream bridge * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * fix(gateway): address redis stream bridge review Redis was imported eagerly through deerflow.runtime and declared as a hard dependency, which made memory-only installs load redis.asyncio at startup and left the lazy factory import ineffective. Move redis behind an optional extra, remove the public eager re-export, and keep make_stream_bridge as the only runtime import path with an actionable install hint when the extra is missing. Because Docker deployments now default the stream bridge to Redis via DEER_FLOW_STREAM_BRIDGE_REDIS_URL, install the redis extra explicitly in Docker/dev container flows and teach the local uv-extra detector to infer redis from both stream_bridge.type and the Redis URL env var. This keeps Docker working while preserving slim non-Docker installs. Harden the Redis bridge by batching XREAD replay, replacing brittle ResponseError string matching with a single fallback to 0-0 for malformed Last-Event-ID values, documenting connection/retention/fail-hard behavior, and adding fake plus opt-in real Redis coverage for XADD/XREAD, replay, invalid IDs, and MAXLEN trimming. * fix(config): bump config version for stream bridge * fix redis stream bridge terminal handling * fix: repair uv.lock, format redis.py, and align Dockerfile extras test The uv.lock file was missing a closing bracket for the redis extras section, redis.py had a formatting issue caught by ruff, and the Dockerfile extras test did not account for the hardcoded --extra redis flag. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
89 lines
2.7 KiB
Python
89 lines
2.7 KiB
Python
"""Async stream bridge factory.
|
|
|
|
Provides an **async context manager** aligned with
|
|
:func:`deerflow.runtime.checkpointer.async_provider.make_checkpointer`.
|
|
|
|
Usage (e.g. FastAPI lifespan)::
|
|
|
|
from deerflow.agents.stream_bridge import make_stream_bridge
|
|
|
|
async with make_stream_bridge() as bridge:
|
|
app.state.stream_bridge = bridge
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import logging
|
|
import os
|
|
from collections.abc import AsyncIterator
|
|
|
|
from deerflow.config.app_config import AppConfig
|
|
from deerflow.config.stream_bridge_config import StreamBridgeConfig, get_stream_bridge_config
|
|
|
|
from .base import StreamBridge
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_ENV_REDIS_URL = "DEER_FLOW_STREAM_BRIDGE_REDIS_URL"
|
|
|
|
|
|
def _resolve_config(app_config: AppConfig | None) -> StreamBridgeConfig | None:
|
|
if app_config is None:
|
|
config = get_stream_bridge_config()
|
|
else:
|
|
config = app_config.stream_bridge
|
|
|
|
if config is None:
|
|
redis_url = os.getenv(_ENV_REDIS_URL)
|
|
if redis_url:
|
|
return StreamBridgeConfig(type="redis", redis_url=redis_url)
|
|
return config
|
|
|
|
|
|
def _resolve_redis_url(config: StreamBridgeConfig) -> str:
|
|
return config.redis_url or os.getenv(_ENV_REDIS_URL) or os.getenv("REDIS_URL") or "redis://localhost:6379/0"
|
|
|
|
|
|
@contextlib.asynccontextmanager
|
|
async def make_stream_bridge(app_config: AppConfig | None = None) -> AsyncIterator[StreamBridge]:
|
|
"""Async context manager that yields a :class:`StreamBridge`.
|
|
|
|
Falls back to :class:`MemoryStreamBridge` when no configuration is
|
|
provided and nothing is set globally.
|
|
"""
|
|
config = _resolve_config(app_config)
|
|
|
|
if config is None or config.type == "memory":
|
|
from deerflow.runtime.stream_bridge.memory import MemoryStreamBridge
|
|
|
|
maxsize = config.queue_maxsize if config is not None else 256
|
|
bridge = MemoryStreamBridge(queue_maxsize=maxsize)
|
|
logger.info("Stream bridge initialised: memory (queue_maxsize=%d)", maxsize)
|
|
try:
|
|
yield bridge
|
|
finally:
|
|
await bridge.close()
|
|
return
|
|
|
|
if config.type == "redis":
|
|
from deerflow.runtime.stream_bridge.redis import RedisStreamBridge
|
|
|
|
redis_url = _resolve_redis_url(config)
|
|
bridge = RedisStreamBridge(
|
|
redis_url=redis_url,
|
|
queue_maxsize=config.queue_maxsize,
|
|
max_connections=config.max_connections,
|
|
)
|
|
logger.info(
|
|
"Stream bridge initialised: redis (queue_maxsize=%d, max_connections=%s)",
|
|
config.queue_maxsize,
|
|
config.max_connections,
|
|
)
|
|
try:
|
|
yield bridge
|
|
finally:
|
|
await bridge.close()
|
|
return
|
|
|
|
raise ValueError(f"Unknown stream bridge type: {config.type!r}")
|