vrr/deer-flow
2
0
Fork 0
mirror of https://github.com/bytedance/deer-flow.git synced 2026-07-09 16:08:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

fix(docker): production Postgres UV extras detection (#3897)

Browse source 
* Fix production postgres UV extras detection

* fix(backend): validate Docker build UV extras
This commit is contained in:
Zhou Kai 2026-07-01 23:40:35 +08:00 • committed by GitHub
parent b431053ef0
commit dd05e1a76d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 314 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

18
backend/Dockerfile
View file

@ -14,7 +14,7 @@ ARG NODE_MAJOR=22
ARG APT_MIRROR
ARG UV_INDEX_URL
# Optional extras to install (e.g. "postgres" for PostgreSQL support)
# Usage: docker build --build-arg UV_EXTRAS=postgres ...
# Usage: docker build --build-arg UV_EXTRAS=postgres,discord ...
ARG UV_EXTRAS
# Optionally override apt mirror for restricted networks (e.g. APT_MIRROR=mirrors.aliyun.com)
@ -46,9 +46,21 @@ WORKDIR /app
COPY backend ./backend
# Install dependencies with cache mount
# When UV_EXTRAS is set (e.g. "postgres"), installs optional dependencies.
# When UV_EXTRAS is set (comma- or whitespace-separated), installs optional dependencies.
RUN --mount=type=cache,target=/root/.cache/uv \
sh -c "cd backend && UV_INDEX_URL=${UV_INDEX_URL:-https://pypi.org/simple} uv sync ${UV_EXTRAS:+--extra $UV_EXTRAS}"
sh -c 'cd backend && \
set -f; \
extras_flags=""; \
for extra in $(printf "%s" "$UV_EXTRAS" | tr "," " "); do \
case "$extra" in \
[!A-Za-z]* | *[!A-Za-z0-9_-]*) \
echo "UV_EXTRAS entry $extra is invalid (must match [A-Za-z][A-Za-z0-9_-]*)" >&2; \
exit 1; \
;; \
esac; \
extras_flags="$extras_flags --extra $extra"; \
done; \
UV_INDEX_URL=${UV_INDEX_URL:-https://pypi.org/simple} uv sync $extras_flags'
# UTF-8 locale prevents UnicodeEncodeError on Chinese/emoji content in minimal
# containers where locale configuration may be missing and the default encoding is not UTF-8.

227
backend/tests/test_deploy_uv_extras.py Normal file
View file

@ -0,0 +1,227 @@
"""Regression coverage for production deploy.sh UV_EXTRAS propagation."""
from __future__ import annotations
import os
import re
import shlex
import shutil
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
def _backend_dockerfile_uv_sync_script() -> str:
dockerfile = (REPO_ROOT / "backend" / "Dockerfile").read_text(encoding="utf-8")
match = re.search(r"""sh -c (?P<quote>["'])(?P<script>.*?uv sync.*?)(?P=quote)""", dockerfile, re.S)
assert match is not None
return match.group("script").replace("\\\n", "\n")
def test_backend_dockerfile_expands_multiple_uv_extras(tmp_path):
"""Dockerfile build args must become repeated uv --extra flags."""
workdir = tmp_path / "work"
backend = workdir / "backend"
backend.mkdir(parents=True)
capture = tmp_path / "uv_args.txt"
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
uv = bin_dir / "uv"
uv.write_text(
'#!/usr/bin/env sh\nfor arg in "$@"; do printf "%s\\n" "$arg"; done > "$CAPTURE_UV_ARGS"\n',
encoding="utf-8",
)
uv.chmod(0o755)
env = os.environ.copy()
env["CAPTURE_UV_ARGS"] = str(capture)
env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
env["UV_EXTRAS"] = "discord,postgres"
subprocess.run(
["sh", "-c", _backend_dockerfile_uv_sync_script()],
cwd=workdir,
env=env,
check=True,
)
assert capture.read_text(encoding="utf-8").splitlines() == [
"sync",
"--extra",
"discord",
"--extra",
"postgres",
]
def test_backend_dockerfile_rejects_glob_uv_extra(tmp_path):
"""Dockerfile extras must reject globs before invoking uv."""
workdir = tmp_path / "work"
backend = workdir / "backend"
backend.mkdir(parents=True)
(backend / "glob-match").touch()
capture = tmp_path / "uv_args.txt"
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
uv = bin_dir / "uv"
uv.write_text(
'#!/usr/bin/env sh\nfor arg in "$@"; do printf "%s\\n" "$arg"; done > "$CAPTURE_UV_ARGS"\n',
encoding="utf-8",
)
uv.chmod(0o755)
env = os.environ.copy()
env["CAPTURE_UV_ARGS"] = str(capture)
env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
env["UV_EXTRAS"] = "postgres,*"
result = subprocess.run(
["sh", "-c", _backend_dockerfile_uv_sync_script()],
cwd=workdir,
env=env,
check=False,
capture_output=True,
text=True,
)
assert result.returncode != 0
assert not capture.exists()
def test_deploy_build_auto_detects_postgres_extra_when_other_extras_are_enabled(tmp_path):
"""Production image builds preserve every detected extra as Docker build tokens."""
worktree = tmp_path / "repo"
shutil.copytree(REPO_ROOT / "scripts", worktree / "scripts")
shutil.copytree(REPO_ROOT / "docker", worktree / "docker")
(worktree / "backend").mkdir()
(worktree / "config.yaml").write_text(
"database:\n backend: postgres\nchannels:\n discord:\n enabled: true\n",
encoding="utf-8",
)
(worktree / "extensions_config.json").write_text('{"mcpServers":{},"skills":{}}\n', encoding="utf-8")
capture = tmp_path / "uv_extras.txt"
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
docker = bin_dir / "docker"
docker.write_text(
'#!/usr/bin/env sh\nprintf "%s" "${UV_EXTRAS:-}" > "$CAPTURE_UV_EXTRAS"\nexit 0\n',
encoding="utf-8",
)
docker.chmod(0o755)
env = os.environ.copy()
env.pop("UV_EXTRAS", None)
env["CAPTURE_UV_EXTRAS"] = str(capture)
env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
subprocess.run(
["bash", str(worktree / "scripts" / "deploy.sh"), "build"],
cwd=worktree,
env=env,
check=True,
text=True,
capture_output=True,
)
assert capture.read_text(encoding="utf-8") == "discord,postgres"
def test_deploy_uses_dotenv_without_sourcing_shell_syntax(tmp_path):
"""Repo-root .env is Docker Compose dotenv, not a shell script."""
worktree = tmp_path / "repo"
shutil.copytree(REPO_ROOT / "scripts", worktree / "scripts")
shutil.copytree(REPO_ROOT / "docker", worktree / "docker")
(worktree / "backend").mkdir()
(worktree / "config.yaml").write_text(
"database:\n backend: postgres\n",
encoding="utf-8",
)
(worktree / "extensions_config.json").write_text('{"mcpServers":{},"skills":{}}\n', encoding="utf-8")
marker = tmp_path / "sourced-marker"
(worktree / ".env").write_text(
f"DATABASE_URL=postgresql://user:pass@localhost/db?sslmode=require&application_name=deer\nUNSAFE=$(touch {shlex.quote(str(marker))})\nUV_EXTRAS=discord\n",
encoding="utf-8",
)
capture_extras = tmp_path / "uv_extras.txt"
capture_args = tmp_path / "docker_args.txt"
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
docker = bin_dir / "docker"
docker.write_text(
'#!/usr/bin/env sh\nprintf "%s" "${UV_EXTRAS:-}" > "$CAPTURE_UV_EXTRAS"\nfor arg in "$@"; do printf "%s\\n" "$arg"; done > "$CAPTURE_DOCKER_ARGS"\nexit 0\n',
encoding="utf-8",
)
docker.chmod(0o755)
env = os.environ.copy()
env.pop("UV_EXTRAS", None)
env["CAPTURE_UV_EXTRAS"] = str(capture_extras)
env["CAPTURE_DOCKER_ARGS"] = str(capture_args)
env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
subprocess.run(
["bash", str(worktree / "scripts" / "deploy.sh"), "build"],
cwd=worktree,
env=env,
check=True,
text=True,
capture_output=True,
)
assert not marker.exists()
assert capture_extras.read_text(encoding="utf-8") == "discord"
args = capture_args.read_text(encoding="utf-8").splitlines()
assert "--env-file" in args
assert str(worktree / ".env") in args
def test_deploy_build_auto_detects_postgres_extra_with_python_fallback(tmp_path):
"""Production deploy hosts may have python but no runnable python3."""
worktree = tmp_path / "repo"
shutil.copytree(REPO_ROOT / "scripts", worktree / "scripts")
shutil.copytree(REPO_ROOT / "docker", worktree / "docker")
(worktree / "backend").mkdir()
(worktree / "config.yaml").write_text(
"database:\n backend: postgres\n",
encoding="utf-8",
)
(worktree / "extensions_config.json").write_text('{"mcpServers":{},"skills":{}}\n', encoding="utf-8")
capture = tmp_path / "uv_extras.txt"
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
docker = bin_dir / "docker"
docker.write_text(
'#!/usr/bin/env sh\nprintf "%s" "${UV_EXTRAS:-}" > "$CAPTURE_UV_EXTRAS"\nexit 0\n',
encoding="utf-8",
)
docker.chmod(0o755)
python3 = bin_dir / "python3"
python3.write_text("#!/usr/bin/env sh\nexit 1\n", encoding="utf-8")
python3.chmod(0o755)
python = bin_dir / "python"
python.write_text(
f'#!/usr/bin/env sh\nexec {shlex.quote(sys.executable)} "$@"\n',
encoding="utf-8",
)
python.chmod(0o755)
env = os.environ.copy()
env.pop("UV_EXTRAS", None)
env["CAPTURE_UV_EXTRAS"] = str(capture)
env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
subprocess.run(
["bash", str(worktree / "scripts" / "deploy.sh"), "build"],
cwd=worktree,
env=env,
check=True,
text=True,
capture_output=True,
)
assert capture.read_text(encoding="utf-8") == "postgres"

6
config.example.yaml
View file

@ -1282,9 +1282,9 @@ skill_evolution:
# UV_EXTRAS=postgres
# Multiple extras (`postgres,ollama`) supported here too — see
# docker/dev-entrypoint.sh.
# Docker img build-arg `UV_EXTRAS=postgres docker compose build` — single
# extra only at build time (backend/Dockerfile passes the value
# as one token to `--extra`).
# Docker img build-arg `UV_EXTRAS=postgres,discord docker compose build`
# supports comma- or whitespace-separated extras at build time
# (backend/Dockerfile expands them to repeated `--extra` flags).
#
# First-time bootstrap (before `make dev`):
# cd backend && uv sync --all-packages --extra postgres

68
scripts/deploy.sh
View file

@ -42,8 +42,36 @@ esac
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$REPO_ROOT"
ENV_FILE="$REPO_ROOT/.env"
DOCKER_DIR="$REPO_ROOT/docker"
COMPOSE_CMD=(docker compose -p deer-flow -f "$DOCKER_DIR/docker-compose.yaml")
if [ -f "$ENV_FILE" ]; then
COMPOSE_CMD=(docker compose --env-file "$ENV_FILE" -p deer-flow -f "$DOCKER_DIR/docker-compose.yaml")
else
COMPOSE_CMD=(docker compose -p deer-flow -f "$DOCKER_DIR/docker-compose.yaml")
fi
load_uv_extras_from_dotenv() {
local line=""
local value=""
[ -f "$ENV_FILE" ] || return 0
[ -z "${UV_EXTRAS+x}" ] || return 0
line="$(grep -E '^[[:space:]]*(export[[:space:]]+)?UV_EXTRAS[[:space:]]*=' "$ENV_FILE" | tail -n 1 || true)"
[ -n "$line" ] || return 0
value="${line#*=}"
value="${value%$'\r'}"
value="${value#"${value%%[![:space:]]*}"}"
value="${value%"${value##*[![:space:]]}"}"
case "$value" in
\"*\") value="${value#\"}"; value="${value%\"}" ;;
\'*\') value="${value#\'}"; value="${value%\'}" ;;
esac
export UV_EXTRAS="$value"
}
load_uv_extras_from_dotenv
# ── Colors ────────────────────────────────────────────────────────────────────
@ -172,6 +200,44 @@ if [ "$CMD" != "down" ] && [ -z "$DEER_FLOW_INTERNAL_AUTH_TOKEN" ]; then
fi
fi
# ── UV_EXTRAS auto-detection ─────────────────────────────────────────────────
# The production Dockerfile accepts UV_EXTRAS as a single build-arg token and
# adds the --extra prefix itself. Convert the detector's uv flag string
# ("--extra postgres --extra discord") to a comma-joined name token.
if [ "$CMD" != "down" ] && [ -z "$UV_EXTRAS" ]; then
_detect_python=""
for _python in python3 python; do
if command -v "$_python" >/dev/null 2>&1 && \
"$_python" -c 'import sys; sys.version_info >= (3, 6) or sys.exit(1)' >/dev/null 2>&1; then
_detect_python="$_python"
break
fi
done
fi
if [ "$CMD" != "down" ] && [ -z "$UV_EXTRAS" ] && [ -n "$_detect_python" ]; then
_uv_extras_flags="$("$_detect_python" "$REPO_ROOT/scripts/detect_uv_extras.py" 2>/dev/null || true)"
_uv_extras=""
set -- $_uv_extras_flags
while [ "$#" -gt 0 ]; do
if [ "$1" = "--extra" ] && [ "$#" -gt 1 ]; then
if [ -z "$_uv_extras" ]; then
_uv_extras="$2"
else
_uv_extras="$_uv_extras,$2"
fi
shift 2
else
shift
fi
done
if [ -n "$_uv_extras" ]; then
export UV_EXTRAS="$_uv_extras"
echo -e "${GREEN}✓ Auto-detected UV_EXTRAS=${UV_EXTRAS} from config.yaml${NC}"
fi
fi
# ── detect_sandbox_mode ───────────────────────────────────────────────────────
detect_sandbox_mode() {

7
scripts/detect_uv_extras.py
View file

@ -5,11 +5,8 @@ Order of resolution:
1. `UV_EXTRAS` env var. Comma- or whitespace-separated names so multiple
extras can be layered (e.g. ``UV_EXTRAS=postgres,ollama``). The same
parsing semantics apply in the Docker dev container via
``docker/dev-entrypoint.sh``. The Docker image-build path
(``backend/Dockerfile``) still treats `UV_EXTRAS` as a single token, so
``UV_EXTRAS=postgres,ollama`` would only install ``postgres,ollama`` as
one (invalid) extra at build time — author build-time values as a
single name.
``docker/dev-entrypoint.sh`` and in the production Docker image build via
``backend/Dockerfile``.
2. Auto-detection from config.yaml — currently maps:
- database.backend == postgres -> postgres
- checkpointer.type == postgres -> postgres