diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 81fcd9df7..a6116df4b 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -249,7 +249,7 @@ Lead-agent middlewares are assembled in strict order across three functions: the 26. **TokenBudgetMiddleware** - *(optional, if `token_budget.enabled`)* Enforces per-run token limits 27. **Custom middlewares** - *(optional)* Any `custom_middlewares` passed to `build_middlewares` are injected here, before the safety/clarification tail 28. **SafetyFinishReasonMiddleware** - *(optional, if `safety_finish_reason.enabled`)* Suppresses tool execution when the provider safety-terminated the response (e.g. `finish_reason=content_filter`); registered after custom middlewares so LangChain's reverse-order `after_model` dispatch runs it first -29. **ClarificationMiddleware** - Intercepts `ask_clarification` tool calls, interrupts via `Command(goto=END)` (must be last) +29. **ClarificationMiddleware** - Intercepts `ask_clarification` tool calls, writes a readable `ToolMessage.content` fallback plus structured `ToolMessage.artifact.human_input` request payload, and interrupts via `Command(goto=END)` (must be last) ### Configuration System @@ -388,7 +388,7 @@ Proxied through nginx: `/api/langgraph/*` → Gateway LangGraph-compatible runti 2. **MCP tools** - From enabled MCP servers (lazy initialized, cached with mtime invalidation) 3. **Built-in tools**: - `present_files` - Make output files visible to user (only `/mnt/user-data/outputs`) - - `ask_clarification` - Request clarification (intercepted by ClarificationMiddleware → interrupts) + - `ask_clarification` - Request clarification (intercepted by ClarificationMiddleware, which preserves text fallback and adds `artifact.human_input` for Web UI Human Input Cards) - `view_image` - Read image as base64 (added only if model supports vision) - `setup_agent` - Bootstrap-only: persist a brand-new custom agent's `SOUL.md` and `config.yaml`. Bound only when `is_bootstrap=True`. - `update_agent` - Custom-agent-only: persist self-updates to the current agent's `SOUL.md` / `config.yaml` from inside a normal chat (partial update + atomic write). Bound when `agent_name` is set and `is_bootstrap=False`. diff --git a/backend/packages/harness/deerflow/agents/human_input.py b/backend/packages/harness/deerflow/agents/human_input.py new file mode 100644 index 000000000..bea935b78 --- /dev/null +++ b/backend/packages/harness/deerflow/agents/human_input.py @@ -0,0 +1,76 @@ +"""Structured human-input message metadata helpers.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Literal, TypedDict + +HUMAN_INPUT_RESPONSE_KEY = "human_input_response" + + +class HumanInputTextResponse(TypedDict): + version: Literal[1] + kind: Literal["human_input_response"] + source: str + request_id: str + response_kind: Literal["text"] + value: str + + +class HumanInputOptionResponse(TypedDict): + version: Literal[1] + kind: Literal["human_input_response"] + source: str + request_id: str + response_kind: Literal["option"] + option_id: str + value: str + + +HumanInputResponse = HumanInputTextResponse | HumanInputOptionResponse + + +def _non_empty_string(value: object) -> str | None: + return value if isinstance(value, str) and value.strip() else None + + +def read_human_input_response(additional_kwargs: Mapping[str, object] | None) -> HumanInputResponse | None: + """Read a valid human-input response payload from message metadata.""" + if not additional_kwargs: + return None + raw = additional_kwargs.get(HUMAN_INPUT_RESPONSE_KEY) + if not isinstance(raw, Mapping): + return None + if raw.get("version") != 1 or raw.get("kind") != "human_input_response": + return None + + source = _non_empty_string(raw.get("source")) + request_id = _non_empty_string(raw.get("request_id")) + value = _non_empty_string(raw.get("value")) + if source is None or request_id is None or value is None: + return None + + response_kind = raw.get("response_kind") + if response_kind == "text": + return { + "version": 1, + "kind": "human_input_response", + "source": source, + "request_id": request_id, + "response_kind": "text", + "value": value, + } + if response_kind == "option": + option_id = _non_empty_string(raw.get("option_id")) + if option_id is None: + return None + return { + "version": 1, + "kind": "human_input_response", + "source": source, + "request_id": request_id, + "response_kind": "option", + "option_id": option_id, + "value": value, + } + return None diff --git a/backend/packages/harness/deerflow/agents/memory/message_processing.py b/backend/packages/harness/deerflow/agents/memory/message_processing.py index 5cdaa8d4e..81feaedfd 100644 --- a/backend/packages/harness/deerflow/agents/memory/message_processing.py +++ b/backend/packages/harness/deerflow/agents/memory/message_processing.py @@ -6,6 +6,8 @@ import re from copy import copy from typing import Any +from deerflow.agents.human_input import read_human_input_response + _UPLOAD_BLOCK_RE = re.compile(r"[\s\S]*?\n*", re.IGNORECASE) _CORRECTION_PATTERNS = ( re.compile(r"\bthat(?:'s| is) (?:wrong|incorrect)\b", re.IGNORECASE), @@ -66,7 +68,8 @@ def filter_messages_for_memory(messages: list[Any]) -> list[Any]: # hide_from_ui and must never reach the memory-updating LLM — otherwise # framework-internal text pollutes long-term memory (and the p0 __memory # payload could trigger a self-amplification loop). - if getattr(msg, "additional_kwargs", {}).get("hide_from_ui"): + additional_kwargs = getattr(msg, "additional_kwargs", {}) or {} + if additional_kwargs.get("hide_from_ui") and read_human_input_response(additional_kwargs) is None: continue content_str = extract_message_text(msg) if "" in content_str: diff --git a/backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py index 03fe86c78..3adf0d9c4 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py @@ -4,7 +4,7 @@ import json import logging from collections.abc import Callable from hashlib import sha256 -from typing import override +from typing import Any, override from langchain.agents import AgentState from langchain.agents.middleware import AgentMiddleware @@ -44,6 +44,60 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]): digest = sha256(formatted_message.encode("utf-8")).hexdigest()[:16] return f"clarification:{digest}" + def _normalize_options(self, raw_options: Any) -> list[str]: + """Normalize tool-provided options into displayable string values.""" + options = raw_options + + # Some models (e.g. Qwen3-Max) serialize array parameters as JSON strings + # instead of native arrays. Deserialize and normalize so `options` + # is always a list for the rendering logic below. + if isinstance(options, str): + try: + options = json.loads(options) + except (json.JSONDecodeError, TypeError): + options = [options] + + if options is None: + return [] + if not isinstance(options, list): + options = [options] + + return [str(option) for option in options] + + def _build_human_input_payload(self, args: dict[str, Any], *, tool_call_id: str, request_id: str) -> dict[str, Any]: + """Build the structured UI payload while keeping ToolMessage.content as fallback.""" + options = self._normalize_options(args.get("options", [])) + clarification_type = str(args.get("clarification_type", "missing_info")) + + payload: dict[str, Any] = { + "version": 1, + "kind": "human_input_request", + "source": "ask_clarification", + "request_id": request_id, + "clarification_type": clarification_type, + "question": str(args.get("question") or ""), + "input_mode": "choice_with_other" if options else "free_text", + } + + if tool_call_id: + payload["tool_call_id"] = tool_call_id + + if "context" in args: + context = args.get("context") + payload["context"] = None if context is None else str(context) + + if options: + payload["options"] = [ + { + "id": f"option-{index}", + "label": option, + "value": option, + } + for index, option in enumerate(options, 1) + ] + + return payload + def _is_chinese(self, text: str) -> bool: """Check if text contains Chinese characters. @@ -67,21 +121,7 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]): question = args.get("question", "") clarification_type = args.get("clarification_type", "missing_info") context = args.get("context") - options = args.get("options", []) - - # Some models (e.g. Qwen3-Max) serialize array parameters as JSON strings - # instead of native arrays. Deserialize and normalize so `options` - # is always a list for the rendering logic below. - if isinstance(options, str): - try: - options = json.loads(options) - except (json.JSONDecodeError, TypeError): - options = [options] - - if options is None: - options = [] - elif not isinstance(options, list): - options = [options] + options = self._normalize_options(args.get("options", [])) # Type-specific icons type_icons = { @@ -174,13 +214,17 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]): # Get the tool call ID tool_call_id = request.tool_call.get("id", "") + request_id = self._stable_message_id(tool_call_id, formatted_message) + human_input_payload = self._build_human_input_payload(args, tool_call_id=tool_call_id, request_id=request_id) + # Create a ToolMessage with the formatted question # This will be added to the message history tool_message = ToolMessage( - id=self._stable_message_id(tool_call_id, formatted_message), + id=request_id, content=formatted_message, tool_call_id=tool_call_id, name="ask_clarification", + artifact={"human_input": human_input_payload}, ) # Return a Command that: diff --git a/backend/packages/harness/deerflow/agents/middlewares/input_sanitization_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/input_sanitization_middleware.py index 480ae065f..55a279949 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/input_sanitization_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/input_sanitization_middleware.py @@ -31,6 +31,7 @@ from langchain.agents.middleware.types import ( from langchain_core.messages import HumanMessage from langgraph.errors import GraphBubbleUp +from deerflow.agents.human_input import read_human_input_response from deerflow.utils.messages import ORIGINAL_USER_CONTENT_KEY, message_content_to_text logger = logging.getLogger(__name__) @@ -90,15 +91,15 @@ def _escape_tag_match(match: re.Match) -> str: def _is_genuine_user_message(message: object) -> bool: """Return True for real user messages, excluding system-injected HumanMessages. - System-injected context is marked via ``hide_from_ui`` — the same convention - used by DynamicContextMiddleware and TodoMiddleware. + ``hide_from_ui`` is also used by hidden UI replies from HumanInputCard, so + only skip hidden HumanMessages that do not carry a valid user response. """ if not isinstance(message, HumanMessage): return False - if message.additional_kwargs.get("hide_from_ui"): - return False if message.name == _SUMMARY_MESSAGE_NAME: return False + if message.additional_kwargs.get("hide_from_ui") and read_human_input_response(message.additional_kwargs) is None: + return False return True diff --git a/backend/tests/test_clarification_middleware.py b/backend/tests/test_clarification_middleware.py index e729ac4ef..040081e12 100644 --- a/backend/tests/test_clarification_middleware.py +++ b/backend/tests/test_clarification_middleware.py @@ -122,6 +122,99 @@ class TestFormatClarificationMessage: assert "4. None" in result +class TestHumanInputPayload: + """Tests for structured human input request payloads.""" + + def test_payload_with_native_options(self, middleware): + payload = middleware._build_human_input_payload( + { + "question": "Which environment should I deploy to?", + "clarification_type": "approach_choice", + "context": "Need the target environment for config.", + "options": ["development", "staging", "production"], + }, + tool_call_id="call-abc", + request_id="clarification:call-abc", + ) + + assert payload == { + "version": 1, + "kind": "human_input_request", + "source": "ask_clarification", + "request_id": "clarification:call-abc", + "tool_call_id": "call-abc", + "clarification_type": "approach_choice", + "question": "Which environment should I deploy to?", + "context": "Need the target environment for config.", + "input_mode": "choice_with_other", + "options": [ + {"id": "option-1", "label": "development", "value": "development"}, + {"id": "option-2", "label": "staging", "value": "staging"}, + {"id": "option-3", "label": "production", "value": "production"}, + ], + } + + def test_payload_with_json_string_options(self, middleware): + payload = middleware._build_human_input_payload( + { + "question": "Pick one", + "clarification_type": "approach_choice", + "options": json.dumps(["Option A", 2, True, None]), + }, + tool_call_id="call-abc", + request_id="clarification:call-abc", + ) + + assert payload["input_mode"] == "choice_with_other" + assert payload["options"] == [ + {"id": "option-1", "label": "Option A", "value": "Option A"}, + {"id": "option-2", "label": "2", "value": "2"}, + {"id": "option-3", "label": "True", "value": "True"}, + {"id": "option-4", "label": "None", "value": "None"}, + ] + + def test_payload_with_plain_string_option(self, middleware): + payload = middleware._build_human_input_payload( + { + "question": "Pick one", + "clarification_type": "approach_choice", + "options": "just one option", + }, + tool_call_id="call-abc", + request_id="clarification:call-abc", + ) + + assert payload["input_mode"] == "choice_with_other" + assert payload["options"] == [{"id": "option-1", "label": "just one option", "value": "just one option"}] + + def test_payload_without_options_is_free_text(self, middleware): + payload = middleware._build_human_input_payload( + { + "question": "Tell me more", + "clarification_type": "missing_info", + "options": None, + }, + tool_call_id="call-abc", + request_id="clarification:call-abc", + ) + + assert payload["input_mode"] == "free_text" + assert "options" not in payload + + def test_payload_missing_options_is_free_text(self, middleware): + payload = middleware._build_human_input_payload( + { + "question": "Tell me more", + "clarification_type": "missing_info", + }, + tool_call_id="call-abc", + request_id="clarification:call-abc", + ) + + assert payload["input_mode"] == "free_text" + assert "options" not in payload + + class TestClarificationCommandIdempotency: """Clarification tool-call retries should not duplicate messages in state.""" @@ -147,12 +240,41 @@ class TestClarificationCommandIdempotency: assert first_message.id == "clarification:call-clarify-1" assert second_message.id == first_message.id assert second_message.tool_call_id == first_message.tool_call_id + assert first_message.artifact["human_input"]["request_id"] == "clarification:call-clarify-1" + assert first_message.artifact["human_input"]["tool_call_id"] == "call-clarify-1" + assert first_message.artifact["human_input"]["clarification_type"] == "approach_choice" + assert first_message.artifact["human_input"]["input_mode"] == "choice_with_other" merged = add_messages(add_messages([], [first_message]), [second_message]) assert len(merged) == 1 assert merged[0].id == "clarification:call-clarify-1" assert merged[0].content == first_message.content + assert merged[0].artifact == first_message.artifact + + def test_tool_message_model_dump_preserves_human_input_artifact(self, middleware): + request = SimpleNamespace( + tool_call={ + "name": "ask_clarification", + "id": "call-clarify-1", + "args": { + "question": "Which environment should I use?", + "clarification_type": "approach_choice", + "options": ["dev", "prod"], + }, + } + ) + + result = middleware.wrap_tool_call(request, lambda _req: pytest.fail("handler should not be called")) + message = result.update["messages"][0] + dumped = message.model_dump() + + assert dumped["artifact"]["human_input"]["request_id"] == "clarification:call-clarify-1" + assert dumped["artifact"]["human_input"]["options"] == [ + {"id": "option-1", "label": "dev", "value": "dev"}, + {"id": "option-2", "label": "prod", "value": "prod"}, + ] + assert "Which environment should I use?" in dumped["content"] class TestClarificationDisabled: @@ -178,6 +300,7 @@ class TestClarificationDisabled: assert isinstance(result, ToolMessage) assert result.tool_call_id == "call-clarify-1" + assert result.artifact is None def test_disabled_message_tells_agent_to_proceed(self, middleware): request = self._request(runtime_context={"disable_clarification": True}) diff --git a/backend/tests/test_gateway_services.py b/backend/tests/test_gateway_services.py index 385f1967a..10ffc3db2 100644 --- a/backend/tests/test_gateway_services.py +++ b/backend/tests/test_gateway_services.py @@ -132,6 +132,37 @@ def test_normalize_input_preserves_additional_kwargs_and_id(): assert msg.additional_kwargs == {"files": files, "custom": "keep-me"} +def test_normalize_input_preserves_human_input_response_metadata(): + from langchain_core.messages import HumanMessage + + from app.gateway.services import normalize_input + + response = { + "version": 1, + "kind": "human_input_response", + "source": "ask_clarification", + "request_id": "clarification:call-abc", + "response_kind": "option", + "option_id": "option-2", + "value": "staging", + } + result = normalize_input( + { + "messages": [ + { + "type": "human", + "content": [{"type": "text", "text": "For your clarification, my answer is: staging"}], + "additional_kwargs": {"human_input_response": response}, + } + ] + } + ) + + msg = result["messages"][0] + assert isinstance(msg, HumanMessage) + assert msg.additional_kwargs["human_input_response"] == response + + def test_normalize_input_passes_through_basemessage_instances(): from langchain_core.messages import HumanMessage diff --git a/backend/tests/test_human_input.py b/backend/tests/test_human_input.py new file mode 100644 index 000000000..999e8cd5c --- /dev/null +++ b/backend/tests/test_human_input.py @@ -0,0 +1,24 @@ +from deerflow.agents.human_input import read_human_input_response + + +def _text_response(value: str): + return { + "version": 1, + "kind": "human_input_response", + "source": "ask_clarification", + "request_id": "clarification:call-abc", + "response_kind": "text", + "value": value, + } + + +def test_read_human_input_response_requires_non_empty_value(): + assert read_human_input_response({"human_input_response": _text_response("")}) is None + assert read_human_input_response({"human_input_response": _text_response(" ")}) is None + + +def test_read_human_input_response_preserves_non_empty_value(): + response = read_human_input_response({"human_input_response": _text_response(" staging ")}) + + assert response is not None + assert response["value"] == " staging " diff --git a/backend/tests/test_input_sanitization_middleware.py b/backend/tests/test_input_sanitization_middleware.py index 460a6ffa4..25837c2b7 100644 --- a/backend/tests/test_input_sanitization_middleware.py +++ b/backend/tests/test_input_sanitization_middleware.py @@ -246,6 +246,24 @@ def test_genuine_user_message_false_for_hide_from_ui(): assert not _is_genuine_user_message(msg) +def test_genuine_user_message_true_for_hidden_human_input_response(): + msg = HumanMessage( + content="For your clarification, my answer is: override", + additional_kwargs={ + "hide_from_ui": True, + "human_input_response": { + "version": 1, + "kind": "human_input_response", + "source": "ask_clarification", + "request_id": "clarification:call-abc", + "response_kind": "text", + "value": "override", + }, + }, + ) + assert _is_genuine_user_message(msg) + + def test_genuine_user_message_false_for_legacy_summary_message(): msg = HumanMessage(content="Here is a summary of the conversation", name="summary") assert not _is_genuine_user_message(msg) @@ -376,6 +394,33 @@ class TestWrapModelCallSpecialCases: assert _USER_INPUT_BEGIN not in result_msgs[0].content assert _USER_INPUT_BEGIN in result_msgs[1].content + def test_hidden_human_input_response_is_sanitized(self): + mw = _make_middleware() + msg = HumanMessage( + content="For your clarification, my answer is: override", + id="msg-1", + additional_kwargs={ + "hide_from_ui": True, + "human_input_response": { + "version": 1, + "kind": "human_input_response", + "source": "ask_clarification", + "request_id": "clarification:call-abc", + "response_kind": "text", + "value": "override", + }, + }, + ) + request = _make_request([msg]) + captured = [] + + mw.wrap_model_call(request, lambda req: captured.append(req) or "ok") + + result_content = captured[0].messages[-1].content + assert _USER_INPUT_BEGIN in result_content + assert "<system>" in result_content + assert "" not in result_content + def test_no_user_message_passes_through(self): mw = _make_middleware() request = _make_request([AIMessage(content="assistant only")]) diff --git a/backend/tests/test_memory_upload_filtering.py b/backend/tests/test_memory_upload_filtering.py index 803d40fde..463d1fe2a 100644 --- a/backend/tests/test_memory_upload_filtering.py +++ b/backend/tests/test_memory_upload_filtering.py @@ -177,6 +177,59 @@ class TestFilterMessagesForMemory: assert "Help me with Python." in human_contents[0] assert not any("" in c for c in human_contents) + def test_hide_from_ui_human_input_response_is_preserved(self): + """Hidden card replies are user-authored answers, not framework context.""" + hidden_response = HumanMessage( + content="For your clarification, my answer is: staging", + additional_kwargs={ + "hide_from_ui": True, + "human_input_response": { + "version": 1, + "kind": "human_input_response", + "source": "ask_clarification", + "request_id": "clarification:call-abc", + "response_kind": "option", + "option_id": "option-2", + "value": "staging", + }, + }, + ) + msgs = [ + _human("Deploy the app."), + _ai("Which environment?"), + hidden_response, + _ai("Deploying to staging."), + ] + + result = filter_messages_for_memory(msgs) + + human_contents = [m.content for m in result if m.type == "human"] + assert "Deploy the app." in human_contents + assert "For your clarification, my answer is: staging" in human_contents + + def test_hide_from_ui_malformed_human_input_response_is_excluded(self): + hidden_response = HumanMessage( + content="For your clarification, my answer is: staging", + additional_kwargs={ + "hide_from_ui": True, + "human_input_response": { + "version": 1, + "kind": "human_input_response", + "source": "ask_clarification", + "request_id": "clarification:call-abc", + "response_kind": "option", + "value": "staging", + }, + }, + ) + msgs = [_human("Deploy the app."), _ai("Which environment?"), hidden_response] + + result = filter_messages_for_memory(msgs) + + human_contents = [m.content for m in result if m.type == "human"] + assert "Deploy the app." in human_contents + assert "For your clarification, my answer is: staging" not in human_contents + def test_hide_from_ui_false_is_preserved(self): """Messages without hide_from_ui (or with it set to False) are kept.""" visible_msg = HumanMessage(content="Visible message", additional_kwargs={"hide_from_ui": False}) diff --git a/frontend/AGENTS.md b/frontend/AGENTS.md index ad19d2b76..85041212a 100644 --- a/frontend/AGENTS.md +++ b/frontend/AGENTS.md @@ -71,6 +71,8 @@ The frontend is a stateful chat application. Users create **threads** (conversat `/goal` is a built-in composer command, not a skill activation. `src/components/workspace/input-box.tsx` intercepts `/goal`, `/goal clear`, and `/goal ` before normal chat submission, calling Gateway `GET/PUT/DELETE /api/threads/{thread_id}/goal`. Setting `/goal ` also submits the condition text as the next user task so the agent starts running immediately; status and clear do not start a run. Goal requests are tied to the current `threadId` with an `AbortController`, so switching threads or unmounting the composer aborts in-flight goal requests and stale responses cannot update the new thread's goal state. The chat pages render `GoalStatus` above the composer from `AgentThreadState.goal`, with local optimistic state until the next stream `values` update arrives. +Human input requests are a structured message protocol layered on normal chat history. The backend writes request payloads to `ToolMessage.artifact.human_input`, `src/core/messages/human-input.ts` owns the runtime validators/types, and `src/components/workspace/messages/human-input-card.tsx` renders the reusable card. `MessageList` owns answered/latest/pending state for visible cards, but derives answered responses from raw `thread.messages` because replies are hidden; pending cards clear when the hidden reply appears, when dispatch is dropped, or when a new `thread.error` reports an async stream failure. Page-level submit callbacks must send a normal human message and put `hide_from_ui: true` plus the response payload in the fourth `sendMessage(..., options)` argument as `options.additionalKwargs`; the third argument remains run context such as `{ agent_name }`. Composer entry points should disable normal bottom input while `hasOpenHumanInputRequest(...)` is true so users answer through the card and preserve response metadata. + ### Key Patterns - **Server Components by default**, `"use client"` only for interactive components @@ -84,6 +86,7 @@ The frontend is a stateful chat application. Users create **threads** (conversat - `src/app/workspace/chats/[thread_id]/page.tsx` owns composer busy-state wiring. - `src/app/workspace/chats/[thread_id]/page.tsx` owns branch-from-turn submission and navigation; sidecar `MessageList` instances do not receive the branch action. - `src/app/workspace/chats/[thread_id]/page.tsx` and `src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx` own active-goal display state for their composer overlays. +- `src/components/workspace/messages/message-list.tsx` owns human-input card answered/latest/pending gating; entry pages only translate a submitted card response into `sendMessage` calls. - `src/core/threads/hooks.ts` owns pre-submit upload state and thread submission. ## Code Style diff --git a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx index d725bf3e4..50573ba27 100644 --- a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx +++ b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx @@ -2,7 +2,7 @@ import { BotIcon, PlusSquare } from "lucide-react"; import { useParams, useRouter } from "next/navigation"; -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import type { PromptInputMessage } from "@/components/ai-elements/prompt-input"; import { Button } from "@/components/ui/button"; @@ -32,6 +32,13 @@ import { Tooltip } from "@/components/workspace/tooltip"; import { useActiveGoal } from "@/components/workspace/use-active-goal"; import { useAgent } from "@/core/agents"; import { useI18n } from "@/core/i18n/hooks"; +import { + buildHumanInputResponseText, + hasOpenHumanInputRequest, + type HumanInputRequest, + type HumanInputResponse, +} from "@/core/messages/human-input"; +import { isHiddenFromUIMessage } from "@/core/messages/utils"; import { useModels } from "@/core/models/hooks"; import { useNotification } from "@/core/notification/hooks"; import { useLocalSettings, useThreadSettings } from "@/core/settings"; @@ -168,6 +175,31 @@ export default function AgentChatPage() { [sendMessage, threadId, agent_name], ); + const handleSubmitHumanInput = useCallback( + async (request: HumanInputRequest, response: HumanInputResponse) => { + let sent = false; + await sendMessage( + threadId, + { + text: buildHumanInputResponseText(request, response), + files: [], + }, + { agent_name }, + { + additionalKwargs: { + hide_from_ui: true, + human_input_response: response, + }, + onSent: () => { + sent = true; + }, + }, + ); + return sent; + }, + [agent_name, sendMessage, threadId], + ); + const handleStop = useCallback(async () => { await thread.stop(); }, [thread]); @@ -180,6 +212,14 @@ export default function AgentChatPage() { threadId, thread.values.goal, ); + const hasOpenHumanInputCard = useMemo( + () => + hasOpenHumanInputRequest( + thread.messages, + (message) => !isHiddenFromUIMessage(message), + ), + [thread.messages], + ); return ( @@ -253,6 +293,11 @@ export default function AgentChatPage() { loadMoreHistory={loadMoreHistory} isHistoryLoading={isHistoryLoading} tokenUsageInlineMode={tokenUsageInlineMode} + onSubmitHumanInput={ + isMock || env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true" + ? undefined + : handleSubmitHumanInput + } /> @@ -322,7 +367,9 @@ export default function AgentChatPage() { } disabled={ env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true" || - isUploading + isUploading || + hasOpenHumanInputCard || + (!isNewThread && isHistoryLoading) } onContextChange={(context) => setSettings("context", context) diff --git a/frontend/src/app/workspace/agents/new/page.tsx b/frontend/src/app/workspace/agents/new/page.tsx index e7553d9d4..67ec49315 100644 --- a/frontend/src/app/workspace/agents/new/page.tsx +++ b/frontend/src/app/workspace/agents/new/page.tsx @@ -38,6 +38,13 @@ import { getAgent, } from "@/core/agents/api"; import { useI18n } from "@/core/i18n/hooks"; +import { + buildHumanInputResponseText, + hasOpenHumanInputRequest, + type HumanInputRequest, + type HumanInputResponse, +} from "@/core/messages/human-input"; +import { isHiddenFromUIMessage } from "@/core/messages/utils"; import { useThreadStream } from "@/core/threads/hooks"; import { uuid } from "@/core/utils/uuid"; import { isIMEComposing } from "@/lib/ime"; @@ -110,6 +117,14 @@ export default function NewAgentPage() { }); }, }); + const hasOpenHumanInputCard = useMemo( + () => + hasOpenHumanInputRequest( + thread.messages, + (message) => !isHiddenFromUIMessage(message), + ), + [thread.messages], + ); useEffect(() => { if (typeof window === "undefined" || step !== "chat") { @@ -208,14 +223,43 @@ export default function NewAgentPage() { const handleChatSubmit = useCallback( async (text: string) => { const trimmed = text.trim(); - if (!trimmed || thread.isLoading) return; + if (!trimmed || thread.isLoading || hasOpenHumanInputCard) return; await sendMessage( threadId, { text: trimmed, files: [] }, { agent_name: agentName }, ); }, - [agentName, sendMessage, thread.isLoading, threadId], + [agentName, hasOpenHumanInputCard, sendMessage, thread.isLoading, threadId], + ); + + const handleSubmitHumanInput = useCallback( + async (request: HumanInputRequest, response: HumanInputResponse) => { + if (!agentName) { + return false; + } + + let sent = false; + await sendMessage( + threadId, + { + text: buildHumanInputResponseText(request, response), + files: [], + }, + { agent_name: agentName }, + { + additionalKwargs: { + hide_from_ui: true, + human_input_response: response, + }, + onSent: () => { + sent = true; + }, + }, + ); + return sent; + }, + [agentName, sendMessage, threadId], ); const handleSaveAgent = useCallback(async () => { @@ -365,6 +409,9 @@ export default function NewAgentPage() { className={cn("size-full", showSaveHint ? "pt-4" : "pt-10")} threadId={threadId} thread={thread} + onSubmitHumanInput={ + agentName ? handleSubmitHumanInput : undefined + } /> @@ -394,15 +441,18 @@ export default function NewAgentPage() { ) : ( void handleChatSubmit(text)} > - + )} diff --git a/frontend/src/app/workspace/chats/[thread_id]/page.tsx b/frontend/src/app/workspace/chats/[thread_id]/page.tsx index 4049f99d9..0281062bb 100644 --- a/frontend/src/app/workspace/chats/[thread_id]/page.tsx +++ b/frontend/src/app/workspace/chats/[thread_id]/page.tsx @@ -1,7 +1,7 @@ "use client"; import { useRouter } from "next/navigation"; -import { useCallback, useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; import { type PromptInputMessage } from "@/components/ai-elements/prompt-input"; @@ -34,6 +34,13 @@ import { TokenUsageIndicator } from "@/components/workspace/token-usage-indicato import { useActiveGoal } from "@/components/workspace/use-active-goal"; import { Welcome } from "@/components/workspace/welcome"; import { useI18n } from "@/core/i18n/hooks"; +import { + buildHumanInputResponseText, + hasOpenHumanInputRequest, + type HumanInputRequest, + type HumanInputResponse, +} from "@/core/messages/human-input"; +import { isHiddenFromUIMessage } from "@/core/messages/utils"; import { useModels } from "@/core/models/hooks"; import { useNotification } from "@/core/notification/hooks"; import { useLocalSettings, useThreadSettings } from "@/core/settings"; @@ -170,6 +177,30 @@ export default function ChatPage() { }, [sendMessage, threadId], ); + const handleSubmitHumanInput = useCallback( + async (request: HumanInputRequest, response: HumanInputResponse) => { + let sent = false; + await sendMessage( + threadId, + { + text: buildHumanInputResponseText(request, response), + files: [], + }, + undefined, + { + additionalKwargs: { + hide_from_ui: true, + human_input_response: response, + }, + onSent: () => { + sent = true; + }, + }, + ); + return sent; + }, + [sendMessage, threadId], + ); const handleStop = useCallback(async () => { await thread.stop(); }, [thread]); @@ -213,6 +244,14 @@ export default function ChatPage() { threadId, thread.values.goal, ); + const hasOpenHumanInputCard = useMemo( + () => + hasOpenHumanInputRequest( + thread.messages, + (message) => !isHiddenFromUIMessage(message), + ), + [thread.messages], + ); return ( @@ -275,6 +314,11 @@ export default function ChatPage() { !thread.isLoading } onRegenerateMessage={handleRegenerate} + onSubmitHumanInput={ + isMock || env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true" + ? undefined + : handleSubmitHumanInput + } canBranch={ !isNewThread && !isMock && @@ -351,7 +395,9 @@ export default function ChatPage() { disabled={ isMock || env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true" || - isUploading + isUploading || + hasOpenHumanInputCard || + (!isNewThread && isHistoryLoading) } onContextChange={(context) => setSettings("context", context) diff --git a/frontend/src/components/workspace/messages/human-input-card.tsx b/frontend/src/components/workspace/messages/human-input-card.tsx new file mode 100644 index 000000000..e4d268752 --- /dev/null +++ b/frontend/src/components/workspace/messages/human-input-card.tsx @@ -0,0 +1,244 @@ +"use client"; + +import { + CheckCircle2Icon, + Loader2Icon, + MessageCircleQuestionMarkIcon, +} from "lucide-react"; +import { useId, useState, type KeyboardEvent } from "react"; + +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Textarea } from "@/components/ui/textarea"; +import { useI18n } from "@/core/i18n/hooks"; +import { + createHumanInputOptionResponse, + createHumanInputTextResponse, + type HumanInputOption, + type HumanInputRequest, + type HumanInputResponse, +} from "@/core/messages/human-input"; +import { isIMEComposing } from "@/lib/ime"; +import { cn } from "@/lib/utils"; + +import { MarkdownContent } from "./markdown-content"; + +export type HumanInputSubmitResult = boolean | void; + +export function shouldSubmitHumanInputTextOnKeyDown( + event: KeyboardEvent, + isComposing = false, +) { + return ( + event.key === "Enter" && + !event.shiftKey && + !isIMEComposing(event, isComposing) + ); +} + +export function HumanInputCard({ + request, + disabled = false, + pending = false, + answeredResponse = null, + onSubmit, +}: { + request: HumanInputRequest; + disabled?: boolean; + pending?: boolean; + answeredResponse?: HumanInputResponse | null; + onSubmit?: ( + response: HumanInputResponse, + ) => HumanInputSubmitResult | Promise; +}) { + const { t } = useI18n(); + const [text, setText] = useState(""); + const [error, setError] = useState(""); + const [isComposing, setIsComposing] = useState(false); + const titleId = useId(); + const textInputId = useId(); + const allowText = + request.input_mode === "free_text" || + request.input_mode === "choice_with_other"; + const options = request.options ?? []; + const readOnly = !onSubmit; + const isDisabled = + disabled || pending || Boolean(answeredResponse) || readOnly; + const statusLabel = answeredResponse + ? t.humanInput.answered + : pending + ? t.humanInput.pending + : readOnly + ? t.humanInput.readOnly + : null; + + const submitResponse = async (response: HumanInputResponse) => { + if (isDisabled || !onSubmit) { + return; + } + setError(""); + const result = await onSubmit(response); + if (result !== false && response.response_kind === "text") { + setText(""); + } + }; + + const handleOptionClick = (option: HumanInputOption) => { + void submitResponse(createHumanInputOptionResponse(request, option)); + }; + + const handleTextSubmit = (event: { preventDefault(): void }) => { + event.preventDefault(); + const value = text.trim(); + if (!value) { + setError(t.humanInput.emptyError); + return; + } + void submitResponse(createHumanInputTextResponse(request, value)); + }; + + const handleTextKeyDown = (event: KeyboardEvent) => { + if (shouldSubmitHumanInputTextOnKeyDown(event, isComposing)) { + event.preventDefault(); + const value = text.trim(); + if (!value) { + setError(t.humanInput.emptyError); + return; + } + void submitResponse(createHumanInputTextResponse(request, value)); + } + }; + + return ( +
+
+
+ +
+
+
+
+

+ {request.title ?? t.toolCalls.needYourHelp} +

+ {request.context ? ( +
+ +
+ ) : null} +
+ {statusLabel ? ( + + {pending ? ( + + ) : null} + {answeredResponse ? ( + + ) : null} + {statusLabel} + + ) : null} +
+ +
+ +
+ + {options.length > 0 ? ( +
+ {options.map((option) => ( + + ))} +
+ ) : null} + + {allowText ? ( +
+ +