fix(middleware): fix positional fallback consuming unrelated todo when same-content list is exhausted (#3709)

* fix(middleware): fix positional fallback consuming unrelated todo when same-content list is exhausted

When next_todos contains the same content string twice (e.g. two "A"
entries), the first iteration pops the matching previous todo from
previous_by_content["A"], leaving an empty list. The second iteration
finds that empty list, treats it as falsy, and sets previous_match=None.
The positional fallback then fires unconditionally — consuming
previous_todos[index] even if it holds a completely different entry
(e.g. "B"). That marks "B" as matched, so the final loop never emits
a todo_remove action for it.

Fix: guard the positional fallback with `content not in previous_by_content`
so it only fires for genuinely new content (the key was never in previous),
not for same-content entries whose list has simply been exhausted.

Add a regression test to _build_todo_actions covering this exact case.

* style: ruff-format the token usage middleware test

Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com>

---------

Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com>
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>

backend/packages/harness/deerflow/agents/middlewares/token_usage_middleware.py

@@ -97,7 +97,7 @@ def _build_todo_actions(previous_todos: list[Todo], next_todos: list[Todo]) -> l
                 previous_index, previous_match = content_matches.pop(0)
                 matched_previous_indices.add(previous_index)
 
-        if previous_match is None and index < len(previous_todos) and index not in matched_previous_indices:
+        if previous_match is None and content not in previous_by_content and index < len(previous_todos) and index not in matched_previous_indices:
             previous_match = previous_todos[index]
             matched_previous_indices.add(index)
 

backend/tests/test_token_usage_middleware.py

@@ -9,6 +9,7 @@ from langchain_core.messages import AIMessage, ToolMessage
 from deerflow.agents.middlewares.token_usage_middleware import (
     TOKEN_USAGE_ATTRIBUTION_KEY,
     TokenUsageMiddleware,
+    _build_todo_actions,
 )
 
 
@@ -279,3 +280,20 @@ class TestTokenUsageMiddleware:
             "output_tokens": 12,
             "total_tokens": 42,
         }
+
+
+class TestBuildTodoActions:
+    def test_duplicate_content_emits_todo_remove(self):
+        """When next_todos has duplicate content entries that exhaust previous_by_content,
+        the positional fallback must not consume an unrelated previous todo as matched.
+        The unrelated previous entry should still produce a todo_remove action."""
+        previous = [
+            {"content": "A", "status": "pending"},
+            {"content": "B", "status": "pending"},
+        ]
+        next_todos = [
+            {"content": "A", "status": "in_progress"},
+            {"content": "A", "status": "completed"},
+        ]
+        actions = _build_todo_actions(previous, next_todos)
+        assert any(a.get("kind") == "todo_remove" and a.get("content") == "B" for a in actions), f"Expected todo_remove for B but got: {actions}"