diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 8ec11fa79..347f469d3 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -7273,7 +7273,13 @@ class LlamaCppBackend: url = f"{self.base_url}/completion" payload = {"prompt": "Hi", "n_predict": 4, "temperature": 0.0, "stream": False} try: - resp = httpx.post(url, json = payload, timeout = timeout, headers = self._auth_headers) + resp = httpx.post( + url, + json = payload, + timeout = timeout, + headers = self._auth_headers, + trust_env = False, + ) except Exception as e: logger.debug(f"MTP decode probe failed: {e}") return False @@ -7425,7 +7431,10 @@ class LlamaCppBackend: return False try: - resp = httpx.get(url, timeout = 2.0) + # trust_env=False: never route the loopback health probe through + # an ambient HTTP(S)_PROXY. A proxy that 503s for 127.0.0.1 makes + # the probe loop until timeout and hangs Studio load. + resp = httpx.get(url, timeout = 2.0, trust_env = False) if resp.status_code == 200: return True except ( @@ -7472,7 +7481,7 @@ class LlamaCppBackend: """ url = f"{self.base_url}/props" try: - resp = httpx.get(url, timeout = 5.0) + resp = httpx.get(url, timeout = 5.0, trust_env = False) if resp.status_code != 200: return None settings = resp.json().get("default_generation_settings") or {} @@ -7552,7 +7561,9 @@ class LlamaCppBackend: which differ only in how they parse the SSE body.""" stream_timeout = httpx.Timeout(connect = 10, read = 0.5, write = 10, pool = 10) with httpx.Client( - timeout = stream_timeout, limits = httpx.Limits(max_keepalive_connections = 0) + timeout = stream_timeout, + limits = httpx.Limits(max_keepalive_connections = 0), + trust_env = False, ) as client: first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_S with self._stream_with_retry( @@ -9044,7 +9055,7 @@ class LlamaCppBackend: system_text = _block_text(system) try: - with httpx.Client(timeout = 10, headers = self._auth_headers) as client: + with httpx.Client(timeout = 10, headers = self._auth_headers, trust_env = False) as client: def _tokenize(text: str) -> int: r = client.post( @@ -9160,7 +9171,7 @@ class LlamaCppBackend: """Codec name on match, None on non-audio, raises on transport/JSON errors.""" if not self.is_loaded: return None - with httpx.Client(timeout = 10, headers = self._auth_headers) as client: + with httpx.Client(timeout = 10, headers = self._auth_headers, trust_env = False) as client: def _detok(tid: int) -> str: # Non-200 means "marker not in vocab" -- keep probing. @@ -9275,7 +9286,9 @@ class LlamaCppBackend: payload["n_probs"] = 1 with httpx.Client( - timeout = httpx.Timeout(300, connect = 10), headers = self._auth_headers + timeout = httpx.Timeout(300, connect = 10), + headers = self._auth_headers, + trust_env = False, ) as client: resp = client.post(f"{self.base_url}/completion", json = payload) if resp.status_code != 200: diff --git a/studio/backend/core/inference/llama_http.py b/studio/backend/core/inference/llama_http.py index b554949c3..8aa072e35 100644 --- a/studio/backend/core/inference/llama_http.py +++ b/studio/backend/core/inference/llama_http.py @@ -22,11 +22,7 @@ _LIMITS = httpx.Limits(max_connections = 64, max_keepalive_connections = 32) def _new_client() -> httpx.AsyncClient: - try: - return httpx.AsyncClient(limits = _LIMITS) - except Exception: - # Mirror external_provider: an unsupported env proxy scheme can raise. - return httpx.AsyncClient(limits = _LIMITS, trust_env = False) + return httpx.AsyncClient(limits = _LIMITS, trust_env = False) # One client per running event loop: an httpx client binds its transport to the diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index d3f56edec..9caacec61 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -6723,7 +6723,10 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge # honor stream_options.include_usage per event, while keeping SSE # framing and token bytes intact. _include_usage = bool((body.get("stream_options") or {}).get("include_usage")) - client = httpx.AsyncClient(timeout = _llama_streaming_generation_timeout()) + client = httpx.AsyncClient( + timeout = _llama_streaming_generation_timeout(), + trust_env = False, + ) resp = None bytes_iter = None disconnect_event = threading.Event() @@ -7798,7 +7801,10 @@ async def _responses_stream( # `async with`, explicit aclose of lines_iter BEFORE resp / client so # the innermost httpcore byte stream is finalised in this task (not via # the asyncgen GC in a sibling task). - client = httpx.AsyncClient(timeout = _llama_streaming_generation_timeout()) + client = httpx.AsyncClient( + timeout = _llama_streaming_generation_timeout(), + trust_env = False, + ) resp = None lines_iter = None disconnect_watcher = None @@ -9308,6 +9314,7 @@ async def _anthropic_passthrough_stream( client = httpx.AsyncClient( timeout = _llama_streaming_generation_timeout(), limits = httpx.Limits(max_keepalive_connections = 0), + trust_env = False, ) resp = None lines_iter = None @@ -9834,6 +9841,7 @@ async def _openai_passthrough_stream( client = httpx.AsyncClient( timeout = _llama_streaming_generation_timeout(), limits = httpx.Limits(max_keepalive_connections = 0), + trust_env = False, ) resp = None _truncate_budget = ( diff --git a/studio/backend/tests/test_api_perf_serialization.py b/studio/backend/tests/test_api_perf_serialization.py index f5ad53306..348e09104 100644 --- a/studio/backend/tests/test_api_perf_serialization.py +++ b/studio/backend/tests/test_api_perf_serialization.py @@ -57,6 +57,15 @@ def test_media_type_and_status(): assert err.status_code == 503 +def test_pooled_client_disables_proxy_env(): + async def _scenario(): + client = llama_http.nonstreaming_client() + assert client.trust_env is False + await llama_http.aclose() + + asyncio.run(_scenario()) + + def test_pooled_client_reused_within_loop_and_recreated_after_close(): async def _scenario(): a = llama_http.nonstreaming_client() diff --git a/studio/backend/tests/test_inference_dispatcher_resilience.py b/studio/backend/tests/test_inference_dispatcher_resilience.py index 2de833db0..60d750348 100644 --- a/studio/backend/tests/test_inference_dispatcher_resilience.py +++ b/studio/backend/tests/test_inference_dispatcher_resilience.py @@ -10,6 +10,7 @@ must be logged and skipped, not fatal. Fakes only. from __future__ import annotations +import ast import queue import sys import threading @@ -89,3 +90,31 @@ def test_dispatcher_survives_mailbox_put_error(): o._dispatcher_stop.set() t.join(timeout = 5) assert not t.is_alive() + + +def test_route_llama_streaming_async_clients_disable_proxy_env(): + """Local llama-server streaming proxies must ignore ambient HTTP_PROXY.""" + source = (Path(__file__).resolve().parent.parent / "routes" / "inference.py").read_text( + encoding = "utf-8" + ) + tree = ast.parse(source) + calls = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + if not ( + isinstance(func, ast.Attribute) + and func.attr == "AsyncClient" + and isinstance(func.value, ast.Name) + and func.value.id == "httpx" + ): + continue + calls.append(node) + + assert len(calls) == 4 + for call in calls: + assert any( + kw.arg == "trust_env" and isinstance(kw.value, ast.Constant) and kw.value.value is False + for kw in call.keywords + ), f"httpx.AsyncClient at line {call.lineno} must set trust_env=False" diff --git a/studio/backend/tests/test_llama_cpp_props_readback.py b/studio/backend/tests/test_llama_cpp_props_readback.py index d87c05f2c..316956325 100644 --- a/studio/backend/tests/test_llama_cpp_props_readback.py +++ b/studio/backend/tests/test_llama_cpp_props_readback.py @@ -113,8 +113,14 @@ def _stub_props( body = None, exc = None, ): - def fake_get(url, timeout = None): + def fake_get( + url, + timeout = None, + trust_env = None, + ): assert url.endswith("/props") + + assert trust_env is False if exc is not None: raise exc return _FakeResponse(status_code, body) diff --git a/studio/backend/tests/test_offline_gguf_cache_fallback.py b/studio/backend/tests/test_offline_gguf_cache_fallback.py index a2a505f47..4499881c4 100644 --- a/studio/backend/tests/test_offline_gguf_cache_fallback.py +++ b/studio/backend/tests/test_offline_gguf_cache_fallback.py @@ -951,7 +951,11 @@ class TestWaitForHealthRetriesOnReadError: calls = {"n": 0} - def fake_get(url, timeout = None): + def fake_get( + url, + timeout = None, + trust_env = None, + ): calls["n"] += 1 if calls["n"] == 1: raise httpx.ReadError("WinError 10054") diff --git a/studio/backend/tests/test_tensor_parallel.py b/studio/backend/tests/test_tensor_parallel.py index 124838602..1f09f9a09 100644 --- a/studio/backend/tests/test_tensor_parallel.py +++ b/studio/backend/tests/test_tensor_parallel.py @@ -554,6 +554,8 @@ def test_probe_mtp_decode_uses_api_key_auth(monkeypatch): backend._api_key = "secret" backend._probe_mtp_decode(timeout = 1.0) assert captured["headers"] == {"Authorization": "Bearer secret"} + + assert captured["trust_env"] is False backend._api_key = None backend._probe_mtp_decode(timeout = 1.0) assert captured["headers"] is None diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx index a05910c29..7fe98b701 100644 --- a/studio/frontend/src/components/assistant-ui/thread.tsx +++ b/studio/frontend/src/components/assistant-ui/thread.tsx @@ -3555,9 +3555,9 @@ const DiffusionCanvas: FC = () => { /** * AssistantMessage handles the display and inline-editing of AI responses. - * - * It utilizes a "Tagged Text" system ( and tags) to allow users - * to edit structured reasoning and tool outputs within a plain-text textarea + * + * It utilizes a "Tagged Text" system ( and tags) to allow users + * to edit structured reasoning and tool outputs within a plain-text textarea * while preserving the underlying data schema and tool-call metadata. */ const AssistantMessage: FC = () => { @@ -3565,7 +3565,7 @@ const AssistantMessage: FC = () => { const messageId = useAuiState(({ message }) => message.id); const messageContent = useAuiState(({ message }) => message.content); const incognito = useChatRuntimeStore((s) => s.incognito); - + // Use global store for editing state to ensure a single source of truth const editingId = useChatRuntimeStore((s) => s.editingMessageId); const setEditingId = useChatRuntimeStore((s) => s.setEditingMessageId); @@ -3588,9 +3588,9 @@ const AssistantMessage: FC = () => { const handleSave = async () => { const finalText = textareaRef.current?.value || ""; - + // Prioritize the specific thread item ID, then fallback to the global active thread ID - const remoteId = aui.threadListItem().getState().remoteId + const remoteId = aui.threadListItem().getState().remoteId || useChatRuntimeStore.getState().activeThreadId; if (!remoteId || remoteId === "" || remoteId === "/") { @@ -3601,9 +3601,9 @@ const AssistantMessage: FC = () => { try { await updateThreadMessage({ - thread: { - export: () => aui.thread().export(), - import: (data) => aui.thread().import(data) + thread: { + export: () => aui.thread().export(), + import: (data) => aui.thread().import(data) }, messageId, remoteId, @@ -3626,14 +3626,14 @@ const AssistantMessage: FC = () => {
{isEditing ? (
-