From d18b40c76e1325754c442ce0028c6f7e7af3a1b3 Mon Sep 17 00:00:00 2001 From: Stanislaw <56738034+anvme@users.noreply.github.com> Date: Wed, 21 Jan 2026 03:14:08 +0100 Subject: [PATCH] Added image processing support to all providers --- backend/app/models/request.py | 52 +++- backend/app/providers/base.py | 26 +- backend/app/providers/claude.py | 19 +- backend/app/providers/gemini.py | 23 +- backend/app/providers/openrouter.py | 11 +- backend/app/routes/chat.py | 25 +- backend/app/services/pipeline.py | 52 +++- backend/app/utils/message_helpers.py | 274 ++++++++++++++++++ backend/tests/test_message_helpers.py | 96 ++++++ frontend/src/components/ChatInput.astro | 130 +++++++-- frontend/src/components/Navbar.astro | 2 +- .../src/components/admin/AdminNavbar.astro | 8 +- frontend/src/layouts/AdminLayout.astro | 10 + frontend/src/layouts/BaseLayout.astro | 12 +- frontend/src/scripts/attachments.ts | 226 +++++++++++---- frontend/src/scripts/chat.ts | 133 ++++++++- frontend/src/scripts/image.ts | 47 ++- frontend/src/scripts/provider-set-selector.ts | 9 +- frontend/src/scripts/storage.ts | 21 +- frontend/src/scripts/types/external.d.ts | 41 ++- nginx/templates/http.conf.template | 3 + nginx/templates/https.conf.template | 3 + 22 files changed, 1068 insertions(+), 155 deletions(-) create mode 100644 backend/app/utils/message_helpers.py create mode 100644 backend/tests/test_message_helpers.py diff --git a/backend/app/models/request.py b/backend/app/models/request.py index 1cfce7c..8b183c5 100644 --- a/backend/app/models/request.py +++ b/backend/app/models/request.py @@ -1,10 +1,56 @@ -from pydantic import BaseModel, Field -from typing import List, Optional +from pydantic import BaseModel, Field, ConfigDict +from typing import List, Optional, Literal, Union + + +class TextContent(BaseModel): + """Text content part of a multimodal message""" + type: Literal["text"] = "text" + text: str + + +class ImageSource(BaseModel): + """Image source with base64 data""" + type: Literal["base64"] = "base64" + media_type: str # e.g., "image/jpeg", "image/png" + data: str # Base64-encoded image data (without data URL prefix) + + +class ImageContent(BaseModel): + """Image content part of a multimodal message""" + type: Literal["image"] = "image" + source: ImageSource class Message(BaseModel): + """Message with either text-only (string) or multimodal (array) content""" role: str = Field(..., pattern="^(user|assistant)$") - content: str + content: Union[str, List[Union[TextContent, ImageContent]]] + + model_config = ConfigDict( + # Allow both formats for backward compatibility + json_schema_extra={ + "examples": [ + { + "role": "user", + "content": "What is the capital of France?" + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": "iVBORw0KGgoAAAANSUhEUgAA..." + } + } + ] + } + ] + } + ) class ChatRequest(BaseModel): diff --git a/backend/app/providers/base.py b/backend/app/providers/base.py index 389735d..274bd23 100644 --- a/backend/app/providers/base.py +++ b/backend/app/providers/base.py @@ -57,6 +57,21 @@ class BaseProvider(ABC): """ return 3.0 if self.is_free_tier else 1.0 + @property + def supports_vision(self) -> bool: + """Whether this provider supports image inputs (vision capabilities).""" + # Override in providers that don't support vision + return True + + def _prepare_message_content(self, message: dict) -> dict: + """ + Prepare message content for this provider. + Override in subclasses for provider-specific formatting. + + Default: pass through as-is (works for Claude format) + """ + return message + @abstractmethod async def stream_chat( self, messages: list[dict], system_prompt: Optional[str] = None @@ -147,12 +162,19 @@ class OpenAIFormatProvider(BaseProvider): async def stream_chat( self, messages: list[dict], system_prompt: Optional[str] = None ) -> AsyncIterator[StreamChunk]: - """Stream chat completion using OpenAI API format.""" + """Stream chat completion using OpenAI API format with vision support.""" try: + # Import here to avoid circular dependency + from app.utils.message_helpers import format_for_openai + formatted_messages = [] if system_prompt: formatted_messages.append({"role": "system", "content": system_prompt}) - formatted_messages.extend(messages) + + # Format messages for OpenAI (converts images to image_url format) + for msg in messages: + formatted_msg = format_for_openai(msg) + formatted_messages.append(formatted_msg) payload = { "model": self.model, diff --git a/backend/app/providers/claude.py b/backend/app/providers/claude.py index ef81ee1..6e19796 100644 --- a/backend/app/providers/claude.py +++ b/backend/app/providers/claude.py @@ -32,11 +32,28 @@ class ClaudeProvider(BaseProvider): async def stream_chat( self, messages: list[dict], system_prompt: Optional[str] = None ) -> AsyncIterator[StreamChunk]: + """Stream chat responses from Claude API with vision support.""" try: + # Prepare messages (Claude accepts our universal format directly) + # Content can be string or array of content blocks + prepared_messages = [] + for msg in messages: + # Ensure content is in correct format + content = msg.get("content") + if isinstance(content, str): + # Text-only message + prepared_messages.append({"role": msg["role"], "content": content}) + elif isinstance(content, list): + # Multimodal message (text + images) + prepared_messages.append({"role": msg["role"], "content": content}) + else: + # Fallback + prepared_messages.append({"role": msg["role"], "content": str(content)}) + payload = { "model": self.model, "max_tokens": 4096, - "messages": messages, + "messages": prepared_messages, "stream": True, } if system_prompt: diff --git a/backend/app/providers/gemini.py b/backend/app/providers/gemini.py index 80120c7..114d3db 100644 --- a/backend/app/providers/gemini.py +++ b/backend/app/providers/gemini.py @@ -27,19 +27,36 @@ class GeminiProvider(BaseProvider): async def stream_chat( self, messages: list[dict], system_prompt: Optional[str] = None ) -> AsyncIterator[StreamChunk]: + """Stream chat responses from Gemini API with vision support.""" try: + # Import here to avoid circular dependency + from app.utils.message_helpers import format_for_gemini + # Convert messages to Gemini format contents = [] for msg in messages: + # Get Gemini-formatted message (with parts array) + formatted = format_for_gemini(msg) + + # Remap role: "assistant" -> "model" role = "user" if msg["role"] == "user" else "model" - contents.append({"role": role, "parts": [{"text": msg["content"]}]}) + + contents.append({ + "role": role, + "parts": formatted["parts"] + }) payload = { "contents": contents, - "generationConfig": {"maxOutputTokens": 4096}, + "generationConfig": { + "maxOutputTokens": 4096, + }, } + if system_prompt: - payload["systemInstruction"] = {"parts": [{"text": system_prompt}]} + payload["systemInstruction"] = { + "parts": [{"text": system_prompt}] + } url = f"/models/{self.model}:streamGenerateContent?key={self.api_key}&alt=sse" diff --git a/backend/app/providers/openrouter.py b/backend/app/providers/openrouter.py index adf029f..d3cc40e 100644 --- a/backend/app/providers/openrouter.py +++ b/backend/app/providers/openrouter.py @@ -47,12 +47,19 @@ class OpenRouterProvider(BaseProvider): async def stream_chat( self, messages: list[dict], system_prompt: Optional[str] = None ): - """Stream chat completion using OpenRouter API (OpenAI-compatible format).""" + """Stream chat completion using OpenRouter API (OpenAI-compatible format) with vision support.""" try: + # Import here to avoid circular dependency + from app.utils.message_helpers import format_for_openai + formatted_messages = [] if system_prompt: formatted_messages.append({"role": "system", "content": system_prompt}) - formatted_messages.extend(messages) + + # Format messages for OpenAI (converts images to image_url format) + for msg in messages: + formatted_msg = format_for_openai(msg) + formatted_messages.append(formatted_msg) payload = { "model": self.model, diff --git a/backend/app/routes/chat.py b/backend/app/routes/chat.py index fdaccbe..efbb47d 100644 --- a/backend/app/routes/chat.py +++ b/backend/app/routes/chat.py @@ -91,7 +91,16 @@ async def chat( question = "" for msg in reversed(request.messages): if msg.role == "user": - question = msg.content + # Extract text from content (handle both string and multimodal) + if isinstance(msg.content, str): + question = msg.content + elif isinstance(msg.content, list): + # Multimodal: extract text parts + text_parts = [] + for block in msg.content: + if isinstance(block, dict) and block.get("type") == "text": + text_parts.append(block.get("text", "")) + question = " ".join(text_parts).strip() or "(image)" break if not question: @@ -100,8 +109,18 @@ async def chat( # Create pipeline orchestrator orchestrator = PipelineOrchestrator(providers=providers) - # Convert messages to dict format - messages = [{"role": m.role, "content": m.content} for m in request.messages] + # Convert messages to dict format (properly serialize Pydantic models) + messages = [] + for m in request.messages: + # If content is a list of Pydantic models, convert to dicts + if isinstance(m.content, list): + content = [ + item.model_dump() if hasattr(item, 'model_dump') else item + for item in m.content + ] + else: + content = m.content + messages.append({"role": m.role, "content": content}) return StreamingResponse( orchestrator.run_pipeline(messages, question), diff --git a/backend/app/services/pipeline.py b/backend/app/services/pipeline.py index bd9b01e..afeb2ce 100644 --- a/backend/app/services/pipeline.py +++ b/backend/app/services/pipeline.py @@ -39,6 +39,7 @@ from app.services.prompts import ( from app.config import settings from app.utils.sse import format_pipeline_sse from app.utils.normalize import normalize_string_list, normalize_to_string, repair_llm_json +from app.utils.message_helpers import extract_text_only, has_images logger = logging.getLogger(__name__) @@ -97,6 +98,9 @@ class PipelineOrchestrator: """ Run the complete 4-step pipeline with SSE streaming. + IMPORTANT: Images are only sent in Step 1. + Steps 2-4 use text-only message history. + Yields SSE events for frontend to display progress. """ # Initialize pipeline state @@ -109,14 +113,39 @@ class PipelineOrchestrator: self.state.provider_to_label[provider.name] = label # ========================================================================= - # STEP 1: Initial Responses + # STEP 1: Initial Responses (with images if present) # ========================================================================= yield format_pipeline_sse("step_start", "system", {"step": 1, "name": "Initial Responses"}) self.state.current_step = 1 + # Check if any messages have images + has_any_images = any(has_images(msg) for msg in messages) + + if has_any_images: + # Filter to only vision-capable providers + vision_providers = [p for p in self.providers if p.supports_vision] + if not vision_providers: + yield format_pipeline_sse("error", "system", { + "message": "No vision-capable providers available for image analysis" + }) + return + logger.info(f"Image(s) detected. Using {len(vision_providers)} vision-capable providers: {[p.name for p in vision_providers]}") + # Temporarily use only vision providers for Step 1 + original_providers = self.providers + original_providers_by_name = self.providers_by_name.copy() + self.providers = vision_providers + # Update provider lookup dictionary + self.providers_by_name = {p.name: p for p in vision_providers} + + # Step 1: Use original messages (WITH images) async for event in self._run_step1(messages): yield event + # Restore original provider list if we filtered + if has_any_images: + self.providers = original_providers + self.providers_by_name = original_providers_by_name + yield format_pipeline_sse( "step_complete", "system", @@ -134,12 +163,21 @@ class PipelineOrchestrator: return # ========================================================================= - # STEP 2: MoA Refinement + # CONVERT TO TEXT-ONLY FOR STEPS 2-4 + # ========================================================================= + # This is CRITICAL: remove images to avoid redundant API calls + text_only_messages = [ + extract_text_only(msg) if msg.get("role") == "user" else msg + for msg in messages + ] + + # ========================================================================= + # STEP 2: MoA Refinement (text-only) # ========================================================================= yield format_pipeline_sse("step_start", "system", {"step": 2, "name": "MoA Refinement"}) self.state.current_step = 2 - async for event in self._run_step2(messages): + async for event in self._run_step2(text_only_messages): yield event yield format_pipeline_sse( @@ -158,12 +196,12 @@ class PipelineOrchestrator: return # ========================================================================= - # STEP 3: Peer Evaluation + # STEP 3: Peer Evaluation (text-only) # ========================================================================= yield format_pipeline_sse("step_start", "system", {"step": 3, "name": "Peer Evaluation"}) self.state.current_step = 3 - async for event in self._run_step3(messages): + async for event in self._run_step3(text_only_messages): yield event yield format_pipeline_sse( @@ -173,12 +211,12 @@ class PipelineOrchestrator: ) # ========================================================================= - # STEP 4: KEA Synthesis + # STEP 4: KEA Synthesis (text-only) # ========================================================================= yield format_pipeline_sse("step_start", "system", {"step": 4, "name": "KEA Synthesis"}) self.state.current_step = 4 - async for event in self._run_step4(messages): + async for event in self._run_step4(text_only_messages): yield event yield format_pipeline_sse( diff --git a/backend/app/utils/message_helpers.py b/backend/app/utils/message_helpers.py new file mode 100644 index 0000000..4e40706 --- /dev/null +++ b/backend/app/utils/message_helpers.py @@ -0,0 +1,274 @@ +"""Message format conversion utilities for multi-provider image support.""" + +from typing import Any +import re + + +def has_images(message: dict[str, Any]) -> bool: + """ + Check if a message contains images. + + Args: + message: Message dict with 'content' field + + Returns: + True if message contains image content blocks + """ + content = message.get("content") + if isinstance(content, str): + return False + if isinstance(content, list): + return any( + isinstance(item, dict) and item.get("type") == "image" + for item in content + ) + return False + + +def extract_text_only(message: dict[str, Any]) -> dict[str, Any]: + """ + Strip images from message, return text-only version. + + Args: + message: Message dict with 'content' field + + Returns: + New message dict with only text content + + Examples: + >>> msg = {"role": "user", "content": [{"type": "text", "text": "Hi"}, {"type": "image", ...}]} + >>> extract_text_only(msg) + {"role": "user", "content": "Hi"} + + >>> msg = {"role": "user", "content": "Plain text"} + >>> extract_text_only(msg) + {"role": "user", "content": "Plain text"} + """ + content = message.get("content") + + # Already text-only + if isinstance(content, str): + return message.copy() + + # Extract text parts from multimodal content + if isinstance(content, list): + text_parts = [] + for item in content: + if isinstance(item, dict) and item.get("type") == "text": + text_parts.append(item.get("text", "")) + + # Join all text parts with newlines + text_content = "\n".join(text_parts).strip() + + return { + "role": message.get("role"), + "content": text_content or "(image)" # Fallback if only images + } + + return message.copy() + + +def format_for_claude(message: dict[str, Any]) -> dict[str, Any]: + """ + Convert message to Claude's format. + + Claude format for images: + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": "iVBORw0KGgoAAAANSUhEUgAA..." # No data URL prefix + } + } + ] + } + + Args: + message: Universal format message + + Returns: + Claude-formatted message + """ + content = message.get("content") + + # Text-only: pass through + if isinstance(content, str): + return message.copy() + + # Multimodal: already in Claude format (our universal format matches Claude) + if isinstance(content, list): + return message.copy() + + return message.copy() + + +def format_for_openai(message: dict[str, Any]) -> dict[str, Any]: + """ + Convert message to OpenAI format (also used by Mistral, Grok, OpenRouter). + + OpenAI format for images: + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAA..." + } + } + ] + } + + Args: + message: Universal format message + + Returns: + OpenAI-formatted message + """ + content = message.get("content") + + # Text-only: pass through + if isinstance(content, str): + return message.copy() + + # Multimodal: convert to OpenAI format + if isinstance(content, list): + openai_content = [] + for item in content: + if isinstance(item, dict): + if item.get("type") == "text": + openai_content.append({ + "type": "text", + "text": item.get("text", "") + }) + elif item.get("type") == "image": + # Convert from Claude format to OpenAI format + source = item.get("source", {}) + media_type = source.get("media_type", "image/jpeg") + data = source.get("data", "") + + # Reconstruct data URL + data_url = f"data:{media_type};base64,{data}" + + openai_content.append({ + "type": "image_url", + "image_url": { + "url": data_url + } + }) + + return { + "role": message.get("role"), + "content": openai_content + } + + return message.copy() + + +def format_for_gemini(message: dict[str, Any]) -> dict[str, Any]: + """ + Convert message to Gemini format. + + Gemini format for images: + { + "role": "user", # Note: will be converted to "user"/"model" by provider + "parts": [ + {"text": "What's in this image?"}, + { + "inline_data": { + "mime_type": "image/jpeg", + "data": "iVBORw0KGgoAAAANSUhEUgAA..." # No data URL prefix + } + } + ] + } + + Args: + message: Universal format message + + Returns: + Gemini-formatted message (intermediate format, provider will finalize) + """ + content = message.get("content") + + # Text-only: return as single text part + if isinstance(content, str): + return { + "role": message.get("role"), + "parts": [{"text": content}] + } + + # Multimodal: convert to Gemini parts format + if isinstance(content, list): + gemini_parts = [] + for item in content: + if isinstance(item, dict): + if item.get("type") == "text": + gemini_parts.append({ + "text": item.get("text", "") + }) + elif item.get("type") == "image": + # Convert from Claude format to Gemini format + source = item.get("source", {}) + gemini_parts.append({ + "inline_data": { + "mime_type": source.get("media_type", "image/jpeg"), + "data": source.get("data", "") + } + }) + + return { + "role": message.get("role"), + "parts": gemini_parts + } + + return message.copy() + + +def get_mime_type_from_data_url(data_url: str) -> str: + """ + Extract MIME type from data URL. + + Args: + data_url: Base64 data URL (e.g., "data:image/jpeg;base64,...") + + Returns: + MIME type string (e.g., "image/jpeg") + + Examples: + >>> get_mime_type_from_data_url("data:image/png;base64,iVBORw0...") + "image/png" + >>> get_mime_type_from_data_url("invalid") + "image/jpeg" # Default fallback + """ + match = re.match(r'data:([^;]+);base64,', data_url) + if match: + return match.group(1) + return "image/jpeg" # Default fallback + + +def split_data_url(data_url: str) -> tuple[str, str]: + """ + Split data URL into MIME type and base64 data. + + Args: + data_url: Base64 data URL + + Returns: + Tuple of (mime_type, base64_data) + + Examples: + >>> split_data_url("data:image/png;base64,iVBORw0...") + ("image/png", "iVBORw0...") + """ + parts = data_url.split(',', 1) + if len(parts) == 2: + mime_type = get_mime_type_from_data_url(data_url) + return mime_type, parts[1] + return "image/jpeg", data_url # Fallback diff --git a/backend/tests/test_message_helpers.py b/backend/tests/test_message_helpers.py new file mode 100644 index 0000000..d3d8c8c --- /dev/null +++ b/backend/tests/test_message_helpers.py @@ -0,0 +1,96 @@ +"""Tests for message helper utilities.""" + +import pytest +from app.utils.message_helpers import ( + has_images, + extract_text_only, + format_for_claude, + format_for_openai, + format_for_gemini, + get_mime_type_from_data_url, + split_data_url, +) + + +def test_has_images_text_only(): + msg = {"role": "user", "content": "Hello"} + assert has_images(msg) == False + + +def test_has_images_with_image(): + msg = { + "role": "user", + "content": [ + {"type": "text", "text": "Hi"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": "abc"}} + ] + } + assert has_images(msg) == True + + +def test_extract_text_only_from_string(): + msg = {"role": "user", "content": "Hello"} + result = extract_text_only(msg) + assert result["content"] == "Hello" + + +def test_extract_text_only_from_multimodal(): + msg = { + "role": "user", + "content": [ + {"type": "text", "text": "Hello"}, + {"type": "image", "source": {}}, + {"type": "text", "text": "World"} + ] + } + result = extract_text_only(msg) + assert result["content"] == "Hello\nWorld" + + +def test_format_for_openai(): + msg = { + "role": "user", + "content": [ + {"type": "text", "text": "What's this?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0" + } + } + ] + } + result = format_for_openai(msg) + assert result["content"][0]["type"] == "text" + assert result["content"][1]["type"] == "image_url" + assert result["content"][1]["image_url"]["url"] == "data:image/png;base64,iVBORw0" + + +def test_format_for_gemini(): + msg = { + "role": "user", + "content": [ + {"type": "text", "text": "What's this?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": "abc123" + } + } + ] + } + result = format_for_gemini(msg) + assert "parts" in result + assert result["parts"][0]["text"] == "What's this?" + assert result["parts"][1]["inline_data"]["mime_type"] == "image/jpeg" + assert result["parts"][1]["inline_data"]["data"] == "abc123" + + +def test_split_data_url(): + mime, data = split_data_url("data:image/png;base64,iVBORw0KGgo") + assert mime == "image/png" + assert data == "iVBORw0KGgo" diff --git a/frontend/src/components/ChatInput.astro b/frontend/src/components/ChatInput.astro index fc37a5a..b2caebd 100644 --- a/frontend/src/components/ChatInput.astro +++ b/frontend/src/components/ChatInput.astro @@ -8,18 +8,21 @@ const { t } = Astro.props;
-
- {/* Gradient border layer - separate from interactive elements */} - - - {/* Draft History Button - only shown when drafts exist, only for new chat */} - diff --git a/frontend/src/layouts/AdminLayout.astro b/frontend/src/layouts/AdminLayout.astro index 456227d..258d8fb 100644 --- a/frontend/src/layouts/AdminLayout.astro +++ b/frontend/src/layouts/AdminLayout.astro @@ -121,6 +121,16 @@ const { title = 'KEA Research Admin' } = Astro.props; .bg-kea-subtle{background-color:rgba(50, 59, 28, 0.15) !important;} [data-bs-theme="dark"] .bg-kea-subtle{background-color:rgba(166, 141, 111, 0.2) !important;} + /* Dropdown active item - use theme-aware colors instead of blue */ + .dropdown-item.active, .dropdown-item:active { + background-color: var(--bs-secondary-bg); + color: var(--bs-body-color); + } + .dropdown-item.active:hover, .dropdown-item.active:focus { + background-color: var(--bs-tertiary-bg); + color: var(--bs-body-color); + } + /* Admin sidebar (future use) */ .admin-sidebar { width: 250px; diff --git a/frontend/src/layouts/BaseLayout.astro b/frontend/src/layouts/BaseLayout.astro index 5ab3fad..e0fc08c 100644 --- a/frontend/src/layouts/BaseLayout.astro +++ b/frontend/src/layouts/BaseLayout.astro @@ -117,6 +117,16 @@ const isRTL = RTL_LOCALES.includes(locale); [data-bs-theme="dark"] #settingsModal .nav-link{color:#a68d6f;} [data-bs-theme="dark"] #settingsModal .nav-link:hover,[data-bs-theme="dark"] #settingsModal .nav-link:focus{color:#9a7c5a;} [data-bs-theme="dark"] #settingsModal .nav-link.active{color:#a68d6f;border-bottom-color:#a68d6f;} + + /* Dropdown active item - use theme-aware colors instead of blue */ + .dropdown-item.active, .dropdown-item:active { + background-color: var(--bs-secondary-bg); + color: var(--bs-body-color); + } + .dropdown-item.active:hover, .dropdown-item.active:focus { + background-color: var(--bs-tertiary-bg); + color: var(--bs-body-color); + } @@ -124,7 +134,7 @@ const isRTL = RTL_LOCALES.includes(locale); {/* Bootstrap JS */} - + {/* Markdown rendering */} {/* PDF and DOCX export */} diff --git a/frontend/src/scripts/attachments.ts b/frontend/src/scripts/attachments.ts index 1b893d8..60aec63 100644 --- a/frontend/src/scripts/attachments.ts +++ b/frontend/src/scripts/attachments.ts @@ -50,7 +50,35 @@ export const AttachmentManager = { }); } - // All attachment features are disabled (Coming Soon) + // === Wire up image attachment button === + const attachImageBtn = document.getElementById('attachImage'); + const imageInput = document.getElementById('imageInput') as HTMLInputElement | null; + + if (attachImageBtn && imageInput) { + // Click button -> trigger file input + attachImageBtn.addEventListener('click', () => { + imageInput.click(); + }); + + // File input change -> process selected files + imageInput.addEventListener('change', async (e) => { + const files = (e.target as HTMLInputElement).files; + if (!files || files.length === 0) return; + + for (const file of Array.from(files)) { + if (file.type.startsWith('image/')) { + await this.addAttachment('image', file); + } else { + console.warn(`Skipping non-image file: ${file.name}`); + } + } + + // Reset input to allow selecting same file again + imageInput.value = ''; + }); + } + + console.log('AttachmentManager initialized (images enabled)'); }, updateIndicator(): void { @@ -69,20 +97,65 @@ export const AttachmentManager = { }, async addAttachment(type: 'image' | 'file', file: File): Promise { - let processedFile = file; + // Validate image type + if (type === 'image') { + const SUPPORTED_TYPES = ['image/jpeg', 'image/png', 'image/webp', 'image/gif']; + if (!SUPPORTED_TYPES.includes(file.type)) { + alert(`Unsupported image format: ${file.type}. Please use JPEG, PNG, WebP, or GIF.`); + return; + } - if (type === 'image' && this.compressImages && file.type.startsWith('image/')) { - try { - console.log(`Compressing image: ${file.name} (${(file.size / 1024).toFixed(1)} KB)`); - const compressedBlob = await ImageCompressor.compressAttachment(file, this.compressionOptions); - processedFile = new File([compressedBlob], file.name, { type: compressedBlob.type }); - console.log(`Compressed to: ${(processedFile.size / 1024).toFixed(1)} KB`); - } catch (error) { - console.error('Image compression failed, using original:', error); + // Check file size (max 10MB before compression) + const MAX_SIZE_BYTES = 10 * 1024 * 1024; + if (file.size > MAX_SIZE_BYTES) { + alert(`Image too large: ${(file.size / 1024 / 1024).toFixed(1)}MB. Maximum is 10MB.`); + return; + } + + // Limit: 5 images per message (Claude's limit) + const MAX_IMAGES = 5; + const currentImageCount = this.attachments.filter((att) => att.type === 'image').length; + if (currentImageCount >= MAX_IMAGES) { + alert(`Maximum ${MAX_IMAGES} images per message. Please remove some images first.`); + return; } } - this.attachments.push({ type, file: processedFile, name: file.name }); + let processedFile = file; + + // Compress image if enabled + if (type === 'image' && this.compressImages && file.type.startsWith('image/')) { + try { + console.log(`Compressing: ${file.name} (${(file.size / 1024).toFixed(1)} KB)`); + + const compressedBlob = await ImageCompressor.compressAttachment( + file, + this.compressionOptions + ); + + // Only use compressed if it's actually smaller + if (compressedBlob.size < file.size) { + processedFile = new File([compressedBlob], file.name, { type: compressedBlob.type }); + console.log( + `Compressed: ${file.name} - ${(file.size / 1024).toFixed(1)}KB → ${(processedFile.size / 1024).toFixed(1)}KB` + ); + } else { + console.warn(`Compression increased size for ${file.name}, using original`); + } + } catch (error) { + console.error('Compression failed, using original:', error); + // Continue with original file + } + } + + // Add to attachments array + this.attachments.push({ + type, + file: processedFile, + name: file.name, + }); + + // Update UI preview this.updateAttachmentPreview(); }, @@ -97,57 +170,94 @@ export const AttachmentManager = { }, updateAttachmentPreview(): void { - let preview = document.getElementById('attachmentPreview'); - if (!preview) { - preview = document.createElement('div'); - preview.id = 'attachmentPreview'; - preview.className = 'd-flex flex-wrap gap-2 mt-2'; - const wrapper = document.getElementById('chatInputWrapper'); - if (wrapper) { - wrapper.after(preview); - } + const previewContainer = document.getElementById('attachmentPreview'); + if (!previewContainer) return; + + if (this.attachments.length === 0) { + previewContainer.innerHTML = ''; + previewContainer.classList.add('d-none'); + return; } - preview.innerHTML = this.attachments.map((att, index) => { - let icon: string, label: string; - switch (att.type) { - case 'image': - icon = ''; - label = att.name || 'Image'; - break; - case 'file': - icon = ''; - label = att.name || 'File'; - break; - case 'youtube': - icon = ''; - label = 'YouTube'; - break; - case 'wikipedia': - icon = ''; - label = 'Wikipedia'; - break; - default: - icon = ''; - label = 'Attachment'; - } - return ` - ${icon} ${label} - - `; - }).join(''); + previewContainer.classList.remove('d-none'); - preview.querySelectorAll('.btn-close').forEach(btn => { + const previewHTML = this.attachments + .map((att, index) => { + let icon = ''; + let preview = ''; + + // Show thumbnail for images + if (att.type === 'image' && att.file) { + const imgUrl = URL.createObjectURL(att.file); + preview = ` + ${att.name} + `; + + // Clean up object URL after a delay + setTimeout(() => URL.revokeObjectURL(imgUrl), 60000); + } else { + preview = icon; + } + + return ` +
+ ${preview} + ${att.name} + +
+ `; + }) + .join(''); + + previewContainer.innerHTML = previewHTML; + + // Wire up remove buttons + previewContainer.querySelectorAll('.btn-close').forEach((btn) => { btn.addEventListener('click', (e) => { - const index = parseInt((e.target as HTMLElement).dataset.index || '0'); - this.attachments.splice(index, 1); - this.updateAttachmentPreview(); + const target = e.target as HTMLElement; + const index = parseInt(target.dataset.attachmentIndex || '0', 10); + this.removeAttachment(index); }); }); + }, - if (this.attachments.length === 0 && preview) { - preview.remove(); - } + removeAttachment(index: number): void { + this.attachments.splice(index, 1); + this.updateAttachmentPreview(); + }, + + /** + * Convert image file to base64 data URL + * @param file Image file + * @returns Promise resolving to base64 data URL (e.g., "data:image/jpeg;base64,...") + */ + async convertImageToBase64(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + + reader.onload = () => { + if (typeof reader.result === 'string') { + resolve(reader.result); + } else { + reject(new Error('FileReader result is not a string')); + } + }; + + reader.onerror = () => { + reject(reader.error || new Error('FileReader error')); + }; + + reader.readAsDataURL(file); + }); }, showYoutubeModal(): void { @@ -168,10 +278,12 @@ export const AttachmentManager = { } }, + /** + * Clear all attachments (called after message is sent) + */ clearAttachments(): void { this.attachments = []; - const preview = document.getElementById('attachmentPreview'); - if (preview) preview.remove(); + this.updateAttachmentPreview(); } }; diff --git a/frontend/src/scripts/chat.ts b/frontend/src/scripts/chat.ts index c283a74..5c23d76 100644 --- a/frontend/src/scripts/chat.ts +++ b/frontend/src/scripts/chat.ts @@ -11,9 +11,27 @@ import { escapeHtml, isMacOS, isTextAreaElement, isInputElement } from './utils' import { fetchStream, ApiRequestError } from './api'; import { getActiveSetId, getActiveSetName, lockSelector, unlockSelector } from './provider-set-selector'; +interface TextContent { + type: 'text'; + text: string; +} + +interface ImageSource { + type: 'base64'; + media_type: string; + data: string; +} + +interface ImageContent { + type: 'image'; + source: ImageSource; +} + +type MessageContent = string | Array; + interface Message { role: 'user' | 'assistant'; - content: string; + content: MessageContent; } const CHAT_EXPANDED_KEY = 'kea_chat_expanded'; @@ -210,7 +228,7 @@ export const ChatManager = { // Lock provider set selector and capture set name for this chat this.currentSetName = getActiveSetName(); - // Create chat with provider set name + // Create chat with provider set name (use text-only for title) this.currentChatId = await StorageUtils.createChat(message, this.currentSetName || undefined); lockSelector(); @@ -228,8 +246,59 @@ export const ChatManager = { } } + // Build message content (text-only or multimodal) + let messageContent: MessageContent; + const imageAttachments = attachments.filter(att => att.type === 'image'); + + if (imageAttachments.length > 0) { + // Multimodal message: array of content blocks + const contentBlocks: Array = []; + + // Add text block (if any) + if (message) { + contentBlocks.push({ type: 'text', text: message }); + } + + // Add image blocks + for (const att of imageAttachments) { + if (!att.file) continue; + + try { + // Convert image to base64 data URL + const dataUrl = await AttachmentManager.convertImageToBase64(att.file); + + // Split data URL into media type and base64 data + // Format: "data:image/jpeg;base64,iVBORw0..." + const match = dataUrl.match(/^data:([^;]+);base64,(.+)$/); + if (!match) { + console.error('Invalid data URL format for image:', att.name); + continue; + } + + const [, mediaType, base64Data] = match; + + // Add image content block + contentBlocks.push({ + type: 'image', + source: { + type: 'base64', + media_type: mediaType, + data: base64Data + } + }); + } catch (error) { + console.error('Failed to convert image to base64:', att.name, error); + } + } + + messageContent = contentBlocks; + } else { + // Text-only message + messageContent = message; + } + // Add user message to history - this.messages.push({ role: 'user', content: message }); + this.messages.push({ role: 'user', content: messageContent }); // Save user message to IndexedDB if (this.currentChatId !== null) { @@ -237,7 +306,7 @@ export const ChatManager = { await StorageUtils.saveMessage({ chatId: this.currentChatId, role: 'user', - content: message, + content: messageContent, timestamp: new Date().toISOString(), }); } catch (error) { @@ -246,7 +315,7 @@ export const ChatManager = { } // Display user message - this.displayUserMessage(message); + this.displayUserMessage(messageContent); // Clear input input.value = ''; @@ -347,8 +416,8 @@ export const ChatManager = { toggleSendStop(streaming: boolean): void { const sendBtn = document.getElementById('sendBtn'); const stopBtn = document.getElementById('stopBtn'); - if (sendBtn) sendBtn.style.display = streaming ? 'none' : 'inline-flex'; - if (stopBtn) stopBtn.style.display = streaming ? 'inline-flex' : 'none'; + if (sendBtn) sendBtn.classList.toggle('d-none', streaming); + if (stopBtn) stopBtn.classList.toggle('d-none', !streaming); }, async runPipeline(): Promise { @@ -431,15 +500,57 @@ export const ChatManager = { } }, - displayUserMessage(message: string): void { + displayUserMessage(content: MessageContent): void { const chatArea = document.getElementById('chatMessages'); if (!chatArea) return; + let contentHTML = ''; + + if (typeof content === 'string') { + // Text-only message + contentHTML = `
${escapeHtml(content)}
`; + } else { + // Multimodal message - separate text and images + const textParts: string[] = []; + const imageParts: string[] = []; + + for (const block of content) { + if (block.type === 'text') { + textParts.push(escapeHtml(block.text)); + } else if (block.type === 'image') { + // Reconstruct data URL for display + const dataUrl = `data:${block.source.media_type};base64,${block.source.data}`; + imageParts.push(` + User image + `); + } + } + + // Display text first (if any) + if (textParts.length > 0) { + contentHTML += `
${textParts.join('
')}
`; + } + + // Display images in horizontal scrollable row (if any) + if (imageParts.length > 0) { + contentHTML += ` +
+ ${imageParts.join('')} +
+ `; + } + } + const messageHTML = `
-
- ${escapeHtml(message)} +
+ ${contentHTML}
@@ -549,7 +660,7 @@ export const ChatManager = {
-
+
${escapeHtml(preview)}
diff --git a/frontend/src/scripts/image.ts b/frontend/src/scripts/image.ts index 5d6d108..d32df02 100644 --- a/frontend/src/scripts/image.ts +++ b/frontend/src/scripts/image.ts @@ -1,23 +1,39 @@ /** - * Image Compression Utility + * Image Compression Utility using Compressor.js */ -declare const imageCompression: any; +declare const Compressor: any; type ImageType = 'avatar' | 'background' | 'attachment'; interface CompressionPreset { - maxSizeMB?: number; - maxWidthOrHeight: number; - initialQuality: number; + maxDimension: number; + quality: number; } const PRESETS: Record = { - avatar: { maxSizeMB: 0.5, maxWidthOrHeight: 200, initialQuality: 0.9 }, - background: { maxSizeMB: 2, maxWidthOrHeight: 1920, initialQuality: 0.85 }, - attachment: { maxWidthOrHeight: 2048, initialQuality: 0.85 } + avatar: { maxDimension: 200, quality: 0.9 }, + background: { maxDimension: 1920, quality: 0.85 }, + attachment: { maxDimension: 2048, quality: 0.85 } }; +/** + * Promise wrapper for Compressor.js callback API + */ +function compressImage(file: File, options: any): Promise { + return new Promise((resolve, reject) => { + new Compressor(file, { + ...options, + success(result: File | Blob) { + resolve(result); + }, + error(err: Error) { + reject(err); + } + }); + }); +} + export const ImageCompressor = { async compress( file: File, @@ -25,7 +41,7 @@ export const ImageCompressor = { customOptions: { quality?: number; maxWidthOrHeight?: number } = {} ): Promise { const preset = PRESETS[type]; - let maxDim = customOptions.maxWidthOrHeight || preset.maxWidthOrHeight; + let maxDim = customOptions.maxWidthOrHeight || preset.maxDimension; // Background: adjust for high-DPI displays if (type === 'background') { @@ -34,15 +50,16 @@ export const ImageCompressor = { } const options = { - maxSizeMB: preset.maxSizeMB, - maxWidthOrHeight: maxDim, - useWebWorker: true, - initialQuality: customOptions.quality ? customOptions.quality / 100 : preset.initialQuality + maxWidth: maxDim, + maxHeight: maxDim, + quality: customOptions.quality ? customOptions.quality / 100 : preset.quality, + checkOrientation: true, // Auto-rotate based on EXIF + mimeType: 'auto' // Preserve original format }; try { - const compressed = await imageCompression(file, options); - return new Blob([await compressed.arrayBuffer()], { type: compressed.type }); + const compressed = await compressImage(file, options); + return compressed instanceof Blob ? compressed : new Blob([compressed]); } catch (error) { console.error(`${type} compression error:`, error); throw error; diff --git a/frontend/src/scripts/provider-set-selector.ts b/frontend/src/scripts/provider-set-selector.ts index 5f8c883..9abd604 100644 --- a/frontend/src/scripts/provider-set-selector.ts +++ b/frontend/src/scripts/provider-set-selector.ts @@ -119,15 +119,16 @@ function renderDropdown(): void { menu.innerHTML = providerSets .map(set => { const isActive = set.id === activeSetId; - const systemBadge = set.is_system ? 'System' : ''; - const countBadge = `${set.provider_count}`; + const systemBadge = set.is_system ? 'System' : ''; + const countBadge = `${set.provider_count}`; return `
    • `; diff --git a/frontend/src/scripts/storage.ts b/frontend/src/scripts/storage.ts index 3fecb40..bcdea88 100644 --- a/frontend/src/scripts/storage.ts +++ b/frontend/src/scripts/storage.ts @@ -46,11 +46,30 @@ export interface PipelineData { errors: Record; } +// Multimodal content types (matches backend format) +export interface TextContent { + type: 'text'; + text: string; +} + +export interface ImageSource { + type: 'base64'; + media_type: string; + data: string; +} + +export interface ImageContent { + type: 'image'; + source: ImageSource; +} + +export type MessageContent = string | Array; + export interface ChatMessage { id?: number; chatId: number; role: 'user' | 'assistant'; - content: string; + content: MessageContent; // Updated to support multimodal content providerResponses?: Record; timestamp: string; pipelineData?: PipelineData; diff --git a/frontend/src/scripts/types/external.d.ts b/frontend/src/scripts/types/external.d.ts index 234b049..4c45f43 100644 --- a/frontend/src/scripts/types/external.d.ts +++ b/frontend/src/scripts/types/external.d.ts @@ -57,22 +57,31 @@ declare namespace bootstrap { } } -// Image Compression Library -interface ImageCompressionOptions { - maxSizeMB?: number; - maxWidthOrHeight?: number; - useWebWorker?: boolean; - initialQuality?: number; - exifOrientation?: number; - fileType?: string; +// Compressor.js - Image Compression Library +interface CompressorOptions { + strict?: boolean; + checkOrientation?: boolean; + retainExif?: boolean; + maxWidth?: number; + maxHeight?: number; + minWidth?: number; + minHeight?: number; + width?: number; + height?: number; + resize?: 'none' | 'contain' | 'cover'; + quality?: number; + mimeType?: string; + convertTypes?: string | string[]; + convertSize?: number; + beforeDraw?(context: CanvasRenderingContext2D, canvas: HTMLCanvasElement): void; + drew?(context: CanvasRenderingContext2D, canvas: HTMLCanvasElement): void; + success?(result: File | Blob): void; + error?(error: Error): void; } -declare function imageCompression( - file: File, - options: ImageCompressionOptions -): Promise; - -declare namespace imageCompression { - function getExifOrientation(file: File): Promise; - function drawImageInCanvas(img: HTMLImageElement): HTMLCanvasElement; +declare class Compressor { + constructor(file: File | Blob, options?: CompressorOptions); + abort(): void; + static noConflict(): typeof Compressor; + static setDefaults(options: CompressorOptions): void; } diff --git a/nginx/templates/http.conf.template b/nginx/templates/http.conf.template index 4e3d839..cdb4f37 100644 --- a/nginx/templates/http.conf.template +++ b/nginx/templates/http.conf.template @@ -6,6 +6,9 @@ server { root /var/www/html; + # Allow larger uploads for image attachments + client_max_body_size 48M; + gzip on; gzip_types text/plain text/css application/json application/javascript text/xml application/xml text/event-stream; diff --git a/nginx/templates/https.conf.template b/nginx/templates/https.conf.template index 9b561f8..7331134 100644 --- a/nginx/templates/https.conf.template +++ b/nginx/templates/https.conf.template @@ -22,6 +22,9 @@ server { add_header X-Frame-Options DENY always; add_header Referrer-Policy strict-origin-when-cross-origin always; + # Allow larger uploads for image attachments + client_max_body_size 48M; + gzip on; gzip_types text/plain text/css application/json application/javascript text/xml application/xml text/event-stream;