mirror of
https://github.com/KeaBase/kea-research.git
synced 2026-07-09 17:38:30 +00:00
Added image processing support to all providers
This commit is contained in:
Stanislaw
parent
35c997742d
commit
d18b40c76e
22 changed files with 1068 additions and 155 deletions
|
|
@ -1,10 +1,56 @@
|
|||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional
|
||||
from pydantic import BaseModel, Field, ConfigDict
|
||||
from typing import List, Optional, Literal, Union
|
||||
|
||||
|
||||
class TextContent(BaseModel):
|
||||
"""Text content part of a multimodal message"""
|
||||
type: Literal["text"] = "text"
|
||||
text: str
|
||||
|
||||
|
||||
class ImageSource(BaseModel):
|
||||
"""Image source with base64 data"""
|
||||
type: Literal["base64"] = "base64"
|
||||
media_type: str # e.g., "image/jpeg", "image/png"
|
||||
data: str # Base64-encoded image data (without data URL prefix)
|
||||
|
||||
|
||||
class ImageContent(BaseModel):
|
||||
"""Image content part of a multimodal message"""
|
||||
type: Literal["image"] = "image"
|
||||
source: ImageSource
|
||||
|
||||
|
||||
class Message(BaseModel):
|
||||
"""Message with either text-only (string) or multimodal (array) content"""
|
||||
role: str = Field(..., pattern="^(user|assistant)$")
|
||||
content: str
|
||||
content: Union[str, List[Union[TextContent, ImageContent]]]
|
||||
|
||||
model_config = ConfigDict(
|
||||
# Allow both formats for backward compatibility
|
||||
json_schema_extra={
|
||||
"examples": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is the capital of France?"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
{
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/jpeg",
|
||||
"data": "iVBORw0KGgoAAAANSUhEUgAA..."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class ChatRequest(BaseModel):
|
||||
|
|
|
|||
|
|
@ -57,6 +57,21 @@ class BaseProvider(ABC):
|
|||
"""
|
||||
return 3.0 if self.is_free_tier else 1.0
|
||||
|
||||
@property
|
||||
def supports_vision(self) -> bool:
|
||||
"""Whether this provider supports image inputs (vision capabilities)."""
|
||||
# Override in providers that don't support vision
|
||||
return True
|
||||
|
||||
def _prepare_message_content(self, message: dict) -> dict:
|
||||
"""
|
||||
Prepare message content for this provider.
|
||||
Override in subclasses for provider-specific formatting.
|
||||
|
||||
Default: pass through as-is (works for Claude format)
|
||||
"""
|
||||
return message
|
||||
|
||||
@abstractmethod
|
||||
async def stream_chat(
|
||||
self, messages: list[dict], system_prompt: Optional[str] = None
|
||||
|
|
@ -147,12 +162,19 @@ class OpenAIFormatProvider(BaseProvider):
|
|||
async def stream_chat(
|
||||
self, messages: list[dict], system_prompt: Optional[str] = None
|
||||
) -> AsyncIterator[StreamChunk]:
|
||||
"""Stream chat completion using OpenAI API format."""
|
||||
"""Stream chat completion using OpenAI API format with vision support."""
|
||||
try:
|
||||
# Import here to avoid circular dependency
|
||||
from app.utils.message_helpers import format_for_openai
|
||||
|
||||
formatted_messages = []
|
||||
if system_prompt:
|
||||
formatted_messages.append({"role": "system", "content": system_prompt})
|
||||
formatted_messages.extend(messages)
|
||||
|
||||
# Format messages for OpenAI (converts images to image_url format)
|
||||
for msg in messages:
|
||||
formatted_msg = format_for_openai(msg)
|
||||
formatted_messages.append(formatted_msg)
|
||||
|
||||
payload = {
|
||||
"model": self.model,
|
||||
|
|
|
|||
|
|
@ -32,11 +32,28 @@ class ClaudeProvider(BaseProvider):
|
|||
async def stream_chat(
|
||||
self, messages: list[dict], system_prompt: Optional[str] = None
|
||||
) -> AsyncIterator[StreamChunk]:
|
||||
"""Stream chat responses from Claude API with vision support."""
|
||||
try:
|
||||
# Prepare messages (Claude accepts our universal format directly)
|
||||
# Content can be string or array of content blocks
|
||||
prepared_messages = []
|
||||
for msg in messages:
|
||||
# Ensure content is in correct format
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str):
|
||||
# Text-only message
|
||||
prepared_messages.append({"role": msg["role"], "content": content})
|
||||
elif isinstance(content, list):
|
||||
# Multimodal message (text + images)
|
||||
prepared_messages.append({"role": msg["role"], "content": content})
|
||||
else:
|
||||
# Fallback
|
||||
prepared_messages.append({"role": msg["role"], "content": str(content)})
|
||||
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"max_tokens": 4096,
|
||||
"messages": messages,
|
||||
"messages": prepared_messages,
|
||||
"stream": True,
|
||||
}
|
||||
if system_prompt:
|
||||
|
|
|
|||
|
|
@ -27,19 +27,36 @@ class GeminiProvider(BaseProvider):
|
|||
async def stream_chat(
|
||||
self, messages: list[dict], system_prompt: Optional[str] = None
|
||||
) -> AsyncIterator[StreamChunk]:
|
||||
"""Stream chat responses from Gemini API with vision support."""
|
||||
try:
|
||||
# Import here to avoid circular dependency
|
||||
from app.utils.message_helpers import format_for_gemini
|
||||
|
||||
# Convert messages to Gemini format
|
||||
contents = []
|
||||
for msg in messages:
|
||||
# Get Gemini-formatted message (with parts array)
|
||||
formatted = format_for_gemini(msg)
|
||||
|
||||
# Remap role: "assistant" -> "model"
|
||||
role = "user" if msg["role"] == "user" else "model"
|
||||
contents.append({"role": role, "parts": [{"text": msg["content"]}]})
|
||||
|
||||
contents.append({
|
||||
"role": role,
|
||||
"parts": formatted["parts"]
|
||||
})
|
||||
|
||||
payload = {
|
||||
"contents": contents,
|
||||
"generationConfig": {"maxOutputTokens": 4096},
|
||||
"generationConfig": {
|
||||
"maxOutputTokens": 4096,
|
||||
},
|
||||
}
|
||||
|
||||
if system_prompt:
|
||||
payload["systemInstruction"] = {"parts": [{"text": system_prompt}]}
|
||||
payload["systemInstruction"] = {
|
||||
"parts": [{"text": system_prompt}]
|
||||
}
|
||||
|
||||
url = f"/models/{self.model}:streamGenerateContent?key={self.api_key}&alt=sse"
|
||||
|
||||
|
|
|
|||
|
|
@ -47,12 +47,19 @@ class OpenRouterProvider(BaseProvider):
|
|||
async def stream_chat(
|
||||
self, messages: list[dict], system_prompt: Optional[str] = None
|
||||
):
|
||||
"""Stream chat completion using OpenRouter API (OpenAI-compatible format)."""
|
||||
"""Stream chat completion using OpenRouter API (OpenAI-compatible format) with vision support."""
|
||||
try:
|
||||
# Import here to avoid circular dependency
|
||||
from app.utils.message_helpers import format_for_openai
|
||||
|
||||
formatted_messages = []
|
||||
if system_prompt:
|
||||
formatted_messages.append({"role": "system", "content": system_prompt})
|
||||
formatted_messages.extend(messages)
|
||||
|
||||
# Format messages for OpenAI (converts images to image_url format)
|
||||
for msg in messages:
|
||||
formatted_msg = format_for_openai(msg)
|
||||
formatted_messages.append(formatted_msg)
|
||||
|
||||
payload = {
|
||||
"model": self.model,
|
||||
|
|
|
|||
|
|
@ -91,7 +91,16 @@ async def chat(
|
|||
question = ""
|
||||
for msg in reversed(request.messages):
|
||||
if msg.role == "user":
|
||||
question = msg.content
|
||||
# Extract text from content (handle both string and multimodal)
|
||||
if isinstance(msg.content, str):
|
||||
question = msg.content
|
||||
elif isinstance(msg.content, list):
|
||||
# Multimodal: extract text parts
|
||||
text_parts = []
|
||||
for block in msg.content:
|
||||
if isinstance(block, dict) and block.get("type") == "text":
|
||||
text_parts.append(block.get("text", ""))
|
||||
question = " ".join(text_parts).strip() or "(image)"
|
||||
break
|
||||
|
||||
if not question:
|
||||
|
|
@ -100,8 +109,18 @@ async def chat(
|
|||
# Create pipeline orchestrator
|
||||
orchestrator = PipelineOrchestrator(providers=providers)
|
||||
|
||||
# Convert messages to dict format
|
||||
messages = [{"role": m.role, "content": m.content} for m in request.messages]
|
||||
# Convert messages to dict format (properly serialize Pydantic models)
|
||||
messages = []
|
||||
for m in request.messages:
|
||||
# If content is a list of Pydantic models, convert to dicts
|
||||
if isinstance(m.content, list):
|
||||
content = [
|
||||
item.model_dump() if hasattr(item, 'model_dump') else item
|
||||
for item in m.content
|
||||
]
|
||||
else:
|
||||
content = m.content
|
||||
messages.append({"role": m.role, "content": content})
|
||||
|
||||
return StreamingResponse(
|
||||
orchestrator.run_pipeline(messages, question),
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from app.services.prompts import (
|
|||
from app.config import settings
|
||||
from app.utils.sse import format_pipeline_sse
|
||||
from app.utils.normalize import normalize_string_list, normalize_to_string, repair_llm_json
|
||||
from app.utils.message_helpers import extract_text_only, has_images
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -97,6 +98,9 @@ class PipelineOrchestrator:
|
|||
"""
|
||||
Run the complete 4-step pipeline with SSE streaming.
|
||||
|
||||
IMPORTANT: Images are only sent in Step 1.
|
||||
Steps 2-4 use text-only message history.
|
||||
|
||||
Yields SSE events for frontend to display progress.
|
||||
"""
|
||||
# Initialize pipeline state
|
||||
|
|
@ -109,14 +113,39 @@ class PipelineOrchestrator:
|
|||
self.state.provider_to_label[provider.name] = label
|
||||
|
||||
# =========================================================================
|
||||
# STEP 1: Initial Responses
|
||||
# STEP 1: Initial Responses (with images if present)
|
||||
# =========================================================================
|
||||
yield format_pipeline_sse("step_start", "system", {"step": 1, "name": "Initial Responses"})
|
||||
self.state.current_step = 1
|
||||
|
||||
# Check if any messages have images
|
||||
has_any_images = any(has_images(msg) for msg in messages)
|
||||
|
||||
if has_any_images:
|
||||
# Filter to only vision-capable providers
|
||||
vision_providers = [p for p in self.providers if p.supports_vision]
|
||||
if not vision_providers:
|
||||
yield format_pipeline_sse("error", "system", {
|
||||
"message": "No vision-capable providers available for image analysis"
|
||||
})
|
||||
return
|
||||
logger.info(f"Image(s) detected. Using {len(vision_providers)} vision-capable providers: {[p.name for p in vision_providers]}")
|
||||
# Temporarily use only vision providers for Step 1
|
||||
original_providers = self.providers
|
||||
original_providers_by_name = self.providers_by_name.copy()
|
||||
self.providers = vision_providers
|
||||
# Update provider lookup dictionary
|
||||
self.providers_by_name = {p.name: p for p in vision_providers}
|
||||
|
||||
# Step 1: Use original messages (WITH images)
|
||||
async for event in self._run_step1(messages):
|
||||
yield event
|
||||
|
||||
# Restore original provider list if we filtered
|
||||
if has_any_images:
|
||||
self.providers = original_providers
|
||||
self.providers_by_name = original_providers_by_name
|
||||
|
||||
yield format_pipeline_sse(
|
||||
"step_complete",
|
||||
"system",
|
||||
|
|
@ -134,12 +163,21 @@ class PipelineOrchestrator:
|
|||
return
|
||||
|
||||
# =========================================================================
|
||||
# STEP 2: MoA Refinement
|
||||
# CONVERT TO TEXT-ONLY FOR STEPS 2-4
|
||||
# =========================================================================
|
||||
# This is CRITICAL: remove images to avoid redundant API calls
|
||||
text_only_messages = [
|
||||
extract_text_only(msg) if msg.get("role") == "user" else msg
|
||||
for msg in messages
|
||||
]
|
||||
|
||||
# =========================================================================
|
||||
# STEP 2: MoA Refinement (text-only)
|
||||
# =========================================================================
|
||||
yield format_pipeline_sse("step_start", "system", {"step": 2, "name": "MoA Refinement"})
|
||||
self.state.current_step = 2
|
||||
|
||||
async for event in self._run_step2(messages):
|
||||
async for event in self._run_step2(text_only_messages):
|
||||
yield event
|
||||
|
||||
yield format_pipeline_sse(
|
||||
|
|
@ -158,12 +196,12 @@ class PipelineOrchestrator:
|
|||
return
|
||||
|
||||
# =========================================================================
|
||||
# STEP 3: Peer Evaluation
|
||||
# STEP 3: Peer Evaluation (text-only)
|
||||
# =========================================================================
|
||||
yield format_pipeline_sse("step_start", "system", {"step": 3, "name": "Peer Evaluation"})
|
||||
self.state.current_step = 3
|
||||
|
||||
async for event in self._run_step3(messages):
|
||||
async for event in self._run_step3(text_only_messages):
|
||||
yield event
|
||||
|
||||
yield format_pipeline_sse(
|
||||
|
|
@ -173,12 +211,12 @@ class PipelineOrchestrator:
|
|||
)
|
||||
|
||||
# =========================================================================
|
||||
# STEP 4: KEA Synthesis
|
||||
# STEP 4: KEA Synthesis (text-only)
|
||||
# =========================================================================
|
||||
yield format_pipeline_sse("step_start", "system", {"step": 4, "name": "KEA Synthesis"})
|
||||
self.state.current_step = 4
|
||||
|
||||
async for event in self._run_step4(messages):
|
||||
async for event in self._run_step4(text_only_messages):
|
||||
yield event
|
||||
|
||||
yield format_pipeline_sse(
|
||||
|
|
|
|||
274
backend/app/utils/message_helpers.py
Normal file
274
backend/app/utils/message_helpers.py
Normal file
|
|
@ -0,0 +1,274 @@
|
|||
"""Message format conversion utilities for multi-provider image support."""
|
||||
|
||||
from typing import Any
|
||||
import re
|
||||
|
||||
|
||||
def has_images(message: dict[str, Any]) -> bool:
|
||||
"""
|
||||
Check if a message contains images.
|
||||
|
||||
Args:
|
||||
message: Message dict with 'content' field
|
||||
|
||||
Returns:
|
||||
True if message contains image content blocks
|
||||
"""
|
||||
content = message.get("content")
|
||||
if isinstance(content, str):
|
||||
return False
|
||||
if isinstance(content, list):
|
||||
return any(
|
||||
isinstance(item, dict) and item.get("type") == "image"
|
||||
for item in content
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def extract_text_only(message: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Strip images from message, return text-only version.
|
||||
|
||||
Args:
|
||||
message: Message dict with 'content' field
|
||||
|
||||
Returns:
|
||||
New message dict with only text content
|
||||
|
||||
Examples:
|
||||
>>> msg = {"role": "user", "content": [{"type": "text", "text": "Hi"}, {"type": "image", ...}]}
|
||||
>>> extract_text_only(msg)
|
||||
{"role": "user", "content": "Hi"}
|
||||
|
||||
>>> msg = {"role": "user", "content": "Plain text"}
|
||||
>>> extract_text_only(msg)
|
||||
{"role": "user", "content": "Plain text"}
|
||||
"""
|
||||
content = message.get("content")
|
||||
|
||||
# Already text-only
|
||||
if isinstance(content, str):
|
||||
return message.copy()
|
||||
|
||||
# Extract text parts from multimodal content
|
||||
if isinstance(content, list):
|
||||
text_parts = []
|
||||
for item in content:
|
||||
if isinstance(item, dict) and item.get("type") == "text":
|
||||
text_parts.append(item.get("text", ""))
|
||||
|
||||
# Join all text parts with newlines
|
||||
text_content = "\n".join(text_parts).strip()
|
||||
|
||||
return {
|
||||
"role": message.get("role"),
|
||||
"content": text_content or "(image)" # Fallback if only images
|
||||
}
|
||||
|
||||
return message.copy()
|
||||
|
||||
|
||||
def format_for_claude(message: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Convert message to Claude's format.
|
||||
|
||||
Claude format for images:
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
{
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/jpeg",
|
||||
"data": "iVBORw0KGgoAAAANSUhEUgAA..." # No data URL prefix
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Args:
|
||||
message: Universal format message
|
||||
|
||||
Returns:
|
||||
Claude-formatted message
|
||||
"""
|
||||
content = message.get("content")
|
||||
|
||||
# Text-only: pass through
|
||||
if isinstance(content, str):
|
||||
return message.copy()
|
||||
|
||||
# Multimodal: already in Claude format (our universal format matches Claude)
|
||||
if isinstance(content, list):
|
||||
return message.copy()
|
||||
|
||||
return message.copy()
|
||||
|
||||
|
||||
def format_for_openai(message: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Convert message to OpenAI format (also used by Mistral, Grok, OpenRouter).
|
||||
|
||||
OpenAI format for images:
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAA..."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Args:
|
||||
message: Universal format message
|
||||
|
||||
Returns:
|
||||
OpenAI-formatted message
|
||||
"""
|
||||
content = message.get("content")
|
||||
|
||||
# Text-only: pass through
|
||||
if isinstance(content, str):
|
||||
return message.copy()
|
||||
|
||||
# Multimodal: convert to OpenAI format
|
||||
if isinstance(content, list):
|
||||
openai_content = []
|
||||
for item in content:
|
||||
if isinstance(item, dict):
|
||||
if item.get("type") == "text":
|
||||
openai_content.append({
|
||||
"type": "text",
|
||||
"text": item.get("text", "")
|
||||
})
|
||||
elif item.get("type") == "image":
|
||||
# Convert from Claude format to OpenAI format
|
||||
source = item.get("source", {})
|
||||
media_type = source.get("media_type", "image/jpeg")
|
||||
data = source.get("data", "")
|
||||
|
||||
# Reconstruct data URL
|
||||
data_url = f"data:{media_type};base64,{data}"
|
||||
|
||||
openai_content.append({
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": data_url
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
"role": message.get("role"),
|
||||
"content": openai_content
|
||||
}
|
||||
|
||||
return message.copy()
|
||||
|
||||
|
||||
def format_for_gemini(message: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Convert message to Gemini format.
|
||||
|
||||
Gemini format for images:
|
||||
{
|
||||
"role": "user", # Note: will be converted to "user"/"model" by provider
|
||||
"parts": [
|
||||
{"text": "What's in this image?"},
|
||||
{
|
||||
"inline_data": {
|
||||
"mime_type": "image/jpeg",
|
||||
"data": "iVBORw0KGgoAAAANSUhEUgAA..." # No data URL prefix
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Args:
|
||||
message: Universal format message
|
||||
|
||||
Returns:
|
||||
Gemini-formatted message (intermediate format, provider will finalize)
|
||||
"""
|
||||
content = message.get("content")
|
||||
|
||||
# Text-only: return as single text part
|
||||
if isinstance(content, str):
|
||||
return {
|
||||
"role": message.get("role"),
|
||||
"parts": [{"text": content}]
|
||||
}
|
||||
|
||||
# Multimodal: convert to Gemini parts format
|
||||
if isinstance(content, list):
|
||||
gemini_parts = []
|
||||
for item in content:
|
||||
if isinstance(item, dict):
|
||||
if item.get("type") == "text":
|
||||
gemini_parts.append({
|
||||
"text": item.get("text", "")
|
||||
})
|
||||
elif item.get("type") == "image":
|
||||
# Convert from Claude format to Gemini format
|
||||
source = item.get("source", {})
|
||||
gemini_parts.append({
|
||||
"inline_data": {
|
||||
"mime_type": source.get("media_type", "image/jpeg"),
|
||||
"data": source.get("data", "")
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
"role": message.get("role"),
|
||||
"parts": gemini_parts
|
||||
}
|
||||
|
||||
return message.copy()
|
||||
|
||||
|
||||
def get_mime_type_from_data_url(data_url: str) -> str:
|
||||
"""
|
||||
Extract MIME type from data URL.
|
||||
|
||||
Args:
|
||||
data_url: Base64 data URL (e.g., "data:image/jpeg;base64,...")
|
||||
|
||||
Returns:
|
||||
MIME type string (e.g., "image/jpeg")
|
||||
|
||||
Examples:
|
||||
>>> get_mime_type_from_data_url("data:image/png;base64,iVBORw0...")
|
||||
"image/png"
|
||||
>>> get_mime_type_from_data_url("invalid")
|
||||
"image/jpeg" # Default fallback
|
||||
"""
|
||||
match = re.match(r'data:([^;]+);base64,', data_url)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return "image/jpeg" # Default fallback
|
||||
|
||||
|
||||
def split_data_url(data_url: str) -> tuple[str, str]:
|
||||
"""
|
||||
Split data URL into MIME type and base64 data.
|
||||
|
||||
Args:
|
||||
data_url: Base64 data URL
|
||||
|
||||
Returns:
|
||||
Tuple of (mime_type, base64_data)
|
||||
|
||||
Examples:
|
||||
>>> split_data_url("data:image/png;base64,iVBORw0...")
|
||||
("image/png", "iVBORw0...")
|
||||
"""
|
||||
parts = data_url.split(',', 1)
|
||||
if len(parts) == 2:
|
||||
mime_type = get_mime_type_from_data_url(data_url)
|
||||
return mime_type, parts[1]
|
||||
return "image/jpeg", data_url # Fallback
|
||||
96
backend/tests/test_message_helpers.py
Normal file
96
backend/tests/test_message_helpers.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
"""Tests for message helper utilities."""
|
||||
|
||||
import pytest
|
||||
from app.utils.message_helpers import (
|
||||
has_images,
|
||||
extract_text_only,
|
||||
format_for_claude,
|
||||
format_for_openai,
|
||||
format_for_gemini,
|
||||
get_mime_type_from_data_url,
|
||||
split_data_url,
|
||||
)
|
||||
|
||||
|
||||
def test_has_images_text_only():
|
||||
msg = {"role": "user", "content": "Hello"}
|
||||
assert has_images(msg) == False
|
||||
|
||||
|
||||
def test_has_images_with_image():
|
||||
msg = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Hi"},
|
||||
{"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": "abc"}}
|
||||
]
|
||||
}
|
||||
assert has_images(msg) == True
|
||||
|
||||
|
||||
def test_extract_text_only_from_string():
|
||||
msg = {"role": "user", "content": "Hello"}
|
||||
result = extract_text_only(msg)
|
||||
assert result["content"] == "Hello"
|
||||
|
||||
|
||||
def test_extract_text_only_from_multimodal():
|
||||
msg = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Hello"},
|
||||
{"type": "image", "source": {}},
|
||||
{"type": "text", "text": "World"}
|
||||
]
|
||||
}
|
||||
result = extract_text_only(msg)
|
||||
assert result["content"] == "Hello\nWorld"
|
||||
|
||||
|
||||
def test_format_for_openai():
|
||||
msg = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's this?"},
|
||||
{
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/png",
|
||||
"data": "iVBORw0"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
result = format_for_openai(msg)
|
||||
assert result["content"][0]["type"] == "text"
|
||||
assert result["content"][1]["type"] == "image_url"
|
||||
assert result["content"][1]["image_url"]["url"] == "data:image/png;base64,iVBORw0"
|
||||
|
||||
|
||||
def test_format_for_gemini():
|
||||
msg = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's this?"},
|
||||
{
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/jpeg",
|
||||
"data": "abc123"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
result = format_for_gemini(msg)
|
||||
assert "parts" in result
|
||||
assert result["parts"][0]["text"] == "What's this?"
|
||||
assert result["parts"][1]["inline_data"]["mime_type"] == "image/jpeg"
|
||||
assert result["parts"][1]["inline_data"]["data"] == "abc123"
|
||||
|
||||
|
||||
def test_split_data_url():
|
||||
mime, data = split_data_url("data:image/png;base64,iVBORw0KGgo")
|
||||
assert mime == "image/png"
|
||||
assert data == "iVBORw0KGgo"
|
||||
|
|
@ -8,18 +8,21 @@ const { t } = Astro.props;
|
|||
|
||||
<div class="d-flex flex-column align-items-center w-100 mt-5" id="chatInputContainer">
|
||||
<div class="chat-input-row" id="chatInputRow">
|
||||
<div class="mb-3 chat-input-wrapper" id="chatInputWrapper">
|
||||
{/* Gradient border layer - separate from interactive elements */}
|
||||
<div class="gradient-border-layer" aria-hidden="true"></div>
|
||||
<textarea class="form-control rounded-4 shadow-sm pe-5" rows="3" placeholder={t('chat.placeholder')} id="chatInput"></textarea>
|
||||
{/* Draft History Button - only shown when drafts exist, only for new chat */}
|
||||
<div class="dropdown position-absolute bottom-0 end-0 mb-2 chat-btn-overlay d-none" id="draftHistoryWrapper" style="right: 5.5rem !important;">
|
||||
{/* Attachment Preview - placed above textarea */}
|
||||
<div id="attachmentPreview" class="d-none attachment-preview-container"></div>
|
||||
<div class="chat-input-content-wrapper">
|
||||
<div class="mb-3 chat-input-wrapper" id="chatInputWrapper">
|
||||
{/* Gradient border layer - separate from interactive elements */}
|
||||
<div class="gradient-border-layer" aria-hidden="true"></div>
|
||||
<textarea class="form-control rounded-4 shadow-sm pe-5" rows="3" placeholder={t('chat.placeholder')} id="chatInput"></textarea>
|
||||
{/* Draft History Button - only shown when drafts exist, only for new chat */}
|
||||
<div class="dropdown position-absolute bottom-0 end-0 mb-2 chat-btn-overlay d-none draft-history-dropdown" id="draftHistoryWrapper">
|
||||
<button class="btn btn-link text-body-secondary p-1" type="button"
|
||||
id="draftHistoryBtn" data-bs-toggle="dropdown" aria-expanded="false"
|
||||
title={t('chat.savedDrafts') || 'Saved drafts'}>
|
||||
<i class="bi bi-chat-dots fs-5"></i>
|
||||
</button>
|
||||
<div class="dropdown-menu dropdown-menu-end p-2" id="draftHistoryDropdown" style="min-width: 320px; max-height: 350px; overflow-y: auto;">
|
||||
<div class="dropdown-menu dropdown-menu-end p-2 draft-history-menu" id="draftHistoryDropdown">
|
||||
<div class="d-flex justify-content-between align-items-center mb-2 px-1">
|
||||
<small class="text-muted fw-semibold">{t('chat.savedDrafts') || 'Saved Drafts'}</small>
|
||||
<button class="btn btn-sm btn-link text-danger p-0 text-decoration-none" id="clearAllDrafts" type="button">
|
||||
|
|
@ -41,11 +44,10 @@ const { t } = Astro.props;
|
|||
<button class="btn btn-link text-body-secondary p-1" type="button" data-bs-toggle="dropdown" aria-expanded="false" title={t('chat.addAttachment')}>
|
||||
<i class="bi bi-plus-circle fs-5"></i>
|
||||
</button>
|
||||
<ul class="dropdown-menu dropdown-menu-end">
|
||||
<ul class="dropdown-menu dropdown-menu-end shadow-sm">
|
||||
<li>
|
||||
<button class="dropdown-item d-flex align-items-center gap-2 disabled" type="button" id="attachImage" disabled>
|
||||
<i class="bi bi-image text-success opacity-50"></i> {t('attachments.attachImage')}
|
||||
<span class="badge bg-secondary ms-auto">{t('common.soon')}</span>
|
||||
<button class="dropdown-item d-flex align-items-center gap-2" type="button" id="attachImage">
|
||||
<i class="bi bi-image text-success"></i> {t('attachments.attachImage')}
|
||||
</button>
|
||||
</li>
|
||||
<li>
|
||||
|
|
@ -82,16 +84,17 @@ const { t } = Astro.props;
|
|||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
{/* Hidden file inputs */}
|
||||
<input type="file" id="imageInput" class="d-none" accept="image/*">
|
||||
<input type="file" id="fileInput" class="d-none">
|
||||
{/* Hidden file inputs */}
|
||||
<input type="file" id="imageInput" class="d-none" accept="image/*" multiple>
|
||||
<input type="file" id="fileInput" class="d-none">
|
||||
</div>
|
||||
<button class="btn btn-kea rounded-4 chat-send-btn" id="sendBtn">
|
||||
<i class="bi bi-search me-1"></i> <span class="send-btn-text">{t('chat.research') || 'Research'}</span>
|
||||
</button>
|
||||
<button class="btn btn-danger rounded-4 chat-send-btn d-none" id="stopBtn">
|
||||
<i class="bi bi-stop-fill"></i> <span class="send-btn-text">Stop</span>
|
||||
</button>
|
||||
</div>
|
||||
<button class="btn btn-kea rounded-4 chat-send-btn" id="sendBtn">
|
||||
<i class="bi bi-search me-1"></i> <span class="send-btn-text">{t('chat.research') || 'Research'}</span>
|
||||
</button>
|
||||
<button class="btn btn-danger rounded-4 chat-send-btn" id="stopBtn" style="display: none;">
|
||||
<i class="bi bi-stop-fill"></i> <span class="send-btn-text">Stop</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -107,6 +110,33 @@ const { t } = Astro.props;
|
|||
to { --gradient-angle: 360deg; }
|
||||
}
|
||||
|
||||
/* Attachment preview positioning */
|
||||
.attachment-preview-container {
|
||||
max-width: 730px;
|
||||
width: 100%;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
/* Draft history dropdown positioning */
|
||||
.draft-history-dropdown {
|
||||
right: 5.5rem !important;
|
||||
}
|
||||
|
||||
/* Draft history menu sizing */
|
||||
.draft-history-menu {
|
||||
min-width: 320px;
|
||||
max-height: 350px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
/* Wrapper for textarea and buttons together */
|
||||
.chat-input-content-wrapper {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
/* Main wrapper */
|
||||
.chat-input-wrapper {
|
||||
position: relative;
|
||||
|
|
@ -173,6 +203,47 @@ const { t } = Astro.props;
|
|||
z-index: 10;
|
||||
}
|
||||
|
||||
/* Attachment preview thumbnail */
|
||||
:global(.attachment-preview-thumbnail) {
|
||||
width: 48px;
|
||||
height: 48px;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
/* Attachment preview item */
|
||||
:global(.attachment-preview-item) {
|
||||
background-color: var(--bs-secondary-bg);
|
||||
border: 1px solid var(--bs-border-color);
|
||||
}
|
||||
|
||||
/* Attachment preview close button */
|
||||
:global(.attachment-preview-close) {
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
/* Chat message image */
|
||||
:global(.chat-message-image) {
|
||||
height: 120px;
|
||||
width: auto;
|
||||
cursor: pointer;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
/* Chat message image container */
|
||||
:global(.chat-message-images) {
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
/* Synthesis result container */
|
||||
:global(.synthesis-result) {
|
||||
max-width: 80%;
|
||||
}
|
||||
|
||||
/* Draft content clickable */
|
||||
:global(.draft-content) {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* Default layout (welcome/new chat) */
|
||||
.chat-input-row {
|
||||
display: flex;
|
||||
|
|
@ -192,14 +263,25 @@ const { t } = Astro.props;
|
|||
|
||||
/* Chat active layout */
|
||||
:global(.chat-active) .chat-input-row {
|
||||
flex-direction: row;
|
||||
align-items: stretch;
|
||||
justify-content: center;
|
||||
gap: 0.5rem;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
width: 100%;
|
||||
max-width: 1100px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
:global(.chat-active) .attachment-preview-container {
|
||||
max-width: 1100px;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
:global(.chat-active) .chat-input-content-wrapper {
|
||||
flex-direction: row;
|
||||
align-items: stretch;
|
||||
gap: 0.5rem;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
:global(.chat-active) .chat-input-wrapper {
|
||||
max-width: none;
|
||||
flex: 1;
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ const { t } = Astro.props;
|
|||
<i class="bi bi-collection"></i>
|
||||
<span id="activeSetName">Cloud AI</span>
|
||||
</button>
|
||||
<ul class="dropdown-menu" id="providerSetMenu" aria-labelledby="providerSetDropdown">
|
||||
<ul class="dropdown-menu shadow-sm" id="providerSetMenu" aria-labelledby="providerSetDropdown">
|
||||
<li class="dropdown-item text-muted">Loading...</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -16,10 +16,10 @@
|
|||
<button class="btn btn-outline-secondary btn-sm dropdown-toggle" type="button" id="themeDropdown" data-bs-toggle="dropdown" aria-expanded="false" title="Theme">
|
||||
<i class="bi bi-circle-half" id="themeIcon"></i>
|
||||
</button>
|
||||
<ul class="dropdown-menu dropdown-menu-end" aria-labelledby="themeDropdown">
|
||||
<li><button class="dropdown-item" data-theme="light"><i class="bi bi-sun me-2"></i>Light</button></li>
|
||||
<li><button class="dropdown-item" data-theme="dark"><i class="bi bi-moon me-2"></i>Dark</button></li>
|
||||
<li><button class="dropdown-item active" data-theme="auto"><i class="bi bi-circle-half me-2"></i>Auto</button></li>
|
||||
<ul class="dropdown-menu dropdown-menu-end shadow-sm" aria-labelledby="themeDropdown">
|
||||
<li><button class="dropdown-item" data-theme="light" type="button"><i class="bi bi-sun me-2"></i>Light</button></li>
|
||||
<li><button class="dropdown-item" data-theme="dark" type="button"><i class="bi bi-moon me-2"></i>Dark</button></li>
|
||||
<li><button class="dropdown-item active" data-theme="auto" type="button"><i class="bi bi-circle-half me-2"></i>Auto</button></li>
|
||||
</ul>
|
||||
</div>
|
||||
<a href="/" class="btn btn-outline-primary btn-sm" title="Go to Research" target="_blank">
|
||||
|
|
|
|||
|
|
@ -121,6 +121,16 @@ const { title = 'KEA Research Admin' } = Astro.props;
|
|||
.bg-kea-subtle{background-color:rgba(50, 59, 28, 0.15) !important;}
|
||||
[data-bs-theme="dark"] .bg-kea-subtle{background-color:rgba(166, 141, 111, 0.2) !important;}
|
||||
|
||||
/* Dropdown active item - use theme-aware colors instead of blue */
|
||||
.dropdown-item.active, .dropdown-item:active {
|
||||
background-color: var(--bs-secondary-bg);
|
||||
color: var(--bs-body-color);
|
||||
}
|
||||
.dropdown-item.active:hover, .dropdown-item.active:focus {
|
||||
background-color: var(--bs-tertiary-bg);
|
||||
color: var(--bs-body-color);
|
||||
}
|
||||
|
||||
/* Admin sidebar (future use) */
|
||||
.admin-sidebar {
|
||||
width: 250px;
|
||||
|
|
|
|||
|
|
@ -117,6 +117,16 @@ const isRTL = RTL_LOCALES.includes(locale);
|
|||
[data-bs-theme="dark"] #settingsModal .nav-link{color:#a68d6f;}
|
||||
[data-bs-theme="dark"] #settingsModal .nav-link:hover,[data-bs-theme="dark"] #settingsModal .nav-link:focus{color:#9a7c5a;}
|
||||
[data-bs-theme="dark"] #settingsModal .nav-link.active{color:#a68d6f;border-bottom-color:#a68d6f;}
|
||||
|
||||
/* Dropdown active item - use theme-aware colors instead of blue */
|
||||
.dropdown-item.active, .dropdown-item:active {
|
||||
background-color: var(--bs-secondary-bg);
|
||||
color: var(--bs-body-color);
|
||||
}
|
||||
.dropdown-item.active:hover, .dropdown-item.active:focus {
|
||||
background-color: var(--bs-tertiary-bg);
|
||||
color: var(--bs-body-color);
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body class="d-flex flex-column vh-100 overflow-hidden">
|
||||
|
|
@ -124,7 +134,7 @@ const isRTL = RTL_LOCALES.includes(locale);
|
|||
|
||||
{/* Bootstrap JS */}
|
||||
<script is:inline src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.8/dist/js/bootstrap.bundle.min.js" crossorigin="anonymous"></script>
|
||||
<script is:inline src="https://cdn.jsdelivr.net/npm/browser-image-compression@2.0.2/dist/browser-image-compression.js"></script>
|
||||
<script is:inline src="https://cdn.jsdelivr.net/npm/compressorjs@1.2.1/dist/compressor.min.js"></script>
|
||||
{/* Markdown rendering */}
|
||||
<script is:inline src="https://cdn.jsdelivr.net/npm/marked@17.0.1/lib/marked.umd.min.js"></script>
|
||||
{/* PDF and DOCX export */}
|
||||
|
|
|
|||
|
|
@ -50,7 +50,35 @@ export const AttachmentManager = {
|
|||
});
|
||||
}
|
||||
|
||||
// All attachment features are disabled (Coming Soon)
|
||||
// === Wire up image attachment button ===
|
||||
const attachImageBtn = document.getElementById('attachImage');
|
||||
const imageInput = document.getElementById('imageInput') as HTMLInputElement | null;
|
||||
|
||||
if (attachImageBtn && imageInput) {
|
||||
// Click button -> trigger file input
|
||||
attachImageBtn.addEventListener('click', () => {
|
||||
imageInput.click();
|
||||
});
|
||||
|
||||
// File input change -> process selected files
|
||||
imageInput.addEventListener('change', async (e) => {
|
||||
const files = (e.target as HTMLInputElement).files;
|
||||
if (!files || files.length === 0) return;
|
||||
|
||||
for (const file of Array.from(files)) {
|
||||
if (file.type.startsWith('image/')) {
|
||||
await this.addAttachment('image', file);
|
||||
} else {
|
||||
console.warn(`Skipping non-image file: ${file.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Reset input to allow selecting same file again
|
||||
imageInput.value = '';
|
||||
});
|
||||
}
|
||||
|
||||
console.log('AttachmentManager initialized (images enabled)');
|
||||
},
|
||||
|
||||
updateIndicator(): void {
|
||||
|
|
@ -69,20 +97,65 @@ export const AttachmentManager = {
|
|||
},
|
||||
|
||||
async addAttachment(type: 'image' | 'file', file: File): Promise<void> {
|
||||
let processedFile = file;
|
||||
// Validate image type
|
||||
if (type === 'image') {
|
||||
const SUPPORTED_TYPES = ['image/jpeg', 'image/png', 'image/webp', 'image/gif'];
|
||||
if (!SUPPORTED_TYPES.includes(file.type)) {
|
||||
alert(`Unsupported image format: ${file.type}. Please use JPEG, PNG, WebP, or GIF.`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (type === 'image' && this.compressImages && file.type.startsWith('image/')) {
|
||||
try {
|
||||
console.log(`Compressing image: ${file.name} (${(file.size / 1024).toFixed(1)} KB)`);
|
||||
const compressedBlob = await ImageCompressor.compressAttachment(file, this.compressionOptions);
|
||||
processedFile = new File([compressedBlob], file.name, { type: compressedBlob.type });
|
||||
console.log(`Compressed to: ${(processedFile.size / 1024).toFixed(1)} KB`);
|
||||
} catch (error) {
|
||||
console.error('Image compression failed, using original:', error);
|
||||
// Check file size (max 10MB before compression)
|
||||
const MAX_SIZE_BYTES = 10 * 1024 * 1024;
|
||||
if (file.size > MAX_SIZE_BYTES) {
|
||||
alert(`Image too large: ${(file.size / 1024 / 1024).toFixed(1)}MB. Maximum is 10MB.`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Limit: 5 images per message (Claude's limit)
|
||||
const MAX_IMAGES = 5;
|
||||
const currentImageCount = this.attachments.filter((att) => att.type === 'image').length;
|
||||
if (currentImageCount >= MAX_IMAGES) {
|
||||
alert(`Maximum ${MAX_IMAGES} images per message. Please remove some images first.`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
this.attachments.push({ type, file: processedFile, name: file.name });
|
||||
let processedFile = file;
|
||||
|
||||
// Compress image if enabled
|
||||
if (type === 'image' && this.compressImages && file.type.startsWith('image/')) {
|
||||
try {
|
||||
console.log(`Compressing: ${file.name} (${(file.size / 1024).toFixed(1)} KB)`);
|
||||
|
||||
const compressedBlob = await ImageCompressor.compressAttachment(
|
||||
file,
|
||||
this.compressionOptions
|
||||
);
|
||||
|
||||
// Only use compressed if it's actually smaller
|
||||
if (compressedBlob.size < file.size) {
|
||||
processedFile = new File([compressedBlob], file.name, { type: compressedBlob.type });
|
||||
console.log(
|
||||
`Compressed: ${file.name} - ${(file.size / 1024).toFixed(1)}KB → ${(processedFile.size / 1024).toFixed(1)}KB`
|
||||
);
|
||||
} else {
|
||||
console.warn(`Compression increased size for ${file.name}, using original`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Compression failed, using original:', error);
|
||||
// Continue with original file
|
||||
}
|
||||
}
|
||||
|
||||
// Add to attachments array
|
||||
this.attachments.push({
|
||||
type,
|
||||
file: processedFile,
|
||||
name: file.name,
|
||||
});
|
||||
|
||||
// Update UI preview
|
||||
this.updateAttachmentPreview();
|
||||
},
|
||||
|
||||
|
|
@ -97,57 +170,94 @@ export const AttachmentManager = {
|
|||
},
|
||||
|
||||
updateAttachmentPreview(): void {
|
||||
let preview = document.getElementById('attachmentPreview');
|
||||
if (!preview) {
|
||||
preview = document.createElement('div');
|
||||
preview.id = 'attachmentPreview';
|
||||
preview.className = 'd-flex flex-wrap gap-2 mt-2';
|
||||
const wrapper = document.getElementById('chatInputWrapper');
|
||||
if (wrapper) {
|
||||
wrapper.after(preview);
|
||||
}
|
||||
const previewContainer = document.getElementById('attachmentPreview');
|
||||
if (!previewContainer) return;
|
||||
|
||||
if (this.attachments.length === 0) {
|
||||
previewContainer.innerHTML = '';
|
||||
previewContainer.classList.add('d-none');
|
||||
return;
|
||||
}
|
||||
|
||||
preview.innerHTML = this.attachments.map((att, index) => {
|
||||
let icon: string, label: string;
|
||||
switch (att.type) {
|
||||
case 'image':
|
||||
icon = '<i class="bi bi-image text-success"></i>';
|
||||
label = att.name || 'Image';
|
||||
break;
|
||||
case 'file':
|
||||
icon = '<i class="bi bi-files text-kea"></i>';
|
||||
label = att.name || 'File';
|
||||
break;
|
||||
case 'youtube':
|
||||
icon = '<i class="bi bi-youtube text-danger"></i>';
|
||||
label = 'YouTube';
|
||||
break;
|
||||
case 'wikipedia':
|
||||
icon = '<i class="bi bi-wikipedia text-body-secondary"></i>';
|
||||
label = 'Wikipedia';
|
||||
break;
|
||||
default:
|
||||
icon = '<i class="bi bi-file"></i>';
|
||||
label = 'Attachment';
|
||||
}
|
||||
return `<span class="badge bg-body-secondary text-body d-flex align-items-center gap-1">
|
||||
${icon} <span class="text-truncate" style="max-width: 100px;">${label}</span>
|
||||
<button type="button" class="btn-close btn-close-sm ms-1" data-index="${index}" aria-label="Remove"></button>
|
||||
</span>`;
|
||||
}).join('');
|
||||
previewContainer.classList.remove('d-none');
|
||||
|
||||
preview.querySelectorAll('.btn-close').forEach(btn => {
|
||||
const previewHTML = this.attachments
|
||||
.map((att, index) => {
|
||||
let icon = '<i class="bi bi-paperclip"></i>';
|
||||
let preview = '';
|
||||
|
||||
// Show thumbnail for images
|
||||
if (att.type === 'image' && att.file) {
|
||||
const imgUrl = URL.createObjectURL(att.file);
|
||||
preview = `
|
||||
<img
|
||||
src="${imgUrl}"
|
||||
class="img-thumbnail me-2 attachment-preview-thumbnail"
|
||||
alt="${att.name}"
|
||||
>
|
||||
`;
|
||||
|
||||
// Clean up object URL after a delay
|
||||
setTimeout(() => URL.revokeObjectURL(imgUrl), 60000);
|
||||
} else {
|
||||
preview = icon;
|
||||
}
|
||||
|
||||
return `
|
||||
<div class="d-inline-flex align-items-center rounded px-2 py-1 me-2 mb-2 attachment-preview-item">
|
||||
${preview}
|
||||
<span class="small me-2 text-body">${att.name}</span>
|
||||
<button
|
||||
type="button"
|
||||
class="btn-close attachment-preview-close"
|
||||
aria-label="Remove"
|
||||
data-attachment-index="${index}"
|
||||
></button>
|
||||
</div>
|
||||
`;
|
||||
})
|
||||
.join('');
|
||||
|
||||
previewContainer.innerHTML = previewHTML;
|
||||
|
||||
// Wire up remove buttons
|
||||
previewContainer.querySelectorAll('.btn-close').forEach((btn) => {
|
||||
btn.addEventListener('click', (e) => {
|
||||
const index = parseInt((e.target as HTMLElement).dataset.index || '0');
|
||||
this.attachments.splice(index, 1);
|
||||
this.updateAttachmentPreview();
|
||||
const target = e.target as HTMLElement;
|
||||
const index = parseInt(target.dataset.attachmentIndex || '0', 10);
|
||||
this.removeAttachment(index);
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
if (this.attachments.length === 0 && preview) {
|
||||
preview.remove();
|
||||
}
|
||||
removeAttachment(index: number): void {
|
||||
this.attachments.splice(index, 1);
|
||||
this.updateAttachmentPreview();
|
||||
},
|
||||
|
||||
/**
|
||||
* Convert image file to base64 data URL
|
||||
* @param file Image file
|
||||
* @returns Promise resolving to base64 data URL (e.g., "data:image/jpeg;base64,...")
|
||||
*/
|
||||
async convertImageToBase64(file: File): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
|
||||
reader.onload = () => {
|
||||
if (typeof reader.result === 'string') {
|
||||
resolve(reader.result);
|
||||
} else {
|
||||
reject(new Error('FileReader result is not a string'));
|
||||
}
|
||||
};
|
||||
|
||||
reader.onerror = () => {
|
||||
reject(reader.error || new Error('FileReader error'));
|
||||
};
|
||||
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
},
|
||||
|
||||
showYoutubeModal(): void {
|
||||
|
|
@ -168,10 +278,12 @@ export const AttachmentManager = {
|
|||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Clear all attachments (called after message is sent)
|
||||
*/
|
||||
clearAttachments(): void {
|
||||
this.attachments = [];
|
||||
const preview = document.getElementById('attachmentPreview');
|
||||
if (preview) preview.remove();
|
||||
this.updateAttachmentPreview();
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -11,9 +11,27 @@ import { escapeHtml, isMacOS, isTextAreaElement, isInputElement } from './utils'
|
|||
import { fetchStream, ApiRequestError } from './api';
|
||||
import { getActiveSetId, getActiveSetName, lockSelector, unlockSelector } from './provider-set-selector';
|
||||
|
||||
interface TextContent {
|
||||
type: 'text';
|
||||
text: string;
|
||||
}
|
||||
|
||||
interface ImageSource {
|
||||
type: 'base64';
|
||||
media_type: string;
|
||||
data: string;
|
||||
}
|
||||
|
||||
interface ImageContent {
|
||||
type: 'image';
|
||||
source: ImageSource;
|
||||
}
|
||||
|
||||
type MessageContent = string | Array<TextContent | ImageContent>;
|
||||
|
||||
interface Message {
|
||||
role: 'user' | 'assistant';
|
||||
content: string;
|
||||
content: MessageContent;
|
||||
}
|
||||
|
||||
const CHAT_EXPANDED_KEY = 'kea_chat_expanded';
|
||||
|
|
@ -210,7 +228,7 @@ export const ChatManager = {
|
|||
// Lock provider set selector and capture set name for this chat
|
||||
this.currentSetName = getActiveSetName();
|
||||
|
||||
// Create chat with provider set name
|
||||
// Create chat with provider set name (use text-only for title)
|
||||
this.currentChatId = await StorageUtils.createChat(message, this.currentSetName || undefined);
|
||||
|
||||
lockSelector();
|
||||
|
|
@ -228,8 +246,59 @@ export const ChatManager = {
|
|||
}
|
||||
}
|
||||
|
||||
// Build message content (text-only or multimodal)
|
||||
let messageContent: MessageContent;
|
||||
const imageAttachments = attachments.filter(att => att.type === 'image');
|
||||
|
||||
if (imageAttachments.length > 0) {
|
||||
// Multimodal message: array of content blocks
|
||||
const contentBlocks: Array<TextContent | ImageContent> = [];
|
||||
|
||||
// Add text block (if any)
|
||||
if (message) {
|
||||
contentBlocks.push({ type: 'text', text: message });
|
||||
}
|
||||
|
||||
// Add image blocks
|
||||
for (const att of imageAttachments) {
|
||||
if (!att.file) continue;
|
||||
|
||||
try {
|
||||
// Convert image to base64 data URL
|
||||
const dataUrl = await AttachmentManager.convertImageToBase64(att.file);
|
||||
|
||||
// Split data URL into media type and base64 data
|
||||
// Format: "data:image/jpeg;base64,iVBORw0..."
|
||||
const match = dataUrl.match(/^data:([^;]+);base64,(.+)$/);
|
||||
if (!match) {
|
||||
console.error('Invalid data URL format for image:', att.name);
|
||||
continue;
|
||||
}
|
||||
|
||||
const [, mediaType, base64Data] = match;
|
||||
|
||||
// Add image content block
|
||||
contentBlocks.push({
|
||||
type: 'image',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: mediaType,
|
||||
data: base64Data
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to convert image to base64:', att.name, error);
|
||||
}
|
||||
}
|
||||
|
||||
messageContent = contentBlocks;
|
||||
} else {
|
||||
// Text-only message
|
||||
messageContent = message;
|
||||
}
|
||||
|
||||
// Add user message to history
|
||||
this.messages.push({ role: 'user', content: message });
|
||||
this.messages.push({ role: 'user', content: messageContent });
|
||||
|
||||
// Save user message to IndexedDB
|
||||
if (this.currentChatId !== null) {
|
||||
|
|
@ -237,7 +306,7 @@ export const ChatManager = {
|
|||
await StorageUtils.saveMessage({
|
||||
chatId: this.currentChatId,
|
||||
role: 'user',
|
||||
content: message,
|
||||
content: messageContent,
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
} catch (error) {
|
||||
|
|
@ -246,7 +315,7 @@ export const ChatManager = {
|
|||
}
|
||||
|
||||
// Display user message
|
||||
this.displayUserMessage(message);
|
||||
this.displayUserMessage(messageContent);
|
||||
|
||||
// Clear input
|
||||
input.value = '';
|
||||
|
|
@ -347,8 +416,8 @@ export const ChatManager = {
|
|||
toggleSendStop(streaming: boolean): void {
|
||||
const sendBtn = document.getElementById('sendBtn');
|
||||
const stopBtn = document.getElementById('stopBtn');
|
||||
if (sendBtn) sendBtn.style.display = streaming ? 'none' : 'inline-flex';
|
||||
if (stopBtn) stopBtn.style.display = streaming ? 'inline-flex' : 'none';
|
||||
if (sendBtn) sendBtn.classList.toggle('d-none', streaming);
|
||||
if (stopBtn) stopBtn.classList.toggle('d-none', !streaming);
|
||||
},
|
||||
|
||||
async runPipeline(): Promise<void> {
|
||||
|
|
@ -431,15 +500,57 @@ export const ChatManager = {
|
|||
}
|
||||
},
|
||||
|
||||
displayUserMessage(message: string): void {
|
||||
displayUserMessage(content: MessageContent): void {
|
||||
const chatArea = document.getElementById('chatMessages');
|
||||
if (!chatArea) return;
|
||||
|
||||
let contentHTML = '';
|
||||
|
||||
if (typeof content === 'string') {
|
||||
// Text-only message
|
||||
contentHTML = `<div>${escapeHtml(content)}</div>`;
|
||||
} else {
|
||||
// Multimodal message - separate text and images
|
||||
const textParts: string[] = [];
|
||||
const imageParts: string[] = [];
|
||||
|
||||
for (const block of content) {
|
||||
if (block.type === 'text') {
|
||||
textParts.push(escapeHtml(block.text));
|
||||
} else if (block.type === 'image') {
|
||||
// Reconstruct data URL for display
|
||||
const dataUrl = `data:${block.source.media_type};base64,${block.source.data}`;
|
||||
imageParts.push(`
|
||||
<img
|
||||
src="${dataUrl}"
|
||||
class="rounded chat-message-image"
|
||||
alt="User image"
|
||||
onclick="window.open(this.src, '_blank')"
|
||||
/>
|
||||
`);
|
||||
}
|
||||
}
|
||||
|
||||
// Display text first (if any)
|
||||
if (textParts.length > 0) {
|
||||
contentHTML += `<div class="mb-2">${textParts.join('<br>')}</div>`;
|
||||
}
|
||||
|
||||
// Display images in horizontal scrollable row (if any)
|
||||
if (imageParts.length > 0) {
|
||||
contentHTML += `
|
||||
<div class="d-flex gap-2 overflow-x-auto pb-1 chat-message-images">
|
||||
${imageParts.join('')}
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
const messageHTML = `
|
||||
<div class="user-message mb-3">
|
||||
<div class="d-flex justify-content-end">
|
||||
<div class="bg-kea text-white rounded-3 p-3" style="max-width: 80%;">
|
||||
${escapeHtml(message)}
|
||||
<div class="bg-kea text-white rounded-3 p-3 synthesis-result">
|
||||
${contentHTML}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -549,7 +660,7 @@ export const ChatManager = {
|
|||
<i class="bi bi-x-lg"></i>
|
||||
</button>
|
||||
</div>
|
||||
<div class="draft-content small text-body cursor-pointer restore-draft-btn" data-content="${escapeHtml(draft.content).replace(/"/g, '"')}" style="cursor: pointer;">
|
||||
<div class="draft-content small text-body restore-draft-btn" data-content="${escapeHtml(draft.content).replace(/"/g, '"')}">
|
||||
${escapeHtml(preview)}
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,23 +1,39 @@
|
|||
/**
|
||||
* Image Compression Utility
|
||||
* Image Compression Utility using Compressor.js
|
||||
*/
|
||||
|
||||
declare const imageCompression: any;
|
||||
declare const Compressor: any;
|
||||
|
||||
type ImageType = 'avatar' | 'background' | 'attachment';
|
||||
|
||||
interface CompressionPreset {
|
||||
maxSizeMB?: number;
|
||||
maxWidthOrHeight: number;
|
||||
initialQuality: number;
|
||||
maxDimension: number;
|
||||
quality: number;
|
||||
}
|
||||
|
||||
const PRESETS: Record<ImageType, CompressionPreset> = {
|
||||
avatar: { maxSizeMB: 0.5, maxWidthOrHeight: 200, initialQuality: 0.9 },
|
||||
background: { maxSizeMB: 2, maxWidthOrHeight: 1920, initialQuality: 0.85 },
|
||||
attachment: { maxWidthOrHeight: 2048, initialQuality: 0.85 }
|
||||
avatar: { maxDimension: 200, quality: 0.9 },
|
||||
background: { maxDimension: 1920, quality: 0.85 },
|
||||
attachment: { maxDimension: 2048, quality: 0.85 }
|
||||
};
|
||||
|
||||
/**
|
||||
* Promise wrapper for Compressor.js callback API
|
||||
*/
|
||||
function compressImage(file: File, options: any): Promise<Blob> {
|
||||
return new Promise((resolve, reject) => {
|
||||
new Compressor(file, {
|
||||
...options,
|
||||
success(result: File | Blob) {
|
||||
resolve(result);
|
||||
},
|
||||
error(err: Error) {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export const ImageCompressor = {
|
||||
async compress(
|
||||
file: File,
|
||||
|
|
@ -25,7 +41,7 @@ export const ImageCompressor = {
|
|||
customOptions: { quality?: number; maxWidthOrHeight?: number } = {}
|
||||
): Promise<Blob> {
|
||||
const preset = PRESETS[type];
|
||||
let maxDim = customOptions.maxWidthOrHeight || preset.maxWidthOrHeight;
|
||||
let maxDim = customOptions.maxWidthOrHeight || preset.maxDimension;
|
||||
|
||||
// Background: adjust for high-DPI displays
|
||||
if (type === 'background') {
|
||||
|
|
@ -34,15 +50,16 @@ export const ImageCompressor = {
|
|||
}
|
||||
|
||||
const options = {
|
||||
maxSizeMB: preset.maxSizeMB,
|
||||
maxWidthOrHeight: maxDim,
|
||||
useWebWorker: true,
|
||||
initialQuality: customOptions.quality ? customOptions.quality / 100 : preset.initialQuality
|
||||
maxWidth: maxDim,
|
||||
maxHeight: maxDim,
|
||||
quality: customOptions.quality ? customOptions.quality / 100 : preset.quality,
|
||||
checkOrientation: true, // Auto-rotate based on EXIF
|
||||
mimeType: 'auto' // Preserve original format
|
||||
};
|
||||
|
||||
try {
|
||||
const compressed = await imageCompression(file, options);
|
||||
return new Blob([await compressed.arrayBuffer()], { type: compressed.type });
|
||||
const compressed = await compressImage(file, options);
|
||||
return compressed instanceof Blob ? compressed : new Blob([compressed]);
|
||||
} catch (error) {
|
||||
console.error(`${type} compression error:`, error);
|
||||
throw error;
|
||||
|
|
|
|||
|
|
@ -119,15 +119,16 @@ function renderDropdown(): void {
|
|||
menu.innerHTML = providerSets
|
||||
.map(set => {
|
||||
const isActive = set.id === activeSetId;
|
||||
const systemBadge = set.is_system ? '<span class="badge bg-secondary ms-2">System</span>' : '';
|
||||
const countBadge = `<span class="badge bg-primary ms-1">${set.provider_count}</span>`;
|
||||
const systemBadge = set.is_system ? '<span class="badge text-bg-secondary ms-2">System</span>' : '';
|
||||
const countBadge = `<span class="badge text-bg-secondary ms-1">${set.provider_count}</span>`;
|
||||
|
||||
return `
|
||||
<li>
|
||||
<button class="dropdown-item d-flex align-items-center justify-content-between ${isActive ? 'active' : ''}"
|
||||
data-set-id="${set.id}">
|
||||
data-set-id="${set.id}"
|
||||
type="button">
|
||||
<span>${set.display_name}${systemBadge}${countBadge}</span>
|
||||
${isActive ? '<i class="bi bi-check2 ms-2"></i>' : ''}
|
||||
${isActive ? '<i class="bi bi-check2 ms-2 text-success"></i>' : ''}
|
||||
</button>
|
||||
</li>
|
||||
`;
|
||||
|
|
|
|||
|
|
@ -46,11 +46,30 @@ export interface PipelineData {
|
|||
errors: Record<string, string[]>;
|
||||
}
|
||||
|
||||
// Multimodal content types (matches backend format)
|
||||
export interface TextContent {
|
||||
type: 'text';
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface ImageSource {
|
||||
type: 'base64';
|
||||
media_type: string;
|
||||
data: string;
|
||||
}
|
||||
|
||||
export interface ImageContent {
|
||||
type: 'image';
|
||||
source: ImageSource;
|
||||
}
|
||||
|
||||
export type MessageContent = string | Array<TextContent | ImageContent>;
|
||||
|
||||
export interface ChatMessage {
|
||||
id?: number;
|
||||
chatId: number;
|
||||
role: 'user' | 'assistant';
|
||||
content: string;
|
||||
content: MessageContent; // Updated to support multimodal content
|
||||
providerResponses?: Record<string, ProviderResponseData>;
|
||||
timestamp: string;
|
||||
pipelineData?: PipelineData;
|
||||
|
|
|
|||
41
frontend/src/scripts/types/external.d.ts
vendored
41
frontend/src/scripts/types/external.d.ts
vendored
|
|
@ -57,22 +57,31 @@ declare namespace bootstrap {
|
|||
}
|
||||
}
|
||||
|
||||
// Image Compression Library
|
||||
interface ImageCompressionOptions {
|
||||
maxSizeMB?: number;
|
||||
maxWidthOrHeight?: number;
|
||||
useWebWorker?: boolean;
|
||||
initialQuality?: number;
|
||||
exifOrientation?: number;
|
||||
fileType?: string;
|
||||
// Compressor.js - Image Compression Library
|
||||
interface CompressorOptions {
|
||||
strict?: boolean;
|
||||
checkOrientation?: boolean;
|
||||
retainExif?: boolean;
|
||||
maxWidth?: number;
|
||||
maxHeight?: number;
|
||||
minWidth?: number;
|
||||
minHeight?: number;
|
||||
width?: number;
|
||||
height?: number;
|
||||
resize?: 'none' | 'contain' | 'cover';
|
||||
quality?: number;
|
||||
mimeType?: string;
|
||||
convertTypes?: string | string[];
|
||||
convertSize?: number;
|
||||
beforeDraw?(context: CanvasRenderingContext2D, canvas: HTMLCanvasElement): void;
|
||||
drew?(context: CanvasRenderingContext2D, canvas: HTMLCanvasElement): void;
|
||||
success?(result: File | Blob): void;
|
||||
error?(error: Error): void;
|
||||
}
|
||||
|
||||
declare function imageCompression(
|
||||
file: File,
|
||||
options: ImageCompressionOptions
|
||||
): Promise<Blob>;
|
||||
|
||||
declare namespace imageCompression {
|
||||
function getExifOrientation(file: File): Promise<number>;
|
||||
function drawImageInCanvas(img: HTMLImageElement): HTMLCanvasElement;
|
||||
declare class Compressor {
|
||||
constructor(file: File | Blob, options?: CompressorOptions);
|
||||
abort(): void;
|
||||
static noConflict(): typeof Compressor;
|
||||
static setDefaults(options: CompressorOptions): void;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,9 @@ server {
|
|||
|
||||
root /var/www/html;
|
||||
|
||||
# Allow larger uploads for image attachments
|
||||
client_max_body_size 48M;
|
||||
|
||||
gzip on;
|
||||
gzip_types text/plain text/css application/json application/javascript text/xml application/xml text/event-stream;
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@ server {
|
|||
add_header X-Frame-Options DENY always;
|
||||
add_header Referrer-Policy strict-origin-when-cross-origin always;
|
||||
|
||||
# Allow larger uploads for image attachments
|
||||
client_max_body_size 48M;
|
||||
|
||||
gzip on;
|
||||
gzip_types text/plain text/css application/json application/javascript text/xml application/xml text/event-stream;
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue