diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 8b64bbe0d..a92f03f53 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1648,7 +1648,7 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec const uint32_t warps = warptile[0] / warptile[10]; const uint32_t load_bufs = (warptile[1] + warptile[2]) * (warptile[3] + bank_conflict_offset) * type_size; - const uint32_t mmid_row_ids = mul_mat_id ? 3072 * sizeof(uint32_t) : 0; + const uint32_t mmid_row_ids = mul_mat_id ? 4096 * sizeof(uint32_t) : 0; const uint32_t coopmat_stage = device->coopmat_support ? warptile[7] * warptile[8] / warps * sizeof(float) : 0; const uint32_t total_size = load_bufs + mmid_row_ids + coopmat_stage + lut_size; @@ -5284,7 +5284,7 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& const uint64_t nei0 = ids->ne[0]; const uint64_t nei1 = ids->ne[1]; - GGML_ASSERT(nei0 * nei1 <= 3072); + GGML_ASSERT(nei0 * nei1 <= 4096); const uint32_t nbi1 = ids->nb[1]; const uint32_t nbi2 = ids->nb[2]; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp index 529ac4d44..7859a1a60 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp @@ -103,7 +103,7 @@ shared FLOAT_TYPE buf_a[BM * SHMEM_STRIDE]; shared FLOAT_TYPE buf_b[BN * SHMEM_STRIDE]; #ifdef MUL_MAT_ID -shared u16vec2 row_ids[3072]; +shared u16vec2 row_ids[4096]; #endif // MUL_MAT_ID #define NUM_WARPS (BLOCK_SIZE / WARP) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp index 344b46610..918465757 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp @@ -92,7 +92,7 @@ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; #ifdef MUL_MAT_ID layout (binding = 3) readonly buffer IDS {int data_ids[];}; -shared u16vec4 row_ids[3072]; +shared u16vec4 row_ids[4096]; layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufB { B_TYPE b[]; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp index 284a35caa..83de90eb7 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp @@ -101,7 +101,7 @@ shared FLOAT_TYPE_VEC2 buf_b_ds[BN]; #define LOAD_VEC_B 4 #ifdef MUL_MAT_ID -shared u16vec2 row_ids[3072]; +shared u16vec2 row_ids[4096]; #endif // MUL_MAT_ID #define NUM_WARPS (BLOCK_SIZE / WARP) diff --git a/tools/server/public/index.html.gz b/tools/server/public/index.html.gz index 260fdcfec..d7363e13e 100644 Binary files a/tools/server/public/index.html.gz and b/tools/server/public/index.html.gz differ diff --git a/tools/server/webui/src/components/useChatExtraContext.tsx b/tools/server/webui/src/components/useChatExtraContext.tsx index 866401db9..7eeff61f5 100644 --- a/tools/server/webui/src/components/useChatExtraContext.tsx +++ b/tools/server/webui/src/components/useChatExtraContext.tsx @@ -37,19 +37,26 @@ export function useChatExtraContext(): ChatExtraContextApi { break; } - if (mimeType.startsWith('image/') && mimeType !== 'image/svg+xml') { - if (!serverProps?.has_multimodal) { + if (mimeType.startsWith('image/')) { + if (!serverProps?.modalities?.vision) { toast.error('Multimodal is not supported by this server or model.'); break; } const reader = new FileReader(); - reader.onload = (event) => { + reader.onload = async (event) => { if (event.target?.result) { + let base64Url = event.target.result as string; + + if (mimeType === 'image/svg+xml') { + // Convert SVG to PNG + base64Url = await svgBase64UrlToPngDataURL(base64Url); + } + addItems([ { type: 'imageFile', name: file.name, - base64Url: event.target.result as string, + base64Url, }, ]); } @@ -172,3 +179,56 @@ export function isLikelyNotBinary(str: string): boolean { const ratio = suspiciousCharCount / sampleLength; return ratio <= options.suspiciousCharThresholdRatio; } + +// WARN: vibe code below +// Converts a Base64URL encoded SVG string to a PNG Data URL using browser Canvas API. +function svgBase64UrlToPngDataURL(base64UrlSvg: string): Promise { + const backgroundColor = 'white'; // Default background color for PNG + + return new Promise((resolve, reject) => { + try { + const img = new Image(); + + img.onload = () => { + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + + if (!ctx) { + reject(new Error('Failed to get 2D canvas context.')); + return; + } + + // Use provided dimensions or SVG's natural dimensions, with fallbacks + // Fallbacks (e.g., 300x300) are for SVGs without explicit width/height + // or when naturalWidth/Height might be 0 before full processing. + const targetWidth = img.naturalWidth || 300; + const targetHeight = img.naturalHeight || 300; + + canvas.width = targetWidth; + canvas.height = targetHeight; + + if (backgroundColor) { + ctx.fillStyle = backgroundColor; + ctx.fillRect(0, 0, canvas.width, canvas.height); + } + + ctx.drawImage(img, 0, 0, targetWidth, targetHeight); + resolve(canvas.toDataURL('image/png')); + }; + + img.onerror = () => { + reject( + new Error('Failed to load SVG image. Ensure the SVG data is valid.') + ); + }; + + // Load SVG string into an Image element + img.src = base64UrlSvg; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const errorMessage = `Error converting SVG to PNG: ${message}`; + toast.error(errorMessage); + reject(new Error(errorMessage)); + } + }); +} diff --git a/tools/server/webui/src/utils/types.ts b/tools/server/webui/src/utils/types.ts index add48be4c..ba673dd94 100644 --- a/tools/server/webui/src/utils/types.ts +++ b/tools/server/webui/src/utils/types.ts @@ -118,6 +118,8 @@ export interface LlamaCppServerProps { build_info: string; model_path: string; n_ctx: number; - has_multimodal: boolean; + modalities?: { + vision: boolean; + }; // TODO: support params }