From 6471e3c090107fb0421bd2d59b83f0bb8df8b58f Mon Sep 17 00:00:00 2001 From: Pascal Date: Fri, 12 Jun 2026 10:20:27 +0200 Subject: [PATCH] UI/jpeg exif orientation (#24196) * ui: bake jpeg exif orientation into uploaded images stb_image in mtmd ignores exif metadata, so rotated smartphone photos reach the model with raw pixel orientation. The webui now reads the exif orientation tag at send time and feeds it into the existing capImageDataURLSize canvas pass: the browser applies the rotation when decoding, so capped images come out upright for free, and images under the cap threshold get a single plain redraw when orientation > 1. At most one re-encode ever happens per image. Upright jpegs with capping disabled pass through untouched, bit perfect. Adds jpeg-orientation.ts with a minimal exif parser working on a bounded base64 prefix (both endianness, returns 1 on any malformed input) and unit tests against handcrafted jpeg byte streams. * ui: move jpeg exif constants into lib/constants * ui: add browser test for jpeg orientation and capping Covers capImageDataURLSize end to end in chromium with real Pillow generated jpeg fixtures across exif orientations 1/3/5/6/8: upright quadrant colors checked pixel-wise, expected dimensions with and without capping, no orientation tag left in the output, and strict passthrough when nothing needs rewriting. --- tools/ui/src/lib/constants/jpeg-exif.ts | 30 ++++ tools/ui/src/lib/services/chat.service.ts | 10 +- tools/ui/src/lib/utils/cap-img-size.ts | 18 ++- tools/ui/src/lib/utils/jpeg-orientation.ts | 146 ++++++++++++++++++ .../tests/client/cap-img-size.svelte.test.ts | 121 +++++++++++++++ tools/ui/tests/unit/jpeg-orientation.test.ts | 100 ++++++++++++ 6 files changed, 419 insertions(+), 6 deletions(-) create mode 100644 tools/ui/src/lib/constants/jpeg-exif.ts create mode 100644 tools/ui/src/lib/utils/jpeg-orientation.ts create mode 100644 tools/ui/tests/client/cap-img-size.svelte.test.ts create mode 100644 tools/ui/tests/unit/jpeg-orientation.test.ts diff --git a/tools/ui/src/lib/constants/jpeg-exif.ts b/tools/ui/src/lib/constants/jpeg-exif.ts new file mode 100644 index 000000000..5b2591b04 --- /dev/null +++ b/tools/ui/src/lib/constants/jpeg-exif.ts @@ -0,0 +1,30 @@ +/** + * JPEG and EXIF binary format constants for orientation parsing. + */ + +/** Bytes of file prefix to scan, the APP1 EXIF segment sits near the start */ +export const EXIF_SCAN_BYTE_LIMIT = 128 * 1024; + +/** JPEG start of image marker */ +export const JPEG_SOI_MARKER = 0xffd8; + +/** APP1 segment marker byte, carries the EXIF payload */ +export const APP1_MARKER = 0xe1; + +/** Start of scan marker byte, compressed data begins and no EXIF follows */ +export const SOS_MARKER = 0xda; + +/** "Exif" signature opening the APP1 payload, big endian uint32 */ +export const EXIF_SIGNATURE = 0x45786966; + +/** TIFF byte order mark for little endian ("II") */ +export const TIFF_LITTLE_ENDIAN = 0x4949; + +/** TIFF magic number following the byte order mark */ +export const TIFF_MAGIC = 42; + +/** EXIF tag id holding the orientation value */ +export const EXIF_ORIENTATION_TAG = 0x0112; + +/** Size in bytes of one IFD directory entry */ +export const IFD_ENTRY_SIZE = 12; diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 468009a54..70844f57e 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -34,7 +34,6 @@ import type { import { modelsStore } from '$lib/stores/models.svelte'; import { settingsStore } from '../stores/settings.svelte'; import { capImageDataURLSize } from '../utils/cap-img-size'; -import { MEGAPIXELS_TO_PIXELS } from '$lib/constants/image-size'; function getAudioInputFormat(mimeType: string): AudioInputFormat { const normalizedMimeType = mimeType.trim().toLowerCase(); @@ -961,10 +960,11 @@ export class ChatService { for (const image of imageFiles) { const maxImageResolution = settingsStore.getConfig(SETTINGS_KEYS.MAX_IMAGE_RESOLUTION); - let base64Url = image.base64Url; - if (maxImageResolution > 1 / MEGAPIXELS_TO_PIXELS) { - base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution); - } + + // Caps the resolution and bakes the jpeg exif orientation in one pass, + // untouched images pass through as is + const base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution); + contentParts.push({ type: ContentPartType.IMAGE_URL, image_url: { url: base64Url } diff --git a/tools/ui/src/lib/utils/cap-img-size.ts b/tools/ui/src/lib/utils/cap-img-size.ts index 1ba218316..c5d9b7306 100644 --- a/tools/ui/src/lib/utils/cap-img-size.ts +++ b/tools/ui/src/lib/utils/cap-img-size.ts @@ -1,11 +1,19 @@ import { MEGAPIXELS_TO_PIXELS } from '$lib/constants/image-size'; import { BASE64_IMAGE_URI_REGEX } from '$lib/constants/uri-template'; +import { getJpegOrientationFromDataURL, isJpegMimeType } from './jpeg-orientation'; import { MimeTypeImage } from '$lib/enums'; /** * Converts an Image base64 data URL to another Image data URL with capped dimensions to reduce file size. + * + * For JPEGs the EXIF orientation is baked into the pixels in the same canvas + * pass, the browser applies the rotation when decoding so naturalWidth and + * naturalHeight already describe the upright image. Backends decoding with + * stb_image ignore EXIF, see ggml-org/llama.cpp#20870. Images that need + * neither capping nor rotation pass through untouched, so at most one + * re-encode ever happens. * @param base64UrlImage - The Image base64 data URL to convert - * @param maxMegapixels - The maximum image size in megapixels for the output Image + * @param maxMegapixels - The maximum image size in megapixels for the output Image, 0 disables capping * @returns Promise resolving to Image data URL */ export function capImageDataURLSize( @@ -26,6 +34,10 @@ export function capImageDataURLSize( return reject(new Error(`Unsupported image MIME type: ${mimeType}`)); } + const orientation = isJpegMimeType(mimeType) + ? getJpegOrientationFromDataURL(base64UrlImage) + : 1; + const img = new Image(); img.onload = () => { @@ -46,6 +58,10 @@ export function capImageDataURLSize( const scaleFactor = Math.sqrt(maxPixels / totalPixels); canvas.width = Math.floor(targetWidth * scaleFactor); canvas.height = Math.floor(targetHeight * scaleFactor); + } else if (orientation > 1) { + // No capping needed but the pixels still need the rotation baked in + canvas.width = targetWidth; + canvas.height = targetHeight; } else { return resolve(base64UrlImage); } diff --git a/tools/ui/src/lib/utils/jpeg-orientation.ts b/tools/ui/src/lib/utils/jpeg-orientation.ts new file mode 100644 index 000000000..15c21017a --- /dev/null +++ b/tools/ui/src/lib/utils/jpeg-orientation.ts @@ -0,0 +1,146 @@ +import { + EXIF_SCAN_BYTE_LIMIT, + JPEG_SOI_MARKER, + APP1_MARKER, + SOS_MARKER, + EXIF_SIGNATURE, + TIFF_LITTLE_ENDIAN, + TIFF_MAGIC, + EXIF_ORIENTATION_TAG, + IFD_ENTRY_SIZE +} from '$lib/constants/jpeg-exif'; +import { MimeTypeImage } from '$lib/enums'; + +/** + * Read the EXIF orientation tag from a JPEG base64 data URL + * + * Only a bounded prefix of the base64 payload is decoded, the APP1 segment + * always sits near the start of the file. + * @param base64UrlJpeg - The JPEG base64 data URL to inspect + * @returns The orientation value (1 to 8), or 1 when absent or unreadable + */ +export function getJpegOrientationFromDataURL(base64UrlJpeg: string): number { + try { + const payloadStart = base64UrlJpeg.indexOf(',') + 1; + + if (payloadStart <= 0) { + return 1; + } + + // Keep the slice a multiple of 4 characters so atob accepts it + const charLimit = Math.ceil(EXIF_SCAN_BYTE_LIMIT / 3) * 4; + const slice = base64UrlJpeg.slice(payloadStart, payloadStart + charLimit); + const binary = atob(slice.slice(0, slice.length - (slice.length % 4))); + const bytes = new Uint8Array(binary.length); + + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + + return findExifOrientation(new DataView(bytes.buffer)); + } catch { + return 1; + } +} + +/** + * Walk the JPEG segments of a header buffer looking for the APP1 EXIF block + * @param view - DataView over the JPEG header bytes + * @returns The orientation value (1 to 8), or 1 when absent or malformed + */ +function findExifOrientation(view: DataView): number { + if (view.byteLength < 4 || view.getUint16(0) !== JPEG_SOI_MARKER) { + return 1; + } + + let offset = 2; + + while (offset + 4 <= view.byteLength) { + if (view.getUint8(offset) !== 0xff) { + return 1; + } + + const marker = view.getUint8(offset + 1); + + // Compressed image data starts here: no EXIF past this point + if (marker === SOS_MARKER) { + return 1; + } + + const segmentLength = view.getUint16(offset + 2); + + if (marker === APP1_MARKER) { + return parseExifOrientation(view, offset + 4, segmentLength); + } + + offset += 2 + segmentLength; + } + + return 1; +} + +/** + * Parse the orientation tag from an APP1 EXIF payload + * @param view - DataView over the JPEG header bytes + * @param start - Offset of the APP1 payload, right after the segment length + * @param segmentLength - Declared APP1 segment length + * @returns The orientation value (1 to 8), or 1 when absent or malformed + */ +function parseExifOrientation(view: DataView, start: number, segmentLength: number): number { + const end = Math.min(start + segmentLength, view.byteLength); + + // The payload opens with the "Exif\0\0" signature + if ( + start + 6 > end || + view.getUint32(start) !== EXIF_SIGNATURE || + view.getUint16(start + 4) !== 0 + ) { + return 1; + } + + const tiff = start + 6; + + if (tiff + 8 > end) { + return 1; + } + + const littleEndian = view.getUint16(tiff) === TIFF_LITTLE_ENDIAN; + + if (view.getUint16(tiff + 2, littleEndian) !== TIFF_MAGIC) { + return 1; + } + + const ifdOffset = view.getUint32(tiff + 4, littleEndian); + + if (tiff + ifdOffset + 2 > end) { + return 1; + } + + const entryCount = view.getUint16(tiff + ifdOffset, littleEndian); + + // Scan IFD0 entries for the orientation tag + for (let i = 0; i < entryCount; i++) { + const entry = tiff + ifdOffset + 2 + i * IFD_ENTRY_SIZE; + + if (entry + IFD_ENTRY_SIZE > end) { + return 1; + } + + if (view.getUint16(entry, littleEndian) === EXIF_ORIENTATION_TAG) { + const orientation = view.getUint16(entry + 8, littleEndian); + + return orientation >= 1 && orientation <= 8 ? orientation : 1; + } + } + + return 1; +} + +/** + * Check if a MIME type represents a JPEG + * @param mimeType - The MIME type to check + * @returns True if the MIME type is a JPEG variant + */ +export function isJpegMimeType(mimeType: string): boolean { + return mimeType === MimeTypeImage.JPEG || mimeType === MimeTypeImage.JPG; +} diff --git a/tools/ui/tests/client/cap-img-size.svelte.test.ts b/tools/ui/tests/client/cap-img-size.svelte.test.ts new file mode 100644 index 000000000..d70494208 --- /dev/null +++ b/tools/ui/tests/client/cap-img-size.svelte.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from 'vitest'; +import { capImageDataURLSize } from '$lib/utils/cap-img-size'; +import { getJpegOrientationFromDataURL } from '$lib/utils/jpeg-orientation'; + +// Real 64x32 jpegs generated with Pillow, quality 90. The upright picture is +// four solid quadrants: top left red, top right green, bottom left blue, +// bottom right yellow. For each exif value the stored pixels are inverse +// transposed so a conforming decoder shows the upright picture, exactly like +// a rotated smartphone photo. +const EXIF1 = `data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAEAAAAAAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAAgAEADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD50ooor8MP9UwooooA9uooor4I/wCcwKKKKAPhSiiiv+gM/qgKKKKAP3Vooor/AJdz+lQooooA/9k=`; +const EXIF3 = `data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAMAAAAAAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAAgAEADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD7Nooor/Mo/XQooooA/Cqiiiv+og/moKKKKAPuuiiiv+fw/lcKKKKAPEaKKK+9P+jMKKKKAP/Z`; +const EXIF5 = `data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAUAAAAAAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCABAACADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD50ooor8MP9UzwKiiiv9xj/HQ99ooor/Dk/wBizwKiiiv9xj/HQ+66KKK/5/D+Vz7qooor+ej/AE9PhWiiiv6FP8wj7qooor+ej/T0/9k=`; +const EXIF6 = `data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAYAAAAAAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCABAACADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDCooor+Tz+Hz7qooor+ej/AE9PhWiiiv6FP8wj7qooor+ej/T0/Keiiiv7CP72PAqKKK/3GP8AHQ99ooor/Dk/2LPAqKKK/wBxj/HQ/9k=`; +const EXIF8 = `data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAgAAAAAAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCABAACADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD896KKK/1TPhz32iiiv8OT/Ys8Cooor/cY/wAdD32iiiv8OT/Ys/Viiiiv49P4JPhWiiiv6FP8wj7qooor+ej/AE9PhWiiiv6FP8wj/9k=`; +const NOEXIF = `data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAAgAEADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD50ooor8MP9UwooooA9uooor4I/wCcwKKKKAPhSiiiv+gM/qgKKKKAP3Vooor/AJdz+lQooooA/9k=`; + +const RED: Rgb = [255, 0, 0]; +const GREEN: Rgb = [0, 200, 0]; +const BLUE: Rgb = [0, 0, 255]; +const YELLOW: Rgb = [255, 220, 0]; + +// Wide tolerance per channel, jpeg compression shifts solid colors a bit +const COLOR_TOLERANCE = 70; + +// 0.000512 megapixels is 512 pixels, a quarter of the area of the 2048 pixel fixtures +const QUARTER_AREA_MEGAPIXELS = 0.000512; + +type Rgb = [number, number, number]; + +function loadImage(dataUrl: string): Promise { + return new Promise((resolve, reject) => { + const img = new Image(); + img.onload = () => resolve(img); + img.onerror = () => reject(new Error('Failed to decode image.')); + img.src = dataUrl; + }); +} + +// Decodes a data URL and samples the center of each quadrant of the picture +async function quadrantColors(dataUrl: string): Promise { + const img = await loadImage(dataUrl); + const canvas = document.createElement('canvas'); + canvas.width = img.naturalWidth; + canvas.height = img.naturalHeight; + const ctx = canvas.getContext('2d')!; + ctx.drawImage(img, 0, 0); + const points = [ + [0.25, 0.25], + [0.75, 0.25], + [0.25, 0.75], + [0.75, 0.75] + ]; + return points.map(([fx, fy]) => { + const d = ctx.getImageData( + Math.floor(canvas.width * fx), + Math.floor(canvas.height * fy), + 1, + 1 + ).data; + return [d[0], d[1], d[2]]; + }); +} + +function expectUpright(colors: Rgb[]) { + const targets = [RED, GREEN, BLUE, YELLOW]; + for (let i = 0; i < 4; i++) { + for (let c = 0; c < 3; c++) { + expect(Math.abs(colors[i][c] - targets[i][c])).toBeLessThan(COLOR_TOLERANCE); + } + } +} + +describe('capImageDataURLSize orientation and capping', () => { + it('passes upright jpegs through untouched when capping is disabled', async () => { + expect(await capImageDataURLSize(EXIF1, 0)).toBe(EXIF1); + expect(await capImageDataURLSize(NOEXIF, 0)).toBe(NOEXIF); + }); + + it('passes upright jpegs through untouched when under the cap threshold', async () => { + expect(await capImageDataURLSize(EXIF1, 1)).toBe(EXIF1); + }); + + it.each([ + ['orientation 3', EXIF3], + ['orientation 5', EXIF5], + ['orientation 6', EXIF6], + ['orientation 8', EXIF8] + ])('bakes %s into upright pixels without capping', async (_label, fixture) => { + const result = await capImageDataURLSize(fixture, 0); + + expect(result).not.toBe(fixture); + + const img = await loadImage(result); + + expect(img.naturalWidth).toBe(64); + expect(img.naturalHeight).toBe(32); + expectUpright(await quadrantColors(result)); + + // The re-encoded jpeg carries no orientation tag anymore + expect(getJpegOrientationFromDataURL(result)).toBe(1); + }); + + it('caps and bakes the orientation in a single output', async () => { + const result = await capImageDataURLSize(EXIF6, QUARTER_AREA_MEGAPIXELS); + const img = await loadImage(result); + + expect(img.naturalWidth).toBe(32); + expect(img.naturalHeight).toBe(16); + expectUpright(await quadrantColors(result)); + expect(getJpegOrientationFromDataURL(result)).toBe(1); + }); + + it('caps upright jpegs without disturbing the picture', async () => { + const result = await capImageDataURLSize(EXIF1, QUARTER_AREA_MEGAPIXELS); + const img = await loadImage(result); + + expect(img.naturalWidth).toBe(32); + expect(img.naturalHeight).toBe(16); + expectUpright(await quadrantColors(result)); + }); +}); diff --git a/tools/ui/tests/unit/jpeg-orientation.test.ts b/tools/ui/tests/unit/jpeg-orientation.test.ts new file mode 100644 index 000000000..b1ad4777d --- /dev/null +++ b/tools/ui/tests/unit/jpeg-orientation.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from 'vitest'; +import { getJpegOrientationFromDataURL, isJpegMimeType } from '$lib/utils/jpeg-orientation'; + +// Builds the TIFF payload of an APP1 segment holding a single IFD0 entry +function buildTiff(littleEndian: boolean, tag: number, value: number): number[] { + const u16 = (v: number) => (littleEndian ? [v & 0xff, v >> 8] : [v >> 8, v & 0xff]); + const u32 = (v: number) => + littleEndian + ? [v & 0xff, (v >> 8) & 0xff, (v >> 16) & 0xff, (v >> 24) & 0xff] + : [(v >> 24) & 0xff, (v >> 16) & 0xff, (v >> 8) & 0xff, v & 0xff]; + + return [ + ...(littleEndian ? [0x49, 0x49] : [0x4d, 0x4d]), + ...u16(42), + ...u32(8), + ...u16(1), + ...u16(tag), + ...u16(3), + ...u32(1), + // SHORT value sits left justified in the 4 byte value field + ...u16(value), + ...u16(0), + ...u32(0) + ]; +} + +// Wraps a TIFF payload into a complete minimal JPEG data URL +function buildJpegDataURL(tiff: number[] | null, prependApp0 = false): string { + const bytes: number[] = [0xff, 0xd8]; + + if (prependApp0) { + // JFIF APP0 segment, irrelevant content the parser walks over + bytes.push(0xff, 0xe0, 0x00, 0x07, 0x4a, 0x46, 0x49, 0x46, 0x00); + } + + if (tiff) { + const payload = [0x45, 0x78, 0x69, 0x66, 0x00, 0x00, ...tiff]; + const length = payload.length + 2; + bytes.push(0xff, 0xe1, length >> 8, length & 0xff, ...payload); + } + + // SOS marker terminates the metadata scan + bytes.push(0xff, 0xda, 0x00, 0x02); + + return `data:image/jpeg;base64,${btoa(String.fromCharCode(...bytes))}`; +} + +describe('getJpegOrientationFromDataURL', () => { + it('returns the orientation from a little endian EXIF block', () => { + expect(getJpegOrientationFromDataURL(buildJpegDataURL(buildTiff(true, 0x0112, 6)))).toBe(6); + }); + + it('returns the orientation from a big endian EXIF block', () => { + expect(getJpegOrientationFromDataURL(buildJpegDataURL(buildTiff(false, 0x0112, 8)))).toBe(8); + }); + + it('walks over a leading APP0 segment', () => { + expect(getJpegOrientationFromDataURL(buildJpegDataURL(buildTiff(true, 0x0112, 3), true))).toBe( + 3 + ); + }); + + it('returns 1 when the EXIF block holds no orientation tag', () => { + expect(getJpegOrientationFromDataURL(buildJpegDataURL(buildTiff(true, 0x0100, 6)))).toBe(1); + }); + + it('returns 1 when the orientation value is out of range', () => { + expect(getJpegOrientationFromDataURL(buildJpegDataURL(buildTiff(true, 0x0112, 9)))).toBe(1); + }); + + it('returns 1 when the JPEG has no APP1 segment', () => { + expect(getJpegOrientationFromDataURL(buildJpegDataURL(null, true))).toBe(1); + }); + + it('returns 1 for a payload that is not a JPEG', () => { + const png = btoa(String.fromCharCode(0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a)); + expect(getJpegOrientationFromDataURL(`data:image/png;base64,${png}`)).toBe(1); + }); + + it('returns 1 for a truncated payload', () => { + const truncated = btoa(String.fromCharCode(0xff, 0xd8, 0xff)); + expect(getJpegOrientationFromDataURL(`data:image/jpeg;base64,${truncated}`)).toBe(1); + }); + + it('returns 1 for a malformed data URL', () => { + expect(getJpegOrientationFromDataURL('not a data url')).toBe(1); + }); +}); + +describe('isJpegMimeType', () => { + it('matches both JPEG MIME variants', () => { + expect(isJpegMimeType('image/jpeg')).toBe(true); + expect(isJpegMimeType('image/jpg')).toBe(true); + }); + + it('rejects other image MIME types', () => { + expect(isJpegMimeType('image/png')).toBe(false); + expect(isJpegMimeType('image/webp')).toBe(false); + }); +});