diff --git a/tools/ui/src/lib/constants/jpeg-exif.ts b/tools/ui/src/lib/constants/jpeg-exif.ts new file mode 100644 index 000000000..5b2591b04 --- /dev/null +++ b/tools/ui/src/lib/constants/jpeg-exif.ts @@ -0,0 +1,30 @@ +/** + * JPEG and EXIF binary format constants for orientation parsing. + */ + +/** Bytes of file prefix to scan, the APP1 EXIF segment sits near the start */ +export const EXIF_SCAN_BYTE_LIMIT = 128 * 1024; + +/** JPEG start of image marker */ +export const JPEG_SOI_MARKER = 0xffd8; + +/** APP1 segment marker byte, carries the EXIF payload */ +export const APP1_MARKER = 0xe1; + +/** Start of scan marker byte, compressed data begins and no EXIF follows */ +export const SOS_MARKER = 0xda; + +/** "Exif" signature opening the APP1 payload, big endian uint32 */ +export const EXIF_SIGNATURE = 0x45786966; + +/** TIFF byte order mark for little endian ("II") */ +export const TIFF_LITTLE_ENDIAN = 0x4949; + +/** TIFF magic number following the byte order mark */ +export const TIFF_MAGIC = 42; + +/** EXIF tag id holding the orientation value */ +export const EXIF_ORIENTATION_TAG = 0x0112; + +/** Size in bytes of one IFD directory entry */ +export const IFD_ENTRY_SIZE = 12; diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 468009a54..70844f57e 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -34,7 +34,6 @@ import type { import { modelsStore } from '$lib/stores/models.svelte'; import { settingsStore } from '../stores/settings.svelte'; import { capImageDataURLSize } from '../utils/cap-img-size'; -import { MEGAPIXELS_TO_PIXELS } from '$lib/constants/image-size'; function getAudioInputFormat(mimeType: string): AudioInputFormat { const normalizedMimeType = mimeType.trim().toLowerCase(); @@ -961,10 +960,11 @@ export class ChatService { for (const image of imageFiles) { const maxImageResolution = settingsStore.getConfig(SETTINGS_KEYS.MAX_IMAGE_RESOLUTION); - let base64Url = image.base64Url; - if (maxImageResolution > 1 / MEGAPIXELS_TO_PIXELS) { - base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution); - } + + // Caps the resolution and bakes the jpeg exif orientation in one pass, + // untouched images pass through as is + const base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution); + contentParts.push({ type: ContentPartType.IMAGE_URL, image_url: { url: base64Url } diff --git a/tools/ui/src/lib/utils/cap-img-size.ts b/tools/ui/src/lib/utils/cap-img-size.ts index 1ba218316..c5d9b7306 100644 --- a/tools/ui/src/lib/utils/cap-img-size.ts +++ b/tools/ui/src/lib/utils/cap-img-size.ts @@ -1,11 +1,19 @@ import { MEGAPIXELS_TO_PIXELS } from '$lib/constants/image-size'; import { BASE64_IMAGE_URI_REGEX } from '$lib/constants/uri-template'; +import { getJpegOrientationFromDataURL, isJpegMimeType } from './jpeg-orientation'; import { MimeTypeImage } from '$lib/enums'; /** * Converts an Image base64 data URL to another Image data URL with capped dimensions to reduce file size. + * + * For JPEGs the EXIF orientation is baked into the pixels in the same canvas + * pass, the browser applies the rotation when decoding so naturalWidth and + * naturalHeight already describe the upright image. Backends decoding with + * stb_image ignore EXIF, see ggml-org/llama.cpp#20870. Images that need + * neither capping nor rotation pass through untouched, so at most one + * re-encode ever happens. * @param base64UrlImage - The Image base64 data URL to convert - * @param maxMegapixels - The maximum image size in megapixels for the output Image + * @param maxMegapixels - The maximum image size in megapixels for the output Image, 0 disables capping * @returns Promise resolving to Image data URL */ export function capImageDataURLSize( @@ -26,6 +34,10 @@ export function capImageDataURLSize( return reject(new Error(`Unsupported image MIME type: ${mimeType}`)); } + const orientation = isJpegMimeType(mimeType) + ? getJpegOrientationFromDataURL(base64UrlImage) + : 1; + const img = new Image(); img.onload = () => { @@ -46,6 +58,10 @@ export function capImageDataURLSize( const scaleFactor = Math.sqrt(maxPixels / totalPixels); canvas.width = Math.floor(targetWidth * scaleFactor); canvas.height = Math.floor(targetHeight * scaleFactor); + } else if (orientation > 1) { + // No capping needed but the pixels still need the rotation baked in + canvas.width = targetWidth; + canvas.height = targetHeight; } else { return resolve(base64UrlImage); } diff --git a/tools/ui/src/lib/utils/jpeg-orientation.ts b/tools/ui/src/lib/utils/jpeg-orientation.ts new file mode 100644 index 000000000..15c21017a --- /dev/null +++ b/tools/ui/src/lib/utils/jpeg-orientation.ts @@ -0,0 +1,146 @@ +import { + EXIF_SCAN_BYTE_LIMIT, + JPEG_SOI_MARKER, + APP1_MARKER, + SOS_MARKER, + EXIF_SIGNATURE, + TIFF_LITTLE_ENDIAN, + TIFF_MAGIC, + EXIF_ORIENTATION_TAG, + IFD_ENTRY_SIZE +} from '$lib/constants/jpeg-exif'; +import { MimeTypeImage } from '$lib/enums'; + +/** + * Read the EXIF orientation tag from a JPEG base64 data URL + * + * Only a bounded prefix of the base64 payload is decoded, the APP1 segment + * always sits near the start of the file. + * @param base64UrlJpeg - The JPEG base64 data URL to inspect + * @returns The orientation value (1 to 8), or 1 when absent or unreadable + */ +export function getJpegOrientationFromDataURL(base64UrlJpeg: string): number { + try { + const payloadStart = base64UrlJpeg.indexOf(',') + 1; + + if (payloadStart <= 0) { + return 1; + } + + // Keep the slice a multiple of 4 characters so atob accepts it + const charLimit = Math.ceil(EXIF_SCAN_BYTE_LIMIT / 3) * 4; + const slice = base64UrlJpeg.slice(payloadStart, payloadStart + charLimit); + const binary = atob(slice.slice(0, slice.length - (slice.length % 4))); + const bytes = new Uint8Array(binary.length); + + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + + return findExifOrientation(new DataView(bytes.buffer)); + } catch { + return 1; + } +} + +/** + * Walk the JPEG segments of a header buffer looking for the APP1 EXIF block + * @param view - DataView over the JPEG header bytes + * @returns The orientation value (1 to 8), or 1 when absent or malformed + */ +function findExifOrientation(view: DataView): number { + if (view.byteLength < 4 || view.getUint16(0) !== JPEG_SOI_MARKER) { + return 1; + } + + let offset = 2; + + while (offset + 4 <= view.byteLength) { + if (view.getUint8(offset) !== 0xff) { + return 1; + } + + const marker = view.getUint8(offset + 1); + + // Compressed image data starts here: no EXIF past this point + if (marker === SOS_MARKER) { + return 1; + } + + const segmentLength = view.getUint16(offset + 2); + + if (marker === APP1_MARKER) { + return parseExifOrientation(view, offset + 4, segmentLength); + } + + offset += 2 + segmentLength; + } + + return 1; +} + +/** + * Parse the orientation tag from an APP1 EXIF payload + * @param view - DataView over the JPEG header bytes + * @param start - Offset of the APP1 payload, right after the segment length + * @param segmentLength - Declared APP1 segment length + * @returns The orientation value (1 to 8), or 1 when absent or malformed + */ +function parseExifOrientation(view: DataView, start: number, segmentLength: number): number { + const end = Math.min(start + segmentLength, view.byteLength); + + // The payload opens with the "Exif\0\0" signature + if ( + start + 6 > end || + view.getUint32(start) !== EXIF_SIGNATURE || + view.getUint16(start + 4) !== 0 + ) { + return 1; + } + + const tiff = start + 6; + + if (tiff + 8 > end) { + return 1; + } + + const littleEndian = view.getUint16(tiff) === TIFF_LITTLE_ENDIAN; + + if (view.getUint16(tiff + 2, littleEndian) !== TIFF_MAGIC) { + return 1; + } + + const ifdOffset = view.getUint32(tiff + 4, littleEndian); + + if (tiff + ifdOffset + 2 > end) { + return 1; + } + + const entryCount = view.getUint16(tiff + ifdOffset, littleEndian); + + // Scan IFD0 entries for the orientation tag + for (let i = 0; i < entryCount; i++) { + const entry = tiff + ifdOffset + 2 + i * IFD_ENTRY_SIZE; + + if (entry + IFD_ENTRY_SIZE > end) { + return 1; + } + + if (view.getUint16(entry, littleEndian) === EXIF_ORIENTATION_TAG) { + const orientation = view.getUint16(entry + 8, littleEndian); + + return orientation >= 1 && orientation <= 8 ? orientation : 1; + } + } + + return 1; +} + +/** + * Check if a MIME type represents a JPEG + * @param mimeType - The MIME type to check + * @returns True if the MIME type is a JPEG variant + */ +export function isJpegMimeType(mimeType: string): boolean { + return mimeType === MimeTypeImage.JPEG || mimeType === MimeTypeImage.JPG; +} diff --git a/tools/ui/tests/client/cap-img-size.svelte.test.ts b/tools/ui/tests/client/cap-img-size.svelte.test.ts new file mode 100644 index 000000000..d70494208 --- /dev/null +++ b/tools/ui/tests/client/cap-img-size.svelte.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from 'vitest'; +import { capImageDataURLSize } from '$lib/utils/cap-img-size'; +import { getJpegOrientationFromDataURL } from '$lib/utils/jpeg-orientation'; + +// Real 64x32 jpegs generated with Pillow, quality 90. The upright picture is +// four solid quadrants: top left red, top right green, bottom left blue, +// bottom right yellow. For each exif value the stored pixels are inverse +// transposed so a conforming decoder shows the upright picture, exactly like +// a rotated smartphone photo. +const EXIF1 = `data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAEAAAAAAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAAgAEADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD50ooor8MP9UwooooA9uooor4I/wCcwKKKKAPhSiiiv+gM/qgKKKKAP3Vooor/AJdz+lQooooA/9k=`; +const EXIF3 = `data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAMAAAAAAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAAgAEADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD7Nooor/Mo/XQooooA/Cqiiiv+og/moKKKKAPuuiiiv+fw/lcKKKKAPEaKKK+9P+jMKKKKAP/Z`; +const EXIF5 = `data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAUAAAAAAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCABAACADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD50ooor8MP9UzwKiiiv9xj/HQ99ooor/Dk/wBizwKiiiv9xj/HQ+66KKK/5/D+Vz7qooor+ej/AE9PhWiiiv6FP8wj7qooor+ej/T0/9k=`; +const EXIF6 = `data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAYAAAAAAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCABAACADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDCooor+Tz+Hz7qooor+ej/AE9PhWiiiv6FP8wj7qooor+ej/T0/Keiiiv7CP72PAqKKK/3GP8AHQ99ooor/Dk/2LPAqKKK/wBxj/HQ/9k=`; +const EXIF8 = `data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAgAAAAAAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCABAACADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD896KKK/1TPhz32iiiv8OT/Ys8Cooor/cY/wAdD32iiiv8OT/Ys/Viiiiv49P4JPhWiiiv6FP8wj7qooor+ej/AE9PhWiiiv6FP8wj/9k=`; +const NOEXIF = `data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAAgAEADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD50ooor8MP9UwooooA9uooor4I/wCcwKKKKAPhSiiiv+gM/qgKKKKAP3Vooor/AJdz+lQooooA/9k=`; + +const RED: Rgb = [255, 0, 0]; +const GREEN: Rgb = [0, 200, 0]; +const BLUE: Rgb = [0, 0, 255]; +const YELLOW: Rgb = [255, 220, 0]; + +// Wide tolerance per channel, jpeg compression shifts solid colors a bit +const COLOR_TOLERANCE = 70; + +// 0.000512 megapixels is 512 pixels, a quarter of the area of the 2048 pixel fixtures +const QUARTER_AREA_MEGAPIXELS = 0.000512; + +type Rgb = [number, number, number]; + +function loadImage(dataUrl: string): Promise { + return new Promise((resolve, reject) => { + const img = new Image(); + img.onload = () => resolve(img); + img.onerror = () => reject(new Error('Failed to decode image.')); + img.src = dataUrl; + }); +} + +// Decodes a data URL and samples the center of each quadrant of the picture +async function quadrantColors(dataUrl: string): Promise { + const img = await loadImage(dataUrl); + const canvas = document.createElement('canvas'); + canvas.width = img.naturalWidth; + canvas.height = img.naturalHeight; + const ctx = canvas.getContext('2d')!; + ctx.drawImage(img, 0, 0); + const points = [ + [0.25, 0.25], + [0.75, 0.25], + [0.25, 0.75], + [0.75, 0.75] + ]; + return points.map(([fx, fy]) => { + const d = ctx.getImageData( + Math.floor(canvas.width * fx), + Math.floor(canvas.height * fy), + 1, + 1 + ).data; + return [d[0], d[1], d[2]]; + }); +} + +function expectUpright(colors: Rgb[]) { + const targets = [RED, GREEN, BLUE, YELLOW]; + for (let i = 0; i < 4; i++) { + for (let c = 0; c < 3; c++) { + expect(Math.abs(colors[i][c] - targets[i][c])).toBeLessThan(COLOR_TOLERANCE); + } + } +} + +describe('capImageDataURLSize orientation and capping', () => { + it('passes upright jpegs through untouched when capping is disabled', async () => { + expect(await capImageDataURLSize(EXIF1, 0)).toBe(EXIF1); + expect(await capImageDataURLSize(NOEXIF, 0)).toBe(NOEXIF); + }); + + it('passes upright jpegs through untouched when under the cap threshold', async () => { + expect(await capImageDataURLSize(EXIF1, 1)).toBe(EXIF1); + }); + + it.each([ + ['orientation 3', EXIF3], + ['orientation 5', EXIF5], + ['orientation 6', EXIF6], + ['orientation 8', EXIF8] + ])('bakes %s into upright pixels without capping', async (_label, fixture) => { + const result = await capImageDataURLSize(fixture, 0); + + expect(result).not.toBe(fixture); + + const img = await loadImage(result); + + expect(img.naturalWidth).toBe(64); + expect(img.naturalHeight).toBe(32); + expectUpright(await quadrantColors(result)); + + // The re-encoded jpeg carries no orientation tag anymore + expect(getJpegOrientationFromDataURL(result)).toBe(1); + }); + + it('caps and bakes the orientation in a single output', async () => { + const result = await capImageDataURLSize(EXIF6, QUARTER_AREA_MEGAPIXELS); + const img = await loadImage(result); + + expect(img.naturalWidth).toBe(32); + expect(img.naturalHeight).toBe(16); + expectUpright(await quadrantColors(result)); + expect(getJpegOrientationFromDataURL(result)).toBe(1); + }); + + it('caps upright jpegs without disturbing the picture', async () => { + const result = await capImageDataURLSize(EXIF1, QUARTER_AREA_MEGAPIXELS); + const img = await loadImage(result); + + expect(img.naturalWidth).toBe(32); + expect(img.naturalHeight).toBe(16); + expectUpright(await quadrantColors(result)); + }); +}); diff --git a/tools/ui/tests/unit/jpeg-orientation.test.ts b/tools/ui/tests/unit/jpeg-orientation.test.ts new file mode 100644 index 000000000..b1ad4777d --- /dev/null +++ b/tools/ui/tests/unit/jpeg-orientation.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from 'vitest'; +import { getJpegOrientationFromDataURL, isJpegMimeType } from '$lib/utils/jpeg-orientation'; + +// Builds the TIFF payload of an APP1 segment holding a single IFD0 entry +function buildTiff(littleEndian: boolean, tag: number, value: number): number[] { + const u16 = (v: number) => (littleEndian ? [v & 0xff, v >> 8] : [v >> 8, v & 0xff]); + const u32 = (v: number) => + littleEndian + ? [v & 0xff, (v >> 8) & 0xff, (v >> 16) & 0xff, (v >> 24) & 0xff] + : [(v >> 24) & 0xff, (v >> 16) & 0xff, (v >> 8) & 0xff, v & 0xff]; + + return [ + ...(littleEndian ? [0x49, 0x49] : [0x4d, 0x4d]), + ...u16(42), + ...u32(8), + ...u16(1), + ...u16(tag), + ...u16(3), + ...u32(1), + // SHORT value sits left justified in the 4 byte value field + ...u16(value), + ...u16(0), + ...u32(0) + ]; +} + +// Wraps a TIFF payload into a complete minimal JPEG data URL +function buildJpegDataURL(tiff: number[] | null, prependApp0 = false): string { + const bytes: number[] = [0xff, 0xd8]; + + if (prependApp0) { + // JFIF APP0 segment, irrelevant content the parser walks over + bytes.push(0xff, 0xe0, 0x00, 0x07, 0x4a, 0x46, 0x49, 0x46, 0x00); + } + + if (tiff) { + const payload = [0x45, 0x78, 0x69, 0x66, 0x00, 0x00, ...tiff]; + const length = payload.length + 2; + bytes.push(0xff, 0xe1, length >> 8, length & 0xff, ...payload); + } + + // SOS marker terminates the metadata scan + bytes.push(0xff, 0xda, 0x00, 0x02); + + return `data:image/jpeg;base64,${btoa(String.fromCharCode(...bytes))}`; +} + +describe('getJpegOrientationFromDataURL', () => { + it('returns the orientation from a little endian EXIF block', () => { + expect(getJpegOrientationFromDataURL(buildJpegDataURL(buildTiff(true, 0x0112, 6)))).toBe(6); + }); + + it('returns the orientation from a big endian EXIF block', () => { + expect(getJpegOrientationFromDataURL(buildJpegDataURL(buildTiff(false, 0x0112, 8)))).toBe(8); + }); + + it('walks over a leading APP0 segment', () => { + expect(getJpegOrientationFromDataURL(buildJpegDataURL(buildTiff(true, 0x0112, 3), true))).toBe( + 3 + ); + }); + + it('returns 1 when the EXIF block holds no orientation tag', () => { + expect(getJpegOrientationFromDataURL(buildJpegDataURL(buildTiff(true, 0x0100, 6)))).toBe(1); + }); + + it('returns 1 when the orientation value is out of range', () => { + expect(getJpegOrientationFromDataURL(buildJpegDataURL(buildTiff(true, 0x0112, 9)))).toBe(1); + }); + + it('returns 1 when the JPEG has no APP1 segment', () => { + expect(getJpegOrientationFromDataURL(buildJpegDataURL(null, true))).toBe(1); + }); + + it('returns 1 for a payload that is not a JPEG', () => { + const png = btoa(String.fromCharCode(0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a)); + expect(getJpegOrientationFromDataURL(`data:image/png;base64,${png}`)).toBe(1); + }); + + it('returns 1 for a truncated payload', () => { + const truncated = btoa(String.fromCharCode(0xff, 0xd8, 0xff)); + expect(getJpegOrientationFromDataURL(`data:image/jpeg;base64,${truncated}`)).toBe(1); + }); + + it('returns 1 for a malformed data URL', () => { + expect(getJpegOrientationFromDataURL('not a data url')).toBe(1); + }); +}); + +describe('isJpegMimeType', () => { + it('matches both JPEG MIME variants', () => { + expect(isJpegMimeType('image/jpeg')).toBe(true); + expect(isJpegMimeType('image/jpg')).toBe(true); + }); + + it('rejects other image MIME types', () => { + expect(isJpegMimeType('image/png')).toBe(false); + expect(isJpegMimeType('image/webp')).toBe(false); + }); +});