From 8b6c26f3887f4cf00a8c133f94b868dbb8e47538 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 6 Sep 2026 08:28:13 -0700 Subject: [PATCH] fix(gateway): honor HTTP dates when revalidating assets and media (#140173) * fix(gateway): honor UTC ordering in HTTP date validators Use one strict singleton-date admission path for static and immutable byte responses. Keep leap-second validators earlier than the following second, while preserving the Retry-After ceiling and existing ETag/range policies. Closes #140153. * test(gateway): include distinct headers in static request fixtures --- docs/web/control-ui.md | 2 + packages/ai/src/internal/retry-after.ts | 29 ++++-- .../control-ui-conditional.http.test.ts | 93 ++++++++++++++++++- src/gateway/control-ui-static.ts | 11 ++- src/gateway/control-ui.auto-root.http.test.ts | 7 +- src/gateway/control-ui.http.test.ts | 12 ++- src/gateway/control-ui.ts | 5 +- src/gateway/http-byte-range.test.ts | 37 ++++++-- src/gateway/http-byte-range.ts | 11 +-- src/gateway/http-conditional.ts | 23 ++++- 10 files changed, 195 insertions(+), 35 deletions(-) diff --git a/docs/web/control-ui.md b/docs/web/control-ui.md index 467f60364919..83913121d64d 100644 --- a/docs/web/control-ui.md +++ b/docs/web/control-ui.md @@ -1377,6 +1377,8 @@ Bundled public assets (themes, fonts, icons, and artwork) use `?v=` UR Non-index static assets use `Last-Modified` for conditional `GET` and `HEAD` requests. `If-None-Match` takes precedence over `If-Modified-Since`: `*` matches an existing asset, while other values receive the normal `200` response because static assets do not emit ETags. Date-only revalidation still returns `304` for unchanged assets. If no available content encoding is acceptable, the Gateway returns `406` before evaluating either condition. +All three HTTP-date formats are interpreted as UTC. Invalid or repeated `If-Modified-Since` fields are ignored, so they cannot suppress the current asset bytes. A leap-second validator remains earlier than the following second. + Static asset URLs support percent-encoded filenames. Contained symlinks retain the requested asset's MIME type, and a symlinked `index.html` receives the same base-path and document preparation as other entry routes. Optional absolute base (fixed asset URLs): diff --git a/packages/ai/src/internal/retry-after.ts b/packages/ai/src/internal/retry-after.ts index 5314a2cc9d7d..5cf0e5e835b8 100644 --- a/packages/ai/src/internal/retry-after.ts +++ b/packages/ai/src/internal/retry-after.ts @@ -29,6 +29,12 @@ type HttpDateComponents = { seconds: number; }; +type HttpDateInstant = { + timestampMs: number; + // Date cannot represent :60; its following timestamp is an exclusive upper bound. + leapSecond: boolean; +}; + function ownDataValue(value: unknown, key: string): unknown { return value && typeof value === "object" ? Object.getOwnPropertyDescriptor(value, key)?.value @@ -106,11 +112,19 @@ export function parseRetryAfterErrorSeconds( } } -/** Parses the three HTTP-date forms accepted for Retry-After without Date.parse normalization. */ +/** Round leap seconds forward for Retry-After so an unrepresentable instant cannot retry early. */ export function parseRetryAfterHttpDateMs(value: string, nowMs = Date.now()): number | undefined { + return parseHttpDateInstant(value, nowMs)?.timestampMs; +} + +/** Parse all three HTTP-date forms while retaining leap-second ordering. */ +export function parseHttpDateInstant( + value: string, + nowMs = Date.now(), +): HttpDateInstant | undefined { const imfFixdate = IMF_FIXDATE_RE.exec(value); if (imfFixdate) { - return parseHttpDateComponentsMs({ + return parseHttpDateComponents({ weekday: HTTP_DATE_SHORT_WEEKDAY_INDEX.get(imfFixdate[1] ?? ""), year: Number.parseInt(imfFixdate[4] ?? "", 10), month: HTTP_DATE_MONTH_INDEX.get(imfFixdate[3] ?? ""), @@ -153,12 +167,12 @@ export function parseRetryAfterHttpDateMs(value: string, nowMs = Date.now()): nu now.getUTCMilliseconds(), ); const resolvedYear = candidate > fiftyYearsFromNow ? candidateYear - 100 : candidateYear; - return parseHttpDateComponentsMs({ year: resolvedYear, ...components }); + return parseHttpDateComponents({ year: resolvedYear, ...components }); } const asctimeDate = OBSOLETE_ASCTIME_DATE_RE.exec(value); if (asctimeDate) { - return parseHttpDateComponentsMs({ + return parseHttpDateComponents({ weekday: HTTP_DATE_SHORT_WEEKDAY_INDEX.get(asctimeDate[1] ?? ""), year: Number.parseInt(asctimeDate[7] ?? "", 10), month: HTTP_DATE_MONTH_INDEX.get(asctimeDate[2] ?? ""), @@ -172,16 +186,17 @@ export function parseRetryAfterHttpDateMs(value: string, nowMs = Date.now()): nu return undefined; } -function parseHttpDateComponentsMs(components: HttpDateComponents): number | undefined { +function parseHttpDateComponents(components: HttpDateComponents): HttpDateInstant | undefined { const timestamp = parseHttpDateCalendarMs(components); if (timestamp === undefined) { return undefined; } - const weekdayTimestamp = components.seconds === 60 ? timestamp - 1_000 : timestamp; + const leapSecond = components.seconds === 60; + const weekdayTimestamp = leapSecond ? timestamp - 1_000 : timestamp; if (new Date(weekdayTimestamp).getUTCDay() !== components.weekday) { return undefined; } - return timestamp; + return { timestampMs: timestamp, leapSecond }; } function parseHttpDateCalendarMs( diff --git a/src/gateway/control-ui-conditional.http.test.ts b/src/gateway/control-ui-conditional.http.test.ts index 309cd528b557..0b8e5c63b258 100644 --- a/src/gateway/control-ui-conditional.http.test.ts +++ b/src/gateway/control-ui-conditional.http.test.ts @@ -11,11 +11,56 @@ const modifiedAt = new Date("2024-01-01T00:00:00.000Z"); const lastModified = modifiedAt.toUTCString(); const laterModifiedSince = new Date("2024-01-02T00:00:00.000Z").toUTCString(); const earlierModifiedSince = new Date("2023-12-31T00:00:00.000Z").toUTCString(); +const beforeLeapSecondAsset = { + filename: "leap-before.js", + modifiedAt: new Date("2016-12-31T23:59:59.000Z"), + body: Buffer.from('console.log("before leap second");\n'), +}; +const afterLeapSecondAsset = { + filename: "leap-after.js", + modifiedAt: new Date("2017-01-01T00:00:00.000Z"), + body: Buffer.from('console.log("after leap second");\n'), +}; +const leapSecondDates = [ + { name: "IMF-fixdate", value: "Sat, 31 Dec 2016 23:59:60 GMT" }, + { name: "RFC 850", value: "Saturday, 31-Dec-16 23:59:60 GMT" }, + { name: "asctime", value: "Sat Dec 31 23:59:60 2016" }, +]; const conditionalCases: { name: string; - headers: Record; + headers: Record; status: 200 | 304; }[] = [ + { + name: "HTTP-date RFC850 matching instant", + headers: { "If-Modified-Since": "Monday, 01-Jan-24 00:00:00 GMT" }, + status: 304, + }, + { + name: "HTTP-date asctime older UTC instant", + headers: { "If-Modified-Since": "Sun Dec 31 20:00:00 2023" }, + status: 200, + }, + { + name: "HTTP-date asctime matching UTC instant", + headers: { "If-Modified-Since": "Mon Jan 1 00:00:00 2024" }, + status: 304, + }, + { + name: "HTTP-date rejects ISO timestamp", + headers: { "If-Modified-Since": "2024-01-02T00:00:00.000Z" }, + status: 200, + }, + { + name: "HTTP-date rejects duplicate fields with later date first", + headers: { "If-Modified-Since": [laterModifiedSince, earlierModifiedSince] }, + status: 200, + }, + { + name: "HTTP-date rejects duplicate fields with earlier date first", + headers: { "If-Modified-Since": [earlierModifiedSince, laterModifiedSince] }, + status: 200, + }, { name: "unconditional request", headers: {}, status: 200 }, { name: "equal If-Modified-Since", @@ -69,7 +114,7 @@ const conditionalCases: { function requestAsset( url: string, method: "GET" | "HEAD", - headers: Record, + headers: Record, ): Promise<{ response: IncomingMessage; body: Buffer }> { // Node HTTP preserves an explicitly empty If-None-Match on the wire. return new Promise((resolve, reject) => { @@ -101,6 +146,11 @@ describe.each([ await fs.writeFile(assetPath, assetBody); await fs.writeFile(`${assetPath}.gz`, gzipSync(assetBody)); await fs.utimes(assetPath, modifiedAt, modifiedAt); + for (const asset of [beforeLeapSecondAsset, afterLeapSecondAsset]) { + const target = path.join(root, "assets", asset.filename); + await fs.writeFile(target, asset.body); + await fs.utimes(target, asset.modifiedAt, asset.modifiedAt); + } for (const publicAsset of [ "themes/absolutely.css", "fonts/test.css", @@ -123,6 +173,10 @@ describe.each([ ``, ); server = createServer((req, res) => { + res.setHeader( + "X-Test-If-Modified-Since-Count", + String(req.headersDistinct["if-modified-since"]?.length ?? 0), + ); void handleControlUiHttpRequest(req, res, { basePath, config: {}, @@ -223,7 +277,10 @@ describe.each([ it.each(conditionalCases)("$name", async ({ headers, status }) => { const { response, body } = await requestAsset(assetUrl, method, headers); - expect(response.statusCode).toBe(status); + if (Array.isArray(headers["If-Modified-Since"])) { + expect(response.headers["x-test-if-modified-since-count"]).toBe("2"); + } + expect(response.statusCode, `TZ=${process.env.TZ ?? "system"}`).toBe(status); expect(response.headers["last-modified"]).toBe(lastModified); expect(response.headers["cache-control"]).toBe(cacheControl); expect(response.headers.vary).toBe("Accept-Encoding"); @@ -234,6 +291,36 @@ describe.each([ expect(body).toEqual(status === 200 && method === "GET" ? assetBody : Buffer.alloc(0)); }); + it.each([ + ...leapSecondDates.flatMap(({ name, value }) => [ + { name: `${name} before midnight`, value, asset: afterLeapSecondAsset, status: 200 }, + { + name: `${name} after the prior second`, + value, + asset: beforeLeapSecondAsset, + status: 304, + }, + ]), + { + name: "the following midnight equality", + value: "Sun, 01 Jan 2017 00:00:00 GMT", + asset: afterLeapSecondAsset, + status: 304, + }, + ])("preserves leap second ordering for $name", async ({ value, asset, status }) => { + const { response, body } = await requestAsset(`${baseUrl}/assets/${asset.filename}`, method, { + "If-Modified-Since": value, + }); + + expect(response.statusCode).toBe(status); + expect(response.headers["last-modified"]).toBe(asset.modifiedAt.toUTCString()); + expect(response.headers["cache-control"]).toBe(cacheControl); + expect(response.headers["content-length"]).toBe( + status === 304 ? undefined : String(asset.body.length), + ); + expect(body).toEqual(status === 200 && method === "GET" ? asset.body : Buffer.alloc(0)); + }); + it.each>([ { "If-Modified-Since": laterModifiedSince }, { "If-None-Match": "*" }, diff --git a/src/gateway/control-ui-static.ts b/src/gateway/control-ui-static.ts index 8a347149e5a7..bb5e51f6f8cf 100644 --- a/src/gateway/control-ui-static.ts +++ b/src/gateway/control-ui-static.ts @@ -6,6 +6,7 @@ import { brotliCompress, constants as zlibConstants, gzip } from "node:zlib"; import { pruneMapToMaxSize } from "../infra/map-size.js"; import { getOrCreatePromise } from "../shared/lazy-promise.js"; import { respondPlainText } from "./control-ui-http-utils.js"; +import { matchesHttpIfModifiedSince } from "./http-conditional.js"; const CONTROL_UI_IMMUTABLE_CACHE_CONTROL = "public, max-age=31536000, immutable"; const CONTROL_UI_HTML_COMPRESSION_CACHE_MAX_ENTRIES = 4; @@ -184,7 +185,11 @@ function setControlUiFileHeaders( } /** Revalidate no-cache static assets without generating entity tags. */ -export function isControlUiFileUnmodified(req: IncomingMessage, lastModifiedMs: number): boolean { +export function isControlUiFileUnmodified( + req: IncomingMessage, + lastModifiedMs: number, + nowMs = Date.now(), +): boolean { if (req.method !== "GET" && req.method !== "HEAD") { return false; } @@ -193,9 +198,7 @@ export function isControlUiFileUnmodified(req: IncomingMessage, lastModifiedMs: if (ifNoneMatch !== undefined) { return ifNoneMatch.trim() === "*"; } - const header = req.headers?.["if-modified-since"]; - const since = typeof header === "string" ? Date.parse(header) : Number.NaN; - return Number.isFinite(since) && Math.floor(lastModifiedMs / 1000) * 1000 <= since; + return matchesHttpIfModifiedSince(req, lastModifiedMs, nowMs); } export function respondControlUiNotModified( diff --git a/src/gateway/control-ui.auto-root.http.test.ts b/src/gateway/control-ui.auto-root.http.test.ts index 55df69be7f69..97a7a60a2ecc 100644 --- a/src/gateway/control-ui.auto-root.http.test.ts +++ b/src/gateway/control-ui.auto-root.http.test.ts @@ -33,7 +33,12 @@ describe("handleControlUiHttpRequest prepared root lifecycle", () => { await fs.link(path.join(assetsDir, "app.js"), path.join(assetsDir, "app.hl.js")); const { res, end } = makeMockHttpResponse(); const handled = await handleControlUiHttpRequest( - { url: "/assets/app.hl.js", method: "GET" } as IncomingMessage, + { + url: "/assets/app.hl.js", + method: "GET", + headers: {}, + headersDistinct: {}, + } as IncomingMessage, res, { root: { kind: "bundled", path: tmp, realPath: await fs.realpath(tmp) } }, ); diff --git a/src/gateway/control-ui.http.test.ts b/src/gateway/control-ui.http.test.ts index 429d209d6bae..9ab47e9aceab 100644 --- a/src/gateway/control-ui.http.test.ts +++ b/src/gateway/control-ui.http.test.ts @@ -212,7 +212,17 @@ describe("handleControlUiHttpRequest", () => { }) { const { res, end, setHeader } = makeMockHttpResponse(); const handled = await handleControlUiHttpRequest( - { url: params.url, method: params.method, headers: params.headers ?? {} } as IncomingMessage, + { + url: params.url, + method: params.method, + headers: params.headers ?? {}, + headersDistinct: Object.fromEntries( + Object.entries(params.headers ?? {}).map(([name, value]) => [ + name, + Array.isArray(value) ? value : [String(value)], + ]), + ), + } as IncomingMessage, res, { ...(params.basePath ? { basePath: params.basePath } : {}), diff --git a/src/gateway/control-ui.ts b/src/gateway/control-ui.ts index b1147ecd294e..70969ae976c2 100644 --- a/src/gateway/control-ui.ts +++ b/src/gateway/control-ui.ts @@ -1241,7 +1241,8 @@ export async function handleControlUiHttpRequest( ) { // Future filesystem clocks must not make later replacements look unmodified; // clamp to response origination as in resolveByteResponse. - const lastModifiedMs = Math.floor(Math.min(safeFile.mtimeMs, Date.now()) / 1_000) * 1_000; + const originatedAtMs = Date.now(); + const lastModifiedMs = Math.floor(Math.min(safeFile.mtimeMs, originatedAtMs) / 1_000) * 1_000; const representation = resolveOpenedControlUiRepresentation({ req, sourceFile: safeFile, @@ -1255,7 +1256,7 @@ export async function handleControlUiHttpRequest( return true; } // Negotiation failures precede preconditions; release the selected representation on 304. - if (isControlUiFileUnmodified(req, lastModifiedMs)) { + if (isControlUiFileUnmodified(req, lastModifiedMs, originatedAtMs)) { fs.closeSync(representation.bodyFile.fd); respondControlUiNotModified(res, { immutable: immutableAsset, lastModifiedMs }); return true; diff --git a/src/gateway/http-byte-range.test.ts b/src/gateway/http-byte-range.test.ts index 8158c0ed45ca..eaad4fcb0233 100644 --- a/src/gateway/http-byte-range.test.ts +++ b/src/gateway/http-byte-range.test.ts @@ -282,27 +282,50 @@ describe("resolveByteResponse", () => { ).toMatchObject({ kind: "not-modified", statusCode: 304 }); }); - it("includes a leap second when applying the RFC 850 rolling-year boundary", () => { + it.each([ + { name: "the prior second", mtimeMs: Date.UTC(1976, 11, 31, 23, 59, 59), statusCode: 304 }, + { name: "the following midnight", mtimeMs: Date.UTC(1977, 0, 1), statusCode: 200 }, + ])("orders the RFC 850 rolling-year leap second against $name", ({ mtimeMs, statusCode }) => { expect( resolveByteResponse({ file: { size: 10 }, - validators: createImmutableFileValidators({ size: 10, mtimeMs: Date.UTC(1977, 0, 1) }), + validators: createImmutableFileValidators({ size: 10, mtimeMs }), nowMs: Date.UTC(2026, 11, 31, 23, 59, 59), method: "GET", request: createByteRequest({ "if-modified-since": "Friday, 31-Dec-76 23:59:60 GMT" }), }), - ).toMatchObject({ kind: "not-modified", statusCode: 304 }); + ).toMatchObject({ kind: statusCode === 304 ? "not-modified" : "full", statusCode }); }); - it("accepts a valid HTTP-date leap second without JavaScript date normalization", () => { + it.each([ + ...[ + { name: "IMF-fixdate", value: "Sat, 31 Dec 2016 23:59:60 GMT" }, + { name: "RFC 850", value: "Saturday, 31-Dec-16 23:59:60 GMT" }, + { name: "asctime", value: "Sat Dec 31 23:59:60 2016" }, + ].flatMap(({ name, value }) => [ + { name: `${name} before midnight`, value, mtimeMs: Date.UTC(2017, 0, 1), statusCode: 200 }, + { + name: `${name} after the prior second`, + value, + mtimeMs: Date.UTC(2016, 11, 31, 23, 59, 59), + statusCode: 304, + }, + ]), + { + name: "the following midnight equality", + value: "Sun, 01 Jan 2017 00:00:00 GMT", + mtimeMs: Date.UTC(2017, 0, 1), + statusCode: 304, + }, + ])("preserves leap second ordering for $name", ({ value, mtimeMs, statusCode }) => { expect( resolveByteResponse({ file: { size: 10 }, - validators: createImmutableFileValidators({ size: 10, mtimeMs: Date.UTC(2017, 0, 1) }), + validators: createImmutableFileValidators({ size: 10, mtimeMs }), method: "GET", - request: createByteRequest({ "if-modified-since": "Sat, 31 Dec 2016 23:59:60 GMT" }), + request: createByteRequest({ "if-modified-since": value }), }), - ).toMatchObject({ kind: "not-modified", statusCode: 304 }); + ).toMatchObject({ kind: statusCode === 304 ? "not-modified" : "full", statusCode }); }); it.each([ diff --git a/src/gateway/http-byte-range.ts b/src/gateway/http-byte-range.ts index c90cda482694..7bd217430f1b 100644 --- a/src/gateway/http-byte-range.ts +++ b/src/gateway/http-byte-range.ts @@ -1,8 +1,7 @@ import { createHash } from "node:crypto"; import type { FileHandle } from "node:fs/promises"; import type { IncomingMessage, ServerResponse } from "node:http"; -import { parseRetryAfterHttpDateMs } from "@openclaw/ai/internal/retry-after"; -import { matchesHttpIfNoneMatch } from "./http-conditional.js"; +import { matchesHttpIfModifiedSince, matchesHttpIfNoneMatch } from "./http-conditional.js"; type FileIdentity = { size: number; @@ -102,19 +101,13 @@ export function resolveByteResponse(params: { lastModifiedMs === undefined ? undefined : new Date(lastModifiedMs).toUTCString(); const headers = params.request?.headers; const ifNoneMatch = headers?.["if-none-match"]; - const ifModifiedSinceValues = params.request?.headersDistinct["if-modified-since"]; - // Node drops duplicate singleton fields from headers; only the distinct list is authoritative. - const ifModifiedSince = - ifModifiedSinceValues?.length === 1 ? ifModifiedSinceValues[0] : undefined; // Any If-None-Match field supersedes If-Modified-Since, even when no ETag matches. if ( (params.method === "GET" || params.method === "HEAD") && (matchesHttpIfNoneMatch(ifNoneMatch, etag) || (ifNoneMatch === undefined && lastModifiedMs !== undefined && - typeof ifModifiedSince === "string" && - (parseRetryAfterHttpDateMs(ifModifiedSince, originatedAtMs) ?? Number.NEGATIVE_INFINITY) >= - lastModifiedMs)) + matchesHttpIfModifiedSince(params.request, lastModifiedMs, originatedAtMs))) ) { // RFC 9110 evaluates representation validators before Range or If-Range. return { kind: "not-modified", statusCode: 304, etag, lastModified }; diff --git a/src/gateway/http-conditional.ts b/src/gateway/http-conditional.ts index ef9ea774c587..92b33acc4425 100644 --- a/src/gateway/http-conditional.ts +++ b/src/gateway/http-conditional.ts @@ -1,4 +1,25 @@ -// HTTP conditional requests use weak entity-tag comparison for representation reuse. +// HTTP validators share strict date admission and weak entity-tag comparison. +import type { IncomingMessage } from "node:http"; +import { parseHttpDateInstant } from "@openclaw/ai/internal/retry-after"; + +export function matchesHttpIfModifiedSince( + request: Pick | undefined, + lastModifiedMs: number, + nowMs = Date.now(), +): boolean { + // Node drops duplicate singleton fields from headers; the distinct list owns admission. + const values = request?.headersDistinct["if-modified-since"]; + const value = values?.length === 1 ? values[0] : undefined; + const since = typeof value === "string" ? parseHttpDateInstant(value, nowMs) : undefined; + const modifiedSecondMs = Math.floor(lastModifiedMs / 1_000) * 1_000; + return ( + since !== undefined && + (since.leapSecond + ? modifiedSecondMs < since.timestampMs + : modifiedSecondMs <= since.timestampMs) + ); +} + export function matchesHttpIfNoneMatch( header: string | string[] | undefined, etag: string | undefined,