From 016b452409b421330333cf430eac2d13bc244a07 Mon Sep 17 00:00:00 2001 From: Kitson Kelly Date: Wed, 12 Apr 2023 21:51:19 +1000 Subject: [PATCH] feat: add http/etag (#3245) --- http/etag.ts | 212 +++++++++++++++++++++++++++++++++++++++ http/etag_test.ts | 129 ++++++++++++++++++++++++ http/file_server.ts | 41 ++------ http/file_server_test.ts | 26 ++--- http/mod.ts | 1 + http/util.ts | 5 +- 6 files changed, 366 insertions(+), 48 deletions(-) create mode 100644 http/etag.ts create mode 100644 http/etag_test.ts diff --git a/http/etag.ts b/http/etag.ts new file mode 100644 index 000000000000..6c81b5dea638 --- /dev/null +++ b/http/etag.ts @@ -0,0 +1,212 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +/** Provides functions for dealing with and matching ETags, including + * {@linkcode calculate} to calculate an etag for a given entity, + * {@linkcode ifMatch} for validating if an ETag matches against a `If-Match` + * header and {@linkcode ifNoneMatch} for validating an Etag against an + * `If-None-Match` header. + * + * See further information on the `ETag` header on + * [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag). + * + * @module + */ + +import { encode as base64Encode } from "../encoding/base64.ts"; + +/** Just the part of `Deno.FileInfo` that is required to calculate an `ETag`, + * so partial or user generated file information can be passed. */ +export interface FileInfo { + mtime: Date | null; + size: number; +} + +type Entity = string | Uint8Array | FileInfo; + +const encoder = new TextEncoder(); + +const DEFAULT_ALGORITHM: AlgorithmIdentifier = "SHA-256"; +const ENV_PERM_STATUS = + Deno.permissions.querySync?.({ name: "env", variable: "DENO_DEPLOYMENT_ID" }) + .state ?? "granted"; // for deno deploy +const DENO_DEPLOYMENT_ID = ENV_PERM_STATUS === "granted" + ? Deno.env.get("DENO_DEPLOYMENT_ID") + : undefined; +const HASHED_DENO_DEPLOYMENT_ID = DENO_DEPLOYMENT_ID + ? crypto.subtle.digest(DEFAULT_ALGORITHM, encoder.encode(DENO_DEPLOYMENT_ID)) + .then((hash) => + `${DENO_DEPLOYMENT_ID.length.toString(16)}-${ + base64Encode(hash).substring(0, 27) + }` + ) + : undefined; + +export interface ETagOptions { + /** A digest algorithm to use to calculate the etag. Defaults to + * `"FNV32A"`. */ + algorithm?: AlgorithmIdentifier; + + /** Override the default behavior of calculating the `ETag`, either forcing + * a tag to be labelled weak or not. */ + weak?: boolean; +} + +function isFileInfo(value: unknown): value is FileInfo { + return Boolean( + value && typeof value === "object" && "mtime" in value && "size" in value, + ); +} + +async function calcEntity( + entity: string | Uint8Array, + { algorithm = DEFAULT_ALGORITHM }: ETagOptions, +) { + // a short circuit for zero length entities + if (entity.length === 0) { + return `0-47DEQpj8HBSa+/TImW+5JCeuQeR`; + } + + if (typeof entity === "string") { + entity = encoder.encode(entity); + } + + const hash = base64Encode(await crypto.subtle.digest(algorithm, entity)) + .substring(0, 27); + + return `${entity.length.toString(16)}-${hash}`; +} + +async function calcFileInfo( + fileInfo: FileInfo, + { algorithm = DEFAULT_ALGORITHM }: ETagOptions, +) { + if (fileInfo.mtime) { + const hash = base64Encode( + await crypto.subtle.digest( + algorithm, + encoder.encode(fileInfo.mtime.toJSON()), + ), + ).substring(0, 27); + return `${fileInfo.size.toString(16)}-${hash}`; + } + return HASHED_DENO_DEPLOYMENT_ID; +} + +/** Calculate an ETag for an entity. When the entity is a specific set of data + * it will be fingerprinted as a "strong" tag, otherwise if it is just file + * information, it will be calculated as a weak tag. + * + * ```ts + * import { calculate } from "https://deno.land/std@$STD_VERSION/http/etag.ts"; + * import { assert } from "https://deno.land/std@$STD_VERSION/testing/asserts.ts" + * + * const body = "hello deno!"; + * + * const etag = await calculate(body); + * assert(etag); + * + * const res = new Response(body, { headers: { etag } }); + * ``` + */ +export async function calculate( + entity: Entity, + options: ETagOptions = {}, +): Promise { + const weak = options.weak ?? isFileInfo(entity); + const tag = + await (isFileInfo(entity) + ? calcFileInfo(entity, options) + : calcEntity(entity, options)); + + return tag ? weak ? `W/"${tag}"` : `"${tag}"` : undefined; +} + +/** A helper function that takes the value from the `If-Match` header and a + * calculated etag for the target. By using strong comparison, return `true` if + * the values match, otherwise `false`. + * + * See MDN's [`If-Match`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Match) + * article for more information on how to use this function. + * + * ```ts + * import { + * calculate, + * ifMatch, + * } from "https://deno.land/std@$STD_VERSION/http/etag.ts"; + * import { serve } from "https://deno.land/std@$STD_VERSION/http/server.ts"; + * import { assert } from "https://deno.land/std@$STD_VERSION/testing/asserts.ts" + * + * const body = "hello deno!"; + * + * await serve(async (req) => { + * const ifMatchValue = req.headers.get("if-match"); + * const etag = await calculate(body); + * assert(etag); + * if (!ifMatchValue || ifMatch(ifMatchValue, etag)) { + * return new Response(body, { status: 200, headers: { etag } }); + * } else { + * return new Response(null, { status: 412, statusText: "Precondition Failed"}); + * } + * }); + * ``` + */ +export function ifMatch( + value: string | null, + etag: string | undefined, +): boolean { + // Weak tags cannot be matched and return false. + if (!value || !etag || etag.startsWith("W/")) { + return false; + } + if (value.trim() === "*") { + return true; + } + const tags = value.split(/\s*,\s*/); + return tags.includes(etag); +} + +/** A helper function that takes the value from the `If-None-Match` header and + * a calculated etag for the target entity and returns `false` if the etag for + * the entity matches the supplied value, otherwise `true`. + * + * See MDN's [`If-None-Match`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match) + * article for more information on how to use this function. + * + * ```ts + * import { + * calculate, + * ifNoneMatch, + * } from "https://deno.land/std@$STD_VERSION/http/etag.ts"; + * import { serve } from "https://deno.land/std@$STD_VERSION/http/server.ts"; + * import { assert } from "https://deno.land/std@$STD_VERSION/testing/asserts.ts" + * + * const body = "hello deno!"; + * + * await serve(async (req) => { + * const ifNoneMatchValue = req.headers.get("if-none-match"); + * const etag = await calculate(body); + * assert(etag); + * if (!ifNoneMatch(ifNoneMatchValue, etag)) { + * return new Response(null, { status: 304, headers: { etag } }); + * } else { + * return new Response(body, { status: 200, headers: { etag } }); + * } + * }); + * ``` + */ +export function ifNoneMatch( + value: string | null, + etag: string | undefined, +): boolean { + if (!value || !etag) { + return true; + } + if (value.trim() === "*") { + return false; + } + etag = etag.startsWith("W/") ? etag.slice(2) : etag; + const tags = value.split(/\s*,\s*/).map((tag) => + tag.startsWith("W/") ? tag.slice(2) : tag + ); + return !tags.includes(etag); +} diff --git a/http/etag_test.ts b/http/etag_test.ts new file mode 100644 index 000000000000..2461241332f8 --- /dev/null +++ b/http/etag_test.ts @@ -0,0 +1,129 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +import { assert, assertEquals } from "../testing/asserts.ts"; + +import { calculate, ifMatch, ifNoneMatch } from "./etag.ts"; + +const encoder = new TextEncoder(); + +Deno.test({ + name: "etag - calculate - string - empty", + async fn() { + const actual = await calculate(""); + assertEquals(actual, `"0-47DEQpj8HBSa+/TImW+5JCeuQeR"`); + }, +}); + +Deno.test({ + name: "etag - calculate - string", + async fn() { + const actual = await calculate("hello deno"); + assertEquals(actual, `"a-YdfmHmj2RiwOVqJupcf3PLK9PuJ"`); + }, +}); + +Deno.test({ + name: "etag - calculate - Uint8Array - empty", + async fn() { + const actual = await calculate(new Uint8Array()); + assertEquals(actual, `"0-47DEQpj8HBSa+/TImW+5JCeuQeR"`); + }, +}); + +Deno.test({ + name: "etag - calculate - Uint8Array", + async fn() { + const actual = await calculate(encoder.encode("hello deno")); + assertEquals(actual, `"a-YdfmHmj2RiwOVqJupcf3PLK9PuJ"`); + }, +}); + +Deno.test({ + name: "etag - calculate - Deno.FileInfo", + async fn() { + const fixture: Deno.FileInfo = { + isFile: true, + isDirectory: false, + isSymlink: false, + size: 1024, + mtime: new Date(Date.UTC(96, 1, 2, 3, 4, 5, 6)), + atime: null, + birthtime: null, + dev: 0, + ino: null, + mode: null, + nlink: null, + uid: null, + gid: null, + rdev: null, + blksize: null, + blocks: null, + }; + const actual = await calculate(fixture); + assertEquals(actual, `W/"400-H0YzXysQPV20qNisAZMuvAEVuHV"`); + }, +}); + +Deno.test({ + name: "etag - ifMatch", + async fn() { + assert(!ifMatch(`"abcdefg"`, await calculate("hello deno"))); + assert( + ifMatch(`"a-YdfmHmj2RiwOVqJupcf3PLK9PuJ"`, await calculate("hello deno")), + ); + assert( + ifMatch( + `"abcdefg", "a-YdfmHmj2RiwOVqJupcf3PLK9PuJ"`, + await calculate("hello deno"), + ), + ); + assert(ifMatch("*", await calculate("hello deno"))); + assert( + !ifMatch( + "*", + await calculate({ + size: 1024, + mtime: new Date(Date.UTC(96, 1, 2, 3, 4, 5, 6)), + }), + ), + ); + }, +}); + +Deno.test({ + name: "etag - ifNoneMatch", + async fn() { + assert(ifNoneMatch(`"abcdefg"`, await calculate("hello deno"))); + assert( + !ifNoneMatch( + `"a-YdfmHmj2RiwOVqJupcf3PLK9PuJ"`, + await calculate("hello deno"), + ), + ); + assert( + !ifNoneMatch( + `"abcdefg", "a-YdfmHmj2RiwOVqJupcf3PLK9PuJ"`, + await calculate("hello deno"), + ), + ); + assert(!ifNoneMatch("*", await calculate("hello deno"))); + assert( + !ifNoneMatch( + `W/"400-H0YzXysQPV20qNisAZMuvAEVuHV"`, + await calculate({ + size: 1024, + mtime: new Date(Date.UTC(96, 1, 2, 3, 4, 5, 6)), + }), + ), + ); + assert( + !ifNoneMatch( + `"400-H0YzXysQPV20qNisAZMuvAEVuHV"`, + await calculate({ + size: 1024, + mtime: new Date(Date.UTC(96, 1, 2, 3, 4, 5, 6)), + }), + ), + ); + }, +}); diff --git a/http/file_server.ts b/http/file_server.ts index fa2347709c60..024522ecceb3 100644 --- a/http/file_server.ts +++ b/http/file_server.ts @@ -8,14 +8,13 @@ import { extname, posix } from "../path/mod.ts"; import { contentType } from "../media_types/content_type.ts"; import { serve, serveTls } from "./server.ts"; +import { calculate, ifNoneMatch } from "./etag.ts"; import { Status } from "./http_status.ts"; import { parse } from "../flags/mod.ts"; import { assert } from "../_util/asserts.ts"; import { red } from "../fmt/colors.ts"; -import { compareEtag, createCommonResponse } from "./util.ts"; +import { createCommonResponse } from "./util.ts"; import { DigestAlgorithm } from "../crypto/crypto.ts"; -import { toHashString } from "../crypto/to_hash_string.ts"; -import { createHash } from "../crypto/_util.ts"; import { VERSION } from "../version.ts"; interface EntryInfo { mode: string; @@ -26,17 +25,6 @@ interface EntryInfo { const encoder = new TextEncoder(); -// avoid top-lebvel-await -const envPermissionStatus = - Deno.permissions.querySync?.({ name: "env", variable: "DENO_DEPLOYMENT_ID" }) - .state ?? "granted"; // for deno deploy -const DENO_DEPLOYMENT_ID = envPermissionStatus === "granted" - ? Deno.env.get("DENO_DEPLOYMENT_ID") - : undefined; -const hashedDenoDeploymentId = DENO_DEPLOYMENT_ID - ? createHash("FNV32A", DENO_DEPLOYMENT_ID).then((hash) => toHashString(hash)) - : undefined; - function modeToString(isDir: boolean, maybeMode: number | null): string { const modeMap = ["---", "--x", "-w-", "-wx", "r--", "r-x", "rw-", "rwx"]; @@ -95,7 +83,7 @@ export interface ServeFileOptions { export async function serveFile( req: Request, filePath: string, - { etagAlgorithm, fileInfo }: ServeFileOptions = {}, + { etagAlgorithm: algorithm, fileInfo }: ServeFileOptions = {}, ): Promise { try { fileInfo ??= await Deno.stat(filePath); @@ -129,15 +117,7 @@ export async function serveFile( headers.set("date", date.toUTCString()); } - // Create a simple etag that is an md5 of the last modified date and filesize concatenated - const etag = fileInfo.mtime - ? toHashString( - await createHash( - etagAlgorithm ?? "FNV32A", - `${fileInfo.mtime.toJSON()}${fileInfo.size}`, - ), - ) - : await hashedDenoDeploymentId; + const etag = await calculate(fileInfo, { algorithm }); // Set last modified header if last modification timestamp is available if (fileInfo.mtime) { @@ -151,14 +131,15 @@ export async function serveFile( // If a `if-none-match` header is present and the value matches the tag or // if a `if-modified-since` header is present and the value is bigger than // the access timestamp value, then return 304 - const ifNoneMatch = req.headers.get("if-none-match"); - const ifModifiedSince = req.headers.get("if-modified-since"); + const ifNoneMatchValue = req.headers.get("if-none-match"); + const ifModifiedSinceValue = req.headers.get("if-modified-since"); if ( - (etag && ifNoneMatch && compareEtag(ifNoneMatch, etag)) || - (ifNoneMatch === null && + (!ifNoneMatch(ifNoneMatchValue, etag)) || + (ifNoneMatchValue === null && fileInfo.mtime && - ifModifiedSince && - fileInfo.mtime.getTime() < new Date(ifModifiedSince).getTime() + 1000) + ifModifiedSinceValue && + fileInfo.mtime.getTime() < + new Date(ifModifiedSinceValue).getTime() + 1000) ) { file.close(); diff --git a/http/file_server_test.ts b/http/file_server_test.ts index 2e09a3fbb688..7f586798ea03 100644 --- a/http/file_server_test.ts +++ b/http/file_server_test.ts @@ -8,10 +8,9 @@ import { iterateReader } from "../streams/iterate_reader.ts"; import { writeAll } from "../streams/write_all.ts"; import { TextLineStream } from "../streams/text_line_stream.ts"; import { serveDir, serveFile } from "./file_server.ts"; +import { calculate } from "./etag.ts"; import { dirname, fromFileUrl, join, resolve, toFileUrl } from "../path/mod.ts"; import { isWindows } from "../_util/os.ts"; -import { toHashString } from "../crypto/to_hash_string.ts"; -import { createHash } from "../crypto/_util.ts"; import { VERSION } from "../version.ts"; import { retry } from "../async/retry.ts"; @@ -660,16 +659,9 @@ const getTestFileStat = async (): Promise => { const getTestFileEtag = async () => { const fileInfo = await getTestFileStat(); - - if (fileInfo.mtime instanceof Date) { - const lastModified = new Date(fileInfo.mtime); - const simpleEtag = toHashString( - await createHash("FNV32A", lastModified.toJSON() + fileInfo.size), - ); - return simpleEtag; - } else { - return ""; - } + const etag = await calculate(fileInfo); + assert(etag); + return etag; }; const getTestFileLastModified = async () => { @@ -933,7 +925,7 @@ Deno.test( try { const res = await fetch("http://localhost:4507/testdata/test%20file.txt"); const expectedEtag = await getTestFileEtag(); - assertEquals(res.headers.get("etag"), `W/${expectedEtag}`); + assertEquals(res.headers.get("etag"), expectedEtag); await res.text(); // Consuming the body so that the test doesn't leak resources } finally { await killFileServer(); @@ -1111,9 +1103,9 @@ Deno.test( "file_server `serveFile` etag value falls back to DENO_DEPLOYMENT_ID if fileInfo.mtime is not available", async () => { const testDenoDeploymentId = "__THIS_IS_DENO_DEPLOYMENT_ID__"; - const hashedDenoDeploymentId = toHashString( - await createHash("FNV32A", testDenoDeploymentId), - ); + const hashedDenoDeploymentId = await calculate(testDenoDeploymentId, { + weak: true, + }); // deno-fmt-ignore const code = ` import { serveFile } from "${import.meta.resolve("./file_server.ts")}"; @@ -1124,7 +1116,7 @@ Deno.test( fileInfo.mtime = null; const req = new Request("http://localhost:4507/testdata/test file.txt"); const res = await serveFile(req, fromFileUrl(testdataPath), { fileInfo }); - assertEquals(res.headers.get("etag"), "${hashedDenoDeploymentId}"); + assertEquals(res.headers.get("etag"), \`${hashedDenoDeploymentId}\`); `; const command = new Deno.Command(Deno.execPath(), { args: ["eval", code], diff --git a/http/mod.ts b/http/mod.ts index daaa2968fbc9..b805487c7282 100644 --- a/http/mod.ts +++ b/http/mod.ts @@ -55,6 +55,7 @@ export * from "./cookie.ts"; export * from "./cookie_map.ts"; +export * from "./etag.ts"; export * from "./http_errors.ts"; export * from "./http_status.ts"; export * from "./negotiation.ts"; diff --git a/http/util.ts b/http/util.ts index 1a52804798c9..b60f9e5a2d01 100644 --- a/http/util.ts +++ b/http/util.ts @@ -4,7 +4,10 @@ import { Status, STATUS_TEXT } from "./http_status.ts"; import { deepMerge } from "../collections/deep_merge.ts"; -/** Returns true if the etags match. Weak etag comparisons are handled. */ +/** Returns true if the etags match. Weak etag comparisons are handled. + * + * @deprecated (will be removed after 0.187.0) use `etag/ifMatch` and `etag/ifNoneMatch` instead. + */ export function compareEtag(a: string, b: string): boolean { if (a === b) { return true;