diff --git a/.changeset/seven-shrimps-hope.md b/.changeset/seven-shrimps-hope.md new file mode 100644 index 000000000000..bebd8bd8a571 --- /dev/null +++ b/.changeset/seven-shrimps-hope.md @@ -0,0 +1,24 @@ +--- +'@astrojs/image': patch +--- + +Adds caching support for transformed images :tada: + +Local images will be cached for 1 year and invalidated when the original image file is changed. + +Remote images will be cached based on the `fetch()` response's cache headers, similar to how a CDN would manage the cache. + +**cacheDir** + +By default, transformed images will be cached to `./node_modules/.astro/image`. This can be configured in the integration's config options. + +``` +export default defineConfig({ + integrations: [image({ + // may be useful if your hosting provider allows caching between CI builds + cacheDir: "./.cache/image" + })] +}); +``` + +Caching can also be disabled by using `cacheDir: false`. diff --git a/packages/integrations/image/README.md b/packages/integrations/image/README.md index 691dff702d80..a1ff9cec3e45 100644 --- a/packages/integrations/image/README.md +++ b/packages/integrations/image/README.md @@ -453,6 +453,25 @@ export default { } ``` +### config.cacheDir + +During static builds, the integration will cache transformed images to avoid rebuilding the same image for every build. This can be particularly helpful if you are using a hosting service that allows you to cache build assets for future deployments. + +Local images will be cached for 1 year and invalidated when the original image file is changed. Remote images will be cached based on the `fetch()` response's cache headers, similar to how a CDN would manage the cache. + +By default, transformed images will be cached to `./node_modules/.astro/image`. This can be configured in the integration's config options. + +``` +export default defineConfig({ + integrations: [image({ + // may be useful if your hosting provider allows caching between CI builds + cacheDir: "./.cache/image" + })] +}); +``` + +Caching can also be disabled by using `cacheDir: false`. + ## Examples ### Local images diff --git a/packages/integrations/image/package.json b/packages/integrations/image/package.json index e1db1fe53744..0ec075cbf357 100644 --- a/packages/integrations/image/package.json +++ b/packages/integrations/image/package.json @@ -49,12 +49,14 @@ "slash": "^4.0.0" }, "devDependencies": { + "@types/http-cache-semantics": "^4.0.1", "@types/mime": "^2.0.3", "@types/sharp": "^0.30.5", "astro": "workspace:*", "astro-scripts": "workspace:*", "chai": "^4.3.6", "cheerio": "^1.0.0-rc.11", + "http-cache-semantics": "^4.1.0", "kleur": "^4.1.4", "mocha": "^9.2.2", "rollup-plugin-copy": "^3.4.0", diff --git a/packages/integrations/image/src/build/cache.ts b/packages/integrations/image/src/build/cache.ts new file mode 100644 index 000000000000..4e0f87e7dd26 --- /dev/null +++ b/packages/integrations/image/src/build/cache.ts @@ -0,0 +1,85 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { debug, error, warn } from '../utils/logger.js'; +import type { LoggerLevel } from '../utils/logger.js'; + +const CACHE_FILE = `cache.json`; + +interface Cache { + [filename: string]: { expires: number } +} + +export class ImageCache { + #cacheDir: URL; + #cacheFile: URL; + #cache: Cache = { } + #logLevel: LoggerLevel; + + constructor(dir: URL, logLevel: LoggerLevel) { + this.#logLevel = logLevel; + this.#cacheDir = dir; + this.#cacheFile = this.#toAbsolutePath(CACHE_FILE); + } + + #toAbsolutePath(file: string) { + return new URL(path.join(this.#cacheDir.toString(), file)); + } + + async init() { + try { + const str = await fs.readFile(this.#cacheFile, 'utf-8'); + this.#cache = JSON.parse(str) as Cache; + } catch { + // noop + debug({ message: 'no cache file found', level: this.#logLevel }); + } + } + + async finalize() { + try { + await fs.mkdir(path.dirname(fileURLToPath(this.#cacheFile)), { recursive: true }); + await fs.writeFile(this.#cacheFile, JSON.stringify(this.#cache)); + } catch { + // noop + warn({ message: 'could not save the cache file', level: this.#logLevel }); + } + } + + async get(file: string): Promise { + if (!this.has(file)) { + return undefined; + } + + try { + const filepath = this.#toAbsolutePath(file); + return await fs.readFile(filepath); + } catch { + warn({ message: `could not load cached file for "${file}"`, level: this.#logLevel }); + return undefined; + } + } + + async set(file: string, buffer: Buffer, opts: Cache['string']): Promise { + try { + const filepath = this.#toAbsolutePath(file); + await fs.mkdir(path.dirname(fileURLToPath(filepath)), { recursive: true }); + await fs.writeFile(filepath, buffer); + + this.#cache[file] = opts; + } catch { + // noop + warn({ message: `could not save cached copy of "${file}"`, level: this.#logLevel }); + } + } + + has(file: string): boolean { + if (!(file in this.#cache)) { + return false; + } + + const { expires } = this.#cache[file]; + + return expires > Date.now(); + } +} diff --git a/packages/integrations/image/src/build/ssg.ts b/packages/integrations/image/src/build/ssg.ts index c3cc75642cfb..bb2e11162519 100644 --- a/packages/integrations/image/src/build/ssg.ts +++ b/packages/integrations/image/src/build/ssg.ts @@ -1,6 +1,7 @@ import { doWork } from '@altano/tiny-async-pool'; import type { AstroConfig } from 'astro'; import { bgGreen, black, cyan, dim, green } from 'kleur/colors'; +import CachePolicy from 'http-cache-semantics'; import fs from 'node:fs/promises'; import OS from 'node:os'; import path from 'node:path'; @@ -8,24 +9,66 @@ import { fileURLToPath } from 'node:url'; import type { SSRImageService, TransformOptions } from '../loaders/index.js'; import { debug, info, LoggerLevel, warn } from '../utils/logger.js'; import { isRemoteImage } from '../utils/paths.js'; +import { ImageCache } from './cache.js'; async function loadLocalImage(src: string | URL) { try { - return await fs.readFile(src); + const data = await fs.readFile(src); + + // Vite's file hash will change if the file is changed at all, + // we can safely cache local images here. + const timeToLive = new Date(); + timeToLive.setFullYear(timeToLive.getFullYear() + 1); + + return { + data, + expires: timeToLive.getTime(), + } } catch { return undefined; } } +function webToCachePolicyRequest({ url, method, headers: _headers }: Request): CachePolicy.Request { + const headers: CachePolicy.Headers = {}; + for (const [key, value] of _headers) { + headers[key] = value; + } + return { + method, + url, + headers, + }; +} + +function webToCachePolicyResponse({ status, headers: _headers }: Response): CachePolicy.Response { + const headers: CachePolicy.Headers = {}; + for (const [key, value] of _headers) { + headers[key] = value; + } + return { + status, + headers, + }; +} + async function loadRemoteImage(src: string) { try { - const res = await fetch(src); + const req = new Request(src); + const res = await fetch(req); if (!res.ok) { return undefined; } - return Buffer.from(await res.arrayBuffer()); + // calculate an expiration date based on the response's TTL + const policy = new CachePolicy(webToCachePolicyRequest(req), webToCachePolicyResponse(res)); + const expires = policy.storable() ? policy.timeToLive() : 0; + + return { + data: Buffer.from(await res.arrayBuffer()), + expires: Date.now() + expires, + }; } catch { return undefined; } @@ -42,9 +85,17 @@ export interface SSGBuildParams { config: AstroConfig; outDir: URL; logLevel: LoggerLevel; + cacheDir?: URL; } -export async function ssgBuild({ loader, staticImages, config, outDir, logLevel }: SSGBuildParams) { +export async function ssgBuild({ loader, staticImages, config, outDir, logLevel, cacheDir }: SSGBuildParams) { + let cache: ImageCache | undefined = undefined; + + if (cacheDir) { + cache = new ImageCache(cacheDir, logLevel); + await cache.init(); + } + const timer = performance.now(); const cpuCount = OS.cpus().length; @@ -67,6 +118,9 @@ export async function ssgBuild({ loader, staticImages, config, outDir, logLevel let inputFile: string | undefined = undefined; let inputBuffer: Buffer | undefined = undefined; + // tracks the cache duration for the original source image + let expires = 0; + // Vite will prefix a hashed image with the base path, we need to strip this // off to find the actual file relative to /dist if (config.base && src.startsWith(config.base)) { @@ -75,11 +129,17 @@ export async function ssgBuild({ loader, staticImages, config, outDir, logLevel if (isRemoteImage(src)) { // try to load the remote image - inputBuffer = await loadRemoteImage(src); + const res = await loadRemoteImage(src); + + inputBuffer = res?.data; + expires = res?.expires || 0; } else { const inputFileURL = new URL(`.${src}`, outDir); inputFile = fileURLToPath(inputFileURL); - inputBuffer = await loadLocalImage(inputFile); + + const res = await loadLocalImage(inputFile); + inputBuffer = res?.data; + expires = res?.expires || 0; } if (!inputBuffer) { @@ -106,14 +166,32 @@ export async function ssgBuild({ loader, staticImages, config, outDir, logLevel outputFile = fileURLToPath(outputFileURL); } - const { data } = await loader.transform(inputBuffer, transform); + const pathRelative = outputFile.replace(fileURLToPath(outDir), ''); + + let data: Buffer | undefined; + + // try to load the transformed image from cache, if available + if (cache?.has(pathRelative)) { + data = await cache.get(pathRelative); + } + + // a valid cache file wasn't found, transform the image and cache it + if (!data) { + const transformed = await loader.transform(inputBuffer, transform); + data = transformed.data; + + // cache the image, if available + if (cache) { + await cache.set(pathRelative, data, { expires }); + } + } await fs.writeFile(outputFile, data); const timeEnd = performance.now(); const timeChange = getTimeStat(timeStart, timeEnd); const timeIncrease = `(+${timeChange})`; - const pathRelative = outputFile.replace(fileURLToPath(outDir), ''); + debug({ level: logLevel, prefix: false, @@ -125,6 +203,11 @@ export async function ssgBuild({ loader, staticImages, config, outDir, logLevel // transform each original image file in batches await doWork(cpuCount, staticImages, processStaticImage); + // saves the cache's JSON manifest to file + if (cache) { + await cache.finalize(); + } + info({ level: logLevel, prefix: false, diff --git a/packages/integrations/image/src/index.ts b/packages/integrations/image/src/index.ts index cc5d0d5187d3..067e8d34f8af 100644 --- a/packages/integrations/image/src/index.ts +++ b/packages/integrations/image/src/index.ts @@ -27,14 +27,16 @@ export interface IntegrationOptions { /** * Entry point for the @type {HostedImageService} or @type {LocalImageService} to be used. */ - serviceEntryPoint?: string; + serviceEntryPoint?: '@astrojs/image/squoosh' | '@astrojs/image/sharp' | string; logLevel?: LoggerLevel; + cacheDir?: false | string; } export default function integration(options: IntegrationOptions = {}): AstroIntegration { const resolvedOptions = { serviceEntryPoint: '@astrojs/image/squoosh', logLevel: 'info' as LoggerLevel, + cacheDir: './node_modules/.astro/image', ...options, }; @@ -127,12 +129,15 @@ export default function integration(options: IntegrationOptions = {}): AstroInte } if (loader && 'transform' in loader && staticImages.size > 0) { + const cacheDir = !!resolvedOptions.cacheDir ? new URL(resolvedOptions.cacheDir, _config.root) : undefined; + await ssgBuild({ loader, staticImages, config: _config, outDir: dir, logLevel: resolvedOptions.logLevel, + cacheDir, }); } }, diff --git a/packages/integrations/image/test/image-ssg.test.js b/packages/integrations/image/test/image-ssg.test.js index 85c25a37a95b..46416f87495b 100644 --- a/packages/integrations/image/test/image-ssg.test.js +++ b/packages/integrations/image/test/image-ssg.test.js @@ -1,6 +1,7 @@ import { expect } from 'chai'; import * as cheerio from 'cheerio'; import sizeOf from 'image-size'; +import fs from 'fs/promises'; import { fileURLToPath } from 'url'; import { loadFixture } from './test-utils.js'; @@ -253,7 +254,7 @@ describe('SSG images - build', function () { size: { width: 544, height: 184, type: 'jpg' }, }, ].forEach(({ title, id, regex, size }) => { - it(title, () => { + it(title, async () => { const image = $(id); expect(image.attr('src')).to.match(regex); @@ -261,6 +262,9 @@ describe('SSG images - build', function () { expect(image.attr('height')).to.equal(size.height.toString()); verifyImage(image.attr('src'), size); + + const url = new URL('./fixtures/basic-image/node_modules/.astro/image' + image.attr('src'), import.meta.url); + expect(await fs.stat(url), 'transformed image was cached').to.not.be.undefined; }); }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ab043f2071e9..933ed4916b76 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2487,12 +2487,14 @@ importers: packages/integrations/image: specifiers: '@altano/tiny-async-pool': ^1.0.2 + '@types/http-cache-semantics': ^4.0.1 '@types/mime': ^2.0.3 '@types/sharp': ^0.30.5 astro: workspace:* astro-scripts: workspace:* chai: ^4.3.6 cheerio: ^1.0.0-rc.11 + http-cache-semantics: ^4.1.0 image-size: ^1.0.2 kleur: ^4.1.4 magic-string: ^0.25.9 @@ -2510,12 +2512,14 @@ importers: mime: 3.0.0 slash: 4.0.0 devDependencies: + '@types/http-cache-semantics': 4.0.1 '@types/mime': 2.0.3 '@types/sharp': 0.30.5 astro: link:../../astro astro-scripts: link:../../../scripts chai: 4.3.6 cheerio: 1.0.0-rc.12 + http-cache-semantics: 4.1.0 kleur: 4.1.5 mocha: 9.2.2 rollup-plugin-copy: 3.4.0 @@ -9439,6 +9443,10 @@ packages: resolution: {integrity: sha512-OcJcvP3Yk8mjYwf/IdXZtTE1tb/u0WF0qa29ER07ZHCYUBZXSN29Z1mBS+/96+kNMGTFUAbSz9X+pHmHpZrTCw==} dev: false + /@types/http-cache-semantics/4.0.1: + resolution: {integrity: sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ==} + dev: true + /@types/is-ci/3.0.0: resolution: {integrity: sha512-Q0Op0hdWbYd1iahB+IFNQcWXFq4O0Q5MwQP7uN0souuQ4rPg1vEYcnIOfr1gY+M+6rc8FGoRaBO1mOOvL29sEQ==} dependencies: