From 08c9ccb56734c64bed5083a099e9876310c81328 Mon Sep 17 00:00:00 2001 From: Jorn Mineur Date: Thu, 22 Jan 2026 19:04:36 +0100 Subject: [PATCH 1/5] Add manifest cache to prevent memory explosion on large builds --- src/image.js | 55 +++++++++++++++++++++++++++- src/manifest-cache.js | 85 +++++++++++++++++++++++++++++++++++++++++++ test/test.js | 21 +++++++++++ 3 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 src/manifest-cache.js diff --git a/src/image.js b/src/image.js index 66dc977..f023686 100644 --- a/src/image.js +++ b/src/image.js @@ -14,6 +14,9 @@ import { generateHTML } from "./generate-html.js"; import { DEFAULTS as GLOBAL_OPTIONS } from "./global-options.js"; import { existsCache, memCache, diskCache } from "./caches.js"; +import ManifestCache from "./manifest-cache.js"; + +let manifestCache = new ManifestCache(); const debug = debugUtil("Eleventy:Image"); const debugAssets = debugUtil("Eleventy:Assets"); @@ -382,7 +385,7 @@ export default class Image { let hashContents = []; if(existsCache.exists(this.src)) { - let fileContents = this.getFileContents(); + let fileContents = fs.readFileSync(this.src); // If the file starts with whitespace or the '<' character, it might be SVG. // Otherwise, skip the expensive buffer.toString() call @@ -842,11 +845,34 @@ export default class Image { return this.getStatsOnly(); } + // For production local files, check manifest cache first + if (this.#canSkipBuffer()) { + const fileStat = fs.statSync(this.src); + const optionsHash = ManifestCache.hashOptions(this.options); + const cached = manifestCache.get( + this.src, + fileStat.mtimeMs, + fileStat.size, + optionsHash + ); + + if (cached && this.#outputFilesExist(cached)) { + return cached; + } + + this.buildLogger.log(`Processing ${this.buildLogger.getFriendlyImageSource(this.src)}`, this.options); + const stats = await this.resize(this.src); + manifestCache.set(this.src, fileStat.mtimeMs, fileStat.size, optionsHash, stats); + return stats; + } + + // Dev mode / dryRun / remote URLs - need the buffer this.buildLogger.log(`Processing ${this.buildLogger.getFriendlyImageSource(this.src)}`, this.options); let input = await this.getInput(); return this.resize(input); + } catch(e) { this.buildLogger.error(`Error: ${e.message} (via ${this.buildLogger.getFriendlyImageSource(this.src)})`, this.options); @@ -924,5 +950,30 @@ export default class Image { let img = Image.create(src, opts); return img.statsByDimensionsSync(width, height); } -} + #canSkipBuffer() { + return typeof this.src === "string" + && !this.isRemoteUrl + && !this.options.dryRun + && !this.options.statsOnly + && !this.options.transformOnRequest + && !this.src.toLowerCase().endsWith(".svg") + && this.options.outputDir; // Must be writing to disk + } + + #outputFilesExist(stats) { + for (const format of Object.keys(stats)) { + for (const stat of stats[format]) { + if (stat.outputPath && !fs.existsSync(stat.outputPath)) { + return false; + } + } + } + return true; + } + + get hasLoadedBuffer() { + return this.#input !== undefined; + } + +} diff --git a/src/manifest-cache.js b/src/manifest-cache.js new file mode 100644 index 0000000..67a42b5 --- /dev/null +++ b/src/manifest-cache.js @@ -0,0 +1,85 @@ +import fs from "node:fs"; +import path from "node:path"; +import { createHashSync } from "@11ty/eleventy-utils"; + +export default class ManifestCache { + #manifest = {}; + #cacheDir; + #filepath; + #loaded = false; + + constructor(cacheDir = ".cache") { + this.#cacheDir = cacheDir; + this.#filepath = path.join(cacheDir, "eleventy-img-manifest.json"); + } + + load() { + if (this.#loaded) return; + + try { + if (fs.existsSync(this.#filepath)) { + const content = fs.readFileSync(this.#filepath, "utf8"); + this.#manifest = JSON.parse(content); + } + } catch { + // Corrupted or unreadable - start fresh + this.#manifest = {}; + } + this.#loaded = true; + } + + save() { + if (!fs.existsSync(this.#cacheDir)) { + fs.mkdirSync(this.#cacheDir, { recursive: true }); + } + fs.writeFileSync(this.#filepath, JSON.stringify(this.#manifest, null, 2)); + } + + #getKey(src, optionsHash) { + return `${src}::${optionsHash}`; + } + + get(src, mtime, size, optionsHash) { + this.load(); + + const key = this.#getKey(src, optionsHash); + const entry = this.#manifest[key]; + + if (!entry) return null; + if (entry.mtime !== mtime || entry.size !== size) return null; + + return entry.stats; + } + + set(src, mtime, size, optionsHash, stats) { + this.load(); + + // Strip buffers before storing + const cleanStats = {}; + for (const format of Object.keys(stats)) { + cleanStats[format] = stats[format].map(stat => { + const copy = { ...stat }; + delete copy.buffer; + return copy; + }); + } + + const key = this.#getKey(src, optionsHash); + this.#manifest[key] = { mtime, size, stats: cleanStats }; + this.save(); + } + + // Hash relevant options that affect output + static hashOptions(options) { + const relevant = { + widths: options.widths, + formats: options.formats, + sharpOptions: options.sharpOptions, + sharpWebpOptions: options.sharpWebpOptions, + sharpPngOptions: options.sharpPngOptions, + sharpJpegOptions: options.sharpJpegOptions, + sharpAvifOptions: options.sharpAvifOptions, + }; + return createHashSync(JSON.stringify(relevant)); + } +} \ No newline at end of file diff --git a/test/test.js b/test/test.js index 55d8688..93c7ef7 100644 --- a/test/test.js +++ b/test/test.js @@ -1277,3 +1277,24 @@ test("#105 Transparent format output filtering (no minimum transparency formats // must include one of: svg, png, or gif t.deepEqual(Object.keys(stats), ["webp", "jpeg"]); }); + +import { memCache } from "../src/caches.js"; + +test("#106 Production run should NOT load buffer into memory", async t => { + // Use unique width to avoid hitting cache from other tests + await eleventyImage("./test/bio-2017.jpg", { + widths: [347], + formats: ["jpeg"], + outputDir: "./test/img/", + }); + + for (let key of Object.keys(memCache.cache)) { + let img = memCache.cache[key].results; + if (key.includes("347") && img.hasLoadedBuffer !== undefined) { + t.false(img.hasLoadedBuffer, "Should not have loaded buffer"); + return; + } + } + t.pass(); +}); + From 7a3d14ea39ca59307d661b5c7d2ca9f1e0f5c45d Mon Sep 17 00:00:00 2001 From: Jorn Mineur Date: Sat, 24 Jan 2026 09:46:19 +0100 Subject: [PATCH 2/5] Use content hash instead of mtime for manifest cache - Replace mtime+size check with content hash for reliable CI cache restoration - Add urlFormat to #canSkipBuffer() exclusions - Style cleanup: use let instead of const per project conventions --- src/image.js | 41 +++++++++++++++++++------------- src/manifest-cache.js | 54 +++++++++++++------------------------------ test/test.js | 12 +++++----- 3 files changed, 47 insertions(+), 60 deletions(-) diff --git a/src/image.js b/src/image.js index f023686..c32b421 100644 --- a/src/image.js +++ b/src/image.js @@ -847,32 +847,28 @@ export default class Image { // For production local files, check manifest cache first if (this.#canSkipBuffer()) { - const fileStat = fs.statSync(this.src); - const optionsHash = ManifestCache.hashOptions(this.options); - const cached = manifestCache.get( - this.src, - fileStat.mtimeMs, - fileStat.size, - optionsHash - ); + let contentHash = this.getHash(); + let optionsHash = this.#getOptionsHash(); + let cacheKey = `${this.src}::${optionsHash}`; + + let cached = manifestCache.get(cacheKey, contentHash); if (cached && this.#outputFilesExist(cached)) { return cached; } this.buildLogger.log(`Processing ${this.buildLogger.getFriendlyImageSource(this.src)}`, this.options); - const stats = await this.resize(this.src); - manifestCache.set(this.src, fileStat.mtimeMs, fileStat.size, optionsHash, stats); + let stats = await this.resize(this.src); + manifestCache.set(cacheKey, contentHash, stats); return stats; } - // Dev mode / dryRun / remote URLs - need the buffer + // Dev mode, dryRun and remote URLs need the buffer this.buildLogger.log(`Processing ${this.buildLogger.getFriendlyImageSource(this.src)}`, this.options); let input = await this.getInput(); return this.resize(input); - } catch(e) { this.buildLogger.error(`Error: ${e.message} (via ${this.buildLogger.getFriendlyImageSource(this.src)})`, this.options); @@ -957,13 +953,14 @@ export default class Image { && !this.options.dryRun && !this.options.statsOnly && !this.options.transformOnRequest + && !this.options.urlFormat && !this.src.toLowerCase().endsWith(".svg") - && this.options.outputDir; // Must be writing to disk + && this.options.outputDir; } #outputFilesExist(stats) { - for (const format of Object.keys(stats)) { - for (const stat of stats[format]) { + for (let format of Object.keys(stats)) { + for (let stat of stats[format]) { if (stat.outputPath && !fs.existsSync(stat.outputPath)) { return false; } @@ -972,8 +969,20 @@ export default class Image { return true; } + #getOptionsHash() { + let relevant = { + widths: this.options.widths, + formats: this.options.formats, + sharpOptions: this.options.sharpOptions, + sharpWebpOptions: this.options.sharpWebpOptions, + sharpPngOptions: this.options.sharpPngOptions, + sharpJpegOptions: this.options.sharpJpegOptions, + sharpAvifOptions: this.options.sharpAvifOptions, + }; + return createHashSync(JSON.stringify(relevant)); + } + get hasLoadedBuffer() { return this.#input !== undefined; } - } diff --git a/src/manifest-cache.js b/src/manifest-cache.js index 67a42b5..d3edca1 100644 --- a/src/manifest-cache.js +++ b/src/manifest-cache.js @@ -1,6 +1,5 @@ import fs from "node:fs"; import path from "node:path"; -import { createHashSync } from "@11ty/eleventy-utils"; export default class ManifestCache { #manifest = {}; @@ -15,14 +14,14 @@ export default class ManifestCache { load() { if (this.#loaded) return; - + try { if (fs.existsSync(this.#filepath)) { - const content = fs.readFileSync(this.#filepath, "utf8"); + let content = fs.readFileSync(this.#filepath, "utf8"); this.#manifest = JSON.parse(content); } } catch { - // Corrupted or unreadable - start fresh + // Corrupted or unreadable, start fresh this.#manifest = {}; } this.#loaded = true; @@ -35,51 +34,30 @@ export default class ManifestCache { fs.writeFileSync(this.#filepath, JSON.stringify(this.#manifest, null, 2)); } - #getKey(src, optionsHash) { - return `${src}::${optionsHash}`; - } - - get(src, mtime, size, optionsHash) { + get(key, hash) { this.load(); - - const key = this.#getKey(src, optionsHash); - const entry = this.#manifest[key]; - + + let entry = this.#manifest[key]; if (!entry) return null; - if (entry.mtime !== mtime || entry.size !== size) return null; - + if (entry.hash !== hash) return null; + return entry.stats; } - set(src, mtime, size, optionsHash, stats) { + set(key, hash, stats) { this.load(); - + // Strip buffers before storing - const cleanStats = {}; - for (const format of Object.keys(stats)) { + let cleanStats = {}; + for (let format of Object.keys(stats)) { cleanStats[format] = stats[format].map(stat => { - const copy = { ...stat }; + let copy = { ...stat }; delete copy.buffer; return copy; }); } - - const key = this.#getKey(src, optionsHash); - this.#manifest[key] = { mtime, size, stats: cleanStats }; - this.save(); - } - // Hash relevant options that affect output - static hashOptions(options) { - const relevant = { - widths: options.widths, - formats: options.formats, - sharpOptions: options.sharpOptions, - sharpWebpOptions: options.sharpWebpOptions, - sharpPngOptions: options.sharpPngOptions, - sharpJpegOptions: options.sharpJpegOptions, - sharpAvifOptions: options.sharpAvifOptions, - }; - return createHashSync(JSON.stringify(relevant)); + this.#manifest[key] = { hash, stats: cleanStats }; + this.save(); } -} \ No newline at end of file +} diff --git a/test/test.js b/test/test.js index 93c7ef7..6d439a8 100644 --- a/test/test.js +++ b/test/test.js @@ -1280,21 +1280,21 @@ test("#105 Transparent format output filtering (no minimum transparency formats import { memCache } from "../src/caches.js"; -test("#106 Production run should NOT load buffer into memory", async t => { - // Use unique width to avoid hitting cache from other tests +test("Production run should not load source buffer into memory", async t => { + // Use unique width to avoid cache collision with other tests await eleventyImage("./test/bio-2017.jpg", { widths: [347], formats: ["jpeg"], outputDir: "./test/img/", }); - + + // Verify the cached Image instance didn't load the source buffer for (let key of Object.keys(memCache.cache)) { let img = memCache.cache[key].results; - if (key.includes("347") && img.hasLoadedBuffer !== undefined) { - t.false(img.hasLoadedBuffer, "Should not have loaded buffer"); + if (key.includes("347") && typeof img.hasLoadedBuffer !== "undefined") { + t.false(img.hasLoadedBuffer, "Source buffer should not be loaded for production local files"); return; } } t.pass(); }); - From f24dc4a1880223e858ad11b78eeacf0cfe89df4b Mon Sep 17 00:00:00 2001 From: Jorn Mineur Date: Sat, 24 Jan 2026 14:47:27 +0100 Subject: [PATCH 3/5] Update test.js Test sequence nr was missing --- test/test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.js b/test/test.js index 6d439a8..27d772f 100644 --- a/test/test.js +++ b/test/test.js @@ -1280,7 +1280,7 @@ test("#105 Transparent format output filtering (no minimum transparency formats import { memCache } from "../src/caches.js"; -test("Production run should not load source buffer into memory", async t => { +test("#106 Production run should not load source buffer into memory", async t => { // Use unique width to avoid cache collision with other tests await eleventyImage("./test/bio-2017.jpg", { widths: [347], From f4d0ce8e8052d449f2c2bbe7d2074ed08428ec12 Mon Sep 17 00:00:00 2001 From: Jorn Mineur Date: Sun, 25 Jan 2026 11:27:54 +0100 Subject: [PATCH 4/5] =?UTF-8?q?The=20presence=20of=20outputDir=20doesn't?= =?UTF-8?q?=20tell=20us=20if=20we're=20actually=20writing=20files=20?= =?UTF-8?q?=E2=80=93=20should=20be=20removed=20as=20a=20condition=20in=20#?= =?UTF-8?q?canSkipBuffer.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/image.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/image.js b/src/image.js index c32b421..f20b0c1 100644 --- a/src/image.js +++ b/src/image.js @@ -954,8 +954,7 @@ export default class Image { && !this.options.statsOnly && !this.options.transformOnRequest && !this.options.urlFormat - && !this.src.toLowerCase().endsWith(".svg") - && this.options.outputDir; + && !this.src.toLowerCase().endsWith(".svg"); } #outputFilesExist(stats) { From f33b5e25c602fafeca5d374339d1e0ace74bea0b Mon Sep 17 00:00:00 2001 From: Jorn Mineur Date: Thu, 29 Jan 2026 15:02:12 +0100 Subject: [PATCH 5/5] Use isCached instead of existsSync so file existence is checked uniformly --- src/image.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/image.js b/src/image.js index f20b0c1..d3622a5 100644 --- a/src/image.js +++ b/src/image.js @@ -960,7 +960,7 @@ export default class Image { #outputFilesExist(stats) { for (let format of Object.keys(stats)) { for (let stat of stats[format]) { - if (stat.outputPath && !fs.existsSync(stat.outputPath)) { + if (stat.outputPath && !diskCache.isCached(stat.outputPath, this.src)) { return false; } }