From bdf2018952e6e7822bdc7da9ce080d01a22acaca Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Mon, 23 Feb 2026 13:49:37 +1300 Subject: [PATCH 1/4] feat: trigger a lambda:start event for extra log context --- packages/lambda-tiler/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/lambda-tiler/src/index.ts b/packages/lambda-tiler/src/index.ts index 475628915..77f350289 100644 --- a/packages/lambda-tiler/src/index.ts +++ b/packages/lambda-tiler/src/index.ts @@ -48,6 +48,8 @@ handler.router.hook('request', (req) => { randomTrace(req); req.set('name', 'LambdaTiler'); + + req.log.debug({ ...req.logContext }, 'Lambda:Start'); }); handler.router.hook('response', (req, res) => { From 3d3887502b15119a730ae7336d3582f3a6e1fb85 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Mon, 23 Feb 2026 20:16:37 +1300 Subject: [PATCH 2/4] fix: lower memory usage when dealing with large tiff requests --- packages/lambda-tiler/src/cli/render.tile.ts | 30 +++++++++++++++----- packages/shared/src/file.system.ts | 25 ++++++++++++++++ packages/shared/src/index.ts | 1 + packages/tiler-sharp/src/index.ts | 11 ++++--- 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/packages/lambda-tiler/src/cli/render.tile.ts b/packages/lambda-tiler/src/cli/render.tile.ts index b99798b18..580c86f3d 100644 --- a/packages/lambda-tiler/src/cli/render.tile.ts +++ b/packages/lambda-tiler/src/cli/render.tile.ts @@ -1,7 +1,7 @@ -import { ConfigProviderMemory } from '@basemaps/config'; +import { ConfigImagery, ConfigProviderMemory, ConfigTileSetRaster } from '@basemaps/config'; import { initConfigFromUrls } from '@basemaps/config-loader'; import { Tile, TileMatrixSet, TileMatrixSets } from '@basemaps/geo'; -import { fsa, LogConfig, setDefaultConfig } from '@basemaps/shared'; +import { fsa, FsaLocalCache, LogConfig, setDefaultConfig } from '@basemaps/shared'; import { LambdaHttpRequest, LambdaUrlRequest, UrlEvent } from '@linzjs/lambda'; import { Context } from 'aws-lambda'; import { extname } from 'path'; @@ -25,6 +25,9 @@ if (sourceRaw == null || tilePath == null) { `); process.exit(1); } + +fsa.middleware.push(FsaLocalCache); + const source = fsa.toUrl(sourceRaw); const tile = fromPath(tilePath); let tileMatrix: TileMatrixSet | null = null; @@ -43,14 +46,27 @@ function fromPath(s: string): Tile & { extension: string } { return tile; } +async function loadConfig( + url: URL, +): Promise<{ tileSet: ConfigTileSetRaster; imagery: ConfigImagery[]; cfg: ConfigProviderMemory }> { + if (url.pathname.endsWith('.json') || url.pathname.endsWith('.json.gz')) { + const cfg = ConfigProviderMemory.fromJson(await fsa.readJson(url), url); + LogConfig.get().info({ url: url.href }, 'ConfigLoaded'); + const imagery = [...cfg.objects.values()].filter((f) => f.id.startsWith('im_')) as ConfigImagery[]; + return { tileSet: cfg.imageryToTileSetByName(imagery[0]), imagery, cfg }; + } + + const cfg = new ConfigProviderMemory(); + const { imagery } = await initConfigFromUrls(cfg, [url]); + return { tileSet: cfg.imageryToTileSetByName(imagery[0]), imagery, cfg }; +} + async function main(): Promise { const log = LogConfig.get(); - log.level = 'trace'; - const provider = new ConfigProviderMemory(); - setDefaultConfig(provider); - const { imagery } = await initConfigFromUrls(provider, [source]); + log.level = 'debug'; - const tileSet = provider.imageryToTileSetByName(imagery[0]); + const { tileSet, imagery, cfg } = await loadConfig(source); + setDefaultConfig(cfg); log.info({ tileSet: tileSet.name, layers: tileSet.layers.length }, 'TileSet:Loaded'); diff --git a/packages/shared/src/file.system.ts b/packages/shared/src/file.system.ts index 5907de024..7c9117d49 100644 --- a/packages/shared/src/file.system.ts +++ b/packages/shared/src/file.system.ts @@ -139,6 +139,31 @@ export const FsaLog = { }; fsa.middleware.push(FsaLog); +/** + * Middleware to cache responses locally into .cache/ + * + * very useful when developing with remote sources as it avoids repeatedly downloading the same data + */ +export const FsaLocalCache = { + name: 'source:local-cache', + remotes: new Set(['s3:', 'http:', 'https:']), + async fetch(req: SourceRequest, next: SourceCallback): Promise { + if (this.remotes.has(req.source.url.protocol) === false) return next(req); + + const requestId = sha256base58( + JSON.stringify({ source: req.source.url.href, offset: req.offset, length: req.length }), + ); + const cacheUrl = fsa.toUrl(`./.cache/${requestId}`); + const bytes = await fsa.read(cacheUrl).catch(() => {}); + if (bytes) return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); + + return next(req).then(async (res) => { + await fsa.write(cacheUrl, Buffer.from(res)).catch(() => {}); + return res; + }); + }, +}; + /** * When chunkd moves to URLs this can be removed * diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 9e592b87c..69089b485 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -12,6 +12,7 @@ export { Fsa as fsa, FsaCache, FsaChunk, + FsaLocalCache, FsaLog, s3Config, signS3Get, diff --git a/packages/tiler-sharp/src/index.ts b/packages/tiler-sharp/src/index.ts index 3ae88a6c3..b368fa1a5 100644 --- a/packages/tiler-sharp/src/index.ts +++ b/packages/tiler-sharp/src/index.ts @@ -108,6 +108,9 @@ export class TileMakerSharp implements TileMaker { } else { todo.push(this.composeTile(comp, ctx.resizeKernel)); } + + // Limit the number of outstanding requests to prevent memory issues + if (todo.length % 128 === 0) await Promise.all(todo); } const overlays = await Promise.all(todo).then((items) => items.filter(notEmpty)); metrics.end('compose:overlay'); @@ -159,13 +162,13 @@ export class TileMakerSharp implements TileMaker { } async composeTilePipeline(comp: CompositionTiff, ctx: TileMakerContext): Promise { - const tile = await comp.asset.images[comp.source.imageId].getTile(comp.source.x, comp.source.y); + let tile = await comp.asset.images[comp.source.imageId].getTile(comp.source.x, comp.source.y); if (tile == null) return null; const tiffTile = { imageId: comp.source.imageId, x: comp.source.x, y: comp.source.y }; - const bytes = await Decompressors[tile.compression]?.bytes(comp.asset, tiffTile, tile.bytes); - if (bytes == null) throw new Error(`Failed to decompress: ${comp.asset.source.url.href}`); + let result = await Decompressors[tile.compression]?.bytes(comp.asset, tiffTile, tile.bytes); + tile = null; + if (result == null) throw new Error(`Failed to decompress: ${comp.asset.source.url.href}`); - let result = bytes; if (ctx.pipeline) { const resizePerf = performance.now(); result = cropResize(comp.asset, result, comp, 'bilinear'); From 550f0f90f9a9fdae7070e922946a27534b5591f3 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Mon, 23 Feb 2026 20:26:59 +1300 Subject: [PATCH 3/4] refactor: use p-limit to reduce concurrency --- package-lock.json | 34 ++++++++++++++++--------------- packages/tiler-sharp/package.json | 4 +++- packages/tiler-sharp/src/index.ts | 26 +++++++++++++++-------- 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index b4c9eb61d..639480732 100644 --- a/package-lock.json +++ b/package-lock.json @@ -422,7 +422,6 @@ "version": "3.470.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.470.0.tgz", "integrity": "sha512-a6wQd0Il9bLFnOY6/ANZl4Lv0UxiIJjrJFkyAuLOMs4heVfCSgDs16AD+Ujm4Qm9byl9/jF6gLxDrRRuN2FrMg==", - "peer": true, "dependencies": { "@aws-crypto/sha256-browser": "3.0.0", "@aws-crypto/sha256-js": "3.0.0", @@ -475,7 +474,6 @@ "version": "3.472.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.472.0.tgz", "integrity": "sha512-Gth+HNhCIjEE6YoeC59ypGqaJ+cZuznuLJ2t3PL+9BQ/2pWXaxhZ5QXFxaJidtzdyebOAgY87/7g3D6PHnWQDg==", - "peer": true, "dependencies": { "@aws-crypto/sha1-browser": "3.0.0", "@aws-crypto/sha256-browser": "3.0.0", @@ -5391,7 +5389,6 @@ "integrity": "sha512-/g2d4sW9nUDJOMz3mabVQvOGhVa4e/BN/Um7yca9Bb2XTzPPnfTWHWQg+IsEYO7M3Vx+EXvaM/I2pJWIMun1bg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@octokit/auth-token": "^4.0.0", "@octokit/graphql": "^7.1.0", @@ -6915,7 +6912,6 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.16.0.tgz", "integrity": "sha512-ar9E+k7CU8rWi2e5ErzQiC93KKEFAXA2Kky0scAlPcxYblLt8+XZuHUZwlyfXILyQa95P6lQg+eZgh/dDs3+Vw==", "dev": true, - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "7.16.0", "@typescript-eslint/types": "7.16.0", @@ -7182,7 +7178,6 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==", "dev": true, - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -7675,7 +7670,6 @@ ], "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "@aws-cdk/asset-awscli-v1": "2.2.242", "@aws-cdk/asset-node-proxy-agent-v6": "^2.1.0", @@ -8951,8 +8945,7 @@ "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.4.3.tgz", "integrity": "sha512-3+ZB67qWGM1vEstNpj6pGaLNN1qz4gxC1CBhEUhZDZk0PqzQWY65IzC1Doq17MGPa9xa2wJ1G/DJ3swU8kWAHQ==", "dev": true, - "license": "Apache-2.0", - "peer": true + "license": "Apache-2.0" }, "node_modules/content-disposition": { "version": "0.5.4", @@ -10651,7 +10644,6 @@ "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", "dev": true, - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -10707,7 +10699,6 @@ "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-9.1.0.tgz", "integrity": "sha512-NSWl5BFQWEPi1j4TjVNItzYV7dZXZ+wP6I6ZhrBGpChQhZRUaElihE9uRRkcbRnNb76UMKDF3r+WTmNcGPKsqw==", "dev": true, - "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -16929,7 +16920,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -17184,7 +17174,6 @@ "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.2.tgz", "integrity": "sha512-rAVeHYMcv8ATV5d508CFdn+8/pHPpXeIid1DdrPwXnaAdH7cqjVbpJaT5eq4yRAFU/lsbwYwSF/n5iNrdJHPQA==", "dev": true, - "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -17510,7 +17499,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", "integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==", "dev": true, - "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -17523,7 +17511,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz", "integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==", "dev": true, - "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.0" @@ -19559,7 +19546,6 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==", "dev": true, - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -20882,7 +20868,8 @@ "@basemaps/tiler": "^8.12.2", "@linzjs/metrics": "^8.0.0", "fzstd": "^0.1.1", - "lerc": "^4.0.4" + "lerc": "^4.0.4", + "p-limit": "^6.2.0" }, "devDependencies": { "@types/pixelmatch": "^5.2.3", @@ -20914,6 +20901,21 @@ "@types/node": "*" } }, + "packages/tiler-sharp/node_modules/p-limit": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-6.2.0.tgz", + "integrity": "sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^1.1.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "packages/tiler/node_modules/@types/pixelmatch": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.4.tgz", diff --git a/packages/tiler-sharp/package.json b/packages/tiler-sharp/package.json index 76fb4ad46..4cb0a8735 100644 --- a/packages/tiler-sharp/package.json +++ b/packages/tiler-sharp/package.json @@ -27,7 +27,9 @@ "@basemaps/tiler": "^8.12.2", "@linzjs/metrics": "^8.0.0", "fzstd": "^0.1.1", - "lerc": "^4.0.4" + "lerc": "^4.0.4", + "p-limit": "^6.2.0" + }, "devDependencies": { "@types/pixelmatch": "^5.2.3", diff --git a/packages/tiler-sharp/src/index.ts b/packages/tiler-sharp/src/index.ts index b368fa1a5..f18e5461c 100644 --- a/packages/tiler-sharp/src/index.ts +++ b/packages/tiler-sharp/src/index.ts @@ -8,6 +8,8 @@ import { TileMakerResizeKernel, } from '@basemaps/tiler'; import { Metrics } from '@linzjs/metrics'; +import type { LimitFunction } from 'p-limit'; +import pLimit from 'p-limit'; import Sharp from 'sharp'; import { Decompressors } from './pipeline/decompressor.lerc.js'; @@ -23,12 +25,23 @@ const EmptyImage = new Map>(); export class TileMakerSharp implements TileMaker { static readonly MaxImageSize = 256 * 2 ** 15; - private width: number; - private height: number; + readonly width: number; + readonly height: number; - public constructor(width: number, height = width) { + /** Limit the number of outstanding tile requests to help reduce maximum memory usage */ + q: LimitFunction; + + /** + * + * @param width Tile output size + * @param height Tile output height (If not supplied it will be the same as width) + * @param limit Number of concurrent tile requests to allow when composing a tile, + * this can help reduce memory usage when there are a large number of layers to compose + */ + public constructor(width: number, height = width, limit = 16) { this.width = width; this.height = height; + this.q = pLimit(limit); } protected isTooLarge(composition: Composition): boolean { @@ -104,13 +117,10 @@ export class TileMakerSharp implements TileMaker { if (this.isTooLarge(comp)) continue; if (ctx.pipeline) { if (comp.type === 'cotar') throw new Error('Cannot use a composition pipeline from cotar'); - todo.push(this.composeTilePipeline(comp, ctx)); + todo.push(this.q(() => this.composeTilePipeline(comp, ctx))); } else { - todo.push(this.composeTile(comp, ctx.resizeKernel)); + todo.push(this.q(() => this.composeTile(comp, ctx.resizeKernel))); } - - // Limit the number of outstanding requests to prevent memory issues - if (todo.length % 128 === 0) await Promise.all(todo); } const overlays = await Promise.all(todo).then((items) => items.filter(notEmpty)); metrics.end('compose:overlay'); From 9bf34e520d3705a2fc51ddd754e1845e72917a80 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Mon, 23 Feb 2026 20:38:51 +1300 Subject: [PATCH 4/4] refactor: cleanup and lint --- packages/lambda-tiler/src/cli/render.tile.ts | 2 +- packages/tiler-sharp/package.json | 1 - packages/tiler-sharp/src/index.ts | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/lambda-tiler/src/cli/render.tile.ts b/packages/lambda-tiler/src/cli/render.tile.ts index 580c86f3d..1a92980aa 100644 --- a/packages/lambda-tiler/src/cli/render.tile.ts +++ b/packages/lambda-tiler/src/cli/render.tile.ts @@ -63,7 +63,7 @@ async function loadConfig( async function main(): Promise { const log = LogConfig.get(); - log.level = 'debug'; + log.level = process.argv.includes('--verbose') ? 'trace' : 'debug'; const { tileSet, imagery, cfg } = await loadConfig(source); setDefaultConfig(cfg); diff --git a/packages/tiler-sharp/package.json b/packages/tiler-sharp/package.json index 4cb0a8735..37b343844 100644 --- a/packages/tiler-sharp/package.json +++ b/packages/tiler-sharp/package.json @@ -29,7 +29,6 @@ "fzstd": "^0.1.1", "lerc": "^4.0.4", "p-limit": "^6.2.0" - }, "devDependencies": { "@types/pixelmatch": "^5.2.3", diff --git a/packages/tiler-sharp/src/index.ts b/packages/tiler-sharp/src/index.ts index f18e5461c..c9f60f279 100644 --- a/packages/tiler-sharp/src/index.ts +++ b/packages/tiler-sharp/src/index.ts @@ -189,6 +189,7 @@ export class TileMakerSharp implements TileMaker { isCrop: comp.crop != null, isResize: comp.resize != null, isExtract: comp.extract != null, + resizeScale: comp.resize?.scale, tiffTile, duration: performance.now() - resizePerf, },