fix: lower memory usage when rendering from large amounts of tiffs BM-1535

packages/lambda-tiler/src/cli/render.tile.ts

@@ -1,7 +1,7 @@
-import { ConfigProviderMemory } from '@basemaps/config';
+import { ConfigImagery, ConfigProviderMemory, ConfigTileSetRaster } from '@basemaps/config';
 import { initConfigFromUrls } from '@basemaps/config-loader';
 import { Tile, TileMatrixSet, TileMatrixSets } from '@basemaps/geo';
-import { fsa, LogConfig, setDefaultConfig } from '@basemaps/shared';
+import { fsa, FsaLocalCache, LogConfig, setDefaultConfig } from '@basemaps/shared';
 import { LambdaHttpRequest, LambdaUrlRequest, UrlEvent } from '@linzjs/lambda';
 import { Context } from 'aws-lambda';
 import { extname } from 'path';
@@ -25,6 +25,9 @@ if (sourceRaw == null || tilePath == null) {
   `);
   process.exit(1);
 }
+
+fsa.middleware.push(FsaLocalCache);
+
 const source = fsa.toUrl(sourceRaw);
 const tile = fromPath(tilePath);
 let tileMatrix: TileMatrixSet | null = null;
@@ -43,14 +46,27 @@ function fromPath(s: string): Tile & { extension: string } {
   return tile;
 }
 
+async function loadConfig(
+  url: URL,
+): Promise<{ tileSet: ConfigTileSetRaster; imagery: ConfigImagery[]; cfg: ConfigProviderMemory }> {
+  if (url.pathname.endsWith('.json') || url.pathname.endsWith('.json.gz')) {
+    const cfg = ConfigProviderMemory.fromJson(await fsa.readJson(url), url);
+    LogConfig.get().info({ url: url.href }, 'ConfigLoaded');
+    const imagery = [...cfg.objects.values()].filter((f) => f.id.startsWith('im_')) as ConfigImagery[];
+    return { tileSet: cfg.imageryToTileSetByName(imagery[0]), imagery, cfg };
+  }
+
+  const cfg = new ConfigProviderMemory();
+  const { imagery } = await initConfigFromUrls(cfg, [url]);
+  return { tileSet: cfg.imageryToTileSetByName(imagery[0]), imagery, cfg };
+}
+
 async function main(): Promise<void> {
   const log = LogConfig.get();
-  log.level = 'trace';
-  const provider = new ConfigProviderMemory();
-  setDefaultConfig(provider);
-  const { imagery } = await initConfigFromUrls(provider, [source]);
+  log.level = process.argv.includes('--verbose') ? 'trace' : 'debug';
 
-  const tileSet = provider.imageryToTileSetByName(imagery[0]);
+  const { tileSet, imagery, cfg } = await loadConfig(source);
+  setDefaultConfig(cfg);
 
   log.info({ tileSet: tileSet.name, layers: tileSet.layers.length }, 'TileSet:Loaded');
 

packages/lambda-tiler/src/index.ts

@@ -48,6 +48,8 @@ handler.router.hook('request', (req) => {
   randomTrace(req);
 
   req.set('name', 'LambdaTiler');
+
+  req.log.debug({ ...req.logContext }, 'Lambda:Start');
 });
 
 handler.router.hook('response', (req, res) => {

packages/shared/src/file.system.ts

@@ -139,6 +139,31 @@ export const FsaLog = {
 };
 fsa.middleware.push(FsaLog);
 
+/**
+ * Middleware to cache responses locally into .cache/
+ *
+ * very useful when developing with remote sources as it avoids repeatedly downloading the same data
+ */
+export const FsaLocalCache = {
+  name: 'source:local-cache',
+  remotes: new Set<string>(['s3:', 'http:', 'https:']),
+  async fetch(req: SourceRequest, next: SourceCallback): Promise<ArrayBuffer> {
+    if (this.remotes.has(req.source.url.protocol) === false) return next(req);
+
+    const requestId = sha256base58(
+      JSON.stringify({ source: req.source.url.href, offset: req.offset, length: req.length }),
+    );
+    const cacheUrl = fsa.toUrl(`./.cache/${requestId}`);
+    const bytes = await fsa.read(cacheUrl).catch(() => {});
+    if (bytes) return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
+
+    return next(req).then(async (res) => {
+      await fsa.write(cacheUrl, Buffer.from(res)).catch(() => {});
+      return res;
+    });
+  },
+};
+
 /**
  * When chunkd moves to URLs this can be removed
  *

packages/shared/src/index.ts

@@ -12,6 +12,7 @@ export {
   Fsa as fsa,
   FsaCache,
   FsaChunk,
+  FsaLocalCache,
   FsaLog,
   s3Config,
   signS3Get,

packages/tiler-sharp/package.json

@@ -27,7 +27,8 @@
     "@basemaps/tiler": "^8.12.2",
     "@linzjs/metrics": "^8.0.0",
     "fzstd": "^0.1.1",
-    "lerc": "^4.0.4"
+    "lerc": "^4.0.4",
+    "p-limit": "^6.2.0"
   },
   "devDependencies": {
     "@types/pixelmatch": "^5.2.3",

packages/tiler-sharp/src/index.ts

@@ -8,6 +8,8 @@ import {
   TileMakerResizeKernel,
 } from '@basemaps/tiler';
 import { Metrics } from '@linzjs/metrics';
+import type { LimitFunction } from 'p-limit';
+import pLimit from 'p-limit';
 import Sharp from 'sharp';
 
 import { Decompressors } from './pipeline/decompressor.lerc.js';
@@ -23,12 +25,23 @@ const EmptyImage = new Map<string, Promise<Buffer>>();
 
 export class TileMakerSharp implements TileMaker {
   static readonly MaxImageSize = 256 * 2 ** 15;
-  private width: number;
-  private height: number;
+  readonly width: number;
+  readonly height: number;
 
-  public constructor(width: number, height = width) {
+  /** Limit the number of outstanding tile requests to help reduce maximum memory usage */
+  q: LimitFunction;
+
+  /**
+   *
+   * @param width Tile output size
+   * @param height Tile output height (If not supplied it will be the same as width)
+   * @param limit Number of concurrent tile requests to allow when composing a tile,
+   *        this can help reduce memory usage when there are a large number of layers to compose
+   */
+  public constructor(width: number, height = width, limit = 16) {
     this.width = width;
     this.height = height;
+    this.q = pLimit(limit);
   }
 
   protected isTooLarge(composition: Composition): boolean {
@@ -104,9 +117,9 @@ export class TileMakerSharp implements TileMaker {
       if (this.isTooLarge(comp)) continue;
       if (ctx.pipeline) {
         if (comp.type === 'cotar') throw new Error('Cannot use a composition pipeline from cotar');
-        todo.push(this.composeTilePipeline(comp, ctx));
+        todo.push(this.q(() => this.composeTilePipeline(comp, ctx)));
       } else {
-        todo.push(this.composeTile(comp, ctx.resizeKernel));
+        todo.push(this.q(() => this.composeTile(comp, ctx.resizeKernel)));
       }
     }
     const overlays = await Promise.all(todo).then((items) => items.filter(notEmpty));
@@ -159,13 +172,13 @@ export class TileMakerSharp implements TileMaker {
   }
 
   async composeTilePipeline(comp: CompositionTiff, ctx: TileMakerContext): Promise<SharpOverlay | null> {
-    const tile = await comp.asset.images[comp.source.imageId].getTile(comp.source.x, comp.source.y);
+    let tile = await comp.asset.images[comp.source.imageId].getTile(comp.source.x, comp.source.y);
     if (tile == null) return null;
     const tiffTile = { imageId: comp.source.imageId, x: comp.source.x, y: comp.source.y };
-    const bytes = await Decompressors[tile.compression]?.bytes(comp.asset, tiffTile, tile.bytes);
-    if (bytes == null) throw new Error(`Failed to decompress: ${comp.asset.source.url.href}`);
+    let result = await Decompressors[tile.compression]?.bytes(comp.asset, tiffTile, tile.bytes);
+    tile = null;
+    if (result == null) throw new Error(`Failed to decompress: ${comp.asset.source.url.href}`);
 
-    let result = bytes;
     if (ctx.pipeline) {
       const resizePerf = performance.now();
       result = cropResize(comp.asset, result, comp, 'bilinear');
@@ -176,6 +189,7 @@ export class TileMakerSharp implements TileMaker {
           isCrop: comp.crop != null,
           isResize: comp.resize != null,
           isExtract: comp.extract != null,
+          resizeScale: comp.resize?.scale,
           tiffTile,
           duration: performance.now() - resizePerf,
         },