From ae00ea9b2704b1323303b7c01fc81a2866d35153 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 19 Mar 2026 22:51:57 +0000 Subject: [PATCH 01/12] Initial plan From 45d309ff2e897289e8f5754b0ec65d6f40bef8e5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 19 Mar 2026 23:08:33 +0000 Subject: [PATCH 02/12] Move Azure blob cache implementation to plugin package - Create new @lage-run/azure-blob-cache-storage plugin package - Add CustomCacheStoragePlugin and CustomCacheStorageConfig types to backfill-config - Remove AzureBlobCacheStorageConfig from CacheStorageConfig union - Remove @azure/storage-blob from backfill-cache dependencies - Remove Azure code from backfill-cache (moved to plugin) - Remove Azure credential handling from @lage-run/cache - Update tests for new custom plugin config format - Map legacy BACKFILL_CACHE_PROVIDER=azure-blob env var to custom plugin config Co-authored-by: ecraig12345 <5864305+ecraig12345@users.noreply.github.com> --- packages/azure-blob-cache-storage/.npmignore | 5 ++ .../azure-blob-cache-storage/package.json | 41 ++++++++++ .../src/AzureBlobCacheStorage.ts | 0 .../src/CacheStorage.ts | 79 +++++++++++++++++++ .../src/CredentialCache.ts | 14 +--- .../azure-blob-cache-storage/src/hashFile.ts | 56 +++++++++++++ .../azure-blob-cache-storage/src/index.ts | 51 ++++++++++++ .../azure-blob-cache-storage/tsconfig.json | 7 ++ packages/backfill-cache/package.json | 1 - .../__tests__/getCacheStorageProvider.test.ts | 33 ++++---- .../src/getCacheStorageProvider.ts | 38 ++++++--- packages/backfill-cache/src/index.ts | 1 + .../src/__tests__/getEnvConfig.test.ts | 18 +++++ packages/backfill-config/src/cacheConfig.ts | 27 ++++++- packages/backfill-config/src/envConfig.ts | 12 ++- packages/backfill-config/src/index.ts | 2 + packages/backfill/src/index.ts | 6 +- packages/cache/package.json | 2 - .../src/__tests__/backfillWrapper.test.ts | 13 +-- packages/cache/src/backfillWrapper.ts | 39 --------- packages/config/src/index.ts | 1 - packages/config/src/types/CacheOptions.ts | 38 +-------- yarn.lock | 21 ++++- 23 files changed, 376 insertions(+), 129 deletions(-) create mode 100644 packages/azure-blob-cache-storage/.npmignore create mode 100644 packages/azure-blob-cache-storage/package.json rename packages/{backfill-cache => azure-blob-cache-storage}/src/AzureBlobCacheStorage.ts (100%) create mode 100644 packages/azure-blob-cache-storage/src/CacheStorage.ts rename packages/{cache => azure-blob-cache-storage}/src/CredentialCache.ts (70%) create mode 100644 packages/azure-blob-cache-storage/src/hashFile.ts create mode 100644 packages/azure-blob-cache-storage/src/index.ts create mode 100644 packages/azure-blob-cache-storage/tsconfig.json diff --git a/packages/azure-blob-cache-storage/.npmignore b/packages/azure-blob-cache-storage/.npmignore new file mode 100644 index 000000000..76f06df88 --- /dev/null +++ b/packages/azure-blob-cache-storage/.npmignore @@ -0,0 +1,5 @@ +**/* +!lib/**/* +lib/**/__tests__/* +lib/**/*.d.ts.map +!bin/**/* diff --git a/packages/azure-blob-cache-storage/package.json b/packages/azure-blob-cache-storage/package.json new file mode 100644 index 000000000..de4965d09 --- /dev/null +++ b/packages/azure-blob-cache-storage/package.json @@ -0,0 +1,41 @@ +{ + "name": "@lage-run/azure-blob-cache-storage", + "version": "0.1.0", + "description": "Azure Blob Storage cache plugin for backfill/lage", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/microsoft/lage" + }, + "homepage": "https://microsoft.github.io/lage/", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "scripts": { + "build": "yarn types && yarn transpile", + "transpile": "monorepo-scripts transpile", + "types": "yarn run -T tsc", + "lint": "monorepo-scripts lint" + }, + "dependencies": { + "@azure/core-auth": "1.9.0", + "@azure/identity": "4.9.1", + "@azure/storage-blob": "12.27.0", + "@lage-run/globby": "workspace:^", + "backfill-config": "workspace:^", + "backfill-logger": "workspace:^", + "fs-extra": "8.1.0", + "tar-fs": "2.1.4" + }, + "devDependencies": { + "@lage-run/monorepo-scripts": "workspace:^", + "@types/fs-extra": "^8.0.0", + "@types/tar-fs": "^2.0.1" + }, + "engines": { + "node": ">=14" + }, + "files": [ + "lib/!(__*)", + "lib/!(__*)/**" + ] +} diff --git a/packages/backfill-cache/src/AzureBlobCacheStorage.ts b/packages/azure-blob-cache-storage/src/AzureBlobCacheStorage.ts similarity index 100% rename from packages/backfill-cache/src/AzureBlobCacheStorage.ts rename to packages/azure-blob-cache-storage/src/AzureBlobCacheStorage.ts diff --git a/packages/azure-blob-cache-storage/src/CacheStorage.ts b/packages/azure-blob-cache-storage/src/CacheStorage.ts new file mode 100644 index 000000000..f615bb2e7 --- /dev/null +++ b/packages/azure-blob-cache-storage/src/CacheStorage.ts @@ -0,0 +1,79 @@ +import { globAsync } from "@lage-run/globby"; + +import type { Logger } from "backfill-logger"; +import type { ICacheStorage } from "backfill-config"; +import { getFileHash } from "./hashFile.js"; + +// First key is the hash, second key is the file relative path +const savedHashes: Map> = new Map(); + +// contract: cwd should be absolute +// The return keys are relative path with posix file separators +async function getHashesFor(cwd: string): Promise> { + const result = new Map(); + + const allFiles = await globAsync(["**/*", "!node_modules"], { cwd }); + //globby returns relative path with posix file separator + await Promise.all( + allFiles.map(async (f) => { + const hash = await getFileHash(cwd, f); + result.set(f, hash); + }) + ); + + return result; +} + +export type { ICacheStorage }; + +export abstract class CacheStorage implements ICacheStorage { + public constructor( + protected logger: Logger, + protected cwd: string, + private incrementalCaching = false + ) {} + public async fetch(hash: string): Promise { + const tracer = this.logger.setTime("fetchTime"); + + const result = await this._fetch(hash); + + tracer.stop(); + + this.logger.setHit(result); + + if (!result && this.incrementalCaching) { + savedHashes.set(hash, await getHashesFor(this.cwd)); + } + + return result; + } + + public async put(hash: string, outputGlob: string[]): Promise { + const tracer = this.logger.setTime("putTime"); + + const filesMatchingOutputGlob = await globAsync(outputGlob, { + cwd: this.cwd, + }); + + let filesToCache = filesMatchingOutputGlob; + if (this.incrementalCaching) { + // Get the list of files that have not changed so we don't need to cache them. + const hashesNow = await getHashesFor(this.cwd); + const hashesThen = + (await savedHashes.get(hash)) || new Map(); + const unchangedFiles = [...hashesThen.keys()].filter( + (s) => hashesThen.get(s) === hashesNow.get(s) + ); + filesToCache = filesMatchingOutputGlob.filter( + (f) => !unchangedFiles.includes(f) + ); + } + + await this._put(hash, filesToCache); + tracer.stop(); + } + + protected abstract _fetch(hash: string): Promise; + + protected abstract _put(hash: string, filesToCache: string[]): Promise; +} diff --git a/packages/cache/src/CredentialCache.ts b/packages/azure-blob-cache-storage/src/CredentialCache.ts similarity index 70% rename from packages/cache/src/CredentialCache.ts rename to packages/azure-blob-cache-storage/src/CredentialCache.ts index 2662c04b4..8f10cd33f 100644 --- a/packages/cache/src/CredentialCache.ts +++ b/packages/azure-blob-cache-storage/src/CredentialCache.ts @@ -6,11 +6,10 @@ import { EnvironmentCredential, WorkloadIdentityCredential, } from "@azure/identity"; -import type { AzureCredentialName } from "@lage-run/config"; -/** - * Exhaustive credential factory map keyed by AzureCredentialName. - * This enforces compile-time alignment with the AzureCredentialName union and provides a single source of truth. - */ + +/** Allowed credential names matching camelCase of @azure/identity credential class names */ +export type AzureCredentialName = "environment" | "workload-identity" | "managed-identity" | "visual-studio-code" | "azure-cli"; + type CredentialFactoryMap = { [K in AzureCredentialName]: () => TokenCredential }; const CREDENTIAL_FACTORY: CredentialFactoryMap = { environment: () => new EnvironmentCredential(), @@ -23,13 +22,8 @@ const CREDENTIAL_FACTORY: CredentialFactoryMap = { export class CredentialCache { private static cache: Map = new Map(); - // Expose the list for runtime validation elsewhere (derived from the exhaustive factory above) public static readonly credentialNames: readonly AzureCredentialName[] = Object.keys(CREDENTIAL_FACTORY) as AzureCredentialName[]; - /** - * Returns a credential instance based on the provided name. Results are cached per name. - * If no name is provided, EnvironmentCredential is used by default. - */ public static getInstance(credentialName?: AzureCredentialName): TokenCredential { const key = (credentialName ?? "environment") as AzureCredentialName; const existing = this.cache.get(key); diff --git a/packages/azure-blob-cache-storage/src/hashFile.ts b/packages/azure-blob-cache-storage/src/hashFile.ts new file mode 100644 index 000000000..3f553bc86 --- /dev/null +++ b/packages/azure-blob-cache-storage/src/hashFile.ts @@ -0,0 +1,56 @@ +import * as path from "path"; +import { promises as fs } from "fs"; +import * as crypto from "crypto"; +import pLimit from "p-limit"; + +let MAX_FILE_OPERATIONS = 5000; + +try { + const maxFileOpEnv = process.env["BACKFILL_MAX_FILE_OP"]; + if (maxFileOpEnv) { + MAX_FILE_OPERATIONS = parseInt(maxFileOpEnv); + } +} catch (_) { + /* The env variable is not an integer, this is fine.*/ +} + +const diskLimit = pLimit(MAX_FILE_OPERATIONS); + +// The first key is the file path, the second key is mtime +const memo = new Map>(); + +async function computeHash(filePath: string): Promise { + const fileBuffer = await diskLimit(() => { + return fs.readFile(filePath); + }); + // We use sha1 for perf reason and because the hashing is not used for security reason. + const hashSum = crypto.createHash("sha1"); + hashSum.update(fileBuffer); + const hash = hashSum.digest("hex"); + return hash; +} + +/* + * Get the hash of a file. + * This function memoizes the hash for files and mtimes. + */ +export async function getFileHash( + cwd: string, + filePath: string +): Promise { + const fileAbsPath = path.join(cwd, filePath); + const stat = await fs.stat(fileAbsPath); + + let memoForFile = memo.get(fileAbsPath); + if (!memoForFile) { + memoForFile = new Map(); + memo.set(fileAbsPath, memoForFile); + } + + let hash = memoForFile.get(stat.mtimeMs); + if (!hash) { + hash = await computeHash(fileAbsPath); + memoForFile.set(stat.mtimeMs, hash); + } + return hash; +} diff --git a/packages/azure-blob-cache-storage/src/index.ts b/packages/azure-blob-cache-storage/src/index.ts new file mode 100644 index 000000000..720d00ff4 --- /dev/null +++ b/packages/azure-blob-cache-storage/src/index.ts @@ -0,0 +1,51 @@ +import type { Logger } from "backfill-logger"; +import type { + ICacheStorage, + CustomCacheStoragePlugin, + AzureBlobCacheStorageOptions, + AzureBlobCacheStorageConnectionStringOptions, +} from "backfill-config"; + +import { AzureBlobCacheStorage } from "./AzureBlobCacheStorage.js"; +import { CredentialCache, type AzureCredentialName } from "./CredentialCache.js"; + +export type { AzureCredentialName } from "./CredentialCache.js"; +export { CredentialCache } from "./CredentialCache.js"; + +export type AzureBlobPluginOptions = AzureBlobCacheStorageOptions & { + /** Optional credential name for Azure Identity authentication. */ + credentialName?: AzureCredentialName; +}; + +function isTokenConnectionString(connectionString: string) { + return connectionString.includes("SharedAccessSignature") || connectionString.includes("AccountKey"); +} + +const plugin: CustomCacheStoragePlugin = { + name: "azure-blob", + getProvider(logger: Logger, cwd: string, options: AzureBlobPluginOptions): ICacheStorage { + // Handle credential injection for connection-string-based options + if ("connectionString" in options && !isTokenConnectionString(options.connectionString)) { + const connStringOptions = options as AzureBlobCacheStorageConnectionStringOptions & { credentialName?: AzureCredentialName }; + if (!connStringOptions.credential) { + const credName = connStringOptions.credentialName + ?? (process.env.AZURE_IDENTITY_CREDENTIAL_NAME as AzureCredentialName | undefined); + + if (credName != null) { + if (!CredentialCache.credentialNames.includes(credName)) { + throw new Error( + `Invalid credentialName: "${credName}". Allowed values: ${CredentialCache.credentialNames.join(", ")}` + ); + } + connStringOptions.credential = CredentialCache.getInstance(credName); + } else { + connStringOptions.credential = CredentialCache.getInstance(); + } + } + } + + return new AzureBlobCacheStorage(options, logger, cwd); + }, +}; + +export default plugin; diff --git a/packages/azure-blob-cache-storage/tsconfig.json b/packages/azure-blob-cache-storage/tsconfig.json new file mode 100644 index 000000000..f1eca65b0 --- /dev/null +++ b/packages/azure-blob-cache-storage/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "@lage-run/monorepo-scripts/config/tsconfig.base.json", + "compilerOptions": { + "outDir": "lib" + }, + "include": ["src"] +} diff --git a/packages/backfill-cache/package.json b/packages/backfill-cache/package.json index 60054566f..0a79a0599 100644 --- a/packages/backfill-cache/package.json +++ b/packages/backfill-cache/package.json @@ -18,7 +18,6 @@ "test": "yarn run -T jest" }, "dependencies": { - "@azure/storage-blob": "12.27.0", "@lage-run/globby": "workspace:^", "backfill-config": "workspace:^", "backfill-logger": "workspace:^", diff --git a/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts b/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts index 538d68c1f..c19c62a9e 100644 --- a/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts +++ b/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts @@ -1,7 +1,6 @@ import { type Logger, makeLogger } from "backfill-logger"; import { getCacheStorageProvider } from "../getCacheStorageProvider.js"; import type { ICacheStorage } from "../CacheStorage.js"; -import { AzureBlobCacheStorage } from "../AzureBlobCacheStorage.js"; import { LocalCacheStorage } from "../LocalCacheStorage.js"; describe("getCacheStorageProvider", () => { @@ -65,23 +64,6 @@ describe("getCacheStorageProvider", () => { expect(provider instanceof LocalCacheStorage).toBeTruthy(); }); - test("can get an azure-blob storage provider", () => { - const provider = getCacheStorageProvider( - { - provider: "azure-blob", - options: { - connectionString: "some connection string", - container: "some container", - }, - }, - "test", - makeLogger("silly"), - "cwd" - ); - - expect(provider instanceof AzureBlobCacheStorage).toBeTruthy(); - }); - test("can get a custom storage provider as a class", () => { const TestProvider = class implements ICacheStorage { constructor( @@ -115,4 +97,19 @@ describe("getCacheStorageProvider", () => { expect(provider.fetch).toBeTruthy(); expect(provider.put).toBeTruthy(); }); + + test("throws when custom plugin cannot be loaded", () => { + expect(() => + getCacheStorageProvider( + { + provider: "custom", + plugin: "nonexistent-plugin-package", + options: {}, + }, + "test", + makeLogger("silly"), + "cwd" + ) + ).toThrow('Failed to load custom cache storage plugin "nonexistent-plugin-package"'); + }); }); diff --git a/packages/backfill-cache/src/getCacheStorageProvider.ts b/packages/backfill-cache/src/getCacheStorageProvider.ts index c6662081d..e0093aa7e 100644 --- a/packages/backfill-cache/src/getCacheStorageProvider.ts +++ b/packages/backfill-cache/src/getCacheStorageProvider.ts @@ -1,8 +1,7 @@ -import type { CacheStorageConfig, CustomStorageConfig } from "backfill-config"; +import type { CacheStorageConfig, CustomStorageConfig, CustomCacheStorageConfig, CustomCacheStoragePlugin } from "backfill-config"; import type { Logger } from "backfill-logger"; import type { ICacheStorage } from "./CacheStorage.js"; -import { AzureBlobCacheStorage } from "./AzureBlobCacheStorage.js"; import { LocalCacheStorage } from "./LocalCacheStorage.js"; import { NpmCacheStorage } from "./NpmCacheStorage.js"; import { LocalSkipCacheStorage } from "./LocalSkipCacheStorage.js"; @@ -13,6 +12,12 @@ export function isCustomProvider( return typeof config.provider === "function"; } +export function isCustomPluginProvider( + config: CacheStorageConfig +): config is CustomCacheStorageConfig { + return typeof config.provider === "string" && config.provider === "custom"; +} + const memo = new Map(); export function getCacheStorageProvider( @@ -32,6 +37,28 @@ export function getCacheStorageProvider( } } + if (isCustomPluginProvider(cacheStorageConfig)) { + const key = `custom:${cacheStorageConfig.plugin}${internalCacheFolder}${cwd}`; + cacheStorage = memo.get(key); + if (cacheStorage) { + return cacheStorage; + } + + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires + const pluginModule = require(cacheStorageConfig.plugin); + const plugin: CustomCacheStoragePlugin = pluginModule.default || pluginModule; + cacheStorage = plugin.getProvider(logger, cwd, cacheStorageConfig.options); + } catch (err) { + throw new Error( + `Failed to load custom cache storage plugin "${cacheStorageConfig.plugin}": ${err}` + ); + } + + memo.set(key, cacheStorage); + return cacheStorage; + } + const key = `${cacheStorageConfig.provider}${internalCacheFolder}${cwd}`; cacheStorage = memo.get(key); if (cacheStorage) { @@ -46,13 +73,6 @@ export function getCacheStorageProvider( cwd, incrementalCaching ); - } else if (cacheStorageConfig.provider === "azure-blob") { - cacheStorage = new AzureBlobCacheStorage( - cacheStorageConfig.options, - logger, - cwd, - incrementalCaching - ); } else if (cacheStorageConfig.provider === "local-skip") { cacheStorage = new LocalSkipCacheStorage( internalCacheFolder, diff --git a/packages/backfill-cache/src/index.ts b/packages/backfill-cache/src/index.ts index 5802014f9..ac992198d 100644 --- a/packages/backfill-cache/src/index.ts +++ b/packages/backfill-cache/src/index.ts @@ -1,5 +1,6 @@ export { getCacheStorageProvider, isCustomProvider, + isCustomPluginProvider, } from "./getCacheStorageProvider.js"; export { type ICacheStorage, CacheStorage } from "./CacheStorage.js"; diff --git a/packages/backfill-config/src/__tests__/getEnvConfig.test.ts b/packages/backfill-config/src/__tests__/getEnvConfig.test.ts index c65dcace8..786c32676 100644 --- a/packages/backfill-config/src/__tests__/getEnvConfig.test.ts +++ b/packages/backfill-config/src/__tests__/getEnvConfig.test.ts @@ -117,6 +117,24 @@ describe("getEnvConfig()", () => { `); }); + it("maps azure-blob to custom plugin config", () => { + process.env["BACKFILL_CACHE_PROVIDER"] = "azure-blob"; + process.env["BACKFILL_CACHE_PROVIDER_OPTIONS"] = JSON.stringify({ + connectionString: "DefaultEndpointsProtocol=https;AccountName=test", + container: "my-container", + }); + + const config = getEnvConfig(logger); + expect(config.cacheStorageConfig).toStrictEqual({ + provider: "custom", + plugin: "@lage-run/azure-blob-cache-storage", + options: { + connectionString: "DefaultEndpointsProtocol=https;AccountName=test", + container: "my-container", + }, + }); + }); + // This should be updated to check for a thrown error once more config // validation is added in a major version it("does not throw on invalid cache provider name", () => { diff --git a/packages/backfill-config/src/cacheConfig.ts b/packages/backfill-config/src/cacheConfig.ts index 674ee78cf..b79ba4eec 100644 --- a/packages/backfill-config/src/cacheConfig.ts +++ b/packages/backfill-config/src/cacheConfig.ts @@ -1,5 +1,4 @@ import type { Logger } from "backfill-logger"; -import type { AzureBlobCacheStorageConfig } from "./azureBlobCacheConfig.js"; import type { NpmCacheStorageConfig } from "./npmCacheConfig.js"; export interface ICacheStorage { @@ -12,6 +11,30 @@ export type CustomStorageConfig = { name?: string; }; +/** + * A plugin that provides a custom cache storage implementation. + * The plugin module should export this as its default export. + */ +export interface CustomCacheStoragePlugin { + name: string; + getProvider: (logger: Logger, cwd: string, options: TOptions) => ICacheStorage; +} + +/** + * Configuration for a custom (plugin-based) cache storage provider. + * The `plugin` field should be a package name or path that exports a + * `CustomCacheStoragePlugin` as its default export. + */ +export type CustomCacheStorageConfig = { + provider: "custom"; + /** + * Package name or path to the plugin module. + * If a package name, it's resolved from node_modules. + */ + plugin: string; + options: TOptions; +}; + export type CacheStorageConfig = | { provider: "local"; @@ -20,7 +43,7 @@ export type CacheStorageConfig = provider: "local-skip"; } | NpmCacheStorageConfig - | AzureBlobCacheStorageConfig + | CustomCacheStorageConfig | CustomStorageConfig; /** diff --git a/packages/backfill-config/src/envConfig.ts b/packages/backfill-config/src/envConfig.ts index 5b4637326..0dcf0e1c6 100644 --- a/packages/backfill-config/src/envConfig.ts +++ b/packages/backfill-config/src/envConfig.ts @@ -8,7 +8,7 @@ import type { Config } from "./Config.js"; import { getAzureBlobConfigFromSerializedOptions } from "./azureBlobCacheConfig.js"; import { getNpmConfigFromSerializedOptions } from "./npmCacheConfig.js"; import { isCorrectMode, modesObject, type BackfillModes } from "./modes.js"; -import type { CacheStorageConfig } from "./cacheConfig.js"; +import type { CacheStorageConfig, CustomCacheStorageConfig } from "./cacheConfig.js"; class BackfillConfigError extends Error { constructor(value: string, envName: string, expected: string) { @@ -84,14 +84,22 @@ export function getEnvConfig(logger: Logger): Partial { const cacheProvider = process.env.BACKFILL_CACHE_PROVIDER as | Exclude // eslint-disable-line + | "azure-blob" // legacy value, mapped to custom plugin | undefined; const serializedCacheProviderOptions = process.env.BACKFILL_CACHE_PROVIDER_OPTIONS; if (cacheProvider === "azure-blob") { - config.cacheStorageConfig = getAzureBlobConfigFromSerializedOptions( + const azureBlobConfig = getAzureBlobConfigFromSerializedOptions( serializedCacheProviderOptions || "{}" ); + // Map the legacy "azure-blob" provider to the new custom plugin config + const customConfig: CustomCacheStorageConfig = { + provider: "custom", + plugin: "@lage-run/azure-blob-cache-storage", + options: azureBlobConfig.options, + }; + config.cacheStorageConfig = customConfig; } else if (cacheProvider === "npm") { config.cacheStorageConfig = getNpmConfigFromSerializedOptions( serializedCacheProviderOptions || "{}" diff --git a/packages/backfill-config/src/index.ts b/packages/backfill-config/src/index.ts index 5916b156d..de853956f 100644 --- a/packages/backfill-config/src/index.ts +++ b/packages/backfill-config/src/index.ts @@ -3,6 +3,8 @@ export type { ICacheStorage, CustomStorageConfig, CacheStorageConfig, + CustomCacheStoragePlugin, + CustomCacheStorageConfig, } from "./cacheConfig.js"; export type { Config } from "./Config.js"; export * from "./createConfig.js"; diff --git a/packages/backfill/src/index.ts b/packages/backfill/src/index.ts index 42325a19c..c294cdd11 100644 --- a/packages/backfill/src/index.ts +++ b/packages/backfill/src/index.ts @@ -1,7 +1,7 @@ import { loadDotenv } from "backfill-utils-dotenv"; import { type Logger, makeLogger } from "backfill-logger"; import { createConfig, type Config } from "backfill-config"; -import { isCustomProvider } from "backfill-cache"; +import { isCustomProvider, isCustomPluginProvider } from "backfill-cache"; import yargs from "yargs"; import { getRawBuildCommand, @@ -46,7 +46,9 @@ export async function backfill( logger.setCacheProvider( isCustomProvider(cacheStorageConfig) ? cacheStorageConfig.name || "custom-storage-provider" - : cacheStorageConfig.provider + : isCustomPluginProvider(cacheStorageConfig) + ? cacheStorageConfig.plugin + : cacheStorageConfig.provider ); const createPackageHash = async () => diff --git a/packages/cache/package.json b/packages/cache/package.json index 37911d577..3830f4e28 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -18,8 +18,6 @@ "lint": "monorepo-scripts lint" }, "dependencies": { - "@azure/core-auth": "1.9.0", - "@azure/identity": "4.9.1", "@lage-run/config": "workspace:^", "@lage-run/logger": "workspace:^", "@lage-run/target-graph": "workspace:^", diff --git a/packages/cache/src/__tests__/backfillWrapper.test.ts b/packages/cache/src/__tests__/backfillWrapper.test.ts index aa2f20d0f..6980f2e83 100644 --- a/packages/cache/src/__tests__/backfillWrapper.test.ts +++ b/packages/cache/src/__tests__/backfillWrapper.test.ts @@ -1,4 +1,4 @@ -import type { AzureBlobCacheStorageConfig, AzureBlobCacheStorageConnectionStringOptions } from "backfill-config"; +import type { CustomCacheStorageConfig } from "backfill-config"; import path from "path"; import { createBackfillLogger, createBackfillCacheConfig } from "../backfillWrapper.js"; @@ -11,10 +11,12 @@ describe("backfill-config", () => { const fixture = path.join(__dirname, "fixtures/backfill-config"); const config = createBackfillCacheConfig(fixture, {}, dummyLogger); - expect(config.cacheStorageConfig.provider).toBe("azure-blob"); + // azure-blob env var is now mapped to the custom plugin config + expect(config.cacheStorageConfig.provider).toBe("custom"); - const cacheStorageConfig = config.cacheStorageConfig as AzureBlobCacheStorageConfig; - const cacheOptions = cacheStorageConfig.options as AzureBlobCacheStorageConnectionStringOptions; + const cacheStorageConfig = config.cacheStorageConfig as CustomCacheStorageConfig; + expect(cacheStorageConfig.plugin).toBe("@lage-run/azure-blob-cache-storage"); + const cacheOptions = cacheStorageConfig.options as { connectionString: string; container: string }; expect(cacheOptions.connectionString).toBe("somestring"); expect(cacheOptions.container).toBe("somecontainer"); @@ -38,7 +40,8 @@ describe("backfill-config", () => { dummyLogger ); - expect(config.cacheStorageConfig.provider).toBe("azure-blob"); + // azure-blob env var is now mapped to the custom plugin config + expect(config.cacheStorageConfig.provider).toBe("custom"); delete process.env.BACKFILL_CACHE_PROVIDER; delete process.env.BACKFILL_CACHE_PROVIDER_OPTIONS; diff --git a/packages/cache/src/backfillWrapper.ts b/packages/cache/src/backfillWrapper.ts index ee8c5f44d..a97aa9a6b 100644 --- a/packages/cache/src/backfillWrapper.ts +++ b/packages/cache/src/backfillWrapper.ts @@ -7,8 +7,6 @@ import { type Config, createDefaultConfig, getEnvConfig } from "backfill-config" import { makeLogger } from "backfill-logger"; import type { Logger as BackfillLogger } from "backfill-logger"; import type { CacheOptions } from "@lage-run/config"; -import type { AzureCredentialName } from "@lage-run/config"; -import { CredentialCache } from "./CredentialCache.js"; export function createBackfillLogger(): BackfillLogger { const stdout = process.stdout; @@ -40,42 +38,5 @@ export function createBackfillCacheConfig( ...envConfig, } as Config; - if (mergedConfig.cacheStorageConfig.provider === "azure-blob") { - const azureOptions = mergedConfig.cacheStorageConfig.options; - if ("connectionString" in azureOptions && !isTokenConnectionString(azureOptions.connectionString)) { - /** Pass through optional credentialName from config to select a specific credential implementation - * Type assertion: only the connection-string variant is augmented with credentialName in @lage-run/config - */ - const name = (azureOptions as { credentialName?: AzureCredentialName }).credentialName as string | undefined; - if (name != null) { - if (!CredentialCache.credentialNames.includes(name as AzureCredentialName)) { - throw new Error( - `Invalid cacheStorageConfig.options.credentialName: "${name}". Allowed values: ${CredentialCache.credentialNames.join(", ")}` - ); - } - azureOptions.credential = CredentialCache.getInstance(name as AzureCredentialName); - } else { - /** No name provided in config: if env var AZURE_IDENTITY_CREDENTIAL_NAME is set, honor it; otherwise default to EnvironmentCredential - */ - const envName = process.env.AZURE_IDENTITY_CREDENTIAL_NAME as string | undefined; - if (envName != null) { - if (!CredentialCache.credentialNames.includes(envName as AzureCredentialName)) { - throw new Error( - `Invalid AZURE_IDENTITY_CREDENTIAL_NAME: "${envName}". Allowed values: ${CredentialCache.credentialNames.join(", ")}` - ); - } - azureOptions.credential = CredentialCache.getInstance(envName as AzureCredentialName); - } else { - // Fall back to EnvironmentCredential - azureOptions.credential = CredentialCache.getInstance(); - } - } - } - } - return mergedConfig; } - -function isTokenConnectionString(connectionString: string) { - return connectionString.includes("SharedAccessSignature") || connectionString.includes("AccountKey"); -} diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index 224d7dbe8..f79f37559 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -5,6 +5,5 @@ export { readConfigFile } from "./readConfigFile.js"; export type { PipelineDefinition } from "./types/PipelineDefinition.js"; export type { ConfigOptions, ConfigFileOptions } from "./types/ConfigOptions.js"; export type { CacheOptions } from "./types/CacheOptions.js"; -export type { AzureCredentialName } from "./types/CacheOptions.js"; export type { LoggerOptions } from "./types/LoggerOptions.js"; export { isRunningFromCI } from "./isRunningFromCI.js"; diff --git a/packages/config/src/types/CacheOptions.ts b/packages/config/src/types/CacheOptions.ts index be3b79094..91a0e1143 100644 --- a/packages/config/src/types/CacheOptions.ts +++ b/packages/config/src/types/CacheOptions.ts @@ -1,44 +1,12 @@ import type { Config as BackfillCacheOptions, CustomStorageConfig } from "backfill-config"; -/** Allowed credential names matching camelCase of @azure/identity credential class names - * @see https://learn.microsoft.com/en-us/azure/developer/javascript/sdk/authentication/credential-chains - */ -export type AzureCredentialName = "environment" | "workload-identity" | "managed-identity" | "visual-studio-code" | "azure-cli"; - -/** Locally augment only the Azure Blob connection-string options by adding an optional `credentialName`. - * This does NOT modify upstream types; it narrows and re-composes the union for our config surface. - */ -type AzureBlobFromBackfill = Extract; - -type AugmentedAzureBlobConfig = AzureBlobFromBackfill extends { - provider: "azure-blob"; - options: infer O; -} - ? { - provider: "azure-blob"; - options: O extends any - ? O extends { connectionString: string } - ? // Assumption: make `credentialName` optional to preserve backward compatibility - O & { credentialName?: AzureCredentialName } // default value is "environment" - : O - : never; - } - : never; - -/** Recompose the cache storage config union to swap in our augmented Azure Blob type - * This is necessary because we want to add the `credentialName` property only to the Azure Blob config, - * without affecting other cache storage configs. - */ -type ExtendedCacheStorageConfig = - | Exclude - | AugmentedAzureBlobConfig; - export type CacheOptions = Omit & { /** - * Use this to specify a remote cache provider such as `'azure-blob'`. + * Use this to specify a remote cache provider. + * For Azure Blob Storage, use `provider: "custom"` with `plugin: "@lage-run/azure-blob-cache-storage"`. * @see https://www.npmjs.com/package/backfill#configuration */ - cacheStorageConfig?: ExtendedCacheStorageConfig; + cacheStorageConfig?: Exclude; /** * Whether to write to the remote cache - useful for continuous integration systems to provide build-over-build cache. diff --git a/yarn.lock b/yarn.lock index 2f6532145..f85413d1a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1537,6 +1537,24 @@ __metadata: languageName: node linkType: hard +"@lage-run/azure-blob-cache-storage@workspace:packages/azure-blob-cache-storage": + version: 0.0.0-use.local + resolution: "@lage-run/azure-blob-cache-storage@workspace:packages/azure-blob-cache-storage" + dependencies: + "@azure/core-auth": "npm:1.9.0" + "@azure/identity": "npm:4.9.1" + "@azure/storage-blob": "npm:12.27.0" + "@lage-run/globby": "workspace:^" + "@lage-run/monorepo-scripts": "workspace:^" + "@types/fs-extra": "npm:^8.0.0" + "@types/tar-fs": "npm:^2.0.1" + backfill-config: "workspace:^" + backfill-logger: "workspace:^" + fs-extra: "npm:8.1.0" + tar-fs: "npm:2.1.4" + languageName: unknown + linkType: soft + "@lage-run/cache-github-actions@workspace:packages/cache-github-actions": version: 0.0.0-use.local resolution: "@lage-run/cache-github-actions@workspace:packages/cache-github-actions" @@ -1553,8 +1571,6 @@ __metadata: version: 0.0.0-use.local resolution: "@lage-run/cache@workspace:packages/cache" dependencies: - "@azure/core-auth": "npm:1.9.0" - "@azure/identity": "npm:4.9.1" "@lage-run/config": "workspace:^" "@lage-run/logger": "workspace:^" "@lage-run/monorepo-fixture": "workspace:^" @@ -3126,7 +3142,6 @@ __metadata: version: 0.0.0-use.local resolution: "backfill-cache@workspace:packages/backfill-cache" dependencies: - "@azure/storage-blob": "npm:12.27.0" "@lage-run/globby": "workspace:^" "@lage-run/monorepo-scripts": "workspace:^" "@types/fs-extra": "npm:^8.0.0" From c86e907b394b205a9e947b2b1bca9223ef5272bb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 19 Mar 2026 23:12:10 +0000 Subject: [PATCH 03/12] Update documentation for Azure blob cache plugin approach - Update docsite remote-cache guide with plugin installation and config - Update backfill README with custom plugin and inline provider sections - Update @lage-run/config reference docs example - Update @lage-run/cache README example Co-authored-by: ecraig12345 <5864305+ecraig12345@users.noreply.github.com> --- docs/docs/guides/remote-cache.md | 37 +++++++++++--- docs/docs/reference/config.md | 7 +-- packages/backfill/README.md | 82 +++++++++++++++++++++++++++----- packages/cache/README.md | 4 +- 4 files changed, 106 insertions(+), 24 deletions(-) diff --git a/docs/docs/guides/remote-cache.md b/docs/docs/guides/remote-cache.md index 378ff70aa..f3871ae69 100644 --- a/docs/docs/guides/remote-cache.md +++ b/docs/docs/guides/remote-cache.md @@ -12,9 +12,15 @@ The theory is that when the CI job runs, it'll produce a "last known good" cache ## Setting up remote cache - Azure Blob Storage -Follow these steps to set up a remote cache. +Azure Blob Storage cache is available as a plugin: `@lage-run/azure-blob-cache-storage`. This plugin must be installed separately. -### 1. Upgrade to latest `lage` +### 1. Install the plugin + +``` +yarn add @lage-run/azure-blob-cache-storage +``` + +### 2. Upgrade to latest `lage` See the [migration guide](../cookbook/migration.mdx) for more details. @@ -22,7 +28,7 @@ See the [migration guide](../cookbook/migration.mdx) for more details. yarn upgrade lage ``` -### 2. Create `.env` and add to `.gitignore` +### 3. Create `.env` and add to `.gitignore` Create the file: @@ -39,7 +45,7 @@ lib dist ``` -### 3. Generate auth tokens from Azure storage account +### 4. Generate auth tokens from Azure storage account Prerequisite is to have a working Storage Account with Blob Storage Container created. Note that container name, it'll be needed for Step 5. @@ -51,7 +57,7 @@ Prerequisite is to have a working Storage Account with Blob Storage Container cr 6. Click "show keys" 7. Save the "connection string" - this is your **read-write** connection string (alternatively, you can create a read-write SAS connection string) -### 4. Modify the `.env` file with the remote cache connection information +### 5. Modify the `.env` file with the remote cache connection information ```txt title=".env" ## This is required as of right now @@ -61,7 +67,24 @@ BACKFILL_CACHE_PROVIDER="azure-blob" BACKFILL_CACHE_PROVIDER_OPTIONS={"connectionString":"the **read-only** connection string","container":"CONTAINER NAME"} ``` -### 5. Create a "secret" in the CI system for a Read/Write token +Alternatively, you can configure it directly in `lage.config.js`: + +```js title="lage.config.js" +module.exports = { + cacheOptions: { + cacheStorageConfig: { + provider: "custom", + plugin: "@lage-run/azure-blob-cache-storage", + options: { + connectionString: "...", + container: "...", + }, + }, + }, +}; +``` + +### 6. Create a "secret" in the CI system for a Read/Write token Here's an example snippet of Github Action with the correct environment variable set: @@ -81,7 +104,7 @@ Create a secret named "BACKFILL_CACHE_PROVIDER_OPTIONS": `process.env.BACKFILL_CACHE_PROVIDER_OPTIONS`is evaluated via backfill (see [`getEnvConfig()`](https://github.com/microsoft/lage/blob/master/packages/backfill-config/src/envConfig.ts#L82) in `backfill-config`). -For "azure-blob" cache provider with a non-sas/key-based `connectionString`(storage account endpoint) requiring azure identity authentication do not use `BACKFILL_CACHE_PROVIDER_OPTIONS`, instead populate the required env variables according to the desired identity/environment. (See [Azure Idenity SDK](https://learn.microsoft.com/en-us/javascript/api/overview/azure/identity-readme)) and set `credentialName` property in the `lage.config.js` under `cacheOptions.cacheStorageConfig.options.credentialName` or via env var `AZURE_IDENTITY_CREDENTIAL_NAME` Supported options are: +For the Azure Blob cache provider with a non-sas/key-based `connectionString` (storage account endpoint) requiring Azure Identity authentication, you can pass a `credentialName` option in the plugin config or via the `AZURE_IDENTITY_CREDENTIAL_NAME` environment variable. Supported options are: - `"azure-cli"` - `"managed-identity"` diff --git a/docs/docs/reference/config.md b/docs/docs/reference/config.md index 4c9495acf..711caeee8 100644 --- a/docs/docs/reference/config.md +++ b/docs/docs/reference/config.md @@ -69,9 +69,10 @@ const config = { cacheOptions: { /** @see https://www.npmjs.com/package/backfill#configuration */ cacheStorageConfig: { - // use this to specify a remote cache provider such as "azure-blob", - provider: "azure-blob", - // there are specific options here for each cache provider + // use this to specify a remote cache plugin such as "@lage-run/azure-blob-cache-storage", + provider: "custom", + plugin: "@lage-run/azure-blob-cache-storage", + // there are specific options here for each cache plugin options: {} }, diff --git a/packages/backfill/README.md b/packages/backfill/README.md index 2acd4a548..7fc8bb936 100644 --- a/packages/backfill/README.md +++ b/packages/backfill/README.md @@ -72,7 +72,8 @@ wish to override. All properties in are optional in the config file. /** @type {Partial} */ const config = { cacheStorageConfig: { - provider: "azure-blob", + provider: "custom", + plugin: "@lage-run/azure-blob-cache-storage", options: { ... } }, outputGlob: ["lib/**/*", "dist/bundles/**/*"] @@ -193,13 +194,20 @@ instance. Backfill supports multiple cache storage providers: - Local folder (`local`), the default option -- [Azure blob storage (`azure-blob`)](#azure-blob-storage) +- [Azure blob storage (plugin)](#azure-blob-storage) - [NPM package (`npm`)](#npm-package) - [Skip cache locally (`local-skip`)](#skipping-cache-locally) -- [Custom (`custom`)](#custom-storage-providers) +- [Custom plugin (`custom`)](#custom-storage-plugins) +- [Custom inline provider](#custom-inline-providers) ### Azure Blob Storage +Azure Blob Storage cache is available as a plugin. First, install the plugin: + +``` +npm install @lage-run/azure-blob-cache-storage +``` + To cache to Microsoft Azure Blob Storage, you need to provide a connection string and the container name. If you are configuring via `backfill.config.js`, use the following syntax: @@ -207,10 +215,11 @@ use the following syntax: ```js module.exports = { cacheStorageConfig: { - provider: "azure-blob", + provider: "custom", + plugin: "@lage-run/azure-blob-cache-storage", options: { connectionString: "...", - container: "..." + container: "...", maxSize: 12345 } } @@ -226,11 +235,12 @@ import { InteractiveBrowserCredential } from '@azure/identity' module.exports = { cacheStorageConfig: { - provider: "azure-blob", + provider: "custom", + plugin: "@lage-run/azure-blob-cache-storage", options: { connectionString: "https://.blob.core.windows.net", - credential: new InteractiveBrowserCredential() - container: "..." + credential: new InteractiveBrowserCredential(), + container: "...", maxSize: 12345 } } @@ -238,15 +248,19 @@ module.exports = { ``` -#### `azure-blob` options +#### Azure Blob Storage options - `connectionString`: retrieve this from the Azure Portal interface - `container`: the name of the blob storage container - `maxSize` (optional): max size of a single package cache, in the number of bytes - `credential` (optional): one of the credential types from `@azure/identity`. +- `credentialName` (optional): name of the credential type to use for Azure + Identity authentication. Supported values: `"azure-cli"`, + `"managed-identity"`, `"visual-studio-code"`, `"environment"`, + `"workload-identity"`. -You can also use environment variables for configuration. +You can also use environment variables for configuration (backward compatible): ``` BACKFILL_CACHE_PROVIDER="azure-blob" @@ -317,9 +331,53 @@ storage strategy: BACKFILL_CACHE_PROVIDER="local-skip" ``` -### Custom storage providers +### Custom storage plugins + +You can use a custom cache storage plugin by setting `provider` to `"custom"` +and specifying the `plugin` package name or path. The plugin module should +export a `CustomCacheStoragePlugin` as its default export. + +```js +// backfill.config.js +/** @type {Partial} */ +const config = { + cacheStorageConfig: { + provider: "custom", + plugin: "my-custom-cache-plugin", + options: { + key1: "value1", + key2: "value2", + }, + }, +}; +module.exports = config; +``` + +A plugin module should have the following structure: + +```js +// my-custom-cache-plugin/index.js +/** @type {import("backfill-config").CustomCacheStoragePlugin} */ +const plugin = { + name: "my-custom-cache", + getProvider(logger, cwd, options) { + return { + async fetch(hash) { + // some fetch logic + return false; + }, + async put(hash, filesToCache) { + // some putting logic + }, + }; + }, +}; +module.exports = plugin; +``` + +### Custom inline providers -It is also possible to use a custom storage provider. This allows ultimate +It is also possible to use a custom storage provider inline. This allows ultimate flexibility in how to handle cache fetching and putting. Configure the custom cache provider this way: diff --git a/packages/cache/README.md b/packages/cache/README.md index bae3d2cc7..e5d6e04b2 100644 --- a/packages/cache/README.md +++ b/packages/cache/README.md @@ -33,10 +33,10 @@ const remoteFallbackCacheProviderOptions = { root, { cacheStorageOptions: { - provider: "azure-blob", + provider: "custom", + plugin: "@lage-run/azure-blob-cache-storage", options: { // This connection string can optionally contain credentials. - // If no credentials are present, see credentialName below. connectionString: "asdfasdfasdfafds"; container: "container"; maxSize?: 150; From 3897f781d93226c96d52cd1715f3507560ec2fa0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 19 Mar 2026 23:13:35 +0000 Subject: [PATCH 04/12] Address code review: improve type safety for credential name validation Co-authored-by: ecraig12345 <5864305+ecraig12345@users.noreply.github.com> --- packages/azure-blob-cache-storage/src/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/azure-blob-cache-storage/src/index.ts b/packages/azure-blob-cache-storage/src/index.ts index 720d00ff4..22793b711 100644 --- a/packages/azure-blob-cache-storage/src/index.ts +++ b/packages/azure-blob-cache-storage/src/index.ts @@ -29,15 +29,15 @@ const plugin: CustomCacheStoragePlugin = { const connStringOptions = options as AzureBlobCacheStorageConnectionStringOptions & { credentialName?: AzureCredentialName }; if (!connStringOptions.credential) { const credName = connStringOptions.credentialName - ?? (process.env.AZURE_IDENTITY_CREDENTIAL_NAME as AzureCredentialName | undefined); + ?? (process.env.AZURE_IDENTITY_CREDENTIAL_NAME || undefined); if (credName != null) { - if (!CredentialCache.credentialNames.includes(credName)) { + if (!CredentialCache.credentialNames.includes(credName as AzureCredentialName)) { throw new Error( `Invalid credentialName: "${credName}". Allowed values: ${CredentialCache.credentialNames.join(", ")}` ); } - connStringOptions.credential = CredentialCache.getInstance(credName); + connStringOptions.credential = CredentialCache.getInstance(credName as AzureCredentialName); } else { connStringOptions.credential = CredentialCache.getInstance(); } From 2a02bfa8cb75ef32c4898274c5aadb98040c80cb Mon Sep 17 00:00:00 2001 From: Elizabeth Craig Date: Thu, 19 Mar 2026 16:23:13 -0700 Subject: [PATCH 05/12] Apply suggestion from @ecraig12345 --- docs/docs/guides/remote-cache.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/guides/remote-cache.md b/docs/docs/guides/remote-cache.md index f3871ae69..7c69520b1 100644 --- a/docs/docs/guides/remote-cache.md +++ b/docs/docs/guides/remote-cache.md @@ -104,7 +104,7 @@ Create a secret named "BACKFILL_CACHE_PROVIDER_OPTIONS": `process.env.BACKFILL_CACHE_PROVIDER_OPTIONS`is evaluated via backfill (see [`getEnvConfig()`](https://github.com/microsoft/lage/blob/master/packages/backfill-config/src/envConfig.ts#L82) in `backfill-config`). -For the Azure Blob cache provider with a non-sas/key-based `connectionString` (storage account endpoint) requiring Azure Identity authentication, you can pass a `credentialName` option in the plugin config or via the `AZURE_IDENTITY_CREDENTIAL_NAME` environment variable. Supported options are: +For the Azure Blob cache provider with a non-sas/key-based `connectionString` (storage account endpoint) requiring [Azure Identity](https://learn.microsoft.com/en-us/javascript/api/overview/azure/identity-readme) authentication, you can pass a `credentialName` option in the plugin config or via the `AZURE_IDENTITY_CREDENTIAL_NAME` environment variable. (Do not use `BACKFILL_CACHE_PROVIDER_OPTIONS` in this case.) Supported options are: - `"azure-cli"` - `"managed-identity"` From db57a98b542ab85bebbdb6e350ea423e80ac3e04 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 19 Mar 2026 23:30:03 +0000 Subject: [PATCH 06/12] Add change file for Azure cache plugin migration Co-authored-by: ecraig12345 <5864305+ecraig12345@users.noreply.github.com> --- ...-922e3e3d-8503-4578-9876-071040126335.json | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 change/change-922e3e3d-8503-4578-9876-071040126335.json diff --git a/change/change-922e3e3d-8503-4578-9876-071040126335.json b/change/change-922e3e3d-8503-4578-9876-071040126335.json new file mode 100644 index 000000000..a54a7081c --- /dev/null +++ b/change/change-922e3e3d-8503-4578-9876-071040126335.json @@ -0,0 +1,46 @@ +{ + "changes": [ + { + "type": "minor", + "comment": "New plugin package for Azure Blob Storage cache", + "packageName": "@lage-run/azure-blob-cache-storage", + "email": "nicolo.ribaudo@gmail.com", + "dependentChangeType": "patch" + }, + { + "type": "major", + "comment": "Move Azure Blob cache to plugin; remove built-in azure-blob provider", + "packageName": "backfill-cache", + "email": "nicolo.ribaudo@gmail.com", + "dependentChangeType": "patch" + }, + { + "type": "major", + "comment": "Remove AzureBlobCacheStorageConfig from CacheStorageConfig union; add CustomCacheStoragePlugin and CustomCacheStorageConfig types", + "packageName": "backfill-config", + "email": "nicolo.ribaudo@gmail.com", + "dependentChangeType": "patch" + }, + { + "type": "minor", + "comment": "Support custom plugin providers via isCustomPluginProvider", + "packageName": "backfill", + "email": "nicolo.ribaudo@gmail.com", + "dependentChangeType": "patch" + }, + { + "type": "major", + "comment": "Remove Azure credential handling; credentials are now managed by the @lage-run/azure-blob-cache-storage plugin", + "packageName": "@lage-run/cache", + "email": "nicolo.ribaudo@gmail.com", + "dependentChangeType": "patch" + }, + { + "type": "major", + "comment": "Remove AzureCredentialName export and Azure-specific type augmentation from CacheOptions", + "packageName": "@lage-run/config", + "email": "nicolo.ribaudo@gmail.com", + "dependentChangeType": "patch" + } + ] +} From 8dda158807aac19c75bd965513447512f8806140 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 20 Mar 2026 00:02:47 +0000 Subject: [PATCH 07/12] Address PR review comments: share code instead of duplicating, remove .npmignore, resolve plugin relative to cwd, remove old CustomStorageConfig, fix emails in change file Co-authored-by: ecraig12345 <5864305+ecraig12345@users.noreply.github.com> --- ...-922e3e3d-8503-4578-9876-071040126335.json | 21 +++-- packages/azure-blob-cache-storage/.npmignore | 5 -- .../azure-blob-cache-storage/package.json | 2 +- .../src/AzureBlobCacheStorage.ts | 2 +- .../src/CacheStorage.ts | 79 ------------------- .../azure-blob-cache-storage/src/hashFile.ts | 56 ------------- .../__tests__/getCacheStorageProvider.test.ts | 37 +-------- .../src/getCacheStorageProvider.ts | 31 ++++---- packages/backfill-cache/src/index.ts | 1 - packages/backfill-config/src/cacheConfig.ts | 8 +- packages/backfill-config/src/index.ts | 1 - packages/backfill/src/__tests__/api.test.ts | 36 ++++++++- packages/backfill/src/index.ts | 10 +-- .../cache-github-actions/src/cacheProvider.ts | 10 +-- packages/cache-github-actions/src/index.ts | 2 +- packages/config/src/types/CacheOptions.ts | 4 +- yarn.lock | 2 +- 17 files changed, 78 insertions(+), 229 deletions(-) delete mode 100644 packages/azure-blob-cache-storage/.npmignore delete mode 100644 packages/azure-blob-cache-storage/src/CacheStorage.ts delete mode 100644 packages/azure-blob-cache-storage/src/hashFile.ts diff --git a/change/change-922e3e3d-8503-4578-9876-071040126335.json b/change/change-922e3e3d-8503-4578-9876-071040126335.json index a54a7081c..14d902738 100644 --- a/change/change-922e3e3d-8503-4578-9876-071040126335.json +++ b/change/change-922e3e3d-8503-4578-9876-071040126335.json @@ -4,42 +4,49 @@ "type": "minor", "comment": "New plugin package for Azure Blob Storage cache", "packageName": "@lage-run/azure-blob-cache-storage", - "email": "nicolo.ribaudo@gmail.com", + "email": "email not defined", "dependentChangeType": "patch" }, { "type": "major", "comment": "Move Azure Blob cache to plugin; remove built-in azure-blob provider", "packageName": "backfill-cache", - "email": "nicolo.ribaudo@gmail.com", + "email": "email not defined", "dependentChangeType": "patch" }, { "type": "major", - "comment": "Remove AzureBlobCacheStorageConfig from CacheStorageConfig union; add CustomCacheStoragePlugin and CustomCacheStorageConfig types", + "comment": "Remove AzureBlobCacheStorageConfig and CustomStorageConfig from CacheStorageConfig union; add CustomCacheStoragePlugin and CustomCacheStorageConfig types", "packageName": "backfill-config", - "email": "nicolo.ribaudo@gmail.com", + "email": "email not defined", "dependentChangeType": "patch" }, { "type": "minor", "comment": "Support custom plugin providers via isCustomPluginProvider", "packageName": "backfill", - "email": "nicolo.ribaudo@gmail.com", + "email": "email not defined", "dependentChangeType": "patch" }, { "type": "major", "comment": "Remove Azure credential handling; credentials are now managed by the @lage-run/azure-blob-cache-storage plugin", "packageName": "@lage-run/cache", - "email": "nicolo.ribaudo@gmail.com", + "email": "email not defined", "dependentChangeType": "patch" }, { "type": "major", "comment": "Remove AzureCredentialName export and Azure-specific type augmentation from CacheOptions", "packageName": "@lage-run/config", - "email": "nicolo.ribaudo@gmail.com", + "email": "email not defined", + "dependentChangeType": "patch" + }, + { + "type": "major", + "comment": "Migrate from CustomStorageConfig to CustomCacheStoragePlugin pattern", + "packageName": "@lage-run/cache-github-actions", + "email": "email not defined", "dependentChangeType": "patch" } ] diff --git a/packages/azure-blob-cache-storage/.npmignore b/packages/azure-blob-cache-storage/.npmignore deleted file mode 100644 index 76f06df88..000000000 --- a/packages/azure-blob-cache-storage/.npmignore +++ /dev/null @@ -1,5 +0,0 @@ -**/* -!lib/**/* -lib/**/__tests__/* -lib/**/*.d.ts.map -!bin/**/* diff --git a/packages/azure-blob-cache-storage/package.json b/packages/azure-blob-cache-storage/package.json index de4965d09..602959772 100644 --- a/packages/azure-blob-cache-storage/package.json +++ b/packages/azure-blob-cache-storage/package.json @@ -20,7 +20,7 @@ "@azure/core-auth": "1.9.0", "@azure/identity": "4.9.1", "@azure/storage-blob": "12.27.0", - "@lage-run/globby": "workspace:^", + "backfill-cache": "workspace:^", "backfill-config": "workspace:^", "backfill-logger": "workspace:^", "fs-extra": "8.1.0", diff --git a/packages/azure-blob-cache-storage/src/AzureBlobCacheStorage.ts b/packages/azure-blob-cache-storage/src/AzureBlobCacheStorage.ts index 30ffe1038..a35e4cf3c 100644 --- a/packages/azure-blob-cache-storage/src/AzureBlobCacheStorage.ts +++ b/packages/azure-blob-cache-storage/src/AzureBlobCacheStorage.ts @@ -7,7 +7,7 @@ import type { AzureBlobCacheStorageOptions } from "backfill-config"; import { stat } from "fs-extra"; import type { ContainerClient } from "@azure/storage-blob"; -import { CacheStorage } from "./CacheStorage.js"; +import { CacheStorage } from "backfill-cache"; const ONE_MEGABYTE = 1024 * 1024; const FOUR_MEGABYTES = 4 * ONE_MEGABYTE; diff --git a/packages/azure-blob-cache-storage/src/CacheStorage.ts b/packages/azure-blob-cache-storage/src/CacheStorage.ts deleted file mode 100644 index f615bb2e7..000000000 --- a/packages/azure-blob-cache-storage/src/CacheStorage.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { globAsync } from "@lage-run/globby"; - -import type { Logger } from "backfill-logger"; -import type { ICacheStorage } from "backfill-config"; -import { getFileHash } from "./hashFile.js"; - -// First key is the hash, second key is the file relative path -const savedHashes: Map> = new Map(); - -// contract: cwd should be absolute -// The return keys are relative path with posix file separators -async function getHashesFor(cwd: string): Promise> { - const result = new Map(); - - const allFiles = await globAsync(["**/*", "!node_modules"], { cwd }); - //globby returns relative path with posix file separator - await Promise.all( - allFiles.map(async (f) => { - const hash = await getFileHash(cwd, f); - result.set(f, hash); - }) - ); - - return result; -} - -export type { ICacheStorage }; - -export abstract class CacheStorage implements ICacheStorage { - public constructor( - protected logger: Logger, - protected cwd: string, - private incrementalCaching = false - ) {} - public async fetch(hash: string): Promise { - const tracer = this.logger.setTime("fetchTime"); - - const result = await this._fetch(hash); - - tracer.stop(); - - this.logger.setHit(result); - - if (!result && this.incrementalCaching) { - savedHashes.set(hash, await getHashesFor(this.cwd)); - } - - return result; - } - - public async put(hash: string, outputGlob: string[]): Promise { - const tracer = this.logger.setTime("putTime"); - - const filesMatchingOutputGlob = await globAsync(outputGlob, { - cwd: this.cwd, - }); - - let filesToCache = filesMatchingOutputGlob; - if (this.incrementalCaching) { - // Get the list of files that have not changed so we don't need to cache them. - const hashesNow = await getHashesFor(this.cwd); - const hashesThen = - (await savedHashes.get(hash)) || new Map(); - const unchangedFiles = [...hashesThen.keys()].filter( - (s) => hashesThen.get(s) === hashesNow.get(s) - ); - filesToCache = filesMatchingOutputGlob.filter( - (f) => !unchangedFiles.includes(f) - ); - } - - await this._put(hash, filesToCache); - tracer.stop(); - } - - protected abstract _fetch(hash: string): Promise; - - protected abstract _put(hash: string, filesToCache: string[]): Promise; -} diff --git a/packages/azure-blob-cache-storage/src/hashFile.ts b/packages/azure-blob-cache-storage/src/hashFile.ts deleted file mode 100644 index 3f553bc86..000000000 --- a/packages/azure-blob-cache-storage/src/hashFile.ts +++ /dev/null @@ -1,56 +0,0 @@ -import * as path from "path"; -import { promises as fs } from "fs"; -import * as crypto from "crypto"; -import pLimit from "p-limit"; - -let MAX_FILE_OPERATIONS = 5000; - -try { - const maxFileOpEnv = process.env["BACKFILL_MAX_FILE_OP"]; - if (maxFileOpEnv) { - MAX_FILE_OPERATIONS = parseInt(maxFileOpEnv); - } -} catch (_) { - /* The env variable is not an integer, this is fine.*/ -} - -const diskLimit = pLimit(MAX_FILE_OPERATIONS); - -// The first key is the file path, the second key is mtime -const memo = new Map>(); - -async function computeHash(filePath: string): Promise { - const fileBuffer = await diskLimit(() => { - return fs.readFile(filePath); - }); - // We use sha1 for perf reason and because the hashing is not used for security reason. - const hashSum = crypto.createHash("sha1"); - hashSum.update(fileBuffer); - const hash = hashSum.digest("hex"); - return hash; -} - -/* - * Get the hash of a file. - * This function memoizes the hash for files and mtimes. - */ -export async function getFileHash( - cwd: string, - filePath: string -): Promise { - const fileAbsPath = path.join(cwd, filePath); - const stat = await fs.stat(fileAbsPath); - - let memoForFile = memo.get(fileAbsPath); - if (!memoForFile) { - memoForFile = new Map(); - memo.set(fileAbsPath, memoForFile); - } - - let hash = memoForFile.get(stat.mtimeMs); - if (!hash) { - hash = await computeHash(fileAbsPath); - memoForFile.set(stat.mtimeMs, hash); - } - return hash; -} diff --git a/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts b/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts index c19c62a9e..4775f180d 100644 --- a/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts +++ b/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts @@ -1,6 +1,5 @@ -import { type Logger, makeLogger } from "backfill-logger"; +import { makeLogger } from "backfill-logger"; import { getCacheStorageProvider } from "../getCacheStorageProvider.js"; -import type { ICacheStorage } from "../CacheStorage.js"; import { LocalCacheStorage } from "../LocalCacheStorage.js"; describe("getCacheStorageProvider", () => { @@ -64,40 +63,6 @@ describe("getCacheStorageProvider", () => { expect(provider instanceof LocalCacheStorage).toBeTruthy(); }); - test("can get a custom storage provider as a class", () => { - const TestProvider = class implements ICacheStorage { - constructor( - private logger: Logger, - private cwd: string - ) {} - - public fetch(hash: string) { - this.logger.silly(`fetching ${this.cwd} ${hash}`); - return Promise.resolve(true); - } - - public put(hash: string, filesToCache: string[]) { - this.logger.silly( - `putting ${this.cwd} ${hash} ${filesToCache.length} files` - ); - return Promise.resolve(); - } - }; - - const provider = getCacheStorageProvider( - { - provider: (logger, cwd) => new TestProvider(logger, cwd), - name: "test-provider", - }, - "test", - makeLogger("silly"), - "cwd" - ); - - expect(provider.fetch).toBeTruthy(); - expect(provider.put).toBeTruthy(); - }); - test("throws when custom plugin cannot be loaded", () => { expect(() => getCacheStorageProvider( diff --git a/packages/backfill-cache/src/getCacheStorageProvider.ts b/packages/backfill-cache/src/getCacheStorageProvider.ts index e0093aa7e..406a2f4e3 100644 --- a/packages/backfill-cache/src/getCacheStorageProvider.ts +++ b/packages/backfill-cache/src/getCacheStorageProvider.ts @@ -1,4 +1,6 @@ -import type { CacheStorageConfig, CustomStorageConfig, CustomCacheStorageConfig, CustomCacheStoragePlugin } from "backfill-config"; +import * as path from "path"; +import { createRequire } from "module"; +import type { CacheStorageConfig, CustomCacheStorageConfig, CustomCacheStoragePlugin } from "backfill-config"; import type { Logger } from "backfill-logger"; import type { ICacheStorage } from "./CacheStorage.js"; @@ -6,12 +8,6 @@ import { LocalCacheStorage } from "./LocalCacheStorage.js"; import { NpmCacheStorage } from "./NpmCacheStorage.js"; import { LocalSkipCacheStorage } from "./LocalSkipCacheStorage.js"; -export function isCustomProvider( - config: CacheStorageConfig -): config is CustomStorageConfig { - return typeof config.provider === "function"; -} - export function isCustomPluginProvider( config: CacheStorageConfig ): config is CustomCacheStorageConfig { @@ -29,14 +25,6 @@ export function getCacheStorageProvider( ): ICacheStorage { let cacheStorage: ICacheStorage | undefined; - if (isCustomProvider(cacheStorageConfig)) { - try { - return cacheStorageConfig.provider(logger, cwd); - } catch { - throw new Error("cacheStorageConfig.provider cannot be creaated"); - } - } - if (isCustomPluginProvider(cacheStorageConfig)) { const key = `custom:${cacheStorageConfig.plugin}${internalCacheFolder}${cwd}`; cacheStorage = memo.get(key); @@ -45,8 +33,19 @@ export function getCacheStorageProvider( } try { + const pluginName = cacheStorageConfig.plugin; + let resolvedPlugin: string; + + if (path.isAbsolute(pluginName)) { + resolvedPlugin = pluginName; + } else { + // Resolve relative paths and package names from cwd + const cwdRequire = createRequire(path.resolve(cwd, "package.json")); + resolvedPlugin = cwdRequire.resolve(pluginName); + } + // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires - const pluginModule = require(cacheStorageConfig.plugin); + const pluginModule = require(resolvedPlugin); const plugin: CustomCacheStoragePlugin = pluginModule.default || pluginModule; cacheStorage = plugin.getProvider(logger, cwd, cacheStorageConfig.options); } catch (err) { diff --git a/packages/backfill-cache/src/index.ts b/packages/backfill-cache/src/index.ts index ac992198d..d91bf5421 100644 --- a/packages/backfill-cache/src/index.ts +++ b/packages/backfill-cache/src/index.ts @@ -1,6 +1,5 @@ export { getCacheStorageProvider, - isCustomProvider, isCustomPluginProvider, } from "./getCacheStorageProvider.js"; export { type ICacheStorage, CacheStorage } from "./CacheStorage.js"; diff --git a/packages/backfill-config/src/cacheConfig.ts b/packages/backfill-config/src/cacheConfig.ts index b79ba4eec..0f6db03d8 100644 --- a/packages/backfill-config/src/cacheConfig.ts +++ b/packages/backfill-config/src/cacheConfig.ts @@ -6,11 +6,6 @@ export interface ICacheStorage { put: (hash: string, filesToCache: string[]) => Promise; } -export type CustomStorageConfig = { - provider: (logger: Logger, cwd: string) => ICacheStorage; - name?: string; -}; - /** * A plugin that provides a custom cache storage implementation. * The plugin module should export this as its default export. @@ -43,8 +38,7 @@ export type CacheStorageConfig = provider: "local-skip"; } | NpmCacheStorageConfig - | CustomCacheStorageConfig - | CustomStorageConfig; + | CustomCacheStorageConfig; /** * Environment variable names for the cache storage config. diff --git a/packages/backfill-config/src/index.ts b/packages/backfill-config/src/index.ts index de853956f..25cc70e52 100644 --- a/packages/backfill-config/src/index.ts +++ b/packages/backfill-config/src/index.ts @@ -1,7 +1,6 @@ export * from "./azureBlobCacheConfig.js"; export type { ICacheStorage, - CustomStorageConfig, CacheStorageConfig, CustomCacheStoragePlugin, CustomCacheStorageConfig, diff --git a/packages/backfill/src/__tests__/api.test.ts b/packages/backfill/src/__tests__/api.test.ts index d98cfff9e..c4e5f6143 100644 --- a/packages/backfill/src/__tests__/api.test.ts +++ b/packages/backfill/src/__tests__/api.test.ts @@ -1,10 +1,26 @@ import path from "path"; +import fs from "fs"; +import os from "os"; import { createDefaultConfig } from "backfill-config"; import { setupFixture } from "backfill-utils-test"; import { fetch, put, makeLogger } from "../api.js"; +function createMockPlugin(provider: { fetch: jest.Mock; put: jest.Mock }) { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "backfill-test-")); + const pluginPath = path.join(tmpDir, "mock-plugin.js"); + // Write a plugin module that returns the provider + fs.writeFileSync( + pluginPath, + `module.exports = { default: { name: "mock", getProvider: () => (${JSON.stringify({})}) } };` + ); + // Override the getProvider at runtime by requiring and patching + const mod = require(pluginPath); + mod.default.getProvider = () => provider; + return { pluginPath, tmpDir }; +} + describe("api", () => { - it("fetch works with custom providers", async () => { + it("fetch works with custom plugin providers", async () => { const packageRoot = await setupFixture("basic"); const logger = makeLogger("silly", process.stdout, process.stderr); @@ -14,8 +30,12 @@ describe("api", () => { put: jest.fn().mockResolvedValue(true), }; + const { pluginPath, tmpDir } = createMockPlugin(provider); + config.cacheStorageConfig = { - provider: () => provider, + provider: "custom", + plugin: pluginPath, + options: {}, }; const fetched = await fetch( @@ -28,9 +48,11 @@ describe("api", () => { expect(fetched).toBeTruthy(); expect(provider.fetch).toHaveBeenCalled(); expect(provider.put).not.toHaveBeenCalled(); + + fs.rmSync(tmpDir, { recursive: true }); }); - it("put works with custom providers", async () => { + it("put works with custom plugin providers", async () => { const packageRoot = await setupFixture("basic"); const logger = makeLogger("silly", process.stdout, process.stderr); @@ -40,8 +62,12 @@ describe("api", () => { put: jest.fn().mockResolvedValue(true), }; + const { pluginPath, tmpDir } = createMockPlugin(provider); + config.cacheStorageConfig = { - provider: () => provider, + provider: "custom", + plugin: pluginPath, + options: {}, }; await put( @@ -53,5 +79,7 @@ describe("api", () => { expect(provider.fetch).not.toHaveBeenCalled(); expect(provider.put).toHaveBeenCalled(); + + fs.rmSync(tmpDir, { recursive: true }); }); }); diff --git a/packages/backfill/src/index.ts b/packages/backfill/src/index.ts index c294cdd11..131efbe5d 100644 --- a/packages/backfill/src/index.ts +++ b/packages/backfill/src/index.ts @@ -1,7 +1,7 @@ import { loadDotenv } from "backfill-utils-dotenv"; import { type Logger, makeLogger } from "backfill-logger"; import { createConfig, type Config } from "backfill-config"; -import { isCustomProvider, isCustomPluginProvider } from "backfill-cache"; +import { isCustomPluginProvider } from "backfill-cache"; import yargs from "yargs"; import { getRawBuildCommand, @@ -44,11 +44,9 @@ export async function backfill( logger.setName(name); logger.setMode(mode, mode === "READ_WRITE" ? "info" : "verbose"); logger.setCacheProvider( - isCustomProvider(cacheStorageConfig) - ? cacheStorageConfig.name || "custom-storage-provider" - : isCustomPluginProvider(cacheStorageConfig) - ? cacheStorageConfig.plugin - : cacheStorageConfig.provider + isCustomPluginProvider(cacheStorageConfig) + ? cacheStorageConfig.plugin + : cacheStorageConfig.provider ); const createPackageHash = async () => diff --git a/packages/cache-github-actions/src/cacheProvider.ts b/packages/cache-github-actions/src/cacheProvider.ts index c1c21fd64..ac3dde5c8 100644 --- a/packages/cache-github-actions/src/cacheProvider.ts +++ b/packages/cache-github-actions/src/cacheProvider.ts @@ -1,13 +1,14 @@ import cache from "@actions/cache"; -import type { CustomStorageConfig } from "backfill-config"; +import type { CustomCacheStoragePlugin, ICacheStorage } from "backfill-config"; import type { Logger } from "backfill-logger"; import path from "path"; import { getWorkspaceManagerRoot } from "workspace-tools"; const root = getWorkspaceManagerRoot(process.cwd())!; -const cacheProvider: CustomStorageConfig = { - provider: (_logger: Logger, cwd: string) => { +const plugin: CustomCacheStoragePlugin = { + name: "github-actions", + getProvider(_logger: Logger, cwd: string): ICacheStorage { return { async fetch(hash: string): Promise { if (!hash) { @@ -33,7 +34,6 @@ const cacheProvider: CustomStorageConfig = { }, }; }, - name: "github-actions", }; -export { cacheProvider }; +export default plugin; diff --git a/packages/cache-github-actions/src/index.ts b/packages/cache-github-actions/src/index.ts index 63569f4f4..414d524f1 100644 --- a/packages/cache-github-actions/src/index.ts +++ b/packages/cache-github-actions/src/index.ts @@ -1 +1 @@ -export { cacheProvider as default } from "./cacheProvider.js"; +export { default } from "./cacheProvider.js"; diff --git a/packages/config/src/types/CacheOptions.ts b/packages/config/src/types/CacheOptions.ts index 91a0e1143..3d3f8baa6 100644 --- a/packages/config/src/types/CacheOptions.ts +++ b/packages/config/src/types/CacheOptions.ts @@ -1,4 +1,4 @@ -import type { Config as BackfillCacheOptions, CustomStorageConfig } from "backfill-config"; +import type { Config as BackfillCacheOptions } from "backfill-config"; export type CacheOptions = Omit & { /** @@ -6,7 +6,7 @@ export type CacheOptions = Omit & { * For Azure Blob Storage, use `provider: "custom"` with `plugin: "@lage-run/azure-blob-cache-storage"`. * @see https://www.npmjs.com/package/backfill#configuration */ - cacheStorageConfig?: Exclude; + cacheStorageConfig?: BackfillCacheOptions["cacheStorageConfig"]; /** * Whether to write to the remote cache - useful for continuous integration systems to provide build-over-build cache. diff --git a/yarn.lock b/yarn.lock index f85413d1a..ab360c010 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1544,10 +1544,10 @@ __metadata: "@azure/core-auth": "npm:1.9.0" "@azure/identity": "npm:4.9.1" "@azure/storage-blob": "npm:12.27.0" - "@lage-run/globby": "workspace:^" "@lage-run/monorepo-scripts": "workspace:^" "@types/fs-extra": "npm:^8.0.0" "@types/tar-fs": "npm:^2.0.1" + backfill-cache: "workspace:^" backfill-config: "workspace:^" backfill-logger: "workspace:^" fs-extra: "npm:8.1.0" From e81201a7ed3ea500ce14eab221eee814137de0bc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 20 Mar 2026 00:25:48 +0000 Subject: [PATCH 08/12] Address follow-up review comments: restore CredentialCache comments, simplify CacheOptions, move plugin impl to separate file, fix mock Co-authored-by: ecraig12345 <5864305+ecraig12345@users.noreply.github.com> --- .../src/CredentialCache.ts | 14 ++++- .../src/createPlugin.ts | 48 +++++++++++++++++ .../azure-blob-cache-storage/src/index.ts | 51 +------------------ packages/backfill/src/__tests__/api.test.ts | 2 +- packages/config/src/types/CacheOptions.ts | 9 +--- 5 files changed, 65 insertions(+), 59 deletions(-) create mode 100644 packages/azure-blob-cache-storage/src/createPlugin.ts diff --git a/packages/azure-blob-cache-storage/src/CredentialCache.ts b/packages/azure-blob-cache-storage/src/CredentialCache.ts index 8f10cd33f..471384c88 100644 --- a/packages/azure-blob-cache-storage/src/CredentialCache.ts +++ b/packages/azure-blob-cache-storage/src/CredentialCache.ts @@ -7,9 +7,16 @@ import { WorkloadIdentityCredential, } from "@azure/identity"; -/** Allowed credential names matching camelCase of @azure/identity credential class names */ +/** + * Allowed credential names matching camelCase of @azure/identity credential class names + * @see https://learn.microsoft.com/en-us/azure/developer/javascript/sdk/authentication/credential-chains + */ export type AzureCredentialName = "environment" | "workload-identity" | "managed-identity" | "visual-studio-code" | "azure-cli"; +/** + * Exhaustive credential factory map keyed by AzureCredentialName. + * This enforces compile-time alignment with the AzureCredentialName union and provides a single source of truth. + */ type CredentialFactoryMap = { [K in AzureCredentialName]: () => TokenCredential }; const CREDENTIAL_FACTORY: CredentialFactoryMap = { environment: () => new EnvironmentCredential(), @@ -22,8 +29,13 @@ const CREDENTIAL_FACTORY: CredentialFactoryMap = { export class CredentialCache { private static cache: Map = new Map(); + // Expose the list for runtime validation elsewhere (derived from the exhaustive factory above) public static readonly credentialNames: readonly AzureCredentialName[] = Object.keys(CREDENTIAL_FACTORY) as AzureCredentialName[]; + /** + * Returns a credential instance based on the provided name. Results are cached per name. + * If no name is provided, EnvironmentCredential is used by default. + */ public static getInstance(credentialName?: AzureCredentialName): TokenCredential { const key = (credentialName ?? "environment") as AzureCredentialName; const existing = this.cache.get(key); diff --git a/packages/azure-blob-cache-storage/src/createPlugin.ts b/packages/azure-blob-cache-storage/src/createPlugin.ts new file mode 100644 index 000000000..e84a19376 --- /dev/null +++ b/packages/azure-blob-cache-storage/src/createPlugin.ts @@ -0,0 +1,48 @@ +import type { Logger } from "backfill-logger"; +import type { + ICacheStorage, + CustomCacheStoragePlugin, + AzureBlobCacheStorageOptions, + AzureBlobCacheStorageConnectionStringOptions, +} from "backfill-config"; + +import { AzureBlobCacheStorage } from "./AzureBlobCacheStorage.js"; +import { CredentialCache, type AzureCredentialName } from "./CredentialCache.js"; + +export type AzureBlobPluginOptions = AzureBlobCacheStorageOptions & { + /** Optional credential name for Azure Identity authentication. */ + credentialName?: AzureCredentialName; +}; + +function isTokenConnectionString(connectionString: string) { + return connectionString.includes("SharedAccessSignature") || connectionString.includes("AccountKey"); +} + +const plugin: CustomCacheStoragePlugin = { + name: "azure-blob", + getProvider(logger: Logger, cwd: string, options: AzureBlobPluginOptions): ICacheStorage { + // Handle credential injection for connection-string-based options + if ("connectionString" in options && !isTokenConnectionString(options.connectionString)) { + const connStringOptions = options as AzureBlobCacheStorageConnectionStringOptions & { credentialName?: AzureCredentialName }; + if (!connStringOptions.credential) { + const credName = connStringOptions.credentialName + ?? (process.env.AZURE_IDENTITY_CREDENTIAL_NAME || undefined); + + if (credName != null) { + if (!CredentialCache.credentialNames.includes(credName as AzureCredentialName)) { + throw new Error( + `Invalid credentialName: "${credName}". Allowed values: ${CredentialCache.credentialNames.join(", ")}` + ); + } + connStringOptions.credential = CredentialCache.getInstance(credName as AzureCredentialName); + } else { + connStringOptions.credential = CredentialCache.getInstance(); + } + } + } + + return new AzureBlobCacheStorage(options, logger, cwd); + }, +}; + +export default plugin; diff --git a/packages/azure-blob-cache-storage/src/index.ts b/packages/azure-blob-cache-storage/src/index.ts index 22793b711..24e5b5e65 100644 --- a/packages/azure-blob-cache-storage/src/index.ts +++ b/packages/azure-blob-cache-storage/src/index.ts @@ -1,51 +1,4 @@ -import type { Logger } from "backfill-logger"; -import type { - ICacheStorage, - CustomCacheStoragePlugin, - AzureBlobCacheStorageOptions, - AzureBlobCacheStorageConnectionStringOptions, -} from "backfill-config"; - -import { AzureBlobCacheStorage } from "./AzureBlobCacheStorage.js"; -import { CredentialCache, type AzureCredentialName } from "./CredentialCache.js"; - export type { AzureCredentialName } from "./CredentialCache.js"; export { CredentialCache } from "./CredentialCache.js"; - -export type AzureBlobPluginOptions = AzureBlobCacheStorageOptions & { - /** Optional credential name for Azure Identity authentication. */ - credentialName?: AzureCredentialName; -}; - -function isTokenConnectionString(connectionString: string) { - return connectionString.includes("SharedAccessSignature") || connectionString.includes("AccountKey"); -} - -const plugin: CustomCacheStoragePlugin = { - name: "azure-blob", - getProvider(logger: Logger, cwd: string, options: AzureBlobPluginOptions): ICacheStorage { - // Handle credential injection for connection-string-based options - if ("connectionString" in options && !isTokenConnectionString(options.connectionString)) { - const connStringOptions = options as AzureBlobCacheStorageConnectionStringOptions & { credentialName?: AzureCredentialName }; - if (!connStringOptions.credential) { - const credName = connStringOptions.credentialName - ?? (process.env.AZURE_IDENTITY_CREDENTIAL_NAME || undefined); - - if (credName != null) { - if (!CredentialCache.credentialNames.includes(credName as AzureCredentialName)) { - throw new Error( - `Invalid credentialName: "${credName}". Allowed values: ${CredentialCache.credentialNames.join(", ")}` - ); - } - connStringOptions.credential = CredentialCache.getInstance(credName as AzureCredentialName); - } else { - connStringOptions.credential = CredentialCache.getInstance(); - } - } - } - - return new AzureBlobCacheStorage(options, logger, cwd); - }, -}; - -export default plugin; +export type { AzureBlobPluginOptions } from "./createPlugin.js"; +export { default } from "./createPlugin.js"; diff --git a/packages/backfill/src/__tests__/api.test.ts b/packages/backfill/src/__tests__/api.test.ts index c4e5f6143..c18d31e2f 100644 --- a/packages/backfill/src/__tests__/api.test.ts +++ b/packages/backfill/src/__tests__/api.test.ts @@ -11,7 +11,7 @@ function createMockPlugin(provider: { fetch: jest.Mock; put: jest.Mock }) { // Write a plugin module that returns the provider fs.writeFileSync( pluginPath, - `module.exports = { default: { name: "mock", getProvider: () => (${JSON.stringify({})}) } };` + `module.exports = { default: { name: "mock", getProvider: () => ({}) } };` ); // Override the getProvider at runtime by requiring and patching const mod = require(pluginPath); diff --git a/packages/config/src/types/CacheOptions.ts b/packages/config/src/types/CacheOptions.ts index 3d3f8baa6..3c2f24fab 100644 --- a/packages/config/src/types/CacheOptions.ts +++ b/packages/config/src/types/CacheOptions.ts @@ -1,13 +1,6 @@ import type { Config as BackfillCacheOptions } from "backfill-config"; -export type CacheOptions = Omit & { - /** - * Use this to specify a remote cache provider. - * For Azure Blob Storage, use `provider: "custom"` with `plugin: "@lage-run/azure-blob-cache-storage"`. - * @see https://www.npmjs.com/package/backfill#configuration - */ - cacheStorageConfig?: BackfillCacheOptions["cacheStorageConfig"]; - +export type CacheOptions = BackfillCacheOptions & { /** * Whether to write to the remote cache - useful for continuous integration systems to provide build-over-build cache. * It is recommended to turn this OFF for local development, turning remote cache to be a build acceleration through remote cache downloads. From 66830c889c8381dbe406062347e611a50acb0dcb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 20 Mar 2026 18:20:01 +0000 Subject: [PATCH 09/12] Remove disallowedChangeTypes from beachball.config.js to allow major version bumps Co-authored-by: ecraig12345 <5864305+ecraig12345@users.noreply.github.com> Agent-Logs-Url: https://github.com/microsoft/lage/sessions/cfdef28c-2024-4a09-9d85-8e96286c05ba --- beachball.config.js | 1 - 1 file changed, 1 deletion(-) diff --git a/beachball.config.js b/beachball.config.js index 0ce2cdd5f..1e7f56fcd 100644 --- a/beachball.config.js +++ b/beachball.config.js @@ -38,6 +38,5 @@ const config = { }, }, ignorePatterns: [".*ignore", "jest.config.js", "**/__*/**/*"], - disallowedChangeTypes: ["major"], }; module.exports = config; From b996507446353249f4c42849d44109108a1b6e45 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 20 Mar 2026 18:31:16 +0000 Subject: [PATCH 10/12] Bring back custom plugin provider test in getCacheStorageProvider.test.ts Co-authored-by: ecraig12345 <5864305+ecraig12345@users.noreply.github.com> Agent-Logs-Url: https://github.com/microsoft/lage/sessions/cff14f63-a48c-4434-9eb9-21adc4c67087 --- .../__tests__/getCacheStorageProvider.test.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts b/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts index 4775f180d..458f2745c 100644 --- a/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts +++ b/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts @@ -1,3 +1,6 @@ +import path from "path"; +import fs from "fs"; +import os from "os"; import { makeLogger } from "backfill-logger"; import { getCacheStorageProvider } from "../getCacheStorageProvider.js"; import { LocalCacheStorage } from "../LocalCacheStorage.js"; @@ -77,4 +80,32 @@ describe("getCacheStorageProvider", () => { ) ).toThrow('Failed to load custom cache storage plugin "nonexistent-plugin-package"'); }); + + test("can get a custom storage provider via plugin", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "backfill-test-")); + const pluginPath = path.join(tmpDir, "mock-plugin.js"); + + fs.writeFileSync( + pluginPath, + `module.exports = { default: { name: "mock", getProvider: () => ({ fetch: () => Promise.resolve(true), put: () => Promise.resolve() }) } };` + ); + + try { + const provider = getCacheStorageProvider( + { + provider: "custom", + plugin: pluginPath, + options: {}, + }, + "test", + makeLogger("silly"), + "cwd" + ); + + expect(provider.fetch).toBeTruthy(); + expect(provider.put).toBeTruthy(); + } finally { + fs.rmSync(tmpDir, { recursive: true }); + } + }); }); From db9c2d88685582473078ce41734d61099acf2d98 Mon Sep 17 00:00:00 2001 From: Elizabeth Craig Date: Fri, 20 Mar 2026 12:06:52 -0700 Subject: [PATCH 11/12] lint --- packages/backfill/src/__tests__/api.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backfill/src/__tests__/api.test.ts b/packages/backfill/src/__tests__/api.test.ts index c18d31e2f..5a9105767 100644 --- a/packages/backfill/src/__tests__/api.test.ts +++ b/packages/backfill/src/__tests__/api.test.ts @@ -14,6 +14,7 @@ function createMockPlugin(provider: { fetch: jest.Mock; put: jest.Mock }) { `module.exports = { default: { name: "mock", getProvider: () => ({}) } };` ); // Override the getProvider at runtime by requiring and patching + // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires const mod = require(pluginPath); mod.default.getProvider = () => provider; return { pluginPath, tmpDir }; From 8771b4bb25f4726c3fc0128ba4ce8ee832fa9b68 Mon Sep 17 00:00:00 2001 From: Elizabeth Craig Date: Fri, 20 Mar 2026 12:57:41 -0700 Subject: [PATCH 12/12] format --- CLAUDE.md | 2 + docs/docs/guides/remote-cache.md | 8 +-- .../src/AzureBlobCacheStorage.ts | 60 +++++-------------- .../src/createPlugin.ts | 7 +-- .../__tests__/getCacheStorageProvider.test.ts | 4 +- .../src/getCacheStorageProvider.ts | 15 ++++- packages/backfill-config/src/cacheConfig.ts | 6 +- packages/backfill-config/src/envConfig.ts | 5 +- packages/backfill/README.md | 18 +++--- 9 files changed, 56 insertions(+), 69 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0a57d2a94..535e3ea79 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -53,6 +53,8 @@ yarn test Before finalizing changes, run `yarn ci` from the root for complete build/test/lint verification. +Also run `yarn format` to correct any formatting issues. + ## Architecture ### Key packages diff --git a/docs/docs/guides/remote-cache.md b/docs/docs/guides/remote-cache.md index 7c69520b1..37d5b3d20 100644 --- a/docs/docs/guides/remote-cache.md +++ b/docs/docs/guides/remote-cache.md @@ -77,10 +77,10 @@ module.exports = { plugin: "@lage-run/azure-blob-cache-storage", options: { connectionString: "...", - container: "...", - }, - }, - }, + container: "..." + } + } + } }; ``` diff --git a/packages/azure-blob-cache-storage/src/AzureBlobCacheStorage.ts b/packages/azure-blob-cache-storage/src/AzureBlobCacheStorage.ts index a35e4cf3c..a5450b483 100644 --- a/packages/azure-blob-cache-storage/src/AzureBlobCacheStorage.ts +++ b/packages/azure-blob-cache-storage/src/AzureBlobCacheStorage.ts @@ -25,11 +25,7 @@ class TimeoutStream extends Transform { this.destroy(new Error(message)); }, timeout); } - public _transform( - chunk: any, - _encoding: BufferEncoding, - callback: TransformCallback - ): void { + public _transform(chunk: any, _encoding: BufferEncoding, callback: TransformCallback): void { clearTimeout(this.timeout); this.push(chunk); callback(); @@ -49,11 +45,7 @@ class SpongeStream extends Transform { readableHighWaterMark: 1024 * 1024 * 1024 * 1024, }); } - public _transform( - chunk: any, - _encoding: BufferEncoding, - callback: TransformCallback - ): void { + public _transform(chunk: any, _encoding: BufferEncoding, callback: TransformCallback): void { this.pause(); this.push(chunk); callback(); @@ -81,8 +73,7 @@ export class AzureBlobCacheStorage extends CacheStorage { super(logger, cwd, incrementalCaching); if ("containerClient" in options) { - this.getContainerClient = () => - Promise.resolve(options.containerClient as ContainerClient); + this.getContainerClient = () => Promise.resolve(options.containerClient as ContainerClient); } else { const { connectionString, container, credential } = options; // This is delay loaded because it's very slow to parse @@ -92,8 +83,7 @@ export class AzureBlobCacheStorage extends CacheStorage { ? new BlobServiceClient(connectionString, credential) : BlobServiceClient.fromConnectionString(connectionString); - const containerClient = - blobServiceClient.getContainerClient(container); + const containerClient = blobServiceClient.getContainerClient(container); return containerClient; }); } @@ -107,13 +97,8 @@ export class AzureBlobCacheStorage extends CacheStorage { if (this.options.maxSize) { const sizeResponse = await blobClient.getProperties(); - if ( - sizeResponse.contentLength && - sizeResponse.contentLength > this.options.maxSize - ) { - this.logger.verbose( - `A blob is too large to be downloaded: ${hash}, size: ${sizeResponse.contentLength} bytes` - ); + if (sizeResponse.contentLength && sizeResponse.contentLength > this.options.maxSize) { + this.logger.verbose(`A blob is too large to be downloaded: ${hash}, size: ${sizeResponse.contentLength} bytes`); return false; } } @@ -129,25 +114,16 @@ export class AzureBlobCacheStorage extends CacheStorage { const spongeStream = new SpongeStream(); - const timeoutStream = new TimeoutStream( - 10 * 60 * 1000, - `The fetch request to ${hash} seems to be hanging` - ); + const timeoutStream = new TimeoutStream(10 * 60 * 1000, `The fetch request to ${hash} seems to be hanging`); const extractionPipeline = new Promise((resolve, reject) => - pipeline( - blobReadableStream, - spongeStream, - timeoutStream, - tarWritableStream, - (err) => { - if (err) { - reject(err); - } else { - resolve(); - } + pipeline(blobReadableStream, spongeStream, timeoutStream, tarWritableStream, (err) => { + if (err) { + reject(err); + } else { + resolve(); } - ) + }) ); await extractionPipeline; @@ -177,17 +153,11 @@ export class AzureBlobCacheStorage extends CacheStorage { } if (total > this.options.maxSize) { - this.logger.verbose( - `The output is too large to be uploaded: ${hash}, size: ${total} bytes` - ); + this.logger.verbose(`The output is too large to be uploaded: ${hash}, size: ${total} bytes`); return; } } - await blockBlobClient.uploadStream( - tarStream, - uploadOptions.bufferSize, - uploadOptions.maxBuffers - ); + await blockBlobClient.uploadStream(tarStream, uploadOptions.bufferSize, uploadOptions.maxBuffers); } } diff --git a/packages/azure-blob-cache-storage/src/createPlugin.ts b/packages/azure-blob-cache-storage/src/createPlugin.ts index e84a19376..1d0bfaa3c 100644 --- a/packages/azure-blob-cache-storage/src/createPlugin.ts +++ b/packages/azure-blob-cache-storage/src/createPlugin.ts @@ -25,14 +25,11 @@ const plugin: CustomCacheStoragePlugin = { if ("connectionString" in options && !isTokenConnectionString(options.connectionString)) { const connStringOptions = options as AzureBlobCacheStorageConnectionStringOptions & { credentialName?: AzureCredentialName }; if (!connStringOptions.credential) { - const credName = connStringOptions.credentialName - ?? (process.env.AZURE_IDENTITY_CREDENTIAL_NAME || undefined); + const credName = connStringOptions.credentialName ?? (process.env.AZURE_IDENTITY_CREDENTIAL_NAME || undefined); if (credName != null) { if (!CredentialCache.credentialNames.includes(credName as AzureCredentialName)) { - throw new Error( - `Invalid credentialName: "${credName}". Allowed values: ${CredentialCache.credentialNames.join(", ")}` - ); + throw new Error(`Invalid credentialName: "${credName}". Allowed values: ${CredentialCache.credentialNames.join(", ")}`); } connStringOptions.credential = CredentialCache.getInstance(credName as AzureCredentialName); } else { diff --git a/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts b/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts index 458f2745c..a616e3aca 100644 --- a/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts +++ b/packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts @@ -78,7 +78,9 @@ describe("getCacheStorageProvider", () => { makeLogger("silly"), "cwd" ) - ).toThrow('Failed to load custom cache storage plugin "nonexistent-plugin-package"'); + ).toThrow( + 'Failed to load custom cache storage plugin "nonexistent-plugin-package"' + ); }); test("can get a custom storage provider via plugin", () => { diff --git a/packages/backfill-cache/src/getCacheStorageProvider.ts b/packages/backfill-cache/src/getCacheStorageProvider.ts index 406a2f4e3..10932cee7 100644 --- a/packages/backfill-cache/src/getCacheStorageProvider.ts +++ b/packages/backfill-cache/src/getCacheStorageProvider.ts @@ -1,6 +1,10 @@ import * as path from "path"; import { createRequire } from "module"; -import type { CacheStorageConfig, CustomCacheStorageConfig, CustomCacheStoragePlugin } from "backfill-config"; +import type { + CacheStorageConfig, + CustomCacheStorageConfig, + CustomCacheStoragePlugin, +} from "backfill-config"; import type { Logger } from "backfill-logger"; import type { ICacheStorage } from "./CacheStorage.js"; @@ -46,8 +50,13 @@ export function getCacheStorageProvider( // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires const pluginModule = require(resolvedPlugin); - const plugin: CustomCacheStoragePlugin = pluginModule.default || pluginModule; - cacheStorage = plugin.getProvider(logger, cwd, cacheStorageConfig.options); + const plugin: CustomCacheStoragePlugin = + pluginModule.default || pluginModule; + cacheStorage = plugin.getProvider( + logger, + cwd, + cacheStorageConfig.options + ); } catch (err) { throw new Error( `Failed to load custom cache storage plugin "${cacheStorageConfig.plugin}": ${err}` diff --git a/packages/backfill-config/src/cacheConfig.ts b/packages/backfill-config/src/cacheConfig.ts index 0f6db03d8..97fd07351 100644 --- a/packages/backfill-config/src/cacheConfig.ts +++ b/packages/backfill-config/src/cacheConfig.ts @@ -12,7 +12,11 @@ export interface ICacheStorage { */ export interface CustomCacheStoragePlugin { name: string; - getProvider: (logger: Logger, cwd: string, options: TOptions) => ICacheStorage; + getProvider: ( + logger: Logger, + cwd: string, + options: TOptions + ) => ICacheStorage; } /** diff --git a/packages/backfill-config/src/envConfig.ts b/packages/backfill-config/src/envConfig.ts index 0dcf0e1c6..a8288ea5d 100644 --- a/packages/backfill-config/src/envConfig.ts +++ b/packages/backfill-config/src/envConfig.ts @@ -8,7 +8,10 @@ import type { Config } from "./Config.js"; import { getAzureBlobConfigFromSerializedOptions } from "./azureBlobCacheConfig.js"; import { getNpmConfigFromSerializedOptions } from "./npmCacheConfig.js"; import { isCorrectMode, modesObject, type BackfillModes } from "./modes.js"; -import type { CacheStorageConfig, CustomCacheStorageConfig } from "./cacheConfig.js"; +import type { + CacheStorageConfig, + CustomCacheStorageConfig, +} from "./cacheConfig.js"; class BackfillConfigError extends Error { constructor(value: string, envName: string, expected: string) { diff --git a/packages/backfill/README.md b/packages/backfill/README.md index 7fc8bb936..bac22caea 100644 --- a/packages/backfill/README.md +++ b/packages/backfill/README.md @@ -220,9 +220,9 @@ module.exports = { options: { connectionString: "...", container: "...", - maxSize: 12345 - } - } + maxSize: 12345, + }, + }, }; ``` @@ -231,21 +231,21 @@ does not have a SAS token. This is useful if you want to use a managed identity or interactive browser login. For example: ```js -import { InteractiveBrowserCredential } from '@azure/identity' +import { InteractiveBrowserCredential } from "@azure/identity"; module.exports = { cacheStorageConfig: { provider: "custom", plugin: "@lage-run/azure-blob-cache-storage", options: { - connectionString: "https://.blob.core.windows.net", + connectionString: + "https://.blob.core.windows.net", credential: new InteractiveBrowserCredential(), container: "...", - maxSize: 12345 - } - } + maxSize: 12345, + }, + }, }; - ``` #### Azure Blob Storage options