From 201ddc275d16727fbe5b9978a42a88879e689476 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Tue, 21 Apr 2026 22:18:59 +0100 Subject: [PATCH 1/9] Retrieve CodeQL versions associated with cached overlay base DBs --- lib/analyze-action.js | 84 +++++++++++++++-------------- lib/init-action.js | 90 ++++++++++++++++--------------- src/overlay/caching.test.ts | 105 ++++++++++++++++++++++++++++++++++++ src/overlay/caching.ts | 102 +++++++++++++++++++++++++++++++---- 4 files changed, 287 insertions(+), 94 deletions(-) diff --git a/lib/analyze-action.js b/lib/analyze-action.js index 8ecb3977bf..4ed11db5be 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -44531,11 +44531,11 @@ var require_valid = __commonJS({ "node_modules/semver/functions/valid.js"(exports2, module2) { "use strict"; var parse2 = require_parse3(); - var valid3 = (version, options) => { + var valid4 = (version, options) => { const v = parse2(version, options); return v ? v.version : null; }; - module2.exports = valid3; + module2.exports = valid4; } }); @@ -44678,8 +44678,8 @@ var require_rcompare = __commonJS({ "node_modules/semver/functions/rcompare.js"(exports2, module2) { "use strict"; var compare3 = require_compare(); - var rcompare = (a, b, loose) => compare3(b, a, loose); - module2.exports = rcompare; + var rcompare2 = (a, b, loose) => compare3(b, a, loose); + module2.exports = rcompare2; } }); @@ -45895,7 +45895,7 @@ var require_semver2 = __commonJS({ var SemVer = require_semver(); var identifiers = require_identifiers(); var parse2 = require_parse3(); - var valid3 = require_valid(); + var valid4 = require_valid(); var clean3 = require_clean(); var inc = require_inc(); var diff = require_diff(); @@ -45904,7 +45904,7 @@ var require_semver2 = __commonJS({ var patch = require_patch(); var prerelease = require_prerelease(); var compare3 = require_compare(); - var rcompare = require_rcompare(); + var rcompare2 = require_rcompare(); var compareLoose = require_compare_loose(); var compareBuild = require_compare_build(); var sort = require_sort(); @@ -45933,7 +45933,7 @@ var require_semver2 = __commonJS({ var subset = require_subset(); module2.exports = { parse: parse2, - valid: valid3, + valid: valid4, clean: clean3, inc, diff, @@ -45942,7 +45942,7 @@ var require_semver2 = __commonJS({ patch, prerelease, compare: compare3, - rcompare, + rcompare: rcompare2, compareLoose, compareBuild, sort, @@ -47732,16 +47732,16 @@ var require_attribute = __commonJS({ var result = new ValidatorResult(instance, schema2, options, ctx); var self2 = this; schema2.allOf.forEach(function(v, i) { - var valid3 = self2.validateSchema(instance, v, options, ctx); - if (!valid3.valid) { + var valid4 = self2.validateSchema(instance, v, options, ctx); + if (!valid4.valid) { var id = v.$id || v.id; var msg = id || v.title && JSON.stringify(v.title) || v["$ref"] && "<" + v["$ref"] + ">" || "[subschema " + i + "]"; result.addError({ name: "allOf", - argument: { id: msg, length: valid3.errors.length, valid: valid3 }, - message: "does not match allOf schema " + msg + " with " + valid3.errors.length + " error[s]:" + argument: { id: msg, length: valid4.errors.length, valid: valid4 }, + message: "does not match allOf schema " + msg + " with " + valid4.errors.length + " error[s]:" }); - result.importErrors(valid3); + result.importErrors(valid4); } }); return result; @@ -48030,8 +48030,8 @@ var require_attribute = __commonJS({ if (typeof schema2.exclusiveMinimum === "boolean") return; if (!this.types.number(instance)) return; var result = new ValidatorResult(instance, schema2, options, ctx); - var valid3 = instance > schema2.exclusiveMinimum; - if (!valid3) { + var valid4 = instance > schema2.exclusiveMinimum; + if (!valid4) { result.addError({ name: "exclusiveMinimum", argument: schema2.exclusiveMinimum, @@ -48044,8 +48044,8 @@ var require_attribute = __commonJS({ if (typeof schema2.exclusiveMaximum === "boolean") return; if (!this.types.number(instance)) return; var result = new ValidatorResult(instance, schema2, options, ctx); - var valid3 = instance < schema2.exclusiveMaximum; - if (!valid3) { + var valid4 = instance < schema2.exclusiveMaximum; + if (!valid4) { result.addError({ name: "exclusiveMaximum", argument: schema2.exclusiveMaximum, @@ -50828,8 +50828,8 @@ var require_semver3 = __commonJS({ return null; } } - exports2.valid = valid3; - function valid3(version, options) { + exports2.valid = valid4; + function valid4(version, options) { var v = parse2(version, options); return v ? v.version : null; } @@ -51129,8 +51129,8 @@ var require_semver3 = __commonJS({ var versionB = new SemVer(b, loose); return versionA.compare(versionB) || versionA.compareBuild(versionB); } - exports2.rcompare = rcompare; - function rcompare(a, b, loose) { + exports2.rcompare = rcompare2; + function rcompare2(a, b, loose) { return compare3(b, a, loose); } exports2.sort = sort; @@ -51958,7 +51958,7 @@ var require_cacheUtils = __commonJS({ var crypto3 = __importStar2(require("crypto")); var fs20 = __importStar2(require("fs")); var path16 = __importStar2(require("path")); - var semver9 = __importStar2(require_semver3()); + var semver10 = __importStar2(require_semver3()); var util = __importStar2(require("util")); var constants_1 = require_constants12(); var versionSalt = "1.0"; @@ -52051,7 +52051,7 @@ var require_cacheUtils = __commonJS({ function getCompressionMethod() { return __awaiter2(this, void 0, void 0, function* () { const versionOutput = yield getVersion("zstd", ["--quiet"]); - const version = semver9.clean(versionOutput); + const version = semver10.clean(versionOutput); core17.debug(`zstd version: ${version}`); if (versionOutput === "") { return constants_1.CompressionMethod.Gzip; @@ -99334,7 +99334,7 @@ var require_manifest = __commonJS({ exports2._findMatch = _findMatch; exports2._getOsVersion = _getOsVersion; exports2._readLinuxVersionFile = _readLinuxVersionFile; - var semver9 = __importStar2(require_semver2()); + var semver10 = __importStar2(require_semver2()); var core_1 = require_core(); var os5 = require("os"); var cp = require("child_process"); @@ -99348,7 +99348,7 @@ var require_manifest = __commonJS({ for (const candidate of candidates) { const version = candidate.version; (0, core_1.debug)(`check ${version} satisfies ${versionSpec}`); - if (semver9.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { + if (semver10.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { file = candidate.files.find((item) => { (0, core_1.debug)(`${item.arch}===${archFilter} && ${item.platform}===${platFilter}`); let chk = item.arch === archFilter && item.platform === platFilter; @@ -99357,7 +99357,7 @@ var require_manifest = __commonJS({ if (osVersion === item.platform_version) { chk = true; } else { - chk = semver9.satisfies(osVersion, item.platform_version); + chk = semver10.satisfies(osVersion, item.platform_version); } } return chk; @@ -99617,7 +99617,7 @@ var require_tool_cache = __commonJS({ var os5 = __importStar2(require("os")); var path16 = __importStar2(require("path")); var httpm = __importStar2(require_lib()); - var semver9 = __importStar2(require_semver2()); + var semver10 = __importStar2(require_semver2()); var stream2 = __importStar2(require("stream")); var util = __importStar2(require("util")); var assert_1 = require("assert"); @@ -99890,7 +99890,7 @@ var require_tool_cache = __commonJS({ } function cacheDir(sourceDir, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver9.clean(version) || version; + version = semver10.clean(version) || version; arch2 = arch2 || os5.arch(); core17.debug(`Caching tool ${tool} ${version} ${arch2}`); core17.debug(`source dir: ${sourceDir}`); @@ -99908,7 +99908,7 @@ var require_tool_cache = __commonJS({ } function cacheFile(sourceFile, targetFile, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver9.clean(version) || version; + version = semver10.clean(version) || version; arch2 = arch2 || os5.arch(); core17.debug(`Caching tool ${tool} ${version} ${arch2}`); core17.debug(`source file: ${sourceFile}`); @@ -99938,7 +99938,7 @@ var require_tool_cache = __commonJS({ } let toolPath = ""; if (versionSpec) { - versionSpec = semver9.clean(versionSpec) || ""; + versionSpec = semver10.clean(versionSpec) || ""; const cachePath = path16.join(_getCacheDirectory(), toolName, versionSpec, arch2); core17.debug(`checking cache: ${cachePath}`); if (fs20.existsSync(cachePath) && fs20.existsSync(`${cachePath}.complete`)) { @@ -100018,7 +100018,7 @@ var require_tool_cache = __commonJS({ } function _createToolPath(tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - const folderPath = path16.join(_getCacheDirectory(), tool, semver9.clean(version) || version, arch2 || ""); + const folderPath = path16.join(_getCacheDirectory(), tool, semver10.clean(version) || version, arch2 || ""); core17.debug(`destination ${folderPath}`); const markerPath = `${folderPath}.complete`; yield io7.rmRF(folderPath); @@ -100028,30 +100028,30 @@ var require_tool_cache = __commonJS({ }); } function _completeToolPath(tool, version, arch2) { - const folderPath = path16.join(_getCacheDirectory(), tool, semver9.clean(version) || version, arch2 || ""); + const folderPath = path16.join(_getCacheDirectory(), tool, semver10.clean(version) || version, arch2 || ""); const markerPath = `${folderPath}.complete`; fs20.writeFileSync(markerPath, ""); core17.debug("finished caching tool"); } function isExplicitVersion(versionSpec) { - const c = semver9.clean(versionSpec) || ""; + const c = semver10.clean(versionSpec) || ""; core17.debug(`isExplicit: ${c}`); - const valid3 = semver9.valid(c) != null; - core17.debug(`explicit? ${valid3}`); - return valid3; + const valid4 = semver10.valid(c) != null; + core17.debug(`explicit? ${valid4}`); + return valid4; } function evaluateVersions(versions, versionSpec) { let version = ""; core17.debug(`evaluating ${versions.length} versions`); versions = versions.sort((a, b) => { - if (semver9.gt(a, b)) { + if (semver10.gt(a, b)) { return 1; } return -1; }); for (let i = versions.length - 1; i >= 0; i--) { const potential = versions[i]; - const satisfied = semver9.satisfies(potential, versionSpec); + const satisfied = semver10.satisfies(potential, versionSpec); if (satisfied) { version = potential; break; @@ -111194,6 +111194,7 @@ async function uploadBundledDatabase(repositoryNwo, language, commitOid, bundled // src/overlay/caching.ts var fs15 = __toESM(require("fs")); var actionsCache4 = __toESM(require_cache5()); +var semver9 = __toESM(require_semver2()); var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; var CACHE_VERSION2 = 1; @@ -111323,13 +111324,16 @@ async function getCacheSaveKey(config, codeQlVersion, checkoutPath, logger) { return `${restoreKeyPrefix}${sha}-${runId}-${attemptId}`; } async function getCacheRestoreKeyPrefix(config, codeQlVersion) { - const languages = [...config.languages].sort().join("_"); + return `${await getCacheKeyPrefixBase(config.languages)}${codeQlVersion}-`; +} +async function getCacheKeyPrefixBase(parsedLanguages) { + const languagesComponent = [...parsedLanguages].sort().join("_"); const cacheKeyComponents = { automationID: await getAutomationID() // Add more components here as needed in the future }; const componentsHash = createCacheKeyHash(cacheKeyComponents); - return `${CACHE_PREFIX}-${CACHE_VERSION2}-${componentsHash}-${languages}-${codeQlVersion}-`; + return `${CACHE_PREFIX}-${CACHE_VERSION2}-${componentsHash}-${languagesComponent}-`; } // src/status-report.ts diff --git a/lib/init-action.js b/lib/init-action.js index 3ce55149db..1ad4f602f6 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -44531,11 +44531,11 @@ var require_valid = __commonJS({ "node_modules/semver/functions/valid.js"(exports2, module2) { "use strict"; var parse2 = require_parse3(); - var valid3 = (version, options) => { + var valid4 = (version, options) => { const v = parse2(version, options); return v ? v.version : null; }; - module2.exports = valid3; + module2.exports = valid4; } }); @@ -44678,8 +44678,8 @@ var require_rcompare = __commonJS({ "node_modules/semver/functions/rcompare.js"(exports2, module2) { "use strict"; var compare2 = require_compare(); - var rcompare = (a, b, loose) => compare2(b, a, loose); - module2.exports = rcompare; + var rcompare2 = (a, b, loose) => compare2(b, a, loose); + module2.exports = rcompare2; } }); @@ -45895,7 +45895,7 @@ var require_semver2 = __commonJS({ var SemVer = require_semver(); var identifiers = require_identifiers(); var parse2 = require_parse3(); - var valid3 = require_valid(); + var valid4 = require_valid(); var clean3 = require_clean(); var inc = require_inc(); var diff = require_diff(); @@ -45904,7 +45904,7 @@ var require_semver2 = __commonJS({ var patch = require_patch(); var prerelease = require_prerelease(); var compare2 = require_compare(); - var rcompare = require_rcompare(); + var rcompare2 = require_rcompare(); var compareLoose = require_compare_loose(); var compareBuild = require_compare_build(); var sort = require_sort(); @@ -45933,7 +45933,7 @@ var require_semver2 = __commonJS({ var subset = require_subset(); module2.exports = { parse: parse2, - valid: valid3, + valid: valid4, clean: clean3, inc, diff, @@ -45942,7 +45942,7 @@ var require_semver2 = __commonJS({ patch, prerelease, compare: compare2, - rcompare, + rcompare: rcompare2, compareLoose, compareBuild, sort, @@ -47732,16 +47732,16 @@ var require_attribute = __commonJS({ var result = new ValidatorResult(instance, schema2, options, ctx); var self2 = this; schema2.allOf.forEach(function(v, i) { - var valid3 = self2.validateSchema(instance, v, options, ctx); - if (!valid3.valid) { + var valid4 = self2.validateSchema(instance, v, options, ctx); + if (!valid4.valid) { var id = v.$id || v.id; var msg = id || v.title && JSON.stringify(v.title) || v["$ref"] && "<" + v["$ref"] + ">" || "[subschema " + i + "]"; result.addError({ name: "allOf", - argument: { id: msg, length: valid3.errors.length, valid: valid3 }, - message: "does not match allOf schema " + msg + " with " + valid3.errors.length + " error[s]:" + argument: { id: msg, length: valid4.errors.length, valid: valid4 }, + message: "does not match allOf schema " + msg + " with " + valid4.errors.length + " error[s]:" }); - result.importErrors(valid3); + result.importErrors(valid4); } }); return result; @@ -48030,8 +48030,8 @@ var require_attribute = __commonJS({ if (typeof schema2.exclusiveMinimum === "boolean") return; if (!this.types.number(instance)) return; var result = new ValidatorResult(instance, schema2, options, ctx); - var valid3 = instance > schema2.exclusiveMinimum; - if (!valid3) { + var valid4 = instance > schema2.exclusiveMinimum; + if (!valid4) { result.addError({ name: "exclusiveMinimum", argument: schema2.exclusiveMinimum, @@ -48044,8 +48044,8 @@ var require_attribute = __commonJS({ if (typeof schema2.exclusiveMaximum === "boolean") return; if (!this.types.number(instance)) return; var result = new ValidatorResult(instance, schema2, options, ctx); - var valid3 = instance < schema2.exclusiveMaximum; - if (!valid3) { + var valid4 = instance < schema2.exclusiveMaximum; + if (!valid4) { result.addError({ name: "exclusiveMaximum", argument: schema2.exclusiveMaximum, @@ -50979,8 +50979,8 @@ var require_semver3 = __commonJS({ return null; } } - exports2.valid = valid3; - function valid3(version, options) { + exports2.valid = valid4; + function valid4(version, options) { var v = parse2(version, options); return v ? v.version : null; } @@ -51280,8 +51280,8 @@ var require_semver3 = __commonJS({ var versionB = new SemVer(b, loose); return versionA.compare(versionB) || versionA.compareBuild(versionB); } - exports2.rcompare = rcompare; - function rcompare(a, b, loose) { + exports2.rcompare = rcompare2; + function rcompare2(a, b, loose) { return compare2(b, a, loose); } exports2.sort = sort; @@ -52109,7 +52109,7 @@ var require_cacheUtils = __commonJS({ var crypto3 = __importStar2(require("crypto")); var fs19 = __importStar2(require("fs")); var path18 = __importStar2(require("path")); - var semver10 = __importStar2(require_semver3()); + var semver11 = __importStar2(require_semver3()); var util = __importStar2(require("util")); var constants_1 = require_constants12(); var versionSalt = "1.0"; @@ -52202,7 +52202,7 @@ var require_cacheUtils = __commonJS({ function getCompressionMethod() { return __awaiter2(this, void 0, void 0, function* () { const versionOutput = yield getVersion("zstd", ["--quiet"]); - const version = semver10.clean(versionOutput); + const version = semver11.clean(versionOutput); core16.debug(`zstd version: ${version}`); if (versionOutput === "") { return constants_1.CompressionMethod.Gzip; @@ -99485,7 +99485,7 @@ var require_manifest = __commonJS({ exports2._findMatch = _findMatch; exports2._getOsVersion = _getOsVersion; exports2._readLinuxVersionFile = _readLinuxVersionFile; - var semver10 = __importStar2(require_semver2()); + var semver11 = __importStar2(require_semver2()); var core_1 = require_core(); var os6 = require("os"); var cp = require("child_process"); @@ -99499,7 +99499,7 @@ var require_manifest = __commonJS({ for (const candidate of candidates) { const version = candidate.version; (0, core_1.debug)(`check ${version} satisfies ${versionSpec}`); - if (semver10.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { + if (semver11.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { file = candidate.files.find((item) => { (0, core_1.debug)(`${item.arch}===${archFilter} && ${item.platform}===${platFilter}`); let chk = item.arch === archFilter && item.platform === platFilter; @@ -99508,7 +99508,7 @@ var require_manifest = __commonJS({ if (osVersion === item.platform_version) { chk = true; } else { - chk = semver10.satisfies(osVersion, item.platform_version); + chk = semver11.satisfies(osVersion, item.platform_version); } } return chk; @@ -99768,7 +99768,7 @@ var require_tool_cache = __commonJS({ var os6 = __importStar2(require("os")); var path18 = __importStar2(require("path")); var httpm = __importStar2(require_lib()); - var semver10 = __importStar2(require_semver2()); + var semver11 = __importStar2(require_semver2()); var stream2 = __importStar2(require("stream")); var util = __importStar2(require("util")); var assert_1 = require("assert"); @@ -100041,7 +100041,7 @@ var require_tool_cache = __commonJS({ } function cacheDir(sourceDir, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver10.clean(version) || version; + version = semver11.clean(version) || version; arch2 = arch2 || os6.arch(); core16.debug(`Caching tool ${tool} ${version} ${arch2}`); core16.debug(`source dir: ${sourceDir}`); @@ -100059,7 +100059,7 @@ var require_tool_cache = __commonJS({ } function cacheFile(sourceFile, targetFile, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver10.clean(version) || version; + version = semver11.clean(version) || version; arch2 = arch2 || os6.arch(); core16.debug(`Caching tool ${tool} ${version} ${arch2}`); core16.debug(`source file: ${sourceFile}`); @@ -100089,7 +100089,7 @@ var require_tool_cache = __commonJS({ } let toolPath = ""; if (versionSpec) { - versionSpec = semver10.clean(versionSpec) || ""; + versionSpec = semver11.clean(versionSpec) || ""; const cachePath = path18.join(_getCacheDirectory(), toolName, versionSpec, arch2); core16.debug(`checking cache: ${cachePath}`); if (fs19.existsSync(cachePath) && fs19.existsSync(`${cachePath}.complete`)) { @@ -100169,7 +100169,7 @@ var require_tool_cache = __commonJS({ } function _createToolPath(tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - const folderPath = path18.join(_getCacheDirectory(), tool, semver10.clean(version) || version, arch2 || ""); + const folderPath = path18.join(_getCacheDirectory(), tool, semver11.clean(version) || version, arch2 || ""); core16.debug(`destination ${folderPath}`); const markerPath = `${folderPath}.complete`; yield io7.rmRF(folderPath); @@ -100179,30 +100179,30 @@ var require_tool_cache = __commonJS({ }); } function _completeToolPath(tool, version, arch2) { - const folderPath = path18.join(_getCacheDirectory(), tool, semver10.clean(version) || version, arch2 || ""); + const folderPath = path18.join(_getCacheDirectory(), tool, semver11.clean(version) || version, arch2 || ""); const markerPath = `${folderPath}.complete`; fs19.writeFileSync(markerPath, ""); core16.debug("finished caching tool"); } function isExplicitVersion(versionSpec) { - const c = semver10.clean(versionSpec) || ""; + const c = semver11.clean(versionSpec) || ""; core16.debug(`isExplicit: ${c}`); - const valid3 = semver10.valid(c) != null; - core16.debug(`explicit? ${valid3}`); - return valid3; + const valid4 = semver11.valid(c) != null; + core16.debug(`explicit? ${valid4}`); + return valid4; } function evaluateVersions(versions, versionSpec) { let version = ""; core16.debug(`evaluating ${versions.length} versions`); versions = versions.sort((a, b) => { - if (semver10.gt(a, b)) { + if (semver11.gt(a, b)) { return 1; } return -1; }); for (let i = versions.length - 1; i >= 0; i--) { const potential = versions[i]; - const satisfied = semver10.satisfies(potential, versionSpec); + const satisfied = semver11.satisfies(potential, versionSpec); if (satisfied) { version = potential; break; @@ -100812,7 +100812,7 @@ var path17 = __toESM(require("path")); var core15 = __toESM(require_core()); var github3 = __toESM(require_github()); var io6 = __toESM(require_io()); -var semver9 = __toESM(require_semver2()); +var semver10 = __toESM(require_semver2()); // node_modules/uuid/dist-node/stringify.js var byteToHex = []; @@ -109578,6 +109578,7 @@ To opt out of this change, ${envVarOptOut}`; // src/overlay/caching.ts var fs16 = __toESM(require("fs")); var actionsCache4 = __toESM(require_cache5()); +var semver9 = __toESM(require_semver2()); var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; var CACHE_VERSION2 = 1; @@ -109719,13 +109720,16 @@ async function downloadOverlayBaseDatabaseFromCache(codeql, config, logger) { }; } async function getCacheRestoreKeyPrefix(config, codeQlVersion) { - const languages = [...config.languages].sort().join("_"); + return `${await getCacheKeyPrefixBase(config.languages)}${codeQlVersion}-`; +} +async function getCacheKeyPrefixBase(parsedLanguages) { + const languagesComponent = [...parsedLanguages].sort().join("_"); const cacheKeyComponents = { automationID: await getAutomationID() // Add more components here as needed in the future }; const componentsHash = createCacheKeyHash(cacheKeyComponents); - return `${CACHE_PREFIX}-${CACHE_VERSION2}-${componentsHash}-${languages}-${codeQlVersion}-`; + return `${CACHE_PREFIX}-${CACHE_VERSION2}-${componentsHash}-${languagesComponent}-`; } // src/status-report.ts @@ -110322,12 +110326,12 @@ async function run(startedAt) { const experimental = "2.19.3"; const publicPreview = "2.22.1"; const actualVer = (await codeql.getVersion()).version; - if (semver9.lt(actualVer, experimental)) { + if (semver10.lt(actualVer, experimental)) { throw new ConfigurationError( `Rust analysis is supported by CodeQL CLI version ${experimental} or higher, but found version ${actualVer}` ); } - if (semver9.lt(actualVer, publicPreview)) { + if (semver10.lt(actualVer, publicPreview)) { core15.exportVariable("CODEQL_ENABLE_EXPERIMENTAL_FEATURES" /* EXPERIMENTAL_FEATURES */, "true"); logger.info("Experimental Rust analysis enabled"); } diff --git a/src/overlay/caching.test.ts b/src/overlay/caching.test.ts index 9c7abc6bd7..7958b8b8d6 100644 --- a/src/overlay/caching.test.ts +++ b/src/overlay/caching.test.ts @@ -23,6 +23,7 @@ import { downloadOverlayBaseDatabaseFromCache, getCacheRestoreKeyPrefix, getCacheSaveKey, + getCodeQlVersionsForOverlayBaseDatabases, } from "./caching"; import { OverlayDatabaseMode } from "./overlay-database-mode"; @@ -285,3 +286,107 @@ test.serial("overlay-base database cache keys remain stable", async (t) => { `Expected save key "${saveKey}" to start with restore key prefix "${restoreKeyPrefix}"`, ); }); + +test.serial( + "getCodeQlVersionsForOverlayBaseDatabases returns unique versions sorted latest first", + async (t) => { + const logger = getRunnerLogger(true); + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(apiClient, "listActionsCaches").resolves([ + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-abc123-1-1", + }, + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.24.1-def456-2-1", + }, + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-ghi789-3-1", + }, + ]); + + const result = await getCodeQlVersionsForOverlayBaseDatabases( + ["javascript", "python"], + logger, + ); + t.deepEqual(result, ["2.24.1", "2.23.0"]); + }, +); + +test.serial( + "getCodeQlVersionsForOverlayBaseDatabases returns empty list when no caches exist", + async (t) => { + const logger = getRunnerLogger(true); + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(apiClient, "listActionsCaches").resolves([]); + + const result = await getCodeQlVersionsForOverlayBaseDatabases( + ["python"], + logger, + ); + t.deepEqual(result, []); + }, +); + +test.serial( + "getCodeQlVersionsForOverlayBaseDatabases returns empty list when cache keys are unparseable", + async (t) => { + const logger = getRunnerLogger(true); + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(apiClient, "listActionsCaches").resolves([ + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-malformed", + }, + { key: undefined }, + ]); + + const result = await getCodeQlVersionsForOverlayBaseDatabases( + ["python"], + logger, + ); + t.deepEqual(result, []); + }, +); + +test.serial( + "getCodeQlVersionsForOverlayBaseDatabases returns the single version when only one cache exists", + async (t) => { + const logger = getRunnerLogger(true); + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(apiClient, "listActionsCaches").resolves([ + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-cpp-2.25.0-abc123-1-1", + }, + ]); + + const result = await getCodeQlVersionsForOverlayBaseDatabases( + ["cpp"], + logger, + ); + t.deepEqual(result, ["2.25.0"]); + }, +); + +test.serial( + "getCodeQlVersionsForOverlayBaseDatabases resolves language aliases", + async (t) => { + const logger = getRunnerLogger(true); + // The alias `c++` should be resolved to "cpp" and match cache entries keyed with "cpp" + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(apiClient, "listActionsCaches").resolves([ + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-cpp-2.25.0-abc123-1-1", + }, + ]); + + const result = await getCodeQlVersionsForOverlayBaseDatabases( + ["c++"], + logger, + ); + t.deepEqual(result, ["2.25.0"]); + }, +); diff --git a/src/overlay/caching.ts b/src/overlay/caching.ts index 2dcb7f837b..04da9f5e4d 100644 --- a/src/overlay/caching.ts +++ b/src/overlay/caching.ts @@ -1,17 +1,19 @@ import * as fs from "fs"; import * as actionsCache from "@actions/cache"; +import * as semver from "semver"; import { getRequiredInput, getWorkflowRunAttempt, getWorkflowRunID, } from "../actions-util"; -import { getAutomationID } from "../api-client"; +import { getAutomationID, listActionsCaches } from "../api-client"; import { createCacheKeyHash } from "../caching-utils"; import { type CodeQL } from "../codeql"; import { type Config } from "../config-utils"; import { getCommitOid } from "../git-utils"; +import { Language, parseBuiltInLanguage } from "../languages"; import { Logger, withGroupAsync } from "../logging"; import { CleanupLevel, @@ -404,7 +406,17 @@ export async function getCacheRestoreKeyPrefix( config: Config, codeQlVersion: string, ): Promise { - const languages = [...config.languages].sort().join("_"); + return `${await getCacheKeyPrefixBase(config.languages)}${codeQlVersion}-`; +} + +/** + * Computes the cache key prefix for overlay-base databases, excluding the + * CodeQL version. + */ +async function getCacheKeyPrefixBase( + parsedLanguages: Language[], +): Promise { + const languagesComponent = [...parsedLanguages].sort().join("_"); const cacheKeyComponents = { automationID: await getAutomationID(), @@ -412,17 +424,85 @@ export async function getCacheRestoreKeyPrefix( }; const componentsHash = createCacheKeyHash(cacheKeyComponents); - // For a cached overlay-base database to be considered compatible for overlay - // analysis, all components in the cache restore key must match: - // // CACHE_PREFIX: distinguishes overlay-base databases from other cache objects // CACHE_VERSION: cache format version // componentsHash: hash of additional components (see above for details) - // languages: the languages included in the overlay-base database - // codeQlVersion: CodeQL bundle version + // languagesComponent: the languages included in the overlay-base database // - // Technically we can also include languages and codeQlVersion in the - // componentsHash, but including them explicitly in the cache key makes it - // easier to debug and understand the cache key structure. - return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`; + // Technically we can also include languages in the componentsHash, but + // including them explicitly in the cache key makes it easier to debug and + // understand the cache key structure. + return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languagesComponent}-`; +} + +/** + * Searches the GitHub Actions cache for overlay-base databases matching the given languages, and + * returns all CodeQL versions found across matching cache entries. + * + * @returns Unique CodeQL versions found in cached overlay-base databases, sorted from latest to + * earliest, or undefined if one of the languages is not a built-in language. + */ +export async function getCodeQlVersionsForOverlayBaseDatabases( + rawLanguages: string[], + logger: Logger, +): Promise { + const languages = rawLanguages.map(parseBuiltInLanguage); + if (languages.includes(undefined)) { + logger.warning( + "One or more provided languages are not recognized as built-in languages. " + + "Skipping searching for overlay-base databases in cache.", + ); + return undefined; + } + const cacheKeyPrefix = await getCacheKeyPrefixBase( + languages.filter((l) => l !== undefined), + ); + + logger.debug( + `Searching for overlay-base databases in Actions cache with ` + + `prefix ${cacheKeyPrefix}`, + ); + + const caches = await listActionsCaches(cacheKeyPrefix); + + if (caches.length === 0) { + logger.info("No overlay-base databases found in Actions cache."); + return []; + } + + logger.info( + `Found ${caches.length} overlay-base ` + + `${caches.length === 1 ? "database" : "databases"} in the Actions cache.`, + ); + + // Parse CodeQL versions from cache keys. + // After the prefix, the remaining key format starts with + // `${codeQlVersion}-`. + const versionRegex = /^([\d.]+)-/; + const versionSet = new Set(); + + for (const cache of caches) { + if (!cache.key) continue; + const suffix = cache.key.substring(cacheKeyPrefix.length); + const match = suffix.match(versionRegex); + if (match && semver.valid(match[1])) { + versionSet.add(match[1]); + } + } + + if (versionSet.size === 0) { + logger.info( + "Could not parse any CodeQL versions from overlay-base database " + + "cache keys.", + ); + return []; + } + + const versions = [...versionSet].sort(semver.rcompare); + + logger.info( + `Found overlay databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`, + ); + + return versions; } From 5026833be54c0331f9e629b7655471b75643ce5a Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Tue, 21 Apr 2026 23:35:29 +0100 Subject: [PATCH 2/9] Document exclusion of nightlies --- src/overlay/caching.test.ts | 27 +++++++++++++++++++++++++++ src/overlay/caching.ts | 15 ++++++++++----- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/src/overlay/caching.test.ts b/src/overlay/caching.test.ts index 7958b8b8d6..c6bc75294c 100644 --- a/src/overlay/caching.test.ts +++ b/src/overlay/caching.test.ts @@ -390,3 +390,30 @@ test.serial( t.deepEqual(result, ["2.25.0"]); }, ); + +test.serial( + "getCodeQlVersionsForOverlayBaseDatabases ignores nightly versions with build metadata", + async (t) => { + const logger = getRunnerLogger(true); + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(apiClient, "listActionsCaches").resolves([ + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.25.0-abc123-1-1", + }, + { + // Nightly release with semver build metadata; should be ignored. + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.26.0+202604211234-def456-2-1", + }, + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.24.0-ghi789-3-1", + }, + ]); + + const result = await getCodeQlVersionsForOverlayBaseDatabases( + ["python"], + logger, + ); + t.deepEqual(result, ["2.25.0", "2.24.0"]); + }, +); diff --git a/src/overlay/caching.ts b/src/overlay/caching.ts index 04da9f5e4d..215bba4b02 100644 --- a/src/overlay/caching.ts +++ b/src/overlay/caching.ts @@ -437,9 +437,9 @@ async function getCacheKeyPrefixBase( /** * Searches the GitHub Actions cache for overlay-base databases matching the given languages, and - * returns all CodeQL versions found across matching cache entries. + * returns all stable CodeQL versions found across matching cache entries. * - * @returns Unique CodeQL versions found in cached overlay-base databases, sorted from latest to + * @returns Unique stable CodeQL versions found in cached overlay-base databases, sorted from latest to * earliest, or undefined if one of the languages is not a built-in language. */ export async function getCodeQlVersionsForOverlayBaseDatabases( @@ -475,9 +475,14 @@ export async function getCodeQlVersionsForOverlayBaseDatabases( `${caches.length === 1 ? "database" : "databases"} in the Actions cache.`, ); - // Parse CodeQL versions from cache keys. - // After the prefix, the remaining key format starts with - // `${codeQlVersion}-`. + // Parse CodeQL versions from cache keys, matching only stable releases. + // + // After the prefix, the remaining key format starts with `${codeQlVersion}-`. Nightlies will have + // a suffix like `+202604201548` that will break the match. + // + // Caveat: this relies on the fact that we haven't released any CodeQL bundles with the + // `x.y.z-` semver format which does not interact well with the current overlay base + // DB cache key format. const versionRegex = /^([\d.]+)-/; const versionSet = new Set(); From af1f6139899520107f44b3291c7a27eea1ac58a5 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Tue, 21 Apr 2026 23:49:37 +0100 Subject: [PATCH 3/9] Use type-only imports --- src/overlay/caching.test.ts | 2 +- src/overlay/caching.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/overlay/caching.test.ts b/src/overlay/caching.test.ts index c6bc75294c..18711fb466 100644 --- a/src/overlay/caching.test.ts +++ b/src/overlay/caching.test.ts @@ -7,7 +7,7 @@ import * as sinon from "sinon"; import * as actionsUtil from "../actions-util"; import * as apiClient from "../api-client"; -import { ResolveDatabaseOutput } from "../codeql"; +import { type ResolveDatabaseOutput } from "../codeql"; import * as gitUtils from "../git-utils"; import { BuiltInLanguage } from "../languages"; import { getRunnerLogger } from "../logging"; diff --git a/src/overlay/caching.ts b/src/overlay/caching.ts index 215bba4b02..367725dfc6 100644 --- a/src/overlay/caching.ts +++ b/src/overlay/caching.ts @@ -13,8 +13,8 @@ import { createCacheKeyHash } from "../caching-utils"; import { type CodeQL } from "../codeql"; import { type Config } from "../config-utils"; import { getCommitOid } from "../git-utils"; -import { Language, parseBuiltInLanguage } from "../languages"; -import { Logger, withGroupAsync } from "../logging"; +import { type Language, parseBuiltInLanguage } from "../languages"; +import { type Logger, withGroupAsync } from "../logging"; import { CleanupLevel, getBaseDatabaseOidsFilePath, From 1279e8d41c608bda4cf183c22c4860d3bfb99cb5 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Wed, 22 Apr 2026 00:04:57 +0100 Subject: [PATCH 4/9] Mitigate caches being evicted before they can be downloaded --- lib/analyze-action.js | 2 ++ lib/init-action.js | 2 ++ src/api-client.ts | 1 + src/overlay/caching.test.ts | 34 ++++++++++++++++++++ src/overlay/caching.ts | 64 ++++++++++++++++++++++++++++++------- 5 files changed, 91 insertions(+), 12 deletions(-) diff --git a/lib/analyze-action.js b/lib/analyze-action.js index 4ed11db5be..4554cd1838 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -111199,6 +111199,8 @@ var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; var CACHE_VERSION2 = 1; var CACHE_PREFIX = "codeql-overlay-base-database"; +var CACHE_ENTRY_MAX_AGE_DAYS = 6; +var CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1e3; var MAX_CACHE_OPERATION_MS2 = 6e5; async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) { const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config); diff --git a/lib/init-action.js b/lib/init-action.js index 1ad4f602f6..499472f2bf 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -109583,6 +109583,8 @@ var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; var CACHE_VERSION2 = 1; var CACHE_PREFIX = "codeql-overlay-base-database"; +var CACHE_ENTRY_MAX_AGE_DAYS = 6; +var CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1e3; var MAX_CACHE_OPERATION_MS3 = 6e5; async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) { const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config); diff --git a/src/api-client.ts b/src/api-client.ts index 4b8cb7b340..8de5058d19 100644 --- a/src/api-client.ts +++ b/src/api-client.ts @@ -249,6 +249,7 @@ export interface ActionsCacheItem { created_at?: string; id?: number; key?: string; + last_accessed_at?: string; size_in_bytes?: number; } diff --git a/src/overlay/caching.test.ts b/src/overlay/caching.test.ts index 18711fb466..2c60fd7893 100644 --- a/src/overlay/caching.test.ts +++ b/src/overlay/caching.test.ts @@ -417,3 +417,37 @@ test.serial( t.deepEqual(result, ["2.25.0", "2.24.0"]); }, ); + +test.serial( + "getCodeQlVersionsForOverlayBaseDatabases ignores cache entries close to eviction", + async (t) => { + const logger = getRunnerLogger(true); + + const now = Date.now(); + const isoDaysAgo = (days: number) => + new Date(now - days * 24 * 60 * 60 * 1000).toISOString(); + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(apiClient, "listActionsCaches").resolves([ + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.25.0-abc123-1-1", + last_accessed_at: isoDaysAgo(1), + }, + { + // Older than the 6-day threshold; close to the 7-day eviction window. + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.26.0-def456-2-1", + last_accessed_at: isoDaysAgo(6.5), + }, + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.24.0-ghi789-3-1", + last_accessed_at: isoDaysAgo(3), + }, + ]); + + const result = await getCodeQlVersionsForOverlayBaseDatabases( + ["python"], + logger, + ); + t.deepEqual(result, ["2.25.0", "2.24.0"]); + }, +); diff --git a/src/overlay/caching.ts b/src/overlay/caching.ts index 367725dfc6..cdde47b754 100644 --- a/src/overlay/caching.ts +++ b/src/overlay/caching.ts @@ -8,7 +8,11 @@ import { getWorkflowRunAttempt, getWorkflowRunID, } from "../actions-util"; -import { getAutomationID, listActionsCaches } from "../api-client"; +import { + type ActionsCacheItem, + getAutomationID, + listActionsCaches, +} from "../api-client"; import { createCacheKeyHash } from "../caching-utils"; import { type CodeQL } from "../codeql"; import { type Config } from "../config-utils"; @@ -48,6 +52,12 @@ const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = const CACHE_VERSION = 1; const CACHE_PREFIX = "codeql-overlay-base-database"; +// The Actions cache evicts entries that have not been accessed in the past 7 +// days. We conservatively set a limit of 6 days to avoid using a cached base DB +// that may be evicted before we can download it. +const CACHE_ENTRY_MAX_AGE_DAYS = 6; +const CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1000; + // The purpose of this ten-minute limit is to guard against the possibility // that the cache service is unresponsive, which would otherwise cause the // entire action to hang. Normally we expect cache operations to complete @@ -435,6 +445,39 @@ async function getCacheKeyPrefixBase( return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languagesComponent}-`; } +/** + * Lists overlay-base database cache entries with the given key prefix, ignoring entries that are + * old enough that they may be evicted by the Actions cache before we attempt to download them. + */ +async function listRecentOverlayBaseDatabaseCaches( + cacheKeyPrefix: string, + logger: Logger, +): Promise { + const allCaches = await listActionsCaches(cacheKeyPrefix); + + if (allCaches.length === 0) { + logger.info("No overlay-base databases found in Actions cache."); + return []; + } + + const cutoffMs = Date.now() - CACHE_ENTRY_MAX_AGE_MS; + const recentCaches = allCaches.filter((cache) => { + if (!cache.last_accessed_at) return true; + const lastAccessedMs = Date.parse(cache.last_accessed_at); + return Number.isNaN(lastAccessedMs) || lastAccessedMs >= cutoffMs; + }); + const numTooOldDatabases = allCaches.length - recentCaches.length; + const tooOldSuffix = + numTooOldDatabases > 0 + ? ` (ignoring ${numTooOldDatabases} that may be evicted soon)` + : ""; + logger.info( + `Found ${allCaches.length} overlay-base ${allCaches.length === 1 ? "database" : "databases"} in the Actions cache${tooOldSuffix}.`, + ); + + return recentCaches; +} + /** * Searches the GitHub Actions cache for overlay-base databases matching the given languages, and * returns all stable CodeQL versions found across matching cache entries. @@ -448,7 +491,7 @@ export async function getCodeQlVersionsForOverlayBaseDatabases( ): Promise { const languages = rawLanguages.map(parseBuiltInLanguage); if (languages.includes(undefined)) { - logger.warning( + logger.info( "One or more provided languages are not recognized as built-in languages. " + "Skipping searching for overlay-base databases in cache.", ); @@ -463,22 +506,19 @@ export async function getCodeQlVersionsForOverlayBaseDatabases( `prefix ${cacheKeyPrefix}`, ); - const caches = await listActionsCaches(cacheKeyPrefix); + const caches = await listRecentOverlayBaseDatabaseCaches( + cacheKeyPrefix, + logger, + ); if (caches.length === 0) { - logger.info("No overlay-base databases found in Actions cache."); return []; } - logger.info( - `Found ${caches.length} overlay-base ` + - `${caches.length === 1 ? "database" : "databases"} in the Actions cache.`, - ); - // Parse CodeQL versions from cache keys, matching only stable releases. // - // After the prefix, the remaining key format starts with `${codeQlVersion}-`. Nightlies will have - // a suffix like `+202604201548` that will break the match. + // After the prefix, the remaining key format starts with `${codeQlVersion}-`. Nightlies have a + // suffix like `+202604201548` that will prevent a match. // // Caveat: this relies on the fact that we haven't released any CodeQL bundles with the // `x.y.z-` semver format which does not interact well with the current overlay base @@ -506,7 +546,7 @@ export async function getCodeQlVersionsForOverlayBaseDatabases( const versions = [...versionSet].sort(semver.rcompare); logger.info( - `Found overlay databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`, + `Found overlay-base databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`, ); return versions; From 7587714d0aadbeca5a7dca63b57dbb40ef2787ab Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Tue, 5 May 2026 18:37:17 +0100 Subject: [PATCH 5/9] Revert "Mitigate caches being evicted before they can be downloaded" This reverts commit 1279e8d41c608bda4cf183c22c4860d3bfb99cb5. --- lib/analyze-action.js | 2 -- lib/init-action.js | 2 -- src/api-client.ts | 1 - src/overlay/caching.test.ts | 34 -------------------- src/overlay/caching.ts | 64 +++++++------------------------------ 5 files changed, 12 insertions(+), 91 deletions(-) diff --git a/lib/analyze-action.js b/lib/analyze-action.js index 4554cd1838..4ed11db5be 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -111199,8 +111199,6 @@ var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; var CACHE_VERSION2 = 1; var CACHE_PREFIX = "codeql-overlay-base-database"; -var CACHE_ENTRY_MAX_AGE_DAYS = 6; -var CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1e3; var MAX_CACHE_OPERATION_MS2 = 6e5; async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) { const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config); diff --git a/lib/init-action.js b/lib/init-action.js index 499472f2bf..1ad4f602f6 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -109583,8 +109583,6 @@ var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; var CACHE_VERSION2 = 1; var CACHE_PREFIX = "codeql-overlay-base-database"; -var CACHE_ENTRY_MAX_AGE_DAYS = 6; -var CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1e3; var MAX_CACHE_OPERATION_MS3 = 6e5; async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) { const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config); diff --git a/src/api-client.ts b/src/api-client.ts index 8de5058d19..4b8cb7b340 100644 --- a/src/api-client.ts +++ b/src/api-client.ts @@ -249,7 +249,6 @@ export interface ActionsCacheItem { created_at?: string; id?: number; key?: string; - last_accessed_at?: string; size_in_bytes?: number; } diff --git a/src/overlay/caching.test.ts b/src/overlay/caching.test.ts index 2c60fd7893..18711fb466 100644 --- a/src/overlay/caching.test.ts +++ b/src/overlay/caching.test.ts @@ -417,37 +417,3 @@ test.serial( t.deepEqual(result, ["2.25.0", "2.24.0"]); }, ); - -test.serial( - "getCodeQlVersionsForOverlayBaseDatabases ignores cache entries close to eviction", - async (t) => { - const logger = getRunnerLogger(true); - - const now = Date.now(); - const isoDaysAgo = (days: number) => - new Date(now - days * 24 * 60 * 60 * 1000).toISOString(); - - sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); - sinon.stub(apiClient, "listActionsCaches").resolves([ - { - key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.25.0-abc123-1-1", - last_accessed_at: isoDaysAgo(1), - }, - { - // Older than the 6-day threshold; close to the 7-day eviction window. - key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.26.0-def456-2-1", - last_accessed_at: isoDaysAgo(6.5), - }, - { - key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.24.0-ghi789-3-1", - last_accessed_at: isoDaysAgo(3), - }, - ]); - - const result = await getCodeQlVersionsForOverlayBaseDatabases( - ["python"], - logger, - ); - t.deepEqual(result, ["2.25.0", "2.24.0"]); - }, -); diff --git a/src/overlay/caching.ts b/src/overlay/caching.ts index cdde47b754..367725dfc6 100644 --- a/src/overlay/caching.ts +++ b/src/overlay/caching.ts @@ -8,11 +8,7 @@ import { getWorkflowRunAttempt, getWorkflowRunID, } from "../actions-util"; -import { - type ActionsCacheItem, - getAutomationID, - listActionsCaches, -} from "../api-client"; +import { getAutomationID, listActionsCaches } from "../api-client"; import { createCacheKeyHash } from "../caching-utils"; import { type CodeQL } from "../codeql"; import { type Config } from "../config-utils"; @@ -52,12 +48,6 @@ const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = const CACHE_VERSION = 1; const CACHE_PREFIX = "codeql-overlay-base-database"; -// The Actions cache evicts entries that have not been accessed in the past 7 -// days. We conservatively set a limit of 6 days to avoid using a cached base DB -// that may be evicted before we can download it. -const CACHE_ENTRY_MAX_AGE_DAYS = 6; -const CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1000; - // The purpose of this ten-minute limit is to guard against the possibility // that the cache service is unresponsive, which would otherwise cause the // entire action to hang. Normally we expect cache operations to complete @@ -445,39 +435,6 @@ async function getCacheKeyPrefixBase( return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languagesComponent}-`; } -/** - * Lists overlay-base database cache entries with the given key prefix, ignoring entries that are - * old enough that they may be evicted by the Actions cache before we attempt to download them. - */ -async function listRecentOverlayBaseDatabaseCaches( - cacheKeyPrefix: string, - logger: Logger, -): Promise { - const allCaches = await listActionsCaches(cacheKeyPrefix); - - if (allCaches.length === 0) { - logger.info("No overlay-base databases found in Actions cache."); - return []; - } - - const cutoffMs = Date.now() - CACHE_ENTRY_MAX_AGE_MS; - const recentCaches = allCaches.filter((cache) => { - if (!cache.last_accessed_at) return true; - const lastAccessedMs = Date.parse(cache.last_accessed_at); - return Number.isNaN(lastAccessedMs) || lastAccessedMs >= cutoffMs; - }); - const numTooOldDatabases = allCaches.length - recentCaches.length; - const tooOldSuffix = - numTooOldDatabases > 0 - ? ` (ignoring ${numTooOldDatabases} that may be evicted soon)` - : ""; - logger.info( - `Found ${allCaches.length} overlay-base ${allCaches.length === 1 ? "database" : "databases"} in the Actions cache${tooOldSuffix}.`, - ); - - return recentCaches; -} - /** * Searches the GitHub Actions cache for overlay-base databases matching the given languages, and * returns all stable CodeQL versions found across matching cache entries. @@ -491,7 +448,7 @@ export async function getCodeQlVersionsForOverlayBaseDatabases( ): Promise { const languages = rawLanguages.map(parseBuiltInLanguage); if (languages.includes(undefined)) { - logger.info( + logger.warning( "One or more provided languages are not recognized as built-in languages. " + "Skipping searching for overlay-base databases in cache.", ); @@ -506,19 +463,22 @@ export async function getCodeQlVersionsForOverlayBaseDatabases( `prefix ${cacheKeyPrefix}`, ); - const caches = await listRecentOverlayBaseDatabaseCaches( - cacheKeyPrefix, - logger, - ); + const caches = await listActionsCaches(cacheKeyPrefix); if (caches.length === 0) { + logger.info("No overlay-base databases found in Actions cache."); return []; } + logger.info( + `Found ${caches.length} overlay-base ` + + `${caches.length === 1 ? "database" : "databases"} in the Actions cache.`, + ); + // Parse CodeQL versions from cache keys, matching only stable releases. // - // After the prefix, the remaining key format starts with `${codeQlVersion}-`. Nightlies have a - // suffix like `+202604201548` that will prevent a match. + // After the prefix, the remaining key format starts with `${codeQlVersion}-`. Nightlies will have + // a suffix like `+202604201548` that will break the match. // // Caveat: this relies on the fact that we haven't released any CodeQL bundles with the // `x.y.z-` semver format which does not interact well with the current overlay base @@ -546,7 +506,7 @@ export async function getCodeQlVersionsForOverlayBaseDatabases( const versions = [...versionSet].sort(semver.rcompare); logger.info( - `Found overlay-base databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`, + `Found overlay databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`, ); return versions; From 5997e25ad9b4bb8bf9d221889e85b99b2d930f3d Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Tue, 5 May 2026 18:43:01 +0100 Subject: [PATCH 6/9] Update `listActionsCaches` doc --- lib/analyze-action.js | 4 ++-- lib/init-action-post.js | 4 ++-- src/api-client.ts | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/analyze-action.js b/lib/analyze-action.js index 4ed11db5be..78c055364b 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -107213,14 +107213,14 @@ function computeAutomationID(analysis_key, environment) { } return automationID; } -async function listActionsCaches(key, ref) { +async function listActionsCaches(keyPrefix, ref) { const repositoryNwo = getRepositoryNwo(); return await getApiClient().paginate( "GET /repos/{owner}/{repo}/actions/caches", { owner: repositoryNwo.owner, repo: repositoryNwo.repo, - key, + key: keyPrefix, ref } ); diff --git a/lib/init-action-post.js b/lib/init-action-post.js index 5e5ed0f603..3bb614ebba 100644 --- a/lib/init-action-post.js +++ b/lib/init-action-post.js @@ -165101,14 +165101,14 @@ function computeAutomationID(analysis_key, environment) { } return automationID; } -async function listActionsCaches(key, ref) { +async function listActionsCaches(keyPrefix, ref) { const repositoryNwo = getRepositoryNwo(); return await getApiClient().paginate( "GET /repos/{owner}/{repo}/actions/caches", { owner: repositoryNwo.owner, repo: repositoryNwo.repo, - key, + key: keyPrefix, ref } ); diff --git a/src/api-client.ts b/src/api-client.ts index 4b8cb7b340..0f907eb9bf 100644 --- a/src/api-client.ts +++ b/src/api-client.ts @@ -252,9 +252,9 @@ export interface ActionsCacheItem { size_in_bytes?: number; } -/** List all Actions cache entries matching the provided key and ref. */ +/** List all Actions cache entries starting with the provided key prefix and matching the provided ref. */ export async function listActionsCaches( - key: string, + keyPrefix: string, ref?: string, ): Promise { const repositoryNwo = getRepositoryNwo(); @@ -264,7 +264,7 @@ export async function listActionsCaches( { owner: repositoryNwo.owner, repo: repositoryNwo.repo, - key, + key: keyPrefix, ref, }, ); From 7fc86e0c371d82b4a6e8d805bafde4d9be0bfd83 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Tue, 5 May 2026 18:43:10 +0100 Subject: [PATCH 7/9] Update type import syntax --- src/overlay/caching.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/overlay/caching.test.ts b/src/overlay/caching.test.ts index 18711fb466..3a2266a4a9 100644 --- a/src/overlay/caching.test.ts +++ b/src/overlay/caching.test.ts @@ -7,7 +7,7 @@ import * as sinon from "sinon"; import * as actionsUtil from "../actions-util"; import * as apiClient from "../api-client"; -import { type ResolveDatabaseOutput } from "../codeql"; +import type { ResolveDatabaseOutput } from "../codeql"; import * as gitUtils from "../git-utils"; import { BuiltInLanguage } from "../languages"; import { getRunnerLogger } from "../logging"; From f64a4491cf3dec11924189e93a28cd5c41b3c514 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Tue, 5 May 2026 18:48:09 +0100 Subject: [PATCH 8/9] Add links to API docs --- src/api-client.ts | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/api-client.ts b/src/api-client.ts index 0f907eb9bf..4a061d4828 100644 --- a/src/api-client.ts +++ b/src/api-client.ts @@ -128,6 +128,8 @@ export async function getGitHubVersionFromApi( // Doesn't strictly have to be the meta endpoint as we're only // using the response headers which are available on every request. + // + // See https://docs.github.com/en/rest/meta/meta#get-github-meta-information. // eslint-disable-next-line @typescript-eslint/no-unsafe-call const response = await apiClient.rest.meta.get(); @@ -164,6 +166,9 @@ export async function getGitHubVersion(): Promise { /** * Get the path of the currently executing workflow relative to the repository root. + * + * See https://docs.github.com/en/rest/actions/workflow-runs#get-a-workflow-run + * and https://docs.github.com/en/rest/actions/workflows#get-a-workflow. */ export async function getWorkflowRelativePath(): Promise { const repo_nwo = getRepositoryNwo(); @@ -252,7 +257,11 @@ export interface ActionsCacheItem { size_in_bytes?: number; } -/** List all Actions cache entries starting with the provided key prefix and matching the provided ref. */ +/** + * List all Actions cache entries starting with the provided key prefix and matching the provided ref. + * + * See https://docs.github.com/en/rest/actions/cache#list-github-actions-caches-for-a-repository. + */ export async function listActionsCaches( keyPrefix: string, ref?: string, @@ -270,7 +279,11 @@ export async function listActionsCaches( ); } -/** Delete an Actions cache item by its ID. */ +/** + * Delete an Actions cache item by its ID. + * + * See https://docs.github.com/en/rest/actions/cache#delete-a-github-actions-cache-for-a-repository-using-a-cache-id. + */ export async function deleteActionsCache(id: number) { const repositoryNwo = getRepositoryNwo(); @@ -281,7 +294,11 @@ export async function deleteActionsCache(id: number) { }); } -/** Retrieve all custom repository properties. */ +/** + * Retrieve all custom repository properties. + * + * See https://docs.github.com/en/rest/repos/custom-properties#get-all-custom-property-values-for-a-repository. + */ export async function getRepositoryProperties(repositoryNwo: RepositoryNwo) { return getApiClient().request("GET /repos/:owner/:repo/properties/values", { owner: repositoryNwo.owner, From 09a1d9ec2ad055a28abb3a54e8f8eebe88581660 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Tue, 5 May 2026 18:54:16 +0100 Subject: [PATCH 9/9] Add note about cache eviction --- src/overlay/caching.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/overlay/caching.ts b/src/overlay/caching.ts index 367725dfc6..268c20c129 100644 --- a/src/overlay/caching.ts +++ b/src/overlay/caching.ts @@ -439,6 +439,13 @@ async function getCacheKeyPrefixBase( * Searches the GitHub Actions cache for overlay-base databases matching the given languages, and * returns all stable CodeQL versions found across matching cache entries. * + * Note that we do not guarantee that the cache entry for these versions of CodeQL will still be + * present by the time we attempt to restore the cache. We could achieve that with a download retry + * loop, but we expect that if there is sufficient Actions cache contention that an overlay-base + * cache entry for a particular CodeQL version is evicted before we can use it, then it is likely + * that the same thing will happen to other overlay-base cache entries, and therefore we will not be + * able to use overlay. + * * @returns Unique stable CodeQL versions found in cached overlay-base databases, sorted from latest to * earliest, or undefined if one of the languages is not a built-in language. */