From a1b52d66ae12d11873cb1adc17dd586fcf9b0795 Mon Sep 17 00:00:00 2001 From: Deepak Kasu Date: Wed, 11 Mar 2026 14:03:31 -0700 Subject: [PATCH] APIGOV-32124 Engage CLI - Migrate & Refactor Caching --- package-lock.json | 51 +++++++ package.json | 3 + src/lib/cache/CacheController.ts | 178 ++++++++++++++++++++++ src/lib/types.ts | 254 +++++++++++++++++++++++++++++++ src/lib/utils/utils.ts | 25 +++ 5 files changed, 511 insertions(+) create mode 100644 src/lib/cache/CacheController.ts create mode 100644 src/lib/types.ts create mode 100644 src/lib/utils/utils.ts diff --git a/package-lock.json b/package-lock.json index 0a8711d1..1778aa0c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "chalk": "^5.6.2", "ci-info": "^4.3.1", "cli-table3": "^0.6.5", + "dayjs": "^1.11.19", "debug": "^4.4.3", "fastest-levenshtein": "^1.0.16", "got": "^14.6.2", @@ -25,6 +26,7 @@ "jose": "^6.1.0", "keytar": "7.9.0", "lodash": "^4.17.21", + "node-cache": "^5.1.2", "pluralize": "^8.0.0", "pretty-bytes": "^7.1.0", "pretty-ms": "^9.3.0", @@ -38,6 +40,7 @@ "devDependencies": { "@koa/router": "^14.0.0", "@oclif/test": "^4.1.14", + "@types/fs-extra": "^11.0.4", "@types/lodash": "^4.17.20", "@types/node": "^22", "@typescript-eslint/eslint-plugin": "^8.46.3", @@ -1073,6 +1076,17 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/fs-extra": { + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.4.tgz", + "integrity": "sha512-yTbItCNreRooED33qjunPthRcSjERP1r4MqCZc7wv0u2sUkzTFp45tgUfS5+r7FrZPdmCCNflLhVSP/o+SemsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/jsonfile": "*", + "@types/node": "*" + } + }, "node_modules/@types/http-cache-semantics": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", @@ -1093,6 +1107,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/jsonfile": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/@types/jsonfile/-/jsonfile-6.1.4.tgz", + "integrity": "sha512-D5qGUYwjvnNNextdU59/+fI+spnwtTFmyQP0h+PfIOSkNfpU6AOICUOkm4i0OnSk+NyjdPJrxCDro0sJsWlRpQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/lodash": { "version": "4.17.20", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.20.tgz", @@ -2289,6 +2313,15 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/clone": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", + "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==", + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, "node_modules/co-body": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/co-body/-/co-body-6.2.0.tgz", @@ -2451,6 +2484,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/dayjs": { + "version": "1.11.19", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.19.tgz", + "integrity": "sha512-t5EcLVS6QPBNqM2z8fakk/NKel+Xzshgt8FFKAn+qwlD1pzZWxh0nVCrvFK7ZDb6XucZeF9z8C7CBWTRIVApAw==", + "license": "MIT" + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -5394,6 +5433,18 @@ "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==", "license": "MIT" }, + "node_modules/node-cache": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz", + "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==", + "license": "MIT", + "dependencies": { + "clone": "2.x" + }, + "engines": { + "node": ">= 8.0.0" + } + }, "node_modules/normalize-url": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.1.0.tgz", diff --git a/package.json b/package.json index 56240c8a..4c4b2190 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "chalk": "^5.6.2", "ci-info": "^4.3.1", "cli-table3": "^0.6.5", + "dayjs": "^1.11.19", "debug": "^4.4.3", "fastest-levenshtein": "^1.0.16", "got": "^14.6.2", @@ -40,6 +41,7 @@ "jose": "^6.1.0", "keytar": "7.9.0", "lodash": "^4.17.21", + "node-cache": "^5.1.2", "pluralize": "^8.0.0", "pretty-bytes": "^7.1.0", "pretty-ms": "^9.3.0", @@ -50,6 +52,7 @@ "devDependencies": { "@koa/router": "^14.0.0", "@oclif/test": "^4.1.14", + "@types/fs-extra": "^11.0.4", "@types/lodash": "^4.17.20", "@types/node": "^22", "@typescript-eslint/eslint-plugin": "^8.46.3", diff --git a/src/lib/cache/CacheController.ts b/src/lib/cache/CacheController.ts new file mode 100644 index 00000000..bf954e97 --- /dev/null +++ b/src/lib/cache/CacheController.ts @@ -0,0 +1,178 @@ +import dayjs from "dayjs"; +import { + lstatSync, + outputJsonSync, + pathExistsSync, + readFileSync, +} from "fs-extra"; +import pkg from "lodash"; +import NodeCache from "node-cache"; +import { homedir } from "os"; +import path from "path"; +import { CACHE_FILE_TTL_MILLISECONDS, MAX_CACHE_FILE_SIZE } from "../types.js"; +import { isValidJson, writeToFile } from "../utils/utils.js"; +import logger from "../logger.js"; + +const { log } = logger("axway-cli: CacheController"); +const { isEmpty } = pkg; + +interface Cache { + set(key: string, value: object): CacheControllerClass; + get(key: string): any; + readFromFile(): CacheControllerClass; + writeToFile(): CacheControllerClass; +} + +interface StoredCache { + data?: object; + metadata?: { + modifyTimestamp?: number; + schemaVersion?: string; + }; +} + +/** + * Note: this file intentionally exporting only a single instance of CacheController, + * since its possible to face a race condition when multiple instances will try to read/write file at the same time + * Please do not use this class directly or rework the logic before. + */ +class CacheControllerClass implements Cache { + public cacheFilePath = path.join( + homedir(), + ".axway", + "central", + "cache.json", + ); + private cache = new NodeCache(); + + constructor() { + // note: init cache fire only once since using only a single instance of the class, remove if this will change + this.initCacheFile(); + this.readFromFile(); + } + + /** + * Inits and validate cache file, should run once before using this class in the code (initialized in cli.ts currently) + * An empty JSON file will be created if it is not exists of the file size is more than some value. + */ + initCacheFile() { + try { + if (pathExistsSync(this.cacheFilePath)) { + log(`init, cache file found at ${this.cacheFilePath}`); + const stats = lstatSync(this.cacheFilePath); + log(`init, cache file size: ${Math.round(stats.size / 1000)} kb`); + if (stats.size >= MAX_CACHE_FILE_SIZE) { + // validating the size + log( + `init, cache size is exceeding the max allowed size of ${Math.round( + MAX_CACHE_FILE_SIZE / 1000, + )} kb, resetting the file`, + ); + outputJsonSync(this.cacheFilePath, {}); + } else if (!isValidJson(readFileSync(this.cacheFilePath, "utf8"))) { + // validating the content + log("init, cache content is invalid, resetting the file "); + outputJsonSync(this.cacheFilePath, {}); + } + } else { + log( + `init, cache file not found, creating an empty one at ${this.cacheFilePath}`, + ); + outputJsonSync(this.cacheFilePath, {}); + } + } catch (e) { + log(`cannot initialize cache file`, e); + } + } + + /** + * Set the key in memory cache. + * @param key cache key to set + * @param value value to set, note that setting "undefined" value will result in "null" value stored + * @returns CacheController instance + */ + set(key: string, value: any): CacheControllerClass { + this.cache.set(key, value); + return this; + } + + /** + * Returns the key value from the memory cache. + * @param key key to get + * @returns key value + */ + get(key: string): any | undefined { + return this.cache.get(key); + } + + /** + * Load stored cache from the file into memory and checks its timestamp. + * If the timestamp is more than X days old it will reset the file without any changes to cache. + * Note: using this method before writeToFile() will override keys in memory cache with the same name. + * @returns CacheController instance + */ + readFromFile() { + try { + log("reading cache from the file"); + const jsonData = readFileSync(this.cacheFilePath, "utf8"); + const storedCache = JSON.parse(jsonData); + + // validate values stored in the cache, reset the content of the file if its not empty already. + if ( + storedCache.data && + storedCache.metadata && + storedCache.metadata.modifyTimestamp && + dayjs().diff(storedCache.metadata.modifyTimestamp, "milliseconds") < + CACHE_FILE_TTL_MILLISECONDS + ) { + for (const [key, val] of Object.entries(storedCache.data)) { + if (storedCache.data.hasOwnProperty(key)) { + this.cache.set(key, val); + } + } + } else if (!isEmpty(storedCache)) { + log( + "timestamp or content is not valid and file is not empty, resetting the cache file", + ); + outputJsonSync(this.cacheFilePath, {}); + } + } catch (e) { + log("cannot read cache from the file", e); + } finally { + return this; + } + } + + /** + * Writes current set of keys to the json file with following structure: + * { + * metadata: { + * modifyTimestamp: current timestamp, used on read for TTL validation + * schemaVersion: indicates the version of cache file structure, can be used later on if changing it. + * }, + * data: {} key-value cache data + * } + * @returns CacheController instance + */ + writeToFile() { + try { + log("writing cache to the file"); + const keys = this.cache.keys(); + const cachedData = this.cache.mget(keys); + const dataToStore: StoredCache = { + metadata: { + modifyTimestamp: Date.now(), + schemaVersion: "1", + }, + data: cachedData, + }; + writeToFile(this.cacheFilePath, JSON.stringify(dataToStore)); + } catch (e) { + log("cannot write cache to the file", e); + } finally { + return this; + } + } +} + +export const CacheController = new CacheControllerClass(); diff --git a/src/lib/types.ts b/src/lib/types.ts new file mode 100644 index 00000000..8ae5ab53 --- /dev/null +++ b/src/lib/types.ts @@ -0,0 +1,254 @@ +export const ABORT_TIMEOUT = + process.env.NODE_ENV === "test" + ? 1e3 + : process.env.DEBUG || process.env.SNOOPLOGG + ? 1e9 + : 30e3; + +export const MAX_TABLE_STRING_LENGTH = 50; +export const MAX_FILE_SIZE = + process.env.NODE_ENV === "test" ? 1e5 : 20 * 1024 * 1024; +export const MAX_CACHE_FILE_SIZE = 5 * 1024 * 1024; + +// 12 hours +export const CACHE_FILE_TTL_MILLISECONDS = + process.env.NODE_ENV === "test" ? 100 : 60000 * 60 * 12; +export const WAIT_TIMEOUT = process.env.NODE_ENV === "test" ? 1e3 : 1e4; + +/** + * Invoked multiple times to indicate progress on something, such as download progress. + * @param progress Value ranging from 0 to 100. + */ +export type ProgressListener = (progress: number) => void; + +/** + * ApiServer backend types + */ +export enum ApiServerVersions { + v1alpha1 = "v1alpha1", +} + +export enum LanguageTypes { + French = "fr-fr", + US = "en-us", + German = "de-de", + Portugese = "pt-br", +} + +export type ApiServerError = { + status: number; + title: string; + detail: string; + source?: object; + meta: { + regexp?: string; + instanceId: string; + tenantId: string; + authenticatedUserId: string; + transactionId: string; + }; +}; + +export type ApiServerErrorResponse = { + errors: ApiServerError[]; +}; + +export interface ResourceDefinition { + apiVersion: ApiServerVersions; + kind: "ResourceDefinition"; + name: string; // "environment" + group: string; //"management" + metadata: { + id: string; //'e4e08f487156b7c8017156b9eef60002'; + audit: { + createTimestamp: string; //'2020-04-07T22:19:18.141+0000'; + modifyTimestamp: string; //'2020-04-07T22:19:18.141+0000' + }; + scope: { + id: string; //'e4e08f487156b7c8017156b9ed930000'; + kind: string; //'ResourceGroup'; + name: string; //'management' + }; + resourceVersion: string; //'1609'; + references: any[]; //[]; + }; + spec: { + kind: string; // "Environment", + plural: string; //"environments", + scope?: { + kind: string; //'Environment' + }; + apiVersions?: { + name: string; + served: boolean; + deprecated: boolean; + }[]; + // note: making it optional for backward-compatible logic. + subResources?: { + names: string[]; + }; + references: { + toResources: { + kind: string; + group: string; + types: ("soft" | "hard")[]; + scopeKind?: string; + from?: { + subResourceName: string; + }; + }[]; + fromResources: { + kind: string; + types: ("soft" | "hard")[]; + scopeKind?: string; + from?: { + subResourceName: string; + }; + }[]; + }; + }; +} + +export interface CommandLineInterfaceColumns { + name: string; //'Name'; + type: string; //'string'; + jsonPath: string; //'.name'; + description: string; //'The name of the environment.'; + hidden: boolean; //false +} + +export interface CommandLineInterface { + apiVersion: ApiServerVersions; + kind: "CommandLineInterface"; + name: string; // "environment" + spec: { + names: { + plural: string; // 'environments'; + // 10/2022 note: "singular" value is not always equal to the "name" value anymore + singular: string; // 'environment'; + shortNames: string[]; // ['env', 'envs']; + shortNamesAlias?: string[]; // ['env'] + }; + columns: CommandLineInterfaceColumns[]; + resourceDefinition: string; //'environment'; + }; + metadata: { + scope: { + name: string; // 'management' + }; + }; +} + +export interface AuditMetadata { + createTimestamp: string; // '2020-08-04T21:05:32.106Z'; + createUserId: string; // '07e6b449-3a31-4a96-8920-e87dd504cb87'; + modifyTimestamp: string; // '2020-08-04T21:05:32.106Z'; + modifyUserId: string; // '07e6b449-3a31-4a96-8920-e87dd504cb87'; +} + +interface Scope { + id: string; + kind: Kind; + name: string; +} + +export enum Kind { + Environment = "Environment", + APIService = "APIService", + APIServiceRevision = "APIServiceRevision", + APIServiceInstance = "APIServiceInstance", + Asset = "Asset", + AssetMapping = "AssetMapping", + Product = "Product", + ReleaseTag = "ReleaseTag", + Secret = "Secret", + Webhook = "Webhook", + ConsumerSubscriptionDefinition = "ConsumerSubscriptionDefinition", + ConsumerInstance = "ConsumerInstance", +} + +export interface Metadata { + audit: AuditMetadata; + resourceVersion?: string; + id: string; + scope?: Scope; + references: { + id: string; // e4e0900570caf70701713be3e36a076e + kind: string; // "Secret" + name: string; // secret1 + types: ["soft", "hard"]; + }[]; +} + +export interface GenericResource { + apiVersion: string; + group: string; + title: string; + name: string; + kind: string; + attributes: object; + tags: string[]; + // note: metadata is not an optional when received from the api-server but + // might be missing in some of our castings and in the resources from a file + metadata?: Metadata; + spec: any; + // note: have to include "any" indexed type for allowing sub-resources + [subresource: string]: any; +} + +export type GenericResourceWithoutName = Omit & { + name?: string; +}; + +/** + * Client's types + */ +export type ApiServerClientListResult = { + data: null | GenericResource[]; + error: null | ApiServerError[]; +}; + +export type ApiServerSubResourceOperation = { + name: string; + operation: () => Promise; +}; + +export type ApiServerClientSingleResult = { + data: null | GenericResource; + updatedSubResourceNames?: string[]; + warning?: boolean; + error: null | ApiServerError[]; + pending?: null | Array; +}; + +export type ApiServerClientApplyResult = { + data?: null | GenericResource; + wasAutoNamed?: boolean; + wasCreated?: boolean; + wasMainResourceChanged?: boolean; + updatedSubResourceNames?: string[]; + error?: { + name: string; + kind: string; + error: ApiServerError | Error | { detail: string; title?: string }; + }[]; +}; + +export type ApiServerClientBulkResult = { + success: GenericResource[]; + error: ApiServerClientError[]; + warning?: GenericResource[]; +}; + +export type ApiServerClientError = { + name: string; + kind: string; + error: ApiServerError | Error | { detail: string; title?: string }; +}; + +export type GetSpecsResult = { + [groupName: string]: { + resources: Map; + cli: Map; + }; +}; diff --git a/src/lib/utils/utils.ts b/src/lib/utils/utils.ts new file mode 100644 index 00000000..300ad9b4 --- /dev/null +++ b/src/lib/utils/utils.ts @@ -0,0 +1,25 @@ +import { writeFileSync } from "fs-extra"; + +export const writeToFile = (path: string, data: any): void => { + try { + writeFileSync(path, data); + } catch (e) { + // if parser is failing, rethrow with our own error + throw new Error(`Error while writing the yaml file to: ${path}`); + } +}; + +/** + * Checks if the passed item can be converted to a JSON or is a valid JSON object. + * @param item item to check + * @returns true if the item can be converted, false otherwise. + */ +export const isValidJson = (item: any) => { + let parsedItem = typeof item !== "string" ? JSON.stringify(item) : item; + try { + parsedItem = JSON.parse(parsedItem); + } catch (e) { + return false; + } + return typeof parsedItem === "object" && item !== null; +};