From 4c8042864db38c9a482774f57d8d9959f9a59e1c Mon Sep 17 00:00:00 2001 From: Rin <58572875+TurtIeSocks@users.noreply.github.com> Date: Tue, 21 Apr 2026 14:38:55 -0400 Subject: [PATCH 1/7] feat: apk caching for speedups --- .gitignore | 3 ++- README.md | 21 ++++++++++++++++++--- devWrapper.ts | 6 ++++++ src/classes/Apk.ts | 38 +++++++++++++++++++++++++++++++++++++- src/index.ts | 35 +++++++++++++++++++++++++++-------- src/typings/inputs.ts | 7 +++++++ 6 files changed, 97 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 82ca258..af8ac51 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ master-latest.json .prettierrc .DS_Store invasions.json -latest.json \ No newline at end of file +latest.json +.cache/ diff --git a/README.md b/README.md index e622eed..7b08634 100644 --- a/README.md +++ b/README.md @@ -39,12 +39,20 @@ Usage: ```js // commonJS -const { generate } = require('pogo-data-generator') -// es6 with invasion function -import { generate, invasions } from 'pogo-data-generator' +const { generate, primeApkCache } = require('pogo-data-generator') +// es6 with exported helpers +import { generate, invasions, primeApkCache } from 'pogo-data-generator' const data = await generate() // returns the default settings +await primeApkCache() +// downloads the APK texts once and saves them to .cache/apk-texts.json + +const cachedData = await generate({ + useApkCache: true, +}) +// reuses the primed APK cache when it is present + const template = { pokemon: { enabled: true, @@ -132,8 +140,15 @@ The generate function accepts an object with the following properties: - `test` (boolean): Writes the masterfile to a local json - `raw` (boolean): Returns the data in its raw format without any template processing - `pokeApi` (boolean): Fetches fresh data from PokeAPI +- `useApkCache` (boolean): Builds and uses the local APK text cache when no cache exists yet +- `apkCachePath` (string): Overrides the default APK cache location (`.cache/apk-texts.json`) - `pokeApiBaseUrl` (string): Overrides the default PokeAPI endpoint (defaults to `https://pokeapi.co/api/v2`) +The package also exports `primeApkCache(options?)`: + +- `force` (boolean): Rebuilds the APK cache even if the cache file already exists +- `apkCachePath` (string): Writes the APK cache to a custom file path + To view some static examples of what this library can create, check out these repos: [Masterfiles](https://github.com/WatWowMap/Masterfile-Generator) [Translations](https://github.com/WatWowMap/pogo-translations) diff --git a/devWrapper.ts b/devWrapper.ts index cf8ca29..3b6b67a 100644 --- a/devWrapper.ts +++ b/devWrapper.ts @@ -13,10 +13,12 @@ const main = async () => { const usePokeApiStaging = process.argv.includes('--pokeapi-staging') const usePokeApi = usePokeApiStaging || process.argv.includes('--pokeapi') + const useApkCache = process.argv.includes('--apk-cache') console.time('Generated in') const data = await generate({ raw: process.argv.includes('--raw'), test: process.argv.includes('--test'), + useApkCache, pokeApi: usePokeApi || { baseStats, tempEvos, @@ -27,6 +29,10 @@ const main = async () => { : undefined, }) + if (useApkCache) { + console.log('APK text cache enabled for this run') + } + if (process.argv.includes('--test')) { if (process.argv.includes('--invasions')) { fs.writeFile( diff --git a/src/classes/Apk.ts b/src/classes/Apk.ts index 44d9bf3..4880933 100644 --- a/src/classes/Apk.ts +++ b/src/classes/Apk.ts @@ -1,11 +1,14 @@ +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import path from 'node:path' import JSZip from 'jszip' export default class ApkReader { texts: Record> codeMap: Record files: JSZip | null + cachePath: string - constructor() { + constructor(cachePath?: string) { this.texts = {} this.codeMap = { 'pt-br': 'pt-br', @@ -25,6 +28,7 @@ export default class ApkReader { 'tr-tr': 'tr', } this.files = null + this.cachePath = path.resolve(cachePath || '.cache/apk-texts.json') } removeEscapes(str: string) { @@ -53,6 +57,37 @@ export default class ApkReader { } } + async loadCachedTexts() { + try { + const cached = await readFile(this.cachePath, 'utf8') + this.texts = JSON.parse(cached) + return true + } catch { + return false + } + } + + async writeCachedTexts() { + try { + await mkdir(path.dirname(this.cachePath), { recursive: true }) + await writeFile(this.cachePath, JSON.stringify(this.texts), 'utf8') + } catch (e) { + console.warn(e, 'Issue with writing APK text cache') + } + } + + async primeCache(force = false) { + if (!force && (await this.loadCachedTexts())) { + return this.texts + } + + this.texts = {} + await this.fetchApk() + await this.extractTexts() + this.cleanup() + return this.texts + } + async extractTexts() { if (!this.files) return try { @@ -85,6 +120,7 @@ export default class ApkReader { } }), ) + await this.writeCachedTexts() } catch (e) { console.warn(e, 'Issue with extracting texts') } diff --git a/src/index.ts b/src/index.ts index fd5532a..8251709 100644 --- a/src/index.ts +++ b/src/index.ts @@ -18,11 +18,20 @@ import type { Input, InvasionsOnly, Locales, + PrimeApkCacheInput, PokemonTemplate, TranslationsTemplate, } from './typings/inputs' import type { InvasionInfo } from './typings/pogoinfo' +export async function primeApkCache({ + force, + apkCachePath, +}: PrimeApkCacheInput = {}) { + const apk = new ApkReader(apkCachePath) + return apk.primeCache(force) +} + export async function generate({ template, url, @@ -30,6 +39,8 @@ export async function generate({ translationRemoteUrl, raw, pokeApi, + useApkCache, + apkCachePath, test, pokeApiBaseUrl, }: Input = {}): Promise { @@ -70,21 +81,29 @@ export async function generate({ translationRemoteUrl, ) const AllPokeApi = new PokeApi(pokeApiBaseUrl) - await AllPokeApi.setMaxPokemonId() - const generations = await AllPokeApi.getGenerations() - AllPokemon.generations = generations const AllMisc = new Misc() const AllLocationCards = new LocationCards(locationCards.options) - const apk = new ApkReader() + const apk = new ApkReader(apkCachePath) + const enabledLocales = Object.values(translations.locales).filter(Boolean) AllMisc.parseRaidLevels() AllMisc.parseRouteTypes() AllMisc.parseTeams() - await apk.fetchApk() - await apk.extractTexts() - apk.cleanup() - AllTranslations.fromApk = apk.texts + if (pokeApi === true) { + await AllPokeApi.setMaxPokemonId() + AllPokemon.generations = await AllPokeApi.getGenerations() + } + + if (translations.enabled && enabledLocales.length > 0) { + const hasCachedApkTexts = await apk.loadCachedTexts() + + if (hasCachedApkTexts) { + AllTranslations.fromApk = apk.texts + } else if (useApkCache) { + AllTranslations.fromApk = await apk.primeCache() + } + } const data: NiaMfObj[] = await AllPokemon.fetch(urlToFetch) diff --git a/src/typings/inputs.ts b/src/typings/inputs.ts index a1cc7eb..dfdde41 100644 --- a/src/typings/inputs.ts +++ b/src/typings/inputs.ts @@ -371,10 +371,17 @@ export interface Input { template?: FullTemplate test?: boolean raw?: boolean + useApkCache?: boolean + apkCachePath?: string pokeApi?: boolean | PokeApi pokeApiBaseUrl?: string } +export interface PrimeApkCacheInput { + force?: boolean + apkCachePath?: string +} + export interface FullTemplate { globalOptions?: { keyJoiner?: string From 6b989bf61bf2bac901227fbc73f2112cee42021e Mon Sep 17 00:00:00 2001 From: Rin <58572875+TurtIeSocks@users.noreply.github.com> Date: Tue, 21 Apr 2026 14:43:37 -0400 Subject: [PATCH 2/7] fix: force cache by filename --- README.md | 9 ++---- devWrapper.ts | 6 ---- src/classes/Apk.ts | 74 +++++++++++++++++++++++++++++++++++++++---- src/index.ts | 9 +----- src/typings/inputs.ts | 1 - 5 files changed, 71 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 7b08634..6d41f83 100644 --- a/README.md +++ b/README.md @@ -48,10 +48,8 @@ const data = await generate() // returns the default settings await primeApkCache() // downloads the APK texts once and saves them to .cache/apk-texts.json -const cachedData = await generate({ - useApkCache: true, -}) -// reuses the primed APK cache when it is present +const cachedData = await generate() +// automatically reuses the primed APK cache when it matches the latest APK filename const template = { pokemon: { @@ -140,13 +138,12 @@ The generate function accepts an object with the following properties: - `test` (boolean): Writes the masterfile to a local json - `raw` (boolean): Returns the data in its raw format without any template processing - `pokeApi` (boolean): Fetches fresh data from PokeAPI -- `useApkCache` (boolean): Builds and uses the local APK text cache when no cache exists yet - `apkCachePath` (string): Overrides the default APK cache location (`.cache/apk-texts.json`) - `pokeApiBaseUrl` (string): Overrides the default PokeAPI endpoint (defaults to `https://pokeapi.co/api/v2`) The package also exports `primeApkCache(options?)`: -- `force` (boolean): Rebuilds the APK cache even if the cache file already exists +- `force` (boolean): Rebuilds the APK cache even if the cache already matches the latest APK filename - `apkCachePath` (string): Writes the APK cache to a custom file path To view some static examples of what this library can create, check out these repos: diff --git a/devWrapper.ts b/devWrapper.ts index 3b6b67a..cf8ca29 100644 --- a/devWrapper.ts +++ b/devWrapper.ts @@ -13,12 +13,10 @@ const main = async () => { const usePokeApiStaging = process.argv.includes('--pokeapi-staging') const usePokeApi = usePokeApiStaging || process.argv.includes('--pokeapi') - const useApkCache = process.argv.includes('--apk-cache') console.time('Generated in') const data = await generate({ raw: process.argv.includes('--raw'), test: process.argv.includes('--test'), - useApkCache, pokeApi: usePokeApi || { baseStats, tempEvos, @@ -29,10 +27,6 @@ const main = async () => { : undefined, }) - if (useApkCache) { - console.log('APK text cache enabled for this run') - } - if (process.argv.includes('--test')) { if (process.argv.includes('--invasions')) { fs.writeFile( diff --git a/src/classes/Apk.ts b/src/classes/Apk.ts index 4880933..79f0ba5 100644 --- a/src/classes/Apk.ts +++ b/src/classes/Apk.ts @@ -2,11 +2,32 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises' import path from 'node:path' import JSZip from 'jszip' +interface ApkTextCache { + apkFilename: string + texts: Record> +} + +const isApkTextCache = (value: unknown): value is ApkTextCache => { + if (!value || typeof value !== 'object') { + return false + } + const candidate = value as { + apkFilename?: unknown + texts?: unknown + } + return ( + typeof candidate.apkFilename === 'string' && + !!candidate.texts && + typeof candidate.texts === 'object' + ) +} + export default class ApkReader { texts: Record> codeMap: Record files: JSZip | null cachePath: string + apkFilename: string | null constructor(cachePath?: string) { this.texts = {} @@ -29,13 +50,14 @@ export default class ApkReader { } this.files = null this.cachePath = path.resolve(cachePath || '.cache/apk-texts.json') + this.apkFilename = null } removeEscapes(str: string) { return str.replace(/\r/g, '').replace(/\n/g, '').replace(/"/g, '”') } - async fetchApk() { + async getLatestApkFilename() { try { const index = await fetch('https://mirror.unownhash.com/index.json') @@ -43,7 +65,21 @@ export default class ApkReader { throw new Error('Unable to fetch index') } const data = await index.json() - const first = data[0].filename + return data[0].filename as string + } catch (e) { + console.warn(e, 'Issue with downloading APK index') + return null + } + } + + async fetchApk(filename?: string) { + try { + const first = filename || (await this.getLatestApkFilename()) + + if (!first) { + throw new Error('Unable to determine latest APK filename') + } + this.apkFilename = first const response = await fetch(`https://mirror.unownhash.com/apks/${first}`) const apk = await response.arrayBuffer() @@ -57,10 +93,18 @@ export default class ApkReader { } } - async loadCachedTexts() { + async loadCachedTexts(expectedFilename?: string) { try { const cached = await readFile(this.cachePath, 'utf8') - this.texts = JSON.parse(cached) + const parsed: unknown = JSON.parse(cached) + if (!isApkTextCache(parsed)) { + return false + } + if (expectedFilename && parsed.apkFilename !== expectedFilename) { + return false + } + this.apkFilename = parsed.apkFilename + this.texts = parsed.texts return true } catch { return false @@ -70,19 +114,35 @@ export default class ApkReader { async writeCachedTexts() { try { await mkdir(path.dirname(this.cachePath), { recursive: true }) - await writeFile(this.cachePath, JSON.stringify(this.texts), 'utf8') + if (!this.apkFilename) { + throw new Error('Missing APK filename for cache write') + } + const payload: ApkTextCache = { + apkFilename: this.apkFilename, + texts: this.texts, + } + await writeFile(this.cachePath, JSON.stringify(payload), 'utf8') } catch (e) { console.warn(e, 'Issue with writing APK text cache') } } async primeCache(force = false) { - if (!force && (await this.loadCachedTexts())) { + const latestFilename = await this.getLatestApkFilename() + + if (!latestFilename) { + if (await this.loadCachedTexts()) { + return this.texts + } + return this.texts + } + + if (!force && (await this.loadCachedTexts(latestFilename))) { return this.texts } this.texts = {} - await this.fetchApk() + await this.fetchApk(latestFilename) await this.extractTexts() this.cleanup() return this.texts diff --git a/src/index.ts b/src/index.ts index 8251709..e67e070 100644 --- a/src/index.ts +++ b/src/index.ts @@ -39,7 +39,6 @@ export async function generate({ translationRemoteUrl, raw, pokeApi, - useApkCache, apkCachePath, test, pokeApiBaseUrl, @@ -96,13 +95,7 @@ export async function generate({ } if (translations.enabled && enabledLocales.length > 0) { - const hasCachedApkTexts = await apk.loadCachedTexts() - - if (hasCachedApkTexts) { - AllTranslations.fromApk = apk.texts - } else if (useApkCache) { - AllTranslations.fromApk = await apk.primeCache() - } + AllTranslations.fromApk = await apk.primeCache() } const data: NiaMfObj[] = await AllPokemon.fetch(urlToFetch) diff --git a/src/typings/inputs.ts b/src/typings/inputs.ts index dfdde41..7e5a549 100644 --- a/src/typings/inputs.ts +++ b/src/typings/inputs.ts @@ -371,7 +371,6 @@ export interface Input { template?: FullTemplate test?: boolean raw?: boolean - useApkCache?: boolean apkCachePath?: string pokeApi?: boolean | PokeApi pokeApiBaseUrl?: string From 6d42d39c1d483a69ba198d178fb1fb7d9f77420f Mon Sep 17 00:00:00 2001 From: Rin <58572875+TurtIeSocks@users.noreply.github.com> Date: Tue, 21 Apr 2026 14:46:33 -0400 Subject: [PATCH 3/7] fix: update the github api urls --- devWrapper.ts | 2 +- src/classes/Invasion.ts | 2 +- src/index.ts | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/devWrapper.ts b/devWrapper.ts index cf8ca29..6b971df 100644 --- a/devWrapper.ts +++ b/devWrapper.ts @@ -6,7 +6,7 @@ import types from './static/types.json' const main = async () => { const mfData = await fetch( - 'https://raw.githubusercontent.com/PokeMiners/game_masters/master/latest/latest.json', + 'https://raw.githubusercontent.com/alexelgt/game_masters/refs/heads/master/GAME_MASTER.json', ) const mf = await mfData.json() fs.writeFileSync('./latest.json', JSON.stringify(mf, null, 2), 'utf8') diff --git a/src/classes/Invasion.ts b/src/classes/Invasion.ts index f4e4b58..3780fc3 100644 --- a/src/classes/Invasion.ts +++ b/src/classes/Invasion.ts @@ -22,7 +22,7 @@ export default class Invasion extends Masterfile { (this.options.customInvasions === undefined && override) ) { return this.fetch( - 'https://raw.githubusercontent.com/WatWowMap/Masterfile-Generator/master/custom-invasions.json', + 'https://raw.githubusercontent.com/WatWowMap/Masterfile-Generator/refs/heads/master/custom-invasions.json', ) } else if (this.options.customInvasions) { return this.options.customInvasions as InvasionInfo diff --git a/src/index.ts b/src/index.ts index e67e070..987a8e5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -18,8 +18,8 @@ import type { Input, InvasionsOnly, Locales, - PrimeApkCacheInput, PokemonTemplate, + PrimeApkCacheInput, TranslationsTemplate, } from './typings/inputs' import type { InvasionInfo } from './typings/pogoinfo' @@ -396,7 +396,7 @@ export async function generate({ if (pokeApi === true) return AllPokeApi[category] if (pokeApi) return pokeApi[category] return AllPokeApi.fetch( - `https://raw.githubusercontent.com/WatWowMap/Pogo-Data-Generator/main/static/${category}.json`, + `https://raw.githubusercontent.com/WatWowMap/Pogo-Data-Generator/refs/heads/main/static/${category}.json`, ) } @@ -430,7 +430,7 @@ export async function generate({ (translations.template as TranslationsTemplate).characters ) { const invasionData: InvasionInfo = await AllInvasions.fetch( - 'https://raw.githubusercontent.com/WatWowMap/event-info/main/grunts/classic.json', + 'https://raw.githubusercontent.com/WatWowMap/event-info/refs/heads/main/grunts/classic.json', ) AllInvasions.invasions( AllInvasions.mergeInvasions( @@ -442,7 +442,7 @@ export async function generate({ if (translations.enabled) { const availableManualTranslations = await AllTranslations.fetch( - 'https://raw.githubusercontent.com/WatWowMap/pogo-translations/master/index.json', + 'https://raw.githubusercontent.com/WatWowMap/pogo-translations/refs/heads/master/index.json', ) await Promise.all( Object.entries(translations.locales).map(async (langCode) => { @@ -678,7 +678,7 @@ export async function invasions({ const finalTemplate = template || base.invasions const AllInvasions = new Invasions(finalTemplate.options) const invasionData: InvasionInfo = await AllInvasions.fetch( - 'https://raw.githubusercontent.com/WatWowMap/event-info/main/grunts/classic.json', + 'https://raw.githubusercontent.com/WatWowMap/event-info/refs/heads/main/grunts/classic.json', ) AllInvasions.invasions( AllInvasions.mergeInvasions( From 35f57e5fc2ab36c9fb7c353a54f34c95f7f3c974 Mon Sep 17 00:00:00 2001 From: Rin <58572875+TurtIeSocks@users.noreply.github.com> Date: Tue, 21 Apr 2026 14:56:31 -0400 Subject: [PATCH 4/7] fix: maintain browser compatibility --- README.md | 17 +++++--- devWrapper.ts | 6 +++ package.json | 11 +++++ src/classes/Apk.ts | 85 ++------------------------------------- src/index.ts | 40 ++++++++++++++----- src/node.ts | 93 +++++++++++++++++++++++++++++++++++++++++++ src/typings/inputs.ts | 19 ++++++++- 7 files changed, 172 insertions(+), 99 deletions(-) create mode 100644 src/node.ts diff --git a/README.md b/README.md index 6d41f83..988b481 100644 --- a/README.md +++ b/README.md @@ -39,16 +39,20 @@ Usage: ```js // commonJS -const { generate, primeApkCache } = require('pogo-data-generator') +const { generate } = require('pogo-data-generator') +const { createNodeApkCache, primeApkCache } = require('pogo-data-generator/node') // es6 with exported helpers -import { generate, invasions, primeApkCache } from 'pogo-data-generator' +import { generate, invasions } from 'pogo-data-generator' +import { createNodeApkCache, primeApkCache } from 'pogo-data-generator/node' const data = await generate() // returns the default settings +const apkCache = createNodeApkCache() + await primeApkCache() -// downloads the APK texts once and saves them to .cache/apk-texts.json +// downloads the APK texts once and saves them to .cache/apk-texts.json in Node environments -const cachedData = await generate() +const cachedData = await generate({ apkCache }) // automatically reuses the primed APK cache when it matches the latest APK filename const template = { @@ -137,12 +141,13 @@ The generate function accepts an object with the following properties: - `url` (string): Custom url to fetch the masterfile from, results not guaranteed - `test` (boolean): Writes the masterfile to a local json - `raw` (boolean): Returns the data in its raw format without any template processing +- `apkCache` (object): Optional server-side cache adapter for APK texts - `pokeApi` (boolean): Fetches fresh data from PokeAPI -- `apkCachePath` (string): Overrides the default APK cache location (`.cache/apk-texts.json`) - `pokeApiBaseUrl` (string): Overrides the default PokeAPI endpoint (defaults to `https://pokeapi.co/api/v2`) -The package also exports `primeApkCache(options?)`: +Node-only cache helpers are exported from `pogo-data-generator/node`: +- `createNodeApkCache(options?)`: Creates a filesystem-backed APK cache adapter for server environments - `force` (boolean): Rebuilds the APK cache even if the cache already matches the latest APK filename - `apkCachePath` (string): Writes the APK cache to a custom file path diff --git a/devWrapper.ts b/devWrapper.ts index 6b971df..1b3e54f 100644 --- a/devWrapper.ts +++ b/devWrapper.ts @@ -1,5 +1,6 @@ import * as fs from 'fs' import { generate, invasions } from './src/index' +import { createNodeApkCache, primeApkCache } from './src/node' import baseStats from './static/baseStats.json' import tempEvos from './static/tempEvos.json' import types from './static/types.json' @@ -13,10 +14,12 @@ const main = async () => { const usePokeApiStaging = process.argv.includes('--pokeapi-staging') const usePokeApi = usePokeApiStaging || process.argv.includes('--pokeapi') + const apkCache = createNodeApkCache() console.time('Generated in') const data = await generate({ raw: process.argv.includes('--raw'), test: process.argv.includes('--test'), + apkCache, pokeApi: usePokeApi || { baseStats, tempEvos, @@ -36,6 +39,9 @@ const main = async () => { () => {}, ) } + if (process.argv.includes('--apk')) { + await primeApkCache() + } if (data?.AllPokeApi) { const { baseStats, tempEvos, types } = data.AllPokeApi fs.writeFile( diff --git a/package.json b/package.json index 4352c4c..c41776e 100644 --- a/package.json +++ b/package.json @@ -17,12 +17,23 @@ ], "main": "dist/index.js", "types": "dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "./node": { + "types": "./dist/node.d.ts", + "default": "./dist/node.js" + } + }, "scripts": { "start": "node .", "generate": "ts-node devWrapper.ts --test", "pokeapi": "ts-node devWrapper.ts --test --pokeapi", "raw": "ts-node devWrapper.ts --test --raw", "invasions": "ts-node devWrapper.ts --test --invasions", + "apk": "ts-node devWrapper.ts --test --apk", "test": "tsc && ./node_modules/.bin/jest", "format": "biome format --write ./src/**/*.ts", "publishBuild": "rm -r dist && tsc" diff --git a/src/classes/Apk.ts b/src/classes/Apk.ts index 79f0ba5..3b98893 100644 --- a/src/classes/Apk.ts +++ b/src/classes/Apk.ts @@ -1,35 +1,13 @@ -import { mkdir, readFile, writeFile } from 'node:fs/promises' -import path from 'node:path' import JSZip from 'jszip' - -interface ApkTextCache { - apkFilename: string - texts: Record> -} - -const isApkTextCache = (value: unknown): value is ApkTextCache => { - if (!value || typeof value !== 'object') { - return false - } - const candidate = value as { - apkFilename?: unknown - texts?: unknown - } - return ( - typeof candidate.apkFilename === 'string' && - !!candidate.texts && - typeof candidate.texts === 'object' - ) -} +import type { ApkTexts } from '../typings/inputs' export default class ApkReader { - texts: Record> + texts: ApkTexts codeMap: Record files: JSZip | null - cachePath: string apkFilename: string | null - constructor(cachePath?: string) { + constructor() { this.texts = {} this.codeMap = { 'pt-br': 'pt-br', @@ -49,7 +27,6 @@ export default class ApkReader { 'tr-tr': 'tr', } this.files = null - this.cachePath = path.resolve(cachePath || '.cache/apk-texts.json') this.apkFilename = null } @@ -93,61 +70,6 @@ export default class ApkReader { } } - async loadCachedTexts(expectedFilename?: string) { - try { - const cached = await readFile(this.cachePath, 'utf8') - const parsed: unknown = JSON.parse(cached) - if (!isApkTextCache(parsed)) { - return false - } - if (expectedFilename && parsed.apkFilename !== expectedFilename) { - return false - } - this.apkFilename = parsed.apkFilename - this.texts = parsed.texts - return true - } catch { - return false - } - } - - async writeCachedTexts() { - try { - await mkdir(path.dirname(this.cachePath), { recursive: true }) - if (!this.apkFilename) { - throw new Error('Missing APK filename for cache write') - } - const payload: ApkTextCache = { - apkFilename: this.apkFilename, - texts: this.texts, - } - await writeFile(this.cachePath, JSON.stringify(payload), 'utf8') - } catch (e) { - console.warn(e, 'Issue with writing APK text cache') - } - } - - async primeCache(force = false) { - const latestFilename = await this.getLatestApkFilename() - - if (!latestFilename) { - if (await this.loadCachedTexts()) { - return this.texts - } - return this.texts - } - - if (!force && (await this.loadCachedTexts(latestFilename))) { - return this.texts - } - - this.texts = {} - await this.fetchApk(latestFilename) - await this.extractTexts() - this.cleanup() - return this.texts - } - async extractTexts() { if (!this.files) return try { @@ -180,7 +102,6 @@ export default class ApkReader { } }), ) - await this.writeCachedTexts() } catch (e) { console.warn(e, 'Issue with extracting texts') } diff --git a/src/index.ts b/src/index.ts index 987a8e5..20fee47 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,21 +15,41 @@ import Weather from './classes/Weather' import type { AllInvasions, FinalResult } from './typings/dataTypes' import type { NiaMfObj } from './typings/general' import type { + ApkTexts, Input, InvasionsOnly, Locales, PokemonTemplate, - PrimeApkCacheInput, TranslationsTemplate, } from './typings/inputs' import type { InvasionInfo } from './typings/pogoinfo' -export async function primeApkCache({ - force, - apkCachePath, -}: PrimeApkCacheInput = {}) { - const apk = new ApkReader(apkCachePath) - return apk.primeCache(force) +async function getApkTexts( + apk: ApkReader, + apkCache?: Input['apkCache'], +): Promise { + const latestFilename = await apk.getLatestApkFilename() + + if (!latestFilename) { + return {} + } + + if (apkCache) { + const cached = await apkCache.load(latestFilename) + if (cached) { + return cached + } + } + + await apk.fetchApk(latestFilename) + await apk.extractTexts() + apk.cleanup() + + if (apkCache && apk.apkFilename) { + await apkCache.save(apk.apkFilename, apk.texts) + } + + return apk.texts } export async function generate({ @@ -39,7 +59,7 @@ export async function generate({ translationRemoteUrl, raw, pokeApi, - apkCachePath, + apkCache, test, pokeApiBaseUrl, }: Input = {}): Promise { @@ -82,7 +102,7 @@ export async function generate({ const AllPokeApi = new PokeApi(pokeApiBaseUrl) const AllMisc = new Misc() const AllLocationCards = new LocationCards(locationCards.options) - const apk = new ApkReader(apkCachePath) + const apk = new ApkReader() const enabledLocales = Object.values(translations.locales).filter(Boolean) AllMisc.parseRaidLevels() @@ -95,7 +115,7 @@ export async function generate({ } if (translations.enabled && enabledLocales.length > 0) { - AllTranslations.fromApk = await apk.primeCache() + AllTranslations.fromApk = await getApkTexts(apk, apkCache) } const data: NiaMfObj[] = await AllPokemon.fetch(urlToFetch) diff --git a/src/node.ts b/src/node.ts new file mode 100644 index 0000000..7d72b1a --- /dev/null +++ b/src/node.ts @@ -0,0 +1,93 @@ +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import path from 'node:path' +import ApkReader from './classes/Apk' +import type { + ApkCacheAdapter, + ApkTexts, + NodeApkCacheInput, + PrimeApkCacheInput, +} from './typings/inputs' + +interface ApkTextCacheFile { + apkFilename: string + texts: ApkTexts +} + +const isApkTextCacheFile = (value: unknown): value is ApkTextCacheFile => { + if (!value || typeof value !== 'object') { + return false + } + const candidate = value as { + apkFilename?: unknown + texts?: unknown + } + return ( + typeof candidate.apkFilename === 'string' && + !!candidate.texts && + typeof candidate.texts === 'object' + ) +} + +export function createNodeApkCache({ + apkCachePath, +}: NodeApkCacheInput = {}): ApkCacheAdapter { + const resolvedPath = path.resolve(apkCachePath || '.cache/apk-texts.json') + + return { + async load(expectedFilename?: string) { + try { + const cached = await readFile(resolvedPath, 'utf8') + const parsed: unknown = JSON.parse(cached) + if (!isApkTextCacheFile(parsed)) { + return null + } + if (expectedFilename && parsed.apkFilename !== expectedFilename) { + return null + } + return parsed.texts + } catch { + return null + } + }, + async save(apkFilename: string, texts: ApkTexts) { + try { + await mkdir(path.dirname(resolvedPath), { recursive: true }) + const payload: ApkTextCacheFile = { + apkFilename, + texts, + } + await writeFile(resolvedPath, JSON.stringify(payload), 'utf8') + } catch (e) { + console.warn(e, 'Issue with writing APK text cache') + } + }, + } +} + +export async function primeApkCache({ + force, + apkCachePath, +}: PrimeApkCacheInput = {}) { + const apk = new ApkReader() + const apkCache = createNodeApkCache({ apkCachePath }) + const latestFilename = await apk.getLatestApkFilename() + + if (!latestFilename) { + return (await apkCache.load()) || {} + } + + if (!force) { + const cached = await apkCache.load(latestFilename) + if (cached) { + return cached + } + } + + await apk.fetchApk(latestFilename) + await apk.extractTexts() + apk.cleanup() + if (apk.apkFilename) { + await apkCache.save(apk.apkFilename, apk.texts) + } + return apk.texts +} diff --git a/src/typings/inputs.ts b/src/typings/inputs.ts index 7e5a549..901ec68 100644 --- a/src/typings/inputs.ts +++ b/src/typings/inputs.ts @@ -364,6 +364,19 @@ export interface TranslationsTemplate { quests?: boolean } +export interface ApkLocaleTexts { + [key: string]: string +} + +export interface ApkTexts { + [locale: string]: ApkLocaleTexts +} + +export interface ApkCacheAdapter { + load(expectedFilename?: string): Promise + save(apkFilename: string, texts: ApkTexts): Promise +} + export interface Input { url?: string translationApkUrl?: string @@ -371,7 +384,7 @@ export interface Input { template?: FullTemplate test?: boolean raw?: boolean - apkCachePath?: string + apkCache?: ApkCacheAdapter pokeApi?: boolean | PokeApi pokeApiBaseUrl?: string } @@ -381,6 +394,10 @@ export interface PrimeApkCacheInput { apkCachePath?: string } +export interface NodeApkCacheInput { + apkCachePath?: string +} + export interface FullTemplate { globalOptions?: { keyJoiner?: string From 8e87a3b0f08dbe094cd1720be25fad4fa6b07a30 Mon Sep 17 00:00:00 2001 From: Rin <58572875+TurtIeSocks@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:01:03 -0400 Subject: [PATCH 5/7] refactor: use os temp dir --- README.md | 4 ++-- src/node.ts | 21 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 988b481..c5c814a 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ const data = await generate() // returns the default settings const apkCache = createNodeApkCache() await primeApkCache() -// downloads the APK texts once and saves them to .cache/apk-texts.json in Node environments +// downloads the APK texts once and saves them to the OS cache directory in Node environments const cachedData = await generate({ apkCache }) // automatically reuses the primed APK cache when it matches the latest APK filename @@ -149,7 +149,7 @@ Node-only cache helpers are exported from `pogo-data-generator/node`: - `createNodeApkCache(options?)`: Creates a filesystem-backed APK cache adapter for server environments - `force` (boolean): Rebuilds the APK cache even if the cache already matches the latest APK filename -- `apkCachePath` (string): Writes the APK cache to a custom file path +- `apkCachePath` (string): Writes the APK cache to a custom file path instead of the OS cache directory To view some static examples of what this library can create, check out these repos: [Masterfiles](https://github.com/WatWowMap/Masterfile-Generator) diff --git a/src/node.ts b/src/node.ts index 7d72b1a..fce75e1 100644 --- a/src/node.ts +++ b/src/node.ts @@ -1,4 +1,5 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises' +import os from 'node:os' import path from 'node:path' import ApkReader from './classes/Apk' import type { @@ -28,10 +29,28 @@ const isApkTextCacheFile = (value: unknown): value is ApkTextCacheFile => { ) } +const defaultNodeApkCachePath = () => { + if (process.platform === 'darwin') { + return path.join(os.homedir(), 'Library', 'Caches', 'pogo-data-generator') + } + if (process.platform === 'win32') { + return path.join( + process.env.LOCALAPPDATA || path.join(os.homedir(), 'AppData', 'Local'), + 'pogo-data-generator', + ) + } + return path.join( + process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache'), + 'pogo-data-generator', + ) +} + export function createNodeApkCache({ apkCachePath, }: NodeApkCacheInput = {}): ApkCacheAdapter { - const resolvedPath = path.resolve(apkCachePath || '.cache/apk-texts.json') + const resolvedPath = path.resolve( + apkCachePath || path.join(defaultNodeApkCachePath(), 'apk-texts.json'), + ) return { async load(expectedFilename?: string) { From 17fa76f59a1617a71e8fb06eee6032821086f80a Mon Sep 17 00:00:00 2001 From: Rin <58572875+TurtIeSocks@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:02:58 -0400 Subject: [PATCH 6/7] chore: persist apk cache for workflows --- .github/workflows/publish.yml | 16 ++++++++++++++++ .github/workflows/test.yml | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bdd423f..87b6042 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,6 +25,14 @@ jobs: node-version: 24 cache: 'yarn' + - name: Restore APK Cache + uses: actions/cache@v4 + with: + path: ~/.cache/pogo-data-generator + key: ${{ runner.os }}-apk-cache-v1-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-apk-cache-v1- + - name: Install Dependencies run: yarn @@ -46,6 +54,14 @@ jobs: with: node-version: 24 registry-url: https://registry.npmjs.org/ + + - name: Restore APK Cache + uses: actions/cache@v4 + with: + path: ~/.cache/pogo-data-generator + key: ${{ runner.os }}-apk-cache-v1-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-apk-cache-v1- # Ensure npm 11.5.1 or later is installed - name: Update npm run: npm install -g npm@latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8a7853c..c868c01 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,14 @@ jobs: node-version: 24 cache: 'yarn' + - name: Restore APK Cache + uses: actions/cache@v4 + with: + path: ~/.cache/pogo-data-generator + key: ${{ runner.os }}-apk-cache-v1-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-apk-cache-v1- + - name: Install Dependencies run: yarn From 755754b746218ca79efb0ca8d95f979f04e48b17 Mon Sep 17 00:00:00 2001 From: Mygod Date: Tue, 21 Apr 2026 15:26:02 -0400 Subject: [PATCH 7/7] fix: harden apk cache refreshes Only persist a downloaded APK filename after base.apk loads successfully so failed refreshes do not poison the cache with empty texts. Refresh the CI APK cache with restore/save actions and remove the unused publish workflow restore so later runs can keep updated cache contents. --- .github/workflows/publish.yml | 15 -------- .github/workflows/test.yml | 13 +++++-- src/classes/Apk.ts | 16 ++++++-- tests/apkCache.test.js | 70 +++++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 22 deletions(-) create mode 100644 tests/apkCache.test.js diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 87b6042..28c3c02 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,14 +25,6 @@ jobs: node-version: 24 cache: 'yarn' - - name: Restore APK Cache - uses: actions/cache@v4 - with: - path: ~/.cache/pogo-data-generator - key: ${{ runner.os }}-apk-cache-v1-${{ hashFiles('yarn.lock') }} - restore-keys: | - ${{ runner.os }}-apk-cache-v1- - - name: Install Dependencies run: yarn @@ -55,13 +47,6 @@ jobs: node-version: 24 registry-url: https://registry.npmjs.org/ - - name: Restore APK Cache - uses: actions/cache@v4 - with: - path: ~/.cache/pogo-data-generator - key: ${{ runner.os }}-apk-cache-v1-${{ hashFiles('yarn.lock') }} - restore-keys: | - ${{ runner.os }}-apk-cache-v1- # Ensure npm 11.5.1 or later is installed - name: Update npm run: npm install -g npm@latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c868c01..e8e51b5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,12 +17,13 @@ jobs: cache: 'yarn' - name: Restore APK Cache - uses: actions/cache@v4 + id: apk-cache + uses: actions/cache/restore@v4 with: path: ~/.cache/pogo-data-generator - key: ${{ runner.os }}-apk-cache-v1-${{ hashFiles('yarn.lock') }} + key: ${{ runner.os }}-apk-cache-v1-${{ hashFiles('yarn.lock') }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | - ${{ runner.os }}-apk-cache-v1- + ${{ runner.os }}-apk-cache-v1-${{ hashFiles('yarn.lock') }}- - name: Install Dependencies run: yarn @@ -30,6 +31,12 @@ jobs: - name: Generate Check run: yarn generate + - name: Save APK Cache + uses: actions/cache/save@v4 + with: + path: ~/.cache/pogo-data-generator + key: ${{ steps.apk-cache.outputs.cache-primary-key }} + - name: Invasion Check run: yarn invasions diff --git a/src/classes/Apk.ts b/src/classes/Apk.ts index 3b98893..dce2ab3 100644 --- a/src/classes/Apk.ts +++ b/src/classes/Apk.ts @@ -50,21 +50,29 @@ export default class ApkReader { } async fetchApk(filename?: string) { + this.apkFilename = null + this.files = null + try { const first = filename || (await this.getLatestApkFilename()) if (!first) { throw new Error('Unable to determine latest APK filename') } - this.apkFilename = first const response = await fetch(`https://mirror.unownhash.com/apks/${first}`) + if (!response.ok) { + throw new Error('Unable to fetch APK') + } const apk = await response.arrayBuffer() - const zip = new JSZip() - const raw = await zip.loadAsync(apk) + const raw = await new JSZip().loadAsync(apk) const file = raw.files['base.apk'] + if (!file) { + throw new Error('Missing base.apk in APK bundle') + } const buffer = await file.async('nodebuffer') - this.files = await zip.loadAsync(buffer) + this.files = await new JSZip().loadAsync(buffer) + this.apkFilename = first } catch (e) { console.warn(e, 'Issue with downloading APK') } diff --git a/tests/apkCache.test.js b/tests/apkCache.test.js new file mode 100644 index 0000000..8e71a0e --- /dev/null +++ b/tests/apkCache.test.js @@ -0,0 +1,70 @@ +const fs = require('node:fs/promises') +const os = require('node:os') +const path = require('node:path') +const JSZip = require('jszip') + +const ApkReader = require('../dist/classes/Apk').default +const { createNodeApkCache, primeApkCache } = require('../dist/node') + +const makeBrokenOuterApk = async () => { + const apk = new JSZip() + apk.file('assets/placeholder.txt', 'broken') + return apk.generateAsync({ type: 'nodebuffer' }) +} + +describe('APK cache refresh safety', () => { + const originalFetch = global.fetch + + afterEach(() => { + global.fetch = originalFetch + jest.restoreAllMocks() + }) + + test('does not keep the latest filename when the APK bundle is malformed', async () => { + const brokenApk = await makeBrokenOuterApk() + jest.spyOn(console, 'warn').mockImplementation(() => {}) + global.fetch = jest.fn().mockResolvedValue({ + ok: true, + arrayBuffer: async () => brokenApk, + }) + + const apk = new ApkReader() + await apk.fetchApk('latest.apk') + + expect(apk.apkFilename).toBeNull() + expect(apk.files).toBeNull() + }) + + test('does not overwrite the persisted cache after a failed refresh', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'pogo-apk-cache-')) + const apkCachePath = path.join(tempDir, 'apk-texts.json') + const cachedTexts = { en: { greeting: 'hello' } } + const brokenApk = await makeBrokenOuterApk() + jest.spyOn(console, 'warn').mockImplementation(() => {}) + const apkCache = createNodeApkCache({ apkCachePath }) + await apkCache.save('cached.apk', cachedTexts) + + global.fetch = jest.fn().mockImplementation(async (url) => { + if (url === 'https://mirror.unownhash.com/index.json') { + return { + ok: true, + json: async () => [{ filename: 'latest.apk' }], + } + } + if (url === 'https://mirror.unownhash.com/apks/latest.apk') { + return { + ok: true, + arrayBuffer: async () => brokenApk, + } + } + throw new Error(`Unexpected fetch: ${url}`) + }) + + const texts = await primeApkCache({ apkCachePath }) + + expect(texts).toEqual({}) + expect(await apkCache.load()).toEqual(cachedTexts) + expect(await apkCache.load('cached.apk')).toEqual(cachedTexts) + expect(await apkCache.load('latest.apk')).toBeNull() + }) +})