diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bdd423f..28c3c02 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -46,6 +46,7 @@ jobs: with: node-version: 24 registry-url: https://registry.npmjs.org/ + # Ensure npm 11.5.1 or later is installed - name: Update npm run: npm install -g npm@latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8a7853c..e8e51b5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,12 +16,27 @@ jobs: node-version: 24 cache: 'yarn' + - name: Restore APK Cache + id: apk-cache + uses: actions/cache/restore@v4 + with: + path: ~/.cache/pogo-data-generator + key: ${{ runner.os }}-apk-cache-v1-${{ hashFiles('yarn.lock') }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + ${{ runner.os }}-apk-cache-v1-${{ hashFiles('yarn.lock') }}- + - name: Install Dependencies run: yarn - name: Generate Check run: yarn generate + - name: Save APK Cache + uses: actions/cache/save@v4 + with: + path: ~/.cache/pogo-data-generator + key: ${{ steps.apk-cache.outputs.cache-primary-key }} + - name: Invasion Check run: yarn invasions diff --git a/.gitignore b/.gitignore index 82ca258..af8ac51 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ master-latest.json .prettierrc .DS_Store invasions.json -latest.json \ No newline at end of file +latest.json +.cache/ diff --git a/README.md b/README.md index e622eed..c5c814a 100644 --- a/README.md +++ b/README.md @@ -40,11 +40,21 @@ Usage: ```js // commonJS const { generate } = require('pogo-data-generator') -// es6 with invasion function +const { createNodeApkCache, primeApkCache } = require('pogo-data-generator/node') +// es6 with exported helpers import { generate, invasions } from 'pogo-data-generator' +import { createNodeApkCache, primeApkCache } from 'pogo-data-generator/node' const data = await generate() // returns the default settings +const apkCache = createNodeApkCache() + +await primeApkCache() +// downloads the APK texts once and saves them to the OS cache directory in Node environments + +const cachedData = await generate({ apkCache }) +// automatically reuses the primed APK cache when it matches the latest APK filename + const template = { pokemon: { enabled: true, @@ -131,9 +141,16 @@ The generate function accepts an object with the following properties: - `url` (string): Custom url to fetch the masterfile from, results not guaranteed - `test` (boolean): Writes the masterfile to a local json - `raw` (boolean): Returns the data in its raw format without any template processing +- `apkCache` (object): Optional server-side cache adapter for APK texts - `pokeApi` (boolean): Fetches fresh data from PokeAPI - `pokeApiBaseUrl` (string): Overrides the default PokeAPI endpoint (defaults to `https://pokeapi.co/api/v2`) +Node-only cache helpers are exported from `pogo-data-generator/node`: + +- `createNodeApkCache(options?)`: Creates a filesystem-backed APK cache adapter for server environments +- `force` (boolean): Rebuilds the APK cache even if the cache already matches the latest APK filename +- `apkCachePath` (string): Writes the APK cache to a custom file path instead of the OS cache directory + To view some static examples of what this library can create, check out these repos: [Masterfiles](https://github.com/WatWowMap/Masterfile-Generator) [Translations](https://github.com/WatWowMap/pogo-translations) diff --git a/devWrapper.ts b/devWrapper.ts index cf8ca29..1b3e54f 100644 --- a/devWrapper.ts +++ b/devWrapper.ts @@ -1,22 +1,25 @@ import * as fs from 'fs' import { generate, invasions } from './src/index' +import { createNodeApkCache, primeApkCache } from './src/node' import baseStats from './static/baseStats.json' import tempEvos from './static/tempEvos.json' import types from './static/types.json' const main = async () => { const mfData = await fetch( - 'https://raw.githubusercontent.com/PokeMiners/game_masters/master/latest/latest.json', + 'https://raw.githubusercontent.com/alexelgt/game_masters/refs/heads/master/GAME_MASTER.json', ) const mf = await mfData.json() fs.writeFileSync('./latest.json', JSON.stringify(mf, null, 2), 'utf8') const usePokeApiStaging = process.argv.includes('--pokeapi-staging') const usePokeApi = usePokeApiStaging || process.argv.includes('--pokeapi') + const apkCache = createNodeApkCache() console.time('Generated in') const data = await generate({ raw: process.argv.includes('--raw'), test: process.argv.includes('--test'), + apkCache, pokeApi: usePokeApi || { baseStats, tempEvos, @@ -36,6 +39,9 @@ const main = async () => { () => {}, ) } + if (process.argv.includes('--apk')) { + await primeApkCache() + } if (data?.AllPokeApi) { const { baseStats, tempEvos, types } = data.AllPokeApi fs.writeFile( diff --git a/package.json b/package.json index 4352c4c..c41776e 100644 --- a/package.json +++ b/package.json @@ -17,12 +17,23 @@ ], "main": "dist/index.js", "types": "dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "./node": { + "types": "./dist/node.d.ts", + "default": "./dist/node.js" + } + }, "scripts": { "start": "node .", "generate": "ts-node devWrapper.ts --test", "pokeapi": "ts-node devWrapper.ts --test --pokeapi", "raw": "ts-node devWrapper.ts --test --raw", "invasions": "ts-node devWrapper.ts --test --invasions", + "apk": "ts-node devWrapper.ts --test --apk", "test": "tsc && ./node_modules/.bin/jest", "format": "biome format --write ./src/**/*.ts", "publishBuild": "rm -r dist && tsc" diff --git a/src/classes/Apk.ts b/src/classes/Apk.ts index 44d9bf3..dce2ab3 100644 --- a/src/classes/Apk.ts +++ b/src/classes/Apk.ts @@ -1,9 +1,11 @@ import JSZip from 'jszip' +import type { ApkTexts } from '../typings/inputs' export default class ApkReader { - texts: Record> + texts: ApkTexts codeMap: Record files: JSZip | null + apkFilename: string | null constructor() { this.texts = {} @@ -25,13 +27,14 @@ export default class ApkReader { 'tr-tr': 'tr', } this.files = null + this.apkFilename = null } removeEscapes(str: string) { return str.replace(/\r/g, '').replace(/\n/g, '').replace(/"/g, '”') } - async fetchApk() { + async getLatestApkFilename() { try { const index = await fetch('https://mirror.unownhash.com/index.json') @@ -39,15 +42,37 @@ export default class ApkReader { throw new Error('Unable to fetch index') } const data = await index.json() - const first = data[0].filename + return data[0].filename as string + } catch (e) { + console.warn(e, 'Issue with downloading APK index') + return null + } + } + + async fetchApk(filename?: string) { + this.apkFilename = null + this.files = null + + try { + const first = filename || (await this.getLatestApkFilename()) + + if (!first) { + throw new Error('Unable to determine latest APK filename') + } const response = await fetch(`https://mirror.unownhash.com/apks/${first}`) + if (!response.ok) { + throw new Error('Unable to fetch APK') + } const apk = await response.arrayBuffer() - const zip = new JSZip() - const raw = await zip.loadAsync(apk) + const raw = await new JSZip().loadAsync(apk) const file = raw.files['base.apk'] + if (!file) { + throw new Error('Missing base.apk in APK bundle') + } const buffer = await file.async('nodebuffer') - this.files = await zip.loadAsync(buffer) + this.files = await new JSZip().loadAsync(buffer) + this.apkFilename = first } catch (e) { console.warn(e, 'Issue with downloading APK') } diff --git a/src/classes/Invasion.ts b/src/classes/Invasion.ts index f4e4b58..3780fc3 100644 --- a/src/classes/Invasion.ts +++ b/src/classes/Invasion.ts @@ -22,7 +22,7 @@ export default class Invasion extends Masterfile { (this.options.customInvasions === undefined && override) ) { return this.fetch( - 'https://raw.githubusercontent.com/WatWowMap/Masterfile-Generator/master/custom-invasions.json', + 'https://raw.githubusercontent.com/WatWowMap/Masterfile-Generator/refs/heads/master/custom-invasions.json', ) } else if (this.options.customInvasions) { return this.options.customInvasions as InvasionInfo diff --git a/src/index.ts b/src/index.ts index fd5532a..20fee47 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,6 +15,7 @@ import Weather from './classes/Weather' import type { AllInvasions, FinalResult } from './typings/dataTypes' import type { NiaMfObj } from './typings/general' import type { + ApkTexts, Input, InvasionsOnly, Locales, @@ -23,6 +24,34 @@ import type { } from './typings/inputs' import type { InvasionInfo } from './typings/pogoinfo' +async function getApkTexts( + apk: ApkReader, + apkCache?: Input['apkCache'], +): Promise { + const latestFilename = await apk.getLatestApkFilename() + + if (!latestFilename) { + return {} + } + + if (apkCache) { + const cached = await apkCache.load(latestFilename) + if (cached) { + return cached + } + } + + await apk.fetchApk(latestFilename) + await apk.extractTexts() + apk.cleanup() + + if (apkCache && apk.apkFilename) { + await apkCache.save(apk.apkFilename, apk.texts) + } + + return apk.texts +} + export async function generate({ template, url, @@ -30,6 +59,7 @@ export async function generate({ translationRemoteUrl, raw, pokeApi, + apkCache, test, pokeApiBaseUrl, }: Input = {}): Promise { @@ -70,21 +100,23 @@ export async function generate({ translationRemoteUrl, ) const AllPokeApi = new PokeApi(pokeApiBaseUrl) - await AllPokeApi.setMaxPokemonId() - const generations = await AllPokeApi.getGenerations() - AllPokemon.generations = generations const AllMisc = new Misc() const AllLocationCards = new LocationCards(locationCards.options) const apk = new ApkReader() + const enabledLocales = Object.values(translations.locales).filter(Boolean) AllMisc.parseRaidLevels() AllMisc.parseRouteTypes() AllMisc.parseTeams() - await apk.fetchApk() - await apk.extractTexts() - apk.cleanup() - AllTranslations.fromApk = apk.texts + if (pokeApi === true) { + await AllPokeApi.setMaxPokemonId() + AllPokemon.generations = await AllPokeApi.getGenerations() + } + + if (translations.enabled && enabledLocales.length > 0) { + AllTranslations.fromApk = await getApkTexts(apk, apkCache) + } const data: NiaMfObj[] = await AllPokemon.fetch(urlToFetch) @@ -384,7 +416,7 @@ export async function generate({ if (pokeApi === true) return AllPokeApi[category] if (pokeApi) return pokeApi[category] return AllPokeApi.fetch( - `https://raw.githubusercontent.com/WatWowMap/Pogo-Data-Generator/main/static/${category}.json`, + `https://raw.githubusercontent.com/WatWowMap/Pogo-Data-Generator/refs/heads/main/static/${category}.json`, ) } @@ -418,7 +450,7 @@ export async function generate({ (translations.template as TranslationsTemplate).characters ) { const invasionData: InvasionInfo = await AllInvasions.fetch( - 'https://raw.githubusercontent.com/WatWowMap/event-info/main/grunts/classic.json', + 'https://raw.githubusercontent.com/WatWowMap/event-info/refs/heads/main/grunts/classic.json', ) AllInvasions.invasions( AllInvasions.mergeInvasions( @@ -430,7 +462,7 @@ export async function generate({ if (translations.enabled) { const availableManualTranslations = await AllTranslations.fetch( - 'https://raw.githubusercontent.com/WatWowMap/pogo-translations/master/index.json', + 'https://raw.githubusercontent.com/WatWowMap/pogo-translations/refs/heads/master/index.json', ) await Promise.all( Object.entries(translations.locales).map(async (langCode) => { @@ -666,7 +698,7 @@ export async function invasions({ const finalTemplate = template || base.invasions const AllInvasions = new Invasions(finalTemplate.options) const invasionData: InvasionInfo = await AllInvasions.fetch( - 'https://raw.githubusercontent.com/WatWowMap/event-info/main/grunts/classic.json', + 'https://raw.githubusercontent.com/WatWowMap/event-info/refs/heads/main/grunts/classic.json', ) AllInvasions.invasions( AllInvasions.mergeInvasions( diff --git a/src/node.ts b/src/node.ts new file mode 100644 index 0000000..fce75e1 --- /dev/null +++ b/src/node.ts @@ -0,0 +1,112 @@ +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import ApkReader from './classes/Apk' +import type { + ApkCacheAdapter, + ApkTexts, + NodeApkCacheInput, + PrimeApkCacheInput, +} from './typings/inputs' + +interface ApkTextCacheFile { + apkFilename: string + texts: ApkTexts +} + +const isApkTextCacheFile = (value: unknown): value is ApkTextCacheFile => { + if (!value || typeof value !== 'object') { + return false + } + const candidate = value as { + apkFilename?: unknown + texts?: unknown + } + return ( + typeof candidate.apkFilename === 'string' && + !!candidate.texts && + typeof candidate.texts === 'object' + ) +} + +const defaultNodeApkCachePath = () => { + if (process.platform === 'darwin') { + return path.join(os.homedir(), 'Library', 'Caches', 'pogo-data-generator') + } + if (process.platform === 'win32') { + return path.join( + process.env.LOCALAPPDATA || path.join(os.homedir(), 'AppData', 'Local'), + 'pogo-data-generator', + ) + } + return path.join( + process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache'), + 'pogo-data-generator', + ) +} + +export function createNodeApkCache({ + apkCachePath, +}: NodeApkCacheInput = {}): ApkCacheAdapter { + const resolvedPath = path.resolve( + apkCachePath || path.join(defaultNodeApkCachePath(), 'apk-texts.json'), + ) + + return { + async load(expectedFilename?: string) { + try { + const cached = await readFile(resolvedPath, 'utf8') + const parsed: unknown = JSON.parse(cached) + if (!isApkTextCacheFile(parsed)) { + return null + } + if (expectedFilename && parsed.apkFilename !== expectedFilename) { + return null + } + return parsed.texts + } catch { + return null + } + }, + async save(apkFilename: string, texts: ApkTexts) { + try { + await mkdir(path.dirname(resolvedPath), { recursive: true }) + const payload: ApkTextCacheFile = { + apkFilename, + texts, + } + await writeFile(resolvedPath, JSON.stringify(payload), 'utf8') + } catch (e) { + console.warn(e, 'Issue with writing APK text cache') + } + }, + } +} + +export async function primeApkCache({ + force, + apkCachePath, +}: PrimeApkCacheInput = {}) { + const apk = new ApkReader() + const apkCache = createNodeApkCache({ apkCachePath }) + const latestFilename = await apk.getLatestApkFilename() + + if (!latestFilename) { + return (await apkCache.load()) || {} + } + + if (!force) { + const cached = await apkCache.load(latestFilename) + if (cached) { + return cached + } + } + + await apk.fetchApk(latestFilename) + await apk.extractTexts() + apk.cleanup() + if (apk.apkFilename) { + await apkCache.save(apk.apkFilename, apk.texts) + } + return apk.texts +} diff --git a/src/typings/inputs.ts b/src/typings/inputs.ts index a1cc7eb..901ec68 100644 --- a/src/typings/inputs.ts +++ b/src/typings/inputs.ts @@ -364,6 +364,19 @@ export interface TranslationsTemplate { quests?: boolean } +export interface ApkLocaleTexts { + [key: string]: string +} + +export interface ApkTexts { + [locale: string]: ApkLocaleTexts +} + +export interface ApkCacheAdapter { + load(expectedFilename?: string): Promise + save(apkFilename: string, texts: ApkTexts): Promise +} + export interface Input { url?: string translationApkUrl?: string @@ -371,10 +384,20 @@ export interface Input { template?: FullTemplate test?: boolean raw?: boolean + apkCache?: ApkCacheAdapter pokeApi?: boolean | PokeApi pokeApiBaseUrl?: string } +export interface PrimeApkCacheInput { + force?: boolean + apkCachePath?: string +} + +export interface NodeApkCacheInput { + apkCachePath?: string +} + export interface FullTemplate { globalOptions?: { keyJoiner?: string diff --git a/tests/apkCache.test.js b/tests/apkCache.test.js new file mode 100644 index 0000000..8e71a0e --- /dev/null +++ b/tests/apkCache.test.js @@ -0,0 +1,70 @@ +const fs = require('node:fs/promises') +const os = require('node:os') +const path = require('node:path') +const JSZip = require('jszip') + +const ApkReader = require('../dist/classes/Apk').default +const { createNodeApkCache, primeApkCache } = require('../dist/node') + +const makeBrokenOuterApk = async () => { + const apk = new JSZip() + apk.file('assets/placeholder.txt', 'broken') + return apk.generateAsync({ type: 'nodebuffer' }) +} + +describe('APK cache refresh safety', () => { + const originalFetch = global.fetch + + afterEach(() => { + global.fetch = originalFetch + jest.restoreAllMocks() + }) + + test('does not keep the latest filename when the APK bundle is malformed', async () => { + const brokenApk = await makeBrokenOuterApk() + jest.spyOn(console, 'warn').mockImplementation(() => {}) + global.fetch = jest.fn().mockResolvedValue({ + ok: true, + arrayBuffer: async () => brokenApk, + }) + + const apk = new ApkReader() + await apk.fetchApk('latest.apk') + + expect(apk.apkFilename).toBeNull() + expect(apk.files).toBeNull() + }) + + test('does not overwrite the persisted cache after a failed refresh', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'pogo-apk-cache-')) + const apkCachePath = path.join(tempDir, 'apk-texts.json') + const cachedTexts = { en: { greeting: 'hello' } } + const brokenApk = await makeBrokenOuterApk() + jest.spyOn(console, 'warn').mockImplementation(() => {}) + const apkCache = createNodeApkCache({ apkCachePath }) + await apkCache.save('cached.apk', cachedTexts) + + global.fetch = jest.fn().mockImplementation(async (url) => { + if (url === 'https://mirror.unownhash.com/index.json') { + return { + ok: true, + json: async () => [{ filename: 'latest.apk' }], + } + } + if (url === 'https://mirror.unownhash.com/apks/latest.apk') { + return { + ok: true, + arrayBuffer: async () => brokenApk, + } + } + throw new Error(`Unexpected fetch: ${url}`) + }) + + const texts = await primeApkCache({ apkCachePath }) + + expect(texts).toEqual({}) + expect(await apkCache.load()).toEqual(cachedTexts) + expect(await apkCache.load('cached.apk')).toEqual(cachedTexts) + expect(await apkCache.load('latest.apk')).toBeNull() + }) +})