Cache APK in Server Envs

.github/workflows/publish.yml

@@ -46,6 +46,7 @@ jobs:
         with:
           node-version: 24
           registry-url: https://registry.npmjs.org/
+
       # Ensure npm 11.5.1 or later is installed
       - name: Update npm
         run: npm install -g npm@latest

.github/workflows/test.yml

@@ -16,12 +16,27 @@ jobs:
           node-version: 24
           cache: 'yarn'
 
+      - name: Restore APK Cache
+        id: apk-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: ~/.cache/pogo-data-generator
+          key: ${{ runner.os }}-apk-cache-v1-${{ hashFiles('yarn.lock') }}-${{ github.run_id }}-${{ github.run_attempt }}
+          restore-keys: |
+            ${{ runner.os }}-apk-cache-v1-${{ hashFiles('yarn.lock') }}-
+
       - name: Install Dependencies
         run: yarn
 
       - name: Generate Check
         run: yarn generate
 
+      - name: Save APK Cache
+        uses: actions/cache/save@v4
+        with:
+          path: ~/.cache/pogo-data-generator
+          key: ${{ steps.apk-cache.outputs.cache-primary-key }}
+
       - name: Invasion Check
         run: yarn invasions
 

.gitignore

@@ -5,4 +5,5 @@ master-latest.json
 .prettierrc
 .DS_Store
 invasions.json
-latest.json
+latest.json
+.cache/

README.md

@@ -40,11 +40,21 @@ Usage:
 ```js
 // commonJS
 const { generate } = require('pogo-data-generator')
-// es6 with invasion function
+const { createNodeApkCache, primeApkCache } = require('pogo-data-generator/node')
+// es6 with exported helpers
 import { generate, invasions } from 'pogo-data-generator'
+import { createNodeApkCache, primeApkCache } from 'pogo-data-generator/node'
 
 const data = await generate() // returns the default settings
 
+const apkCache = createNodeApkCache()
+
+await primeApkCache()
+// downloads the APK texts once and saves them to the OS cache directory in Node environments
+
+const cachedData = await generate({ apkCache })
+// automatically reuses the primed APK cache when it matches the latest APK filename
+
 const template = {
   pokemon: {
     enabled: true,
@@ -131,9 +141,16 @@ The generate function accepts an object with the following properties:
 - `url` (string): Custom url to fetch the masterfile from, results not guaranteed
 - `test` (boolean): Writes the masterfile to a local json
 - `raw` (boolean): Returns the data in its raw format without any template processing
+- `apkCache` (object): Optional server-side cache adapter for APK texts
 - `pokeApi` (boolean): Fetches fresh data from PokeAPI
 - `pokeApiBaseUrl` (string): Overrides the default PokeAPI endpoint (defaults to `https://pokeapi.co/api/v2`)
 
+Node-only cache helpers are exported from `pogo-data-generator/node`:
+
+- `createNodeApkCache(options?)`: Creates a filesystem-backed APK cache adapter for server environments
+- `force` (boolean): Rebuilds the APK cache even if the cache already matches the latest APK filename
+- `apkCachePath` (string): Writes the APK cache to a custom file path instead of the OS cache directory
+
 To view some static examples of what this library can create, check out these repos:
 [Masterfiles](https://github.com/WatWowMap/Masterfile-Generator)
 [Translations](https://github.com/WatWowMap/pogo-translations)

devWrapper.ts

@@ -1,22 +1,25 @@
 import * as fs from 'fs'
 import { generate, invasions } from './src/index'
+import { createNodeApkCache, primeApkCache } from './src/node'
 import baseStats from './static/baseStats.json'
 import tempEvos from './static/tempEvos.json'
 import types from './static/types.json'
 
 const main = async () => {
   const mfData = await fetch(
-    'https://raw.githubusercontent.com/PokeMiners/game_masters/master/latest/latest.json',
+    'https://raw.githubusercontent.com/alexelgt/game_masters/refs/heads/master/GAME_MASTER.json',
   )
   const mf = await mfData.json()
   fs.writeFileSync('./latest.json', JSON.stringify(mf, null, 2), 'utf8')
 
   const usePokeApiStaging = process.argv.includes('--pokeapi-staging')
   const usePokeApi = usePokeApiStaging || process.argv.includes('--pokeapi')
+  const apkCache = createNodeApkCache()
   console.time('Generated in')
   const data = await generate({
     raw: process.argv.includes('--raw'),
     test: process.argv.includes('--test'),
+    apkCache,
     pokeApi: usePokeApi || {
       baseStats,
       tempEvos,
@@ -36,6 +39,9 @@ const main = async () => {
         () => {},
       )
     }
+    if (process.argv.includes('--apk')) {
+      await primeApkCache()
+    }
     if (data?.AllPokeApi) {
       const { baseStats, tempEvos, types } = data.AllPokeApi
       fs.writeFile(

package.json

@@ -17,12 +17,23 @@
   ],
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./node": {
+      "types": "./dist/node.d.ts",
+      "default": "./dist/node.js"
+    }
+  },
   "scripts": {
     "start": "node .",
     "generate": "ts-node devWrapper.ts --test",
     "pokeapi": "ts-node devWrapper.ts --test --pokeapi",
     "raw": "ts-node devWrapper.ts --test --raw",
     "invasions": "ts-node devWrapper.ts --test --invasions",
+    "apk": "ts-node devWrapper.ts --test --apk",
     "test": "tsc && ./node_modules/.bin/jest",
     "format": "biome format --write ./src/**/*.ts",
     "publishBuild": "rm -r dist && tsc"

src/classes/Apk.ts

@@ -1,9 +1,11 @@
 import JSZip from 'jszip'
+import type { ApkTexts } from '../typings/inputs'
 
 export default class ApkReader {
-  texts: Record<string, Record<string, string>>
+  texts: ApkTexts
   codeMap: Record<string, string>
   files: JSZip | null
+  apkFilename: string | null
 
   constructor() {
     this.texts = {}
@@ -25,29 +27,52 @@ export default class ApkReader {
       'tr-tr': 'tr',
     }
     this.files = null
+    this.apkFilename = null
   }
 
   removeEscapes(str: string) {
     return str.replace(/\r/g, '').replace(/\n/g, '').replace(/"/g, '”')
   }
 
-  async fetchApk() {
+  async getLatestApkFilename() {
     try {
       const index = await fetch('https://mirror.unownhash.com/index.json')
 
       if (!index.ok) {
         throw new Error('Unable to fetch index')
       }
       const data = await index.json()
-      const first = data[0].filename
+      return data[0].filename as string
+    } catch (e) {
+      console.warn(e, 'Issue with downloading APK index')
+      return null
+    }
+  }
+
+  async fetchApk(filename?: string) {
+    this.apkFilename = null
+    this.files = null
+
+    try {
+      const first = filename || (await this.getLatestApkFilename())
+
+      if (!first) {
+        throw new Error('Unable to determine latest APK filename')
+      }
 
       const response = await fetch(`https://mirror.unownhash.com/apks/${first}`)
+      if (!response.ok) {
+        throw new Error('Unable to fetch APK')
+      }
       const apk = await response.arrayBuffer()
-      const zip = new JSZip()
-      const raw = await zip.loadAsync(apk)
+      const raw = await new JSZip().loadAsync(apk)
       const file = raw.files['base.apk']
+      if (!file) {
+        throw new Error('Missing base.apk in APK bundle')
+      }
       const buffer = await file.async('nodebuffer')
-      this.files = await zip.loadAsync(buffer)
+      this.files = await new JSZip().loadAsync(buffer)
+      this.apkFilename = first
     } catch (e) {
       console.warn(e, 'Issue with downloading APK')
     }

src/classes/Invasion.ts

@@ -22,7 +22,7 @@ export default class Invasion extends Masterfile {
         (this.options.customInvasions === undefined && override)
       ) {
         return this.fetch(
-          'https://raw.githubusercontent.com/WatWowMap/Masterfile-Generator/master/custom-invasions.json',
+          'https://raw.githubusercontent.com/WatWowMap/Masterfile-Generator/refs/heads/master/custom-invasions.json',
         )
       } else if (this.options.customInvasions) {
         return this.options.customInvasions as InvasionInfo

src/index.ts

@@ -15,6 +15,7 @@ import Weather from './classes/Weather'
 import type { AllInvasions, FinalResult } from './typings/dataTypes'
 import type { NiaMfObj } from './typings/general'
 import type {
+  ApkTexts,
   Input,
   InvasionsOnly,
   Locales,
@@ -23,13 +24,42 @@ import type {
 } from './typings/inputs'
 import type { InvasionInfo } from './typings/pogoinfo'
 
+async function getApkTexts(
+  apk: ApkReader,
+  apkCache?: Input['apkCache'],
+): Promise<ApkTexts> {
+  const latestFilename = await apk.getLatestApkFilename()
+
+  if (!latestFilename) {
+    return {}
+  }
+
+  if (apkCache) {
+    const cached = await apkCache.load(latestFilename)
+    if (cached) {
+      return cached
+    }
+  }
+
+  await apk.fetchApk(latestFilename)
+  await apk.extractTexts()
+  apk.cleanup()
+
+  if (apkCache && apk.apkFilename) {
+    await apkCache.save(apk.apkFilename, apk.texts)
+  }
+
+  return apk.texts
+}
+
 export async function generate({
   template,
   url,
   translationApkUrl,
   translationRemoteUrl,
   raw,
   pokeApi,
+  apkCache,
   test,
   pokeApiBaseUrl,
 }: Input = {}): Promise<FinalResult> {
@@ -70,21 +100,23 @@ export async function generate({
     translationRemoteUrl,
   )
   const AllPokeApi = new PokeApi(pokeApiBaseUrl)
-  await AllPokeApi.setMaxPokemonId()
-  const generations = await AllPokeApi.getGenerations()
-  AllPokemon.generations = generations
   const AllMisc = new Misc()
   const AllLocationCards = new LocationCards(locationCards.options)
   const apk = new ApkReader()
+  const enabledLocales = Object.values(translations.locales).filter(Boolean)
 
   AllMisc.parseRaidLevels()
   AllMisc.parseRouteTypes()
   AllMisc.parseTeams()
 
-  await apk.fetchApk()
-  await apk.extractTexts()
-  apk.cleanup()
-  AllTranslations.fromApk = apk.texts
+  if (pokeApi === true) {
+    await AllPokeApi.setMaxPokemonId()
+    AllPokemon.generations = await AllPokeApi.getGenerations()
+  }
+
+  if (translations.enabled && enabledLocales.length > 0) {
+    AllTranslations.fromApk = await getApkTexts(apk, apkCache)
+  }
 
   const data: NiaMfObj[] = await AllPokemon.fetch(urlToFetch)
 
@@ -384,7 +416,7 @@ export async function generate({
     if (pokeApi === true) return AllPokeApi[category]
     if (pokeApi) return pokeApi[category]
     return AllPokeApi.fetch(
-      `https://raw.githubusercontent.com/WatWowMap/Pogo-Data-Generator/main/static/${category}.json`,
+      `https://raw.githubusercontent.com/WatWowMap/Pogo-Data-Generator/refs/heads/main/static/${category}.json`,
     )
   }
 
@@ -418,7 +450,7 @@ export async function generate({
     (translations.template as TranslationsTemplate).characters
   ) {
     const invasionData: InvasionInfo = await AllInvasions.fetch(
-      'https://raw.githubusercontent.com/WatWowMap/event-info/main/grunts/classic.json',
+      'https://raw.githubusercontent.com/WatWowMap/event-info/refs/heads/main/grunts/classic.json',
     )
     AllInvasions.invasions(
       AllInvasions.mergeInvasions(
@@ -430,7 +462,7 @@ export async function generate({
 
   if (translations.enabled) {
     const availableManualTranslations = await AllTranslations.fetch(
-      'https://raw.githubusercontent.com/WatWowMap/pogo-translations/master/index.json',
+      'https://raw.githubusercontent.com/WatWowMap/pogo-translations/refs/heads/master/index.json',
     )
     await Promise.all(
       Object.entries(translations.locales).map(async (langCode) => {
@@ -666,7 +698,7 @@ export async function invasions({
   const finalTemplate = template || base.invasions
   const AllInvasions = new Invasions(finalTemplate.options)
   const invasionData: InvasionInfo = await AllInvasions.fetch(
-    'https://raw.githubusercontent.com/WatWowMap/event-info/main/grunts/classic.json',
+    'https://raw.githubusercontent.com/WatWowMap/event-info/refs/heads/main/grunts/classic.json',
   )
   AllInvasions.invasions(
     AllInvasions.mergeInvasions(