From 203ec7d578bb4ef86516f3933df5bd06cfaeae47 Mon Sep 17 00:00:00 2001 From: matthiasschalk Date: Sun, 10 May 2026 03:33:49 +0200 Subject: [PATCH 1/3] feat: add updater and speed up dashboard startup --- .github/workflows/release.yml | 2 + .gitignore | 3 + AGENTS.md | 32 +++ package-lock.json | 18 +- src/loaders/amp.ts | 14 +- src/loaders/cache.test.ts | 52 +++++ src/loaders/cache.ts | 134 +++++++++++ src/loaders/claude.ts | 109 +++++---- src/loaders/codex.ts | 11 +- src/loaders/gemini.ts | 16 +- src/loaders/opencode.ts | 6 + src/loaders/pi.ts | 14 +- src/store.ts | 133 ++++++++++- widget/index.html | 16 ++ widget/package-lock.json | 44 +++- widget/package.json | 4 +- widget/src-tauri/Cargo.lock | 255 ++++++++++++++++++++- widget/src-tauri/Cargo.toml | 2 + widget/src-tauri/capabilities/default.json | 2 + widget/src-tauri/src/lib.rs | 2 + widget/src-tauri/tauri.conf.json | 13 +- widget/src/main.ts | 3 + widget/src/styles.css | 46 +++- widget/src/update.ts | 143 ++++++++++++ 24 files changed, 979 insertions(+), 95 deletions(-) create mode 100644 AGENTS.md create mode 100644 src/loaders/cache.test.ts create mode 100644 src/loaders/cache.ts create mode 100644 widget/src/update.ts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aa2c40c..15f8dbd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -101,6 +101,8 @@ jobs: uses: tauri-apps/tauri-action@v0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }} + TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }} with: projectPath: widget tagName: ${{ github.ref_name }} diff --git a/.gitignore b/.gitignore index 21c1ac4..fccf3dc 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,9 @@ Thumbs.db .env .env.local +# Local Tauri updater signing keys +.tauri/ + # Claude Code local state .claude/ CLAUDE.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..b37ab93 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,32 @@ +# Repository Guidelines + +## Project Structure & Module Organization + +TokenBBQ is a TypeScript ESM CLI and dashboard. Core source lives in `src/`: `index.ts` is the CLI entry point, `loaders/` normalizes usage data from supported tools, `aggregator.ts` prepares summaries, `pricing.ts` handles model pricing, and `server.ts`/`dashboard.ts` serve the web UI. Tests sit next to source as `*.test.ts`; loader fixtures live in `src/loaders/__fixtures__/`. Build helpers are in `scripts/`. The desktop widget is a separate Tauri/Vite app under `widget/`, with Rust backend code in `widget/src-tauri/` and frontend assets in `widget/src/`. Generated output belongs in `dist/`. + +## Build, Test, and Development Commands + +- `npm install`: install root dependencies. +- `npm run dev`: inline required assets, then run the CLI locally through `tsx`. +- `npm run lint`: run TypeScript type checking with `tsc --noEmit`. +- `npm run test`: run Node's test runner against `src/**/*.test.ts`. +- `npm run build`: produce the publishable CLI bundle in `dist/`. +- `npm run widget:install`: install widget dependencies. +- `npm run widget:dev`: build the sidecar and launch the Tauri widget. +- `npm run widget:build`: build the CLI, sidecar, and desktop widget package. + +## Coding Style & Naming Conventions + +Use strict TypeScript, ESM `import`/`export`, and Node 20+ APIs. Keep module names lowercase and descriptive, for example `event-merge.ts` or `platform-paths.ts`. Tests should mirror the target module name, such as `pricing.test.ts`. Prefer small functions with explicit types at module boundaries. The codebase currently uses two-space indentation in TypeScript files; avoid unrelated formatting churn. + +## Testing Guidelines + +Tests use `node:test` and `node:assert/strict`, executed via `scripts/run-tests.mjs` so glob expansion works across supported Node versions. Add focused tests beside changed code when modifying aggregation, pricing, persistence, or loader behavior. For new loaders, include representative fixture data under `src/loaders/__fixtures__/` when practical and verify missing data directories return empty results rather than throwing. + +## Commit & Pull Request Guidelines + +Recent history follows Conventional Commit-style subjects, for example `fix(audit): ...`, `docs: ...`, and `chore(release): ...`. Keep subjects imperative and scoped when useful. Pull requests should fill out `.github/PULL_REQUEST_TEMPLATE.md`: explain what changed, why it is needed, confirm `npm run build`, note loader registration changes, and update `README.md` for user-facing behavior. Include screenshots or recordings for widget/dashboard UI changes. + +## Security & Configuration Tips + +Do not commit local usage databases, credentials, or generated `dist/` artifacts unless release packaging requires them. Preserve cross-platform path handling; CI runs Linux, macOS, and Windows. diff --git a/package-lock.json b/package-lock.json index a8aa994..b9d54ac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -585,9 +585,9 @@ } }, "node_modules/@hono/node-server": { - "version": "1.19.9", - "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.9.tgz", - "integrity": "sha512-vHL6w3ecZsky+8P5MD+eFfaGTyCeOHUIFYMGpQGbrBTSmNNoxv0if69rEZ5giu36weC5saFuznL411gRX7bJDw==", + "version": "1.19.14", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz", + "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==", "license": "MIT", "engines": { "node": ">=18.14.1" @@ -1236,9 +1236,9 @@ } }, "node_modules/hono": { - "version": "4.12.3", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.3.tgz", - "integrity": "sha512-SFsVSjp8sj5UumXOOFlkZOG6XS9SJDKw0TbwFeV+AJ8xlST8kxK5Z/5EYa111UY8732lK2S/xB653ceuaoGwpg==", + "version": "4.12.18", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.18.tgz", + "integrity": "sha512-RWzP96k/yv0PQfyXnWjs6zot20TqfpfsNXhOnev8d1InAxubW93L11/oNUc3tQqn2G0bSdAOBpX+2uDFHV7kdQ==", "license": "MIT", "engines": { "node": ">=16.9.0" @@ -1374,9 +1374,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "license": "MIT", "engines": { "node": ">=12" diff --git a/src/loaders/amp.ts b/src/loaders/amp.ts index acfe5bd..27bcedf 100644 --- a/src/loaders/amp.ts +++ b/src/loaders/amp.ts @@ -5,6 +5,7 @@ import { glob } from 'tinyglobby'; import type { UnifiedTokenEvent } from '../types.js'; import { isValidTimestamp } from '../types.js'; import { getPlatformDataDirs } from '../platform-paths.js'; +import { loadCachedFileEvents } from './cache.js'; function getAmpPath(): string | null { const envPath = (process.env.AMP_DATA_DIR ?? '').trim(); @@ -31,21 +32,21 @@ export async function loadAmpEvents(): Promise { if (!existsSync(threadsDir)) return []; const files = await glob('**/*.json', { cwd: threadsDir, absolute: true }); - const events: UnifiedTokenEvent[] = []; - for (const file of files) { + const events = await loadCachedFileEvents('amp', files, async (file) => { + const fileEvents: UnifiedTokenEvent[] = []; let content: string; try { content = await readFile(file, 'utf-8'); } catch { - continue; + return fileEvents; } let thread: Record; try { thread = JSON.parse(content); } catch { - continue; + return fileEvents; } const threadId = String(thread.id ?? path.basename(file, '.json')); @@ -85,7 +86,7 @@ export async function loadAmpEvents(): Promise { } } - events.push({ + fileEvents.push({ source: 'amp', timestamp: evt.timestamp, sessionId: threadId, @@ -100,7 +101,8 @@ export async function loadAmpEvents(): Promise { costUSD: 0, }); } - } + return fileEvents; + }); events.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()); return events; diff --git a/src/loaders/cache.test.ts b/src/loaders/cache.test.ts new file mode 100644 index 0000000..8879f03 --- /dev/null +++ b/src/loaders/cache.test.ts @@ -0,0 +1,52 @@ +import { afterEach, beforeEach, describe, test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { loadCachedFileEvents } from './cache.js'; +import type { UnifiedTokenEvent } from '../types.js'; + +let tmp: string; + +beforeEach(() => { + tmp = mkdtempSync(path.join(tmpdir(), 'tbq-loader-cache-')); + process.env.TOKENBBQ_DATA_DIR = path.join(tmp, 'data'); +}); + +afterEach(() => { + delete process.env.TOKENBBQ_DATA_DIR; + delete process.env.TOKENBBQ_DISABLE_LOADER_CACHE; + rmSync(tmp, { recursive: true, force: true }); +}); + +function event(sessionId: string): UnifiedTokenEvent { + return { + source: 'codex', + timestamp: '2026-04-22T14:02:11.812Z', + sessionId, + model: 'gpt-5', + tokens: { input: 1, output: 2, cacheCreation: 0, cacheRead: 0, reasoning: 0 }, + costUSD: 0, + }; +} + +describe('loadCachedFileEvents', () => { + test('reuses parsed events while file mtime and size are unchanged', async () => { + const file = path.join(tmp, 'session.jsonl'); + writeFileSync(file, 'first', 'utf-8'); + let parses = 0; + + const parseFile = async (target: string): Promise => { + parses++; + return [event(readFileSync(target, 'utf-8'))]; + }; + + assert.equal((await loadCachedFileEvents('codex', [file], parseFile))[0]?.sessionId, 'first'); + assert.equal((await loadCachedFileEvents('codex', [file], parseFile))[0]?.sessionId, 'first'); + assert.equal(parses, 1); + + writeFileSync(file, 'second-value', 'utf-8'); + assert.equal((await loadCachedFileEvents('codex', [file], parseFile))[0]?.sessionId, 'second-value'); + assert.equal(parses, 2); + }); +}); diff --git a/src/loaders/cache.ts b/src/loaders/cache.ts new file mode 100644 index 0000000..e063fd9 --- /dev/null +++ b/src/loaders/cache.ts @@ -0,0 +1,134 @@ +import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { getStoreDir } from '../store.js'; +import type { Source, UnifiedTokenEvent } from '../types.js'; + +const CACHE_VERSION = 1; + +interface FileCacheEntry { + mtimeMs: number; + size: number; + records: T[]; +} + +interface LoaderCacheFile { + v: number; + files: Record>; +} + +function cacheEnabled(): boolean { + return process.env.TOKENBBQ_DISABLE_LOADER_CACHE !== '1'; +} + +function cachePath(source: Source): string { + return path.join(getStoreDir(), 'cache', 'loaders', `${source}.json`); +} + +function isValidEvent(v: unknown): v is UnifiedTokenEvent { + if (!v || typeof v !== 'object') return false; + const e = v as Record; + const tokens = e.tokens as Record | undefined; + return ( + typeof e.source === 'string' && + typeof e.timestamp === 'string' && + typeof e.sessionId === 'string' && + typeof e.model === 'string' && + !!tokens && + typeof tokens === 'object' + ); +} + +function isValidEntry(v: unknown, isValidRecord: (v: unknown) => v is T): v is FileCacheEntry { + if (!v || typeof v !== 'object') return false; + const e = v as Record; + return ( + typeof e.mtimeMs === 'number' && + typeof e.size === 'number' && + Array.isArray(e.records) && + e.records.every(isValidRecord) + ); +} + +async function readCache( + source: Source, + isValidRecord: (v: unknown) => v is T, +): Promise> { + try { + const parsed = JSON.parse(await readFile(cachePath(source), 'utf-8')) as unknown; + if (!parsed || typeof parsed !== 'object') return { v: CACHE_VERSION, files: {} }; + const obj = parsed as Record; + if (obj.v !== CACHE_VERSION || !obj.files || typeof obj.files !== 'object') { + return { v: CACHE_VERSION, files: {} }; + } + const files: Record> = {}; + for (const [file, entry] of Object.entries(obj.files as Record)) { + if (isValidEntry(entry, isValidRecord)) files[file] = entry; + } + return { v: CACHE_VERSION, files }; + } catch { + return { v: CACHE_VERSION, files: {} }; + } +} + +async function writeCache(source: Source, cache: LoaderCacheFile): Promise { + const file = cachePath(source); + const dir = path.dirname(file); + const tmp = path.join(dir, `${path.basename(file)}.${process.pid}.${Date.now()}.tmp`); + try { + await mkdir(dir, { recursive: true }); + await writeFile(tmp, JSON.stringify(cache), 'utf-8'); + await rename(tmp, file); + } catch { + // Loader caches are performance-only. A failed write must never make scans fail. + } +} + +export async function loadCachedFileRecords( + source: Source, + files: string[], + parseFile: (file: string) => Promise, + isValidRecord: (v: unknown) => v is T, +): Promise { + if (!cacheEnabled()) { + const records: T[] = []; + for (const file of files) records.push(...await parseFile(file)); + return records; + } + + const cache = await readCache(source, isValidRecord); + const nextFiles: Record> = {}; + const records: T[] = []; + + for (const file of files) { + let info: { mtimeMs: number; size: number }; + try { + const s = await stat(file); + info = { mtimeMs: s.mtimeMs, size: s.size }; + } catch { + continue; + } + + const hit = cache.files[file]; + if (hit && hit.mtimeMs === info.mtimeMs && hit.size === info.size) { + nextFiles[file] = hit; + records.push(...hit.records); + continue; + } + + const parsed = await parseFile(file); + const entry = { ...info, records: parsed }; + nextFiles[file] = entry; + records.push(...parsed); + } + + await writeCache(source, { v: CACHE_VERSION, files: nextFiles }); + return records; +} + +export async function loadCachedFileEvents( + source: Source, + files: string[], + parseFile: (file: string) => Promise, +): Promise { + return loadCachedFileRecords(source, files, parseFile, isValidEvent); +} diff --git a/src/loaders/claude.ts b/src/loaders/claude.ts index 91e39d2..aa0404a 100644 --- a/src/loaders/claude.ts +++ b/src/loaders/claude.ts @@ -6,6 +6,7 @@ import { glob } from 'tinyglobby'; import type { UnifiedTokenEvent } from '../types.js'; import { isValidTimestamp } from '../types.js'; import { resolveProjectRoot } from '../project.js'; +import { loadCachedFileRecords } from './cache.js'; const HOME = homedir(); @@ -58,6 +59,27 @@ function parseLine(raw: Record): UnifiedTokenEvent | null { }; } +type CachedClaudeEvent = { + dedupeKey: string; + event: UnifiedTokenEvent; +}; + +function isCachedClaudeEvent(value: unknown): value is CachedClaudeEvent { + if (!value || typeof value !== 'object') return false; + const record = value as Record; + const event = record.event as Record | undefined; + return ( + typeof record.dedupeKey === 'string' && + !!event && + typeof event.source === 'string' && + typeof event.timestamp === 'string' && + typeof event.sessionId === 'string' && + typeof event.model === 'string' && + !!event.tokens && + typeof event.tokens === 'object' + ); +} + export function getClaudeWatchPaths(): string[] { return getClaudePaths().map((p) => path.join(p, 'projects')); } @@ -66,58 +88,65 @@ export async function loadClaudeEvents(): Promise { const claudePaths = getClaudePaths(); if (claudePaths.length === 0) return []; - const events: UnifiedTokenEvent[] = []; - const seen = new Set(); + const allFiles: string[] = []; for (const claudePath of claudePaths) { const projectsDir = path.join(claudePath, 'projects'); const files = await glob('**/*.jsonl', { cwd: projectsDir, absolute: true }); + allFiles.push(...files); + } + + const records = await loadCachedFileRecords('claude-code', allFiles, async (file) => { + const fileEvents: CachedClaudeEvent[] = []; + let content: string; + try { + content = await readFile(file, 'utf-8'); + } catch { + return fileEvents; + } + + const sessionId = path.basename(file, '.jsonl'); - for (const file of files) { - let content: string; + for (const line of content.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed) continue; + + let parsed: Record; try { - content = await readFile(file, 'utf-8'); + parsed = JSON.parse(trimmed); } catch { continue; } - const sessionId = path.basename(file, '.jsonl'); - - for (const line of content.split(/\r?\n/)) { - const trimmed = line.trim(); - if (!trimmed) continue; - - let parsed: Record; - try { - parsed = JSON.parse(trimmed); - } catch { - continue; - } - - const event = parseLine(parsed); - if (!event) continue; - - event.sessionId = sessionId; - // cwd can change mid-session (user cd's); we honor the cwd at each event. - const cwd = typeof parsed.cwd === 'string' ? parsed.cwd : undefined; - if (cwd) { - event.project = resolveProjectRoot(cwd).name; - } - // No fallback: if cwd is absent, event.project stays undefined and the event - // is excluded from per-project aggregation (but still counts toward totals). - - const requestId = String(parsed.requestId ?? ''); - const messageId = String((parsed.message as Record)?.id ?? ''); - const dedupeKey = requestId && messageId - ? `${messageId}:${requestId}` - : `${event.timestamp}:${event.model}:${event.tokens.input}:${event.tokens.output}`; - if (seen.has(dedupeKey)) continue; - seen.add(dedupeKey); - - events.push(event); + const event = parseLine(parsed); + if (!event) continue; + + event.sessionId = sessionId; + // cwd can change mid-session (user cd's); we honor the cwd at each event. + const cwd = typeof parsed.cwd === 'string' ? parsed.cwd : undefined; + if (cwd) { + event.project = resolveProjectRoot(cwd).name; } + // No fallback: if cwd is absent, event.project stays undefined and the event + // is excluded from per-project aggregation (but still counts toward totals). + + const requestId = String(parsed.requestId ?? ''); + const messageId = String((parsed.message as Record)?.id ?? ''); + const dedupeKey = requestId && messageId + ? `${messageId}:${requestId}` + : `${event.timestamp}:${event.model}:${event.tokens.input}:${event.tokens.output}`; + + fileEvents.push({ dedupeKey, event }); } - } + return fileEvents; + }, isCachedClaudeEvent); + + const seen = new Set(); + const events = records.flatMap((record) => { + if (seen.has(record.dedupeKey)) return []; + seen.add(record.dedupeKey); + return [record.event]; + }); events.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()); return events; diff --git a/src/loaders/codex.ts b/src/loaders/codex.ts index 3478999..39cf3d9 100644 --- a/src/loaders/codex.ts +++ b/src/loaders/codex.ts @@ -6,6 +6,7 @@ import { glob } from 'tinyglobby'; import type { UnifiedTokenEvent, CodexRateLimits, CodexWindowUsage } from '../types.js'; import { isValidTimestamp } from '../types.js'; import { resolveProjectRoot } from '../project.js'; +import { loadCachedFileEvents } from './cache.js'; const HOME = homedir(); const FALLBACK_MODEL = 'gpt-5'; @@ -79,15 +80,14 @@ export async function loadCodexEvents(): Promise { const sessionsDir = path.join(codexDir, 'sessions'); const files = await glob('**/*.jsonl', { cwd: sessionsDir, absolute: true }); - const events: UnifiedTokenEvent[] = []; - - for (const file of files) { + const events = await loadCachedFileEvents('codex', files, async (file) => { + const events: UnifiedTokenEvent[] = []; const sessionId = path.relative(sessionsDir, file).replace(/\.jsonl$/i, '').replace(/\\/g, '/'); let content: string; try { content = await readFile(file, 'utf-8'); } catch { - continue; + return events; } let prevTotals: RawUsage | null = null; @@ -169,7 +169,8 @@ export async function loadCodexEvents(): Promise { project: sessionProject, }); } - } + return events; + }); events.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()); return events; diff --git a/src/loaders/gemini.ts b/src/loaders/gemini.ts index ef0fb19..52bbc49 100644 --- a/src/loaders/gemini.ts +++ b/src/loaders/gemini.ts @@ -5,6 +5,7 @@ import path from 'node:path'; import { glob } from 'tinyglobby'; import type { UnifiedTokenEvent } from '../types.js'; import { isValidTimestamp } from '../types.js'; +import { loadCachedFileEvents } from './cache.js'; const HOME = homedir(); const FALLBACK_MODEL = 'gemini'; @@ -47,22 +48,21 @@ export async function loadGeminiEvents(): Promise { const tmpDir = path.join(geminiDir, 'tmp'); const files = await glob('**/chats/session-*.json', { cwd: tmpDir, absolute: true }); - const events: UnifiedTokenEvent[] = []; - const seen = new Set(); - for (const file of files) { + const events = await loadCachedFileEvents('gemini', files, async (file) => { + const fileEvents: UnifiedTokenEvent[] = []; let content: string; try { content = await readFile(file, 'utf-8'); } catch { - continue; + return fileEvents; } let session: Record; try { session = JSON.parse(content); } catch { - continue; + return fileEvents; } const sessionId = String(session.sessionId ?? path.basename(file, '.json')); @@ -70,6 +70,7 @@ export async function loadGeminiEvents(): Promise { const messages = Array.isArray(session.messages) ? (session.messages as Record[]) : []; + const seen = new Set(); for (const msg of messages) { const tokens = msg.tokens as Record | undefined; @@ -112,7 +113,7 @@ export async function loadGeminiEvents(): Promise { ? msg.model : FALLBACK_MODEL; - events.push({ + fileEvents.push({ source: 'gemini', timestamp, sessionId, @@ -128,7 +129,8 @@ export async function loadGeminiEvents(): Promise { project, }); } - } + return fileEvents; + }); events.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()); return events; diff --git a/src/loaders/opencode.ts b/src/loaders/opencode.ts index a6ecd70..b4476e7 100644 --- a/src/loaders/opencode.ts +++ b/src/loaders/opencode.ts @@ -6,6 +6,7 @@ import type { LoaderOptions } from './index.js'; import { resolveProjectRoot } from '../project.js'; import { getPlatformDataDirs } from '../platform-paths.js'; import { SQL_WASM_BASE64 } from './sql-wasm-inline.js'; +import { loadCachedFileEvents } from './cache.js'; // sql.js's default loader fopen()s `sql-wasm.wasm` from the path Emscripten // recorded at sql.js build time. After Bun --compile (or any other deploy @@ -48,7 +49,12 @@ export async function loadOpenCodeEvents(opts: LoaderOptions = { quiet: false }) if (!dir) return []; const dbFile = path.join(dir, 'opencode.db'); const warn = opts.quiet ? () => {} : console.warn.bind(console); + const cached = await loadCachedFileEvents('opencode', [dbFile], async () => parseOpenCodeDb(dbFile, warn)); + cached.sort((a, b) => a.timestamp.localeCompare(b.timestamp)); + return cached; +} +async function parseOpenCodeDb(dbFile: string, warn: (...args: unknown[]) => void): Promise { let SQL: Awaited>; try { SQL = await initSqlJs({ wasmBinary: SQL_WASM_BINARY }); diff --git a/src/loaders/pi.ts b/src/loaders/pi.ts index b3e41c5..ccdf336 100644 --- a/src/loaders/pi.ts +++ b/src/loaders/pi.ts @@ -5,6 +5,7 @@ import path from 'node:path'; import { glob } from 'tinyglobby'; import type { UnifiedTokenEvent } from '../types.js'; import { isValidTimestamp } from '../types.js'; +import { loadCachedFileEvents } from './cache.js'; const HOME = homedir(); @@ -29,10 +30,10 @@ export async function loadPiEvents(): Promise { if (!piDir) return []; const files = await glob('**/*.jsonl', { cwd: piDir, absolute: true }); - const events: UnifiedTokenEvent[] = []; - const seen = new Set(); - for (const file of files) { + const events = await loadCachedFileEvents('pi', files, async (file) => { + const fileEvents: UnifiedTokenEvent[] = []; + const seen = new Set(); const relPath = path.relative(piDir, file); const segments = relPath.split(path.sep); const project = segments.length >= 2 ? segments[0] : 'unknown'; @@ -44,7 +45,7 @@ export async function loadPiEvents(): Promise { try { content = await readFile(file, 'utf-8'); } catch { - continue; + return fileEvents; } for (const line of content.split(/\r?\n/)) { @@ -84,7 +85,7 @@ export async function loadPiEvents(): Promise { const rawModel = String(message.model ?? 'unknown'); const cost = usage.cost as Record | undefined; - events.push({ + fileEvents.push({ source: 'pi', timestamp, sessionId, @@ -100,7 +101,8 @@ export async function loadPiEvents(): Promise { project, }); } - } + return fileEvents; + }); events.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()); return events; diff --git a/src/store.ts b/src/store.ts index 5243643..ff6e37a 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1,4 +1,4 @@ -import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync } from 'node:fs'; +import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, statSync, writeFileSync } from 'node:fs'; import { createHash } from 'node:crypto'; import { homedir, hostname } from 'node:os'; import path from 'node:path'; @@ -23,6 +23,10 @@ function getEventsDir(): string { return path.join(getStoreDir(), 'events'); } +function getStoreCachePath(): string { + return path.join(getStoreDir(), 'cache', 'store-v1.json'); +} + function sanitizeForFilename(s: string): string { return s.replace(/[^A-Za-z0-9._-]/g, '_'); } @@ -66,6 +70,123 @@ interface LoadOutcome { futureSeen: number; } +interface StoreFileMeta { + path: string; + mtimeMs: number; + size: number; +} + +interface StoreReadCache { + v: number; + files: StoreFileMeta[]; + events: UnifiedTokenEvent[]; +} + +function fileMeta(file: string): StoreFileMeta | null { + try { + const s = statSync(file); + if (!s.isFile() || s.size === 0) return null; + return { path: file, mtimeMs: s.mtimeMs, size: s.size }; + } catch { + return null; + } +} + +function listStoreFiles(eventsDir: string): StoreFileMeta[] { + const files: StoreFileMeta[] = []; + const legacy = fileMeta(getLegacyFilePath()); + if (legacy) files.push(legacy); + + let entries: string[] = []; + try { + entries = readdirSync(eventsDir); + } catch { + // ignore - fresh install with empty dir + } + + for (const name of entries) { + if (!name.endsWith('.ndjson')) continue; + const meta = fileMeta(path.join(eventsDir, name)); + if (meta) files.push(meta); + } + + files.sort((a, b) => a.path.localeCompare(b.path)); + return files; +} + +function sameFileSet(a: StoreFileMeta[], b: StoreFileMeta[]): boolean { + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) { + if (a[i]!.path !== b[i]!.path || a[i]!.mtimeMs !== b[i]!.mtimeMs || a[i]!.size !== b[i]!.size) { + return false; + } + } + return true; +} + +function isTokenCounts(v: unknown): v is UnifiedTokenEvent['tokens'] { + if (!v || typeof v !== 'object') return false; + const t = v as Record; + return ( + typeof t.input === 'number' && + typeof t.output === 'number' && + typeof t.cacheCreation === 'number' && + typeof t.cacheRead === 'number' && + typeof t.reasoning === 'number' + ); +} + +function isStoreEvent(v: unknown): v is UnifiedTokenEvent { + if (!v || typeof v !== 'object') return false; + const e = v as Record; + return ( + typeof e.source === 'string' && + typeof e.timestamp === 'string' && + typeof e.sessionId === 'string' && + typeof e.model === 'string' && + isTokenCounts(e.tokens) && + typeof e.costUSD === 'number' + ); +} + +function outcomeFromEvents(events: UnifiedTokenEvent[]): LoadOutcome { + const outcome: LoadOutcome = { events: [], hashes: new Set(), badSeen: 0, futureSeen: 0 }; + for (const event of events) { + const hash = hashEvent(event); + if (outcome.hashes.has(hash)) continue; + outcome.hashes.add(hash); + outcome.events.push(event); + } + return outcome; +} + +function readStoreCache(files: StoreFileMeta[]): LoadOutcome | null { + try { + const parsed = JSON.parse(readFileSync(getStoreCachePath(), 'utf-8')) as unknown; + if (!parsed || typeof parsed !== 'object') return null; + const cache = parsed as StoreReadCache; + if (cache.v !== CURRENT_VERSION || !Array.isArray(cache.files) || !Array.isArray(cache.events)) return null; + if (!sameFileSet(cache.files, files)) return null; + if (!cache.events.every(isStoreEvent)) return null; + return outcomeFromEvents(cache.events); + } catch { + return null; + } +} + +function writeStoreCache(files: StoreFileMeta[], events: UnifiedTokenEvent[]): void { + const target = getStoreCachePath(); + const dir = path.dirname(target); + const tmp = path.join(dir, `${path.basename(target)}.${process.pid}.${Date.now()}.tmp`); + try { + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + writeFileSync(tmp, JSON.stringify({ v: CURRENT_VERSION, files, events }), 'utf-8'); + renameSync(tmp, target); + } catch { + // Performance-only cache. Store reads must keep working if this fails. + } +} + function loadFile(file: string, into: LoadOutcome): void { let raw: string; try { @@ -141,6 +262,10 @@ export function loadStore(): StoreState { if (!existsSync(eventsDir)) mkdirSync(eventsDir, { recursive: true }); if (!existsSync(ownFile)) appendFileSync(ownFile, ''); + const files = listStoreFiles(eventsDir); + const cached = readStoreCache(files); + if (cached) return { events: cached.events, hashes: cached.hashes, path: ownFile }; + const outcome: LoadOutcome = { events: [], hashes: new Set(), @@ -171,6 +296,7 @@ export function loadStore(): StoreState { if (outcome.badSeen > 0) console.warn(`tokenbbq: skipped ${outcome.badSeen} malformed line(s) in store`); if (outcome.futureSeen > 0) console.warn(`tokenbbq: skipped ${outcome.futureSeen} line(s) with future schema version`); + writeStoreCache(files, outcome.events); return { events: outcome.events, hashes: outcome.hashes, path: ownFile }; } @@ -192,6 +318,9 @@ export function appendEvents(state: StoreState, events: UnifiedTokenEvent[]): Un // contention. Two processes that race to scan the same upstream tool can // each persist the same event into their own file; loadStore unions and // dedupes them on the next read. Slightly redundant on disk, lossless. - if (buffer) appendFileSync(state.path, buffer); + if (buffer) { + appendFileSync(state.path, buffer); + writeStoreCache(listStoreFiles(getEventsDir()), state.events); + } return added; } diff --git a/widget/index.html b/widget/index.html index 0e5e49b..8b10156 100644 --- a/widget/index.html +++ b/widget/index.html @@ -153,6 +153,22 @@ +
+ +
+ +
+
+ +
+ + +
+
+
diff --git a/widget/package-lock.json b/widget/package-lock.json index b6e171b..215232e 100644 --- a/widget/package-lock.json +++ b/widget/package-lock.json @@ -1,17 +1,19 @@ { "name": "tokenbbq-widget", - "version": "0.5.0", + "version": "0.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "tokenbbq-widget", - "version": "0.5.0", + "version": "0.6.0", "license": "MIT", "dependencies": { "@tauri-apps/api": "^2", "@tauri-apps/plugin-autostart": "^2.5.1", - "@tauri-apps/plugin-store": "^2" + "@tauri-apps/plugin-process": "^2.3.1", + "@tauri-apps/plugin-store": "^2", + "@tauri-apps/plugin-updater": "^2.10.1" }, "devDependencies": { "@tailwindcss/vite": "^4", @@ -1435,6 +1437,15 @@ "@tauri-apps/api": "^2.8.0" } }, + "node_modules/@tauri-apps/plugin-process": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@tauri-apps/plugin-process/-/plugin-process-2.3.1.tgz", + "integrity": "sha512-nCa4fGVaDL/B9ai03VyPOjfAHRHSBz5v6F/ObsB73r/dA3MHHhZtldaDMIc0V/pnUw9ehzr2iEG+XkSEyC0JJA==", + "license": "MIT OR Apache-2.0", + "dependencies": { + "@tauri-apps/api": "^2.8.0" + } + }, "node_modules/@tauri-apps/plugin-store": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/@tauri-apps/plugin-store/-/plugin-store-2.4.2.tgz", @@ -1444,6 +1455,15 @@ "@tauri-apps/api": "^2.8.0" } }, + "node_modules/@tauri-apps/plugin-updater": { + "version": "2.10.1", + "resolved": "https://registry.npmjs.org/@tauri-apps/plugin-updater/-/plugin-updater-2.10.1.tgz", + "integrity": "sha512-NFYMg+tWOZPJdzE/PpFj2qfqwAWwNS3kXrb1tm1gnBJ9mYzZ4WDRrwy8udzWoAnfGCHLuePNLY1WVCNHnh3eRA==", + "license": "MIT OR Apache-2.0", + "dependencies": { + "@tauri-apps/api": "^2.10.1" + } + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -1865,9 +1885,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", "engines": { @@ -1878,9 +1898,9 @@ } }, "node_modules/postcss": { - "version": "8.5.8", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", - "integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==", + "version": "8.5.14", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.14.tgz", + "integrity": "sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==", "dev": true, "funding": [ { @@ -2014,9 +2034,9 @@ } }, "node_modules/vite": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz", - "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.2.tgz", + "integrity": "sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ==", "dev": true, "license": "MIT", "dependencies": { diff --git a/widget/package.json b/widget/package.json index 2ae824f..54e281e 100644 --- a/widget/package.json +++ b/widget/package.json @@ -15,7 +15,9 @@ "dependencies": { "@tauri-apps/api": "^2", "@tauri-apps/plugin-autostart": "^2.5.1", - "@tauri-apps/plugin-store": "^2" + "@tauri-apps/plugin-process": "^2.3.1", + "@tauri-apps/plugin-store": "^2", + "@tauri-apps/plugin-updater": "^2.10.1" }, "devDependencies": { "@tailwindcss/vite": "^4", diff --git a/widget/src-tauri/Cargo.lock b/widget/src-tauri/Cargo.lock index ac2ce4d..83aaba4 100644 --- a/widget/src-tauri/Cargo.lock +++ b/widget/src-tauri/Cargo.lock @@ -47,6 +47,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "atk" version = "0.18.2" @@ -563,6 +572,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "derive_more" version = "0.99.20" @@ -782,6 +802,16 @@ dependencies = [ "typeid", ] +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -807,6 +837,16 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "filetime" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5b2eef6fafbf69f877e55509ce5b11a760690ac9700a2921be067aa6afaef6" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1817,6 +1857,12 @@ dependencies = [ "libc", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.1" @@ -1913,6 +1959,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minisign-verify" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f9645cb765ea72b8111f36c522475d2daa0d22c957a9826437e97534bc4e9e" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2104,6 +2156,7 @@ checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ "bitflags 2.11.0", "block2", + "libc", "objc2", "objc2-core-foundation", ] @@ -2119,6 +2172,18 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-osa-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f112d1746737b0da274ef79a23aac283376f335f4095a083a267a082f21db0c0" +dependencies = [ + "bitflags 2.11.0", + "objc2", + "objc2-app-kit", + "objc2-foundation", +] + [[package]] name = "objc2-quartz-core" version = "0.3.2" @@ -2163,12 +2228,32 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "osakit" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "732c71caeaa72c065bb69d7ea08717bd3f4863a4f451402fc9513e29dbd5261b" +dependencies = [ + "objc2", + "objc2-foundation", + "objc2-osa-kit", + "serde", + "serde_json", + "thiserror 2.0.18", +] + [[package]] name = "pango" version = "0.18.3" @@ -2895,15 +2980,20 @@ dependencies = [ "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", "serde", "serde_json", "sync_wrapper", "tokio", + "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -2944,6 +3034,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.11.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.37" @@ -2958,6 +3061,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.5.1", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -2968,11 +3083,38 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework 3.5.1", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "ring", "rustls-pki-types", @@ -3000,6 +3142,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "schemars" version = "0.8.22" @@ -3606,6 +3757,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tar" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.16" @@ -3757,6 +3919,16 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "tauri-plugin-process" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55511a7bf6cd70c8767b02c97bf8134fa434daf3926cfc1be0a0f94132d165a" +dependencies = [ + "tauri", + "tauri-plugin", +] + [[package]] name = "tauri-plugin-store" version = "2.4.2" @@ -3773,6 +3945,39 @@ dependencies = [ "tracing", ] +[[package]] +name = "tauri-plugin-updater" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "806d9dac662c2e4594ff03c647a552f2c9bd544e7d0f683ec58f872f952ce4af" +dependencies = [ + "base64 0.22.1", + "dirs 6.0.0", + "flate2", + "futures-util", + "http", + "infer", + "log", + "minisign-verify", + "osakit", + "percent-encoding", + "reqwest 0.13.2", + "rustls", + "semver", + "serde", + "serde_json", + "tar", + "tauri", + "tauri-plugin", + "tempfile", + "thiserror 2.0.18", + "time", + "tokio", + "url", + "windows-sys 0.60.2", + "zip", +] + [[package]] name = "tauri-runtime" version = "2.10.1" @@ -3873,6 +4078,19 @@ dependencies = [ "toml 0.9.12+spec-1.1.0", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "tendril" version = "0.4.3" @@ -4002,7 +4220,9 @@ dependencies = [ "tauri", "tauri-build", "tauri-plugin-autostart", + "tauri-plugin-process", "tauri-plugin-store", + "tauri-plugin-updater", "tokio", ] @@ -4653,6 +4873,15 @@ dependencies = [ "system-deps", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.6" @@ -5334,6 +5563,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "yoke" version = "0.8.1" @@ -5451,6 +5690,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zip" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa8cd6af31c3b31c6631b8f483848b91589021b28fffe50adada48d4f4d2ed1" +dependencies = [ + "arbitrary", + "crc32fast", + "indexmap 2.13.0", + "memchr", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/widget/src-tauri/Cargo.toml b/widget/src-tauri/Cargo.toml index 4ca57ae..f9b057f 100644 --- a/widget/src-tauri/Cargo.toml +++ b/widget/src-tauri/Cargo.toml @@ -19,6 +19,8 @@ serde_json = "1" reqwest = { version = "0.12", features = ["json", "rustls-tls"], default-features = false } tokio = { version = "1", features = ["rt-multi-thread", "macros", "fs"] } chrono = { version = "0.4", default-features = false, features = ["alloc", "now"] } +tauri-plugin-updater = "2" +tauri-plugin-process = "2" [target.'cfg(windows)'.dependencies] keyring = { version = "3", features = ["windows-native"] } diff --git a/widget/src-tauri/capabilities/default.json b/widget/src-tauri/capabilities/default.json index 339cf19..457bf42 100644 --- a/widget/src-tauri/capabilities/default.json +++ b/widget/src-tauri/capabilities/default.json @@ -14,6 +14,8 @@ "core:window:allow-close", "core:window:allow-start-dragging", "store:default", + "process:default", + "updater:default", "autostart:default", "autostart:allow-is-enabled", "autostart:allow-enable", diff --git a/widget/src-tauri/src/lib.rs b/widget/src-tauri/src/lib.rs index dd44398..fe47911 100644 --- a/widget/src-tauri/src/lib.rs +++ b/widget/src-tauri/src/lib.rs @@ -57,6 +57,8 @@ fn save_widget_position(app: &AppHandle, x: i32, y: i32) { pub fn run() { tauri::Builder::default() .plugin(tauri_plugin_store::Builder::new().build()) + .plugin(tauri_plugin_process::init()) + .plugin(tauri_plugin_updater::Builder::new().build()) .plugin(tauri_plugin_autostart::init( tauri_plugin_autostart::MacosLauncher::LaunchAgent, None, diff --git a/widget/src-tauri/tauri.conf.json b/widget/src-tauri/tauri.conf.json index d7b8c29..ef5013f 100644 --- a/widget/src-tauri/tauri.conf.json +++ b/widget/src-tauri/tauri.conf.json @@ -30,6 +30,7 @@ }, "bundle": { "active": true, + "createUpdaterArtifacts": true, "targets": ["nsis", "msi", "dmg", "app"], "icon": [ "icons/32x32.png", @@ -43,5 +44,15 @@ "minimumSystemVersion": "11.0" } }, - "plugins": {} + "plugins": { + "updater": { + "pubkey": "dW50cnVzdGVkIGNvbW1lbnQ6IG1pbmlzaWduIHB1YmxpYyBrZXk6IDNGN0Y2M0Y1RDRFQ0YxRjgKUldUNDhlelU5V04vUHp3ZnNaNTNSV3NsMHUxbWFTcW92YS92aDVzYncvMmhIRnBiTGc5S2RxalIK", + "endpoints": [ + "https://github.com/offbyone1/tokenbbq/releases/latest/download/latest.json" + ], + "windows": { + "installMode": "passive" + } + } + } } diff --git a/widget/src/main.ts b/widget/src/main.ts index 48ce6c2..36bea73 100644 --- a/widget/src/main.ts +++ b/widget/src/main.ts @@ -6,6 +6,7 @@ import { getCurrentWebview } from "@tauri-apps/api/webview"; import type { ClaudeUsageResponse, LocalUsageSummary, SettingsDisplay } from "./types"; import { loadToggleState, saveToggleState, resolveMode, type SourceToggleState } from "./source-toggle"; import { renderCompact, renderExpanded, renderError, renderLocalCompact, setViewState, getWorkAreaPhysical, currentFrameInsetLogical, clampWindowToWorkAreaOnce, refreshPillPositionIfPillMode, setMonitorWorkAreaPhysical, refitExpandedHeight } from "./ui"; +import { scheduleAutoUpdateCheck, setupUpdateControls } from "./update"; const LOCAL_POLL_INTERVAL_MS = 5 * 60 * 1000; // Persistent cache of the last successful fetchLocalUsage result. Codex / @@ -157,6 +158,7 @@ async function init(): Promise { // user can open settings via the gear icon. await setViewState("compact", currentMode()); startPolling(); + scheduleAutoUpdateCheck(); const win = getCurrentWindow(); win.onCloseRequested(async (event) => { @@ -469,6 +471,7 @@ function setupEventListeners(): void { document.getElementById("btn-save-settings")!.addEventListener("click", saveSettings); document.getElementById("btn-cancel-settings")!.addEventListener("click", closeSettings); document.getElementById("btn-cancel-settings-2")!.addEventListener("click", closeSettings); + setupUpdateControls(); // Delegated: the button is re-rendered with the expanded panel on every // refresh, so a static handle would go stale. diff --git a/widget/src/styles.css b/widget/src/styles.css index de1b866..225769d 100644 --- a/widget/src/styles.css +++ b/widget/src/styles.css @@ -638,7 +638,8 @@ body.view-transitioning { padding: 0 14px 14px; } -.settings-footer .footer-btn { +.settings-footer .footer-btn, +.update-actions .footer-btn { flex: 1; height: 34px; border-radius: 9px; @@ -656,19 +657,22 @@ body.view-transitioning { gap: 5px; } -.settings-footer .footer-btn:hover { +.settings-footer .footer-btn:hover, +.update-actions .footer-btn:hover { background: var(--card-elevated); border-color: var(--accent); color: var(--text-primary); } -.settings-footer .footer-btn.primary { +.settings-footer .footer-btn.primary, +.update-actions .footer-btn.primary { background: var(--accent); border-color: var(--accent); color: #fff; } -.settings-footer .footer-btn.primary:hover { +.settings-footer .footer-btn.primary:hover, +.update-actions .footer-btn.primary:hover { background: #d06b28; box-shadow: 0 0 16px var(--accent-glow); } @@ -879,6 +883,40 @@ body.view-transitioning { font-weight: 500; } +.update-actions { + display: flex; + gap: 8px; + margin-top: 12px; +} + +.update-actions .footer-btn { + flex: 1; + height: 28px; +} + +.update-actions .footer-btn[hidden] { + display: none; +} + +.update-status { + min-height: 18px; + margin-top: 6px; + font-size: 11px; + color: var(--text-tertiary); +} + +.update-status.success { + color: var(--green); +} + +.update-status.error { + color: var(--red); +} + +.update-status.loading { + color: var(--text-secondary); +} + .theme-switch { position: relative; width: 36px; diff --git a/widget/src/update.ts b/widget/src/update.ts new file mode 100644 index 0000000..a2a920e --- /dev/null +++ b/widget/src/update.ts @@ -0,0 +1,143 @@ +import { check, type Update } from "@tauri-apps/plugin-updater"; +import { relaunch } from "@tauri-apps/plugin-process"; + +const AUTO_UPDATE_KEY = "tokenbbq-auto-update-checks"; +const LAST_UPDATE_CHECK_KEY = "tokenbbq-last-update-check-at"; +const AUTO_CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000; +const AUTO_CHECK_DELAY_MS = 30_000; + +let availableUpdate: Update | null = null; +let isChecking = false; +let isInstalling = false; + +export function autoUpdateChecksEnabled(): boolean { + return localStorage.getItem(AUTO_UPDATE_KEY) !== "0"; +} + +function saveAutoUpdateChecksEnabled(enabled: boolean): void { + localStorage.setItem(AUTO_UPDATE_KEY, enabled ? "1" : "0"); +} + +function shouldRunAutomaticCheck(): boolean { + const lastRaw = localStorage.getItem(LAST_UPDATE_CHECK_KEY); + const last = lastRaw ? Number(lastRaw) : 0; + return !Number.isFinite(last) || Date.now() - last >= AUTO_CHECK_INTERVAL_MS; +} + +function markAutomaticCheckAttempted(): void { + localStorage.setItem(LAST_UPDATE_CHECK_KEY, String(Date.now())); +} + +function setStatus(message: string, kind: "idle" | "success" | "error" | "loading" = "idle"): void { + const el = document.getElementById("update-status"); + if (!el) return; + el.textContent = message; + el.className = `update-status ${kind}`; +} + +function setInstallVisible(visible: boolean): void { + const btn = document.getElementById("btn-install-update") as HTMLButtonElement | null; + if (!btn) return; + btn.hidden = !visible; + btn.disabled = !visible || isInstalling; +} + +async function checkForUpdates(manual: boolean): Promise { + if (isChecking || isInstalling) return; + isChecking = true; + availableUpdate = null; + setInstallVisible(false); + + const checkBtn = document.getElementById("btn-check-updates") as HTMLButtonElement | null; + if (checkBtn) checkBtn.disabled = true; + if (manual) setStatus("Checking for updates...", "loading"); + + try { + const update = await check({ timeout: 15_000 }); + if (!manual) markAutomaticCheckAttempted(); + + if (!update) { + if (manual) setStatus("TokenBBQ is up to date.", "success"); + return; + } + + availableUpdate = update; + setStatus(`Version ${update.version} is available.`, "success"); + setInstallVisible(true); + } catch (err) { + if (!manual) markAutomaticCheckAttempted(); + const message = err instanceof Error ? err.message : String(err); + if (manual) setStatus(`Update check failed: ${message}`, "error"); + console.warn("update check failed:", err); + } finally { + isChecking = false; + if (checkBtn) checkBtn.disabled = false; + } +} + +async function installAvailableUpdate(): Promise { + if (!availableUpdate || isInstalling) return; + isInstalling = true; + setInstallVisible(true); + + const installBtn = document.getElementById("btn-install-update") as HTMLButtonElement | null; + if (installBtn) installBtn.disabled = true; + + let downloaded = 0; + try { + setStatus("Downloading update...", "loading"); + await availableUpdate.downloadAndInstall((event) => { + if (event.event === "Started") { + downloaded = 0; + setStatus("Downloading update...", "loading"); + } else if (event.event === "Progress") { + downloaded += event.data.chunkLength; + const mb = (downloaded / 1024 / 1024).toFixed(1); + setStatus(`Downloading update... ${mb} MB`, "loading"); + } else if (event.event === "Finished") { + setStatus("Installing update...", "loading"); + } + }); + setStatus("Update installed. Restarting...", "success"); + await relaunch(); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + setStatus(`Update install failed: ${message}`, "error"); + console.warn("update install failed:", err); + isInstalling = false; + if (installBtn) installBtn.disabled = false; + } +} + +export function scheduleAutoUpdateCheck(): void { + if (!autoUpdateChecksEnabled() || !shouldRunAutomaticCheck()) return; + window.setTimeout(() => { + if (!autoUpdateChecksEnabled()) return; + void checkForUpdates(false); + }, AUTO_CHECK_DELAY_MS); +} + +export function setupUpdateControls(): void { + const toggle = document.getElementById("auto-update-toggle") as HTMLInputElement | null; + const checkBtn = document.getElementById("btn-check-updates") as HTMLButtonElement | null; + const installBtn = document.getElementById("btn-install-update") as HTMLButtonElement | null; + + if (toggle) { + toggle.checked = autoUpdateChecksEnabled(); + toggle.addEventListener("change", () => { + saveAutoUpdateChecksEnabled(toggle.checked); + setStatus( + toggle.checked ? "Automatic checks are on." : "Automatic checks are off.", + "idle", + ); + }); + } + + checkBtn?.addEventListener("click", () => { + void checkForUpdates(true); + }); + installBtn?.addEventListener("click", () => { + void installAvailableUpdate(); + }); + setInstallVisible(false); +} From ab4dd6c20bdcbbebfb1784d26a17e5c65cada62a Mon Sep 17 00:00:00 2001 From: matthiasschalk Date: Sat, 16 May 2026 22:01:42 +0200 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20store?= =?UTF-8?q?-cache=20race,=20gemini/pi=20dedup,=20local=20widget=20build?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finding 1: appendEvents() paired a fresh on-disk file snapshot with this process's stale in-memory event list, persisting a read-cache that looked complete (sameFileSet matched) but dropped events another process had appended. Stop refreshing the read-cache in appendEvents(); the append already invalidates the cache via the changed file mtime/size, so the next loadStore() does a correct full rescan and rewrites it. Finding 2: the loader-cache refactor moved gemini/pi dedupe into the per-file parse callback, so duplicate sessions/messages spanning files were emitted more than once. Mirror the claude.ts pattern: carry a dedupeKey out of the per-file parse and dedupe globally after the cache merge. Finding 3: split the widget build. widget:build now builds locally with updater artifacts disabled (widget/src-tauri/tauri.dev.conf.json), so no signing key is required. widget:build:release is the signed path (TAURI_SIGNING_PRIVATE_KEY), matching release CI tauri-action. Adds regression tests for findings 1 and 2 (store, gemini, pi). Co-Authored-By: Claude Opus 4.7 (1M context) --- AGENTS.md | 3 +- package.json | 3 +- src/loaders/gemini.test.ts | 50 +++++++++++++++++++++ src/loaders/gemini.ts | 66 +++++++++++++++++++-------- src/loaders/pi.test.ts | 45 +++++++++++++++++++ src/loaders/pi.ts | 67 ++++++++++++++++++++-------- src/store.test.ts | 22 +++++++++ src/store.ts | 10 ++++- widget/src-tauri/tauri.dev.conf.json | 5 +++ 9 files changed, 232 insertions(+), 39 deletions(-) create mode 100644 src/loaders/gemini.test.ts create mode 100644 src/loaders/pi.test.ts create mode 100644 widget/src-tauri/tauri.dev.conf.json diff --git a/AGENTS.md b/AGENTS.md index b37ab93..8261592 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,8 @@ TokenBBQ is a TypeScript ESM CLI and dashboard. Core source lives in `src/`: `in - `npm run build`: produce the publishable CLI bundle in `dist/`. - `npm run widget:install`: install widget dependencies. - `npm run widget:dev`: build the sidecar and launch the Tauri widget. -- `npm run widget:build`: build the CLI, sidecar, and desktop widget package. +- `npm run widget:build`: build the CLI, sidecar, and desktop widget package locally. Updater artifacts are disabled (via `widget/src-tauri/tauri.dev.conf.json`), so no signing key is needed. +- `npm run widget:build:release`: full signed build with updater artifacts. Requires `TAURI_SIGNING_PRIVATE_KEY` (and `TAURI_SIGNING_PRIVATE_KEY_PASSWORD`); release CI builds this path via `tauri-action`. ## Coding Style & Naming Conventions diff --git a/package.json b/package.json index ca244d2..928c203 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,8 @@ "build:sidecar": "node scripts/inline-wasm.mjs && node scripts/inline-dashboard-icon.mjs && node scripts/build-sidecar.mjs", "widget:install": "npm install --prefix widget", "widget:dev": "node scripts/build-sidecar.mjs --skip-if-no-bun && npm run --prefix widget tauri dev", - "widget:build": "npm run build && npm run build:sidecar && npm run --prefix widget tauri build" + "widget:build": "npm run build && npm run build:sidecar && npm run --prefix widget tauri build -- --config src-tauri/tauri.dev.conf.json", + "widget:build:release": "npm run build && npm run build:sidecar && npm run --prefix widget tauri build" }, "dependencies": { "@hono/node-server": "^1.13.0", diff --git a/src/loaders/gemini.test.ts b/src/loaders/gemini.test.ts new file mode 100644 index 0000000..2400f35 --- /dev/null +++ b/src/loaders/gemini.test.ts @@ -0,0 +1,50 @@ +import { test, describe, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { loadGeminiEvents } from './gemini.js'; + +let tmp: string; +beforeEach(() => { + tmp = mkdtempSync(path.join(tmpdir(), 'tbq-gemini-')); + process.env.GEMINI_DIR = tmp; + process.env.TOKENBBQ_DATA_DIR = path.join(tmp, 'data'); +}); +afterEach(() => { + delete process.env.GEMINI_DIR; + delete process.env.TOKENBBQ_DATA_DIR; + rmSync(tmp, { recursive: true, force: true }); +}); + +function writeSession(name: string, sessionId: string, messageId: string): void { + const dir = path.join(tmp, 'tmp', 'proj', 'chats'); + mkdirSync(dir, { recursive: true }); + writeFileSync( + path.join(dir, name), + JSON.stringify({ + sessionId, + messages: [ + { + id: messageId, + timestamp: '2026-04-22T14:02:11.812Z', + model: 'gemini-2.0', + tokens: { input: 10, output: 20 }, + }, + ], + }), + 'utf-8', + ); +} + +describe('loadGeminiEvents', () => { + test('dedupes the same logical event across multiple session files', async () => { + // Same sessionId + message id appearing in two files must collapse to + // one event — dedup is global across files, not per-file. + writeSession('session-1.json', 'sess', 'm1'); + writeSession('session-2.json', 'sess', 'm1'); + + const events = await loadGeminiEvents(); + assert.equal(events.length, 1); + }); +}); diff --git a/src/loaders/gemini.ts b/src/loaders/gemini.ts index 52bbc49..9068432 100644 --- a/src/loaders/gemini.ts +++ b/src/loaders/gemini.ts @@ -5,11 +5,32 @@ import path from 'node:path'; import { glob } from 'tinyglobby'; import type { UnifiedTokenEvent } from '../types.js'; import { isValidTimestamp } from '../types.js'; -import { loadCachedFileEvents } from './cache.js'; +import { loadCachedFileRecords } from './cache.js'; const HOME = homedir(); const FALLBACK_MODEL = 'gemini'; +type CachedGeminiEvent = { + dedupeKey: string; + event: UnifiedTokenEvent; +}; + +function isCachedGeminiEvent(value: unknown): value is CachedGeminiEvent { + if (!value || typeof value !== 'object') return false; + const record = value as Record; + const event = record.event as Record | undefined; + return ( + typeof record.dedupeKey === 'string' && + !!event && + typeof event.source === 'string' && + typeof event.timestamp === 'string' && + typeof event.sessionId === 'string' && + typeof event.model === 'string' && + !!event.tokens && + typeof event.tokens === 'object' + ); +} + function getGeminiDir(): string | null { const envPath = (process.env.GEMINI_DIR ?? '').trim(); if (envPath !== '') { @@ -49,8 +70,8 @@ export async function loadGeminiEvents(): Promise { const tmpDir = path.join(geminiDir, 'tmp'); const files = await glob('**/chats/session-*.json', { cwd: tmpDir, absolute: true }); - const events = await loadCachedFileEvents('gemini', files, async (file) => { - const fileEvents: UnifiedTokenEvent[] = []; + const records = await loadCachedFileRecords('gemini', files, async (file) => { + const fileEvents: CachedGeminiEvent[] = []; let content: string; try { content = await readFile(file, 'utf-8'); @@ -70,7 +91,6 @@ export async function loadGeminiEvents(): Promise { const messages = Array.isArray(session.messages) ? (session.messages as Record[]) : []; - const seen = new Set(); for (const msg of messages) { const tokens = msg.tokens as Record | undefined; @@ -105,8 +125,6 @@ export async function loadGeminiEvents(): Promise { const dedupeKey = id ? `gemini:${sessionId}:${id}` : `gemini:${sessionId}:${timestamp}:${input}:${output}:${cacheRead}:${reasoning}`; - if (seen.has(dedupeKey)) continue; - seen.add(dedupeKey); const model = typeof msg.model === 'string' && msg.model.trim() !== '' @@ -114,22 +132,34 @@ export async function loadGeminiEvents(): Promise { : FALLBACK_MODEL; fileEvents.push({ - source: 'gemini', - timestamp, - sessionId, - model, - tokens: { - input, - output, - cacheCreation, - cacheRead, - reasoning, + dedupeKey, + event: { + source: 'gemini', + timestamp, + sessionId, + model, + tokens: { + input, + output, + cacheCreation, + cacheRead, + reasoning, + }, + costUSD: 0, + project, }, - costUSD: 0, - project, }); } return fileEvents; + }, isCachedGeminiEvent); + + // Dedup globally across all files (cached or freshly parsed), not per-file: + // the same logical message can appear in more than one session file. + const seen = new Set(); + const events = records.flatMap((record) => { + if (seen.has(record.dedupeKey)) return []; + seen.add(record.dedupeKey); + return [record.event]; }); events.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()); diff --git a/src/loaders/pi.test.ts b/src/loaders/pi.test.ts new file mode 100644 index 0000000..ee39054 --- /dev/null +++ b/src/loaders/pi.test.ts @@ -0,0 +1,45 @@ +import { test, describe, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { loadPiEvents } from './pi.js'; + +let tmp: string; +beforeEach(() => { + tmp = mkdtempSync(path.join(tmpdir(), 'tbq-pi-')); + process.env.PI_AGENT_DIR = tmp; + process.env.TOKENBBQ_DATA_DIR = path.join(tmp, 'data'); +}); +afterEach(() => { + delete process.env.PI_AGENT_DIR; + delete process.env.TOKENBBQ_DATA_DIR; + rmSync(tmp, { recursive: true, force: true }); +}); + +function writeJsonl(rel: string): void { + const file = path.join(tmp, rel); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync( + file, + JSON.stringify({ + type: 'message', + timestamp: '2026-04-22T14:02:11.812Z', + message: { role: 'assistant', model: 'pi-1', usage: { input: 10, output: 20 } }, + }) + '\n', + 'utf-8', + ); +} + +describe('loadPiEvents', () => { + test('dedupes the same logical event across multiple session files', async () => { + // pi's dedupe key is timestamp + token total (no file/session in it), + // so the same event in two files must collapse to one — global, not + // per-file. + writeJsonl(path.join('proj', 'a_session-1.jsonl')); + writeJsonl(path.join('proj', 'b_session-2.jsonl')); + + const events = await loadPiEvents(); + assert.equal(events.length, 1); + }); +}); diff --git a/src/loaders/pi.ts b/src/loaders/pi.ts index ccdf336..171b2f1 100644 --- a/src/loaders/pi.ts +++ b/src/loaders/pi.ts @@ -5,10 +5,31 @@ import path from 'node:path'; import { glob } from 'tinyglobby'; import type { UnifiedTokenEvent } from '../types.js'; import { isValidTimestamp } from '../types.js'; -import { loadCachedFileEvents } from './cache.js'; +import { loadCachedFileRecords } from './cache.js'; const HOME = homedir(); +type CachedPiEvent = { + dedupeKey: string; + event: UnifiedTokenEvent; +}; + +function isCachedPiEvent(value: unknown): value is CachedPiEvent { + if (!value || typeof value !== 'object') return false; + const record = value as Record; + const event = record.event as Record | undefined; + return ( + typeof record.dedupeKey === 'string' && + !!event && + typeof event.source === 'string' && + typeof event.timestamp === 'string' && + typeof event.sessionId === 'string' && + typeof event.model === 'string' && + !!event.tokens && + typeof event.tokens === 'object' + ); +} + function getPiAgentDir(): string | null { const envPath = (process.env.PI_AGENT_DIR ?? '').trim(); if (envPath !== '') { @@ -31,9 +52,8 @@ export async function loadPiEvents(): Promise { const files = await glob('**/*.jsonl', { cwd: piDir, absolute: true }); - const events = await loadCachedFileEvents('pi', files, async (file) => { - const fileEvents: UnifiedTokenEvent[] = []; - const seen = new Set(); + const records = await loadCachedFileRecords('pi', files, async (file) => { + const fileEvents: CachedPiEvent[] = []; const relPath = path.relative(piDir, file); const segments = relPath.split(path.sep); const project = segments.length >= 2 ? segments[0] : 'unknown'; @@ -79,29 +99,40 @@ export async function loadPiEvents(): Promise { if (!isValidTimestamp(parsed.timestamp)) continue; const timestamp = parsed.timestamp; const dedupeKey = `pi:${timestamp}:${input + output}`; - if (seen.has(dedupeKey)) continue; - seen.add(dedupeKey); const rawModel = String(message.model ?? 'unknown'); const cost = usage.cost as Record | undefined; fileEvents.push({ - source: 'pi', - timestamp, - sessionId, - model: `[pi] ${rawModel}`, - tokens: { - input, - output, - cacheCreation: Number(usage.cacheWrite ?? 0), - cacheRead: Number(usage.cacheRead ?? 0), - reasoning: 0, + dedupeKey, + event: { + source: 'pi', + timestamp, + sessionId, + model: `[pi] ${rawModel}`, + tokens: { + input, + output, + cacheCreation: Number(usage.cacheWrite ?? 0), + cacheRead: Number(usage.cacheRead ?? 0), + reasoning: 0, + }, + costUSD: typeof cost?.total === 'number' ? cost.total : 0, + project, }, - costUSD: typeof cost?.total === 'number' ? cost.total : 0, - project, }); } return fileEvents; + }, isCachedPiEvent); + + // Dedup globally across all files, not per-file: pi's dedupe key is + // timestamp + token total (no file/session component), so the same logical + // event recorded in more than one session file must collapse to one. + const seen = new Set(); + const events = records.flatMap((record) => { + if (seen.has(record.dedupeKey)) return []; + seen.add(record.dedupeKey); + return [record.event]; }); events.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()); diff --git a/src/store.test.ts b/src/store.test.ts index d4dc561..555766c 100644 --- a/src/store.test.ts +++ b/src/store.test.ts @@ -139,4 +139,26 @@ describe('appendEvents', () => { assert.ok(!existsSync(legacyPath()), 'legacy events.ndjson should not be created'); assert.ok(state.path !== legacyPath(), 'state.path must be the per-process file'); }); + + test('does not persist a stale read-cache that hides another state\'s events', () => { + // Two loaded store states race on appends. A stale state writing the + // read-cache must not drop events a fresher state already persisted. + const a = ev({ sessionId: 'a' }); + const b = ev({ sessionId: 'b' }); + const c = ev({ sessionId: 'c' }); + + const stateA = loadStore(); + appendEvents(stateA, [a]); + + const stateB = loadStore(); // sees a + appendEvents(stateB, [b]); + + appendEvents(stateA, [c]); // stale stateA appends; must not bury b + + const reread = loadStore(); + assert.deepEqual( + reread.events.map((e) => e.sessionId).sort(), + ['a', 'b', 'c'], + ); + }); }); diff --git a/src/store.ts b/src/store.ts index ff6e37a..174cb4b 100644 --- a/src/store.ts +++ b/src/store.ts @@ -320,7 +320,15 @@ export function appendEvents(state: StoreState, events: UnifiedTokenEvent[]): Un // dedupes them on the next read. Slightly redundant on disk, lossless. if (buffer) { appendFileSync(state.path, buffer); - writeStoreCache(listStoreFiles(getEventsDir()), state.events); + // Deliberately do NOT refresh the read-cache here. state.events is this + // process's view as of its last loadStore() plus its own appends — it does + // not include events another process appended to its own file since then. + // Pairing that stale list with a fresh on-disk file snapshot would persist + // a cache that looks complete (sameFileSet matches) but silently drops the + // other process's events. Appending changed this process's own file + // (mtime/size), so the existing cache no longer matches the file set and + // the next loadStore() does a correct full rescan and rewrites the cache + // from the true on-disk union. } return added; } diff --git a/widget/src-tauri/tauri.dev.conf.json b/widget/src-tauri/tauri.dev.conf.json new file mode 100644 index 0000000..55a9946 --- /dev/null +++ b/widget/src-tauri/tauri.dev.conf.json @@ -0,0 +1,5 @@ +{ + "bundle": { + "createUpdaterArtifacts": false + } +} From fa539dbf6ab8fee0c2f7f8b2914d799c8e7e9064 Mon Sep 17 00:00:00 2001 From: matthiasschalk Date: Sat, 16 May 2026 22:01:42 +0200 Subject: [PATCH 3/3] fix: invalidate store read-cache poisoned by the pre-fix appendEvents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second-opinion review (Codex) surfaced a gap in the Finding 1 fix: a store-v1.json written by the OLD buggy appendEvents() (a fresh file snapshot paired with an incomplete event list) is still accepted after upgrade whenever the file set is unchanged, so incomplete data can persist until the first new event is captured. Bumping the shared CURRENT_VERSION would have been wrong — it also versions the on-disk ndjson event lines, so older tokenbbq builds would skip v2 lines as "future". Introduce a separate STORE_CACHE_VERSION (2) used only by the store read-cache, and move the cache file to store-v2.json. Any pre-fix v1 cache is now ignored (different filename and version field) and rebuilt correctly on the next load. Regression test added: a poisoned cache at the old path and at the new path with the old version field is ignored; loadStore() returns the true on-disk union. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/store.test.ts | 38 +++++++++++++++++++++++++++++++++++++- src/store.ts | 15 ++++++++++++--- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/store.test.ts b/src/store.test.ts index 555766c..1bff687 100644 --- a/src/store.test.ts +++ b/src/store.test.ts @@ -1,6 +1,6 @@ import { test, describe, beforeEach, afterEach } from 'node:test'; import assert from 'node:assert/strict'; -import { mkdtempSync, rmSync, readFileSync, appendFileSync, existsSync, mkdirSync } from 'node:fs'; +import { mkdtempSync, rmSync, readFileSync, appendFileSync, existsSync, mkdirSync, statSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { loadStore, appendEvents, hashEvent, getStoreDir } from './store.js'; @@ -100,6 +100,42 @@ describe('loadStore', () => { const state = loadStore(); assert.equal(state.events.length, 1); }); + + test('ignores a poisoned cache written by the pre-fix version', () => { + const eventsDir = path.join(tmp, 'events'); + mkdirSync(eventsDir, { recursive: true }); + + const a = ev({ sessionId: 'a' }); + const b = ev({ sessionId: 'b' }); + const file = path.join(eventsDir, 'events-host-1.ndjson'); + appendFileSync( + file, + JSON.stringify({ v: 1, ...a, eventHash: hashEvent(a) }) + '\n' + + JSON.stringify({ v: 1, ...b, eventHash: hashEvent(b) }) + '\n', + ); + + // The old buggy appendEvents() could persist a cache whose file metadata + // matches the real file but whose event list is incomplete (missing b). + // Such a cache must not be trusted after upgrade — neither at the old + // store-v1 path nor via the version field. + const s = statSync(file); + const meta = [{ path: file, mtimeMs: s.mtimeMs, size: s.size }]; + const cacheDir = path.join(tmp, 'cache'); + mkdirSync(cacheDir, { recursive: true }); + writeFileSync( + path.join(cacheDir, 'store-v1.json'), + JSON.stringify({ v: 1, files: meta, events: [a] }), + 'utf-8', + ); + writeFileSync( + path.join(cacheDir, 'store-v2.json'), + JSON.stringify({ v: 1, files: meta, events: [a] }), + 'utf-8', + ); + + const state = loadStore(); + assert.deepEqual(state.events.map((e) => e.sessionId).sort(), ['a', 'b']); + }); }); describe('appendEvents', () => { diff --git a/src/store.ts b/src/store.ts index 174cb4b..e7b655b 100644 --- a/src/store.ts +++ b/src/store.ts @@ -6,6 +6,15 @@ import type { UnifiedTokenEvent } from './types.js'; const CURRENT_VERSION = 1; +// Version of the store read-cache file, independent of the event-line schema +// version (CURRENT_VERSION) on purpose: bumping CURRENT_VERSION would change +// the on-disk ndjson `v` and make older tokenbbq builds skip those lines as +// "future". Bumped to 2 to invalidate any store-v1 cache written by the +// pre-fix appendEvents(), which could persist a file snapshot paired with an +// incomplete event list. A stale v1 cache is now ignored (different filename +// and version) and rebuilt correctly on the next load. +const STORE_CACHE_VERSION = 2; + export interface StoreState { events: UnifiedTokenEvent[]; hashes: Set; @@ -24,7 +33,7 @@ function getEventsDir(): string { } function getStoreCachePath(): string { - return path.join(getStoreDir(), 'cache', 'store-v1.json'); + return path.join(getStoreDir(), 'cache', `store-v${STORE_CACHE_VERSION}.json`); } function sanitizeForFilename(s: string): string { @@ -165,7 +174,7 @@ function readStoreCache(files: StoreFileMeta[]): LoadOutcome | null { const parsed = JSON.parse(readFileSync(getStoreCachePath(), 'utf-8')) as unknown; if (!parsed || typeof parsed !== 'object') return null; const cache = parsed as StoreReadCache; - if (cache.v !== CURRENT_VERSION || !Array.isArray(cache.files) || !Array.isArray(cache.events)) return null; + if (cache.v !== STORE_CACHE_VERSION || !Array.isArray(cache.files) || !Array.isArray(cache.events)) return null; if (!sameFileSet(cache.files, files)) return null; if (!cache.events.every(isStoreEvent)) return null; return outcomeFromEvents(cache.events); @@ -180,7 +189,7 @@ function writeStoreCache(files: StoreFileMeta[], events: UnifiedTokenEvent[]): v const tmp = path.join(dir, `${path.basename(target)}.${process.pid}.${Date.now()}.tmp`); try { if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); - writeFileSync(tmp, JSON.stringify({ v: CURRENT_VERSION, files, events }), 'utf-8'); + writeFileSync(tmp, JSON.stringify({ v: STORE_CACHE_VERSION, files, events }), 'utf-8'); renameSync(tmp, target); } catch { // Performance-only cache. Store reads must keep working if this fails.