From 0e0a0e72796e1d69986b1e6374fb2c96d1db931f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Me=CC=81de=CC=81de=CC=81=20Raymond=20KPATCHAA?= Date: Mon, 11 May 2026 13:58:25 +0100 Subject: [PATCH 1/9] feat: add ft web browser dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a browser-based bookmark observatory accessible via `ft web`. Uses Node.js built-in HTTP — no new npm dependencies. - src/web.ts: HTTP server with REST API (/api/overview, /api/bookmarks, /api/count, /api/bookmarks/:id) and an inline SPA (Tailwind + Chart.js + Alpine.js via CDN). Binds to 127.0.0.1 only. Exports createWebServer() for testing. - src/bookmarks-viz.ts: export buildVizData() and VizData interface so the web server can reuse the terminal viz aggregation logic. - src/cli.ts: register `ft web [--port ] [--no-open]` command. - tests/web.test.ts: 18 integration tests covering all routes, filters, pagination, and error paths. - tests/bookmarks-viz.test.ts: add buildVizData() unit test. - README.md: document ft web in quick start and commands table. --- README.md | 2 + src/bookmarks-viz.ts | 40 +- src/cli.ts | 13 + src/web.ts | 975 ++++++++++++++++++++++++++++++++++++ tests/bookmarks-viz.test.ts | 67 ++- tests/web.test.ts | 255 ++++++++++ 6 files changed, 1331 insertions(+), 21 deletions(-) create mode 100644 src/web.ts create mode 100644 tests/web.test.ts diff --git a/README.md b/README.md index f7ddb85..d97730d 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ ft search "distributed systems" # 3. Explore ft viz +ft web # browser-based dashboard ft categories ft stats ``` @@ -58,6 +59,7 @@ On first run, `ft sync` extracts your X session from your browser and downloads | `ft sample ` | Random sample from a category | | `ft stats` | Top authors, languages, date range | | `ft viz` | Terminal dashboard with sparklines, categories, and domains | +| `ft web` | Browser-based bookmark dashboard with charts, search, and filters (default port 4321) | | `ft categories` | Show category distribution | | `ft domains` | Subject domain distribution | | `ft folders` | Show X bookmark folder distribution (requires `ft sync --folders` first) | diff --git a/src/bookmarks-viz.ts b/src/bookmarks-viz.ts index 2f4b29c..aa33ea6 100644 --- a/src/bookmarks-viz.ts +++ b/src/bookmarks-viz.ts @@ -13,16 +13,16 @@ const rgb = (r: number, g: number, b: number) => `${ESC}38;2;${r};${g};${b}m`; // Palette — muted, tasteful const C = { - title: rgb(199, 146, 234), // soft lavender - accent: rgb(130, 170, 255), // periwinkle - warm: rgb(255, 180, 120), // peach - green: rgb(120, 220, 170), // mint - dim: rgb(100, 100, 120), // muted gray - text: rgb(200, 200, 210), // light gray - hot: rgb(255, 120, 140), // coral - gold: rgb(240, 200, 100), // amber - cyan: rgb(100, 220, 230), // teal - violet: rgb(170, 130, 255), // violet + title: rgb(199, 146, 234), // soft lavender + accent: rgb(130, 170, 255), // periwinkle + warm: rgb(255, 180, 120), // peach + green: rgb(120, 220, 170), // mint + dim: rgb(100, 100, 120), // muted gray + text: rgb(200, 200, 210), // light gray + hot: rgb(255, 120, 140), // coral + gold: rgb(240, 200, 100), // amber + cyan: rgb(100, 220, 230), // teal + violet: rgb(170, 130, 255), // violet }; // ── Block characters for bar charts ────────────────────────────────────────── @@ -116,14 +116,14 @@ function lerpColor( // ── Data queries ───────────────────────────────────────────────────────────── -interface GemBookmark { +export interface GemBookmark { author: string; text: string; tweetId: string; postedAt: string; } -interface VizData { +export interface VizData { total: number; uniqueAuthors: number; dateRange: { earliest: string; latest: string }; @@ -255,10 +255,10 @@ function aggregateTimelineData(rows: TimelineAggregateRow[]): { const risingVoices = latestPostedMonth == null ? [] : [...authorPostedCounts.entries()] - .filter(([handle, count]) => count >= 3 && authorPostedMonths.get(handle)?.size === 1 && authorPostedMonths.get(handle)?.has(latestPostedMonth)) - .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) - .slice(0, 8) - .map(([handle, count]) => ({ handle, count })); + .filter(([handle, count]) => count >= 3 && authorPostedMonths.get(handle)?.size === 1 && authorPostedMonths.get(handle)?.has(latestPostedMonth)) + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .slice(0, 8) + .map(([handle, count]) => ({ handle, count })); return { dateRange: { earliest, latest }, @@ -270,7 +270,7 @@ function aggregateTimelineData(rows: TimelineAggregateRow[]): { }; } -async function queryVizData(): Promise { +export async function buildVizData(): Promise { const db = await openDb(twitterBookmarksIndexPath()); try { @@ -310,9 +310,9 @@ async function queryVizData(): Promise { if (domain && domain !== 'x.com' && domain !== 't.co') { domainCounts.set(domain, (domainCounts.get(domain) ?? 0) + 1); } - } catch {} + } catch { } } - } catch {} + } catch { } } const topDomains = [...domainCounts.entries()] .sort((a, b) => b[1] - a[1]) @@ -788,7 +788,7 @@ function renderRisingVoices(data: VizData): string[] { // ── Main render ────────────────────────────────────────────────────────────── export async function renderViz(): Promise { - const data = await queryVizData(); + const data = await buildVizData(); const sections = [ ...renderHiddenGems(data), diff --git a/src/cli.ts b/src/cli.ts index b32621c..c48dcd8 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -31,6 +31,7 @@ import { askMd } from './md-ask.js'; import { lintMd, fixLintIssues } from './md-lint.js'; import { exportBookmarks } from './md-export.js'; import { renderViz } from './bookmarks-viz.js'; +import { startWeb } from './web.js'; import { listBrowserIds } from './browsers.js'; import { configureHttpProxyFromEnv } from './http-proxy.js'; import { dataDir, ensureDataDir, isFirstRun, migrateLegacyIdeasData, twitterBookmarksIndexPath, twitterBackfillStatePath, mdDir, bookmarkMediaDir, bookmarkMediaManifestPath } from './paths.js'; @@ -1233,6 +1234,18 @@ export function buildCli() { console.log(await renderViz()); })); + // ── web ────────────────────────────────────────────────────────────────── + + program + .command('web') + .description('Open a browser-based dashboard for your bookmarks') + .option('--port ', 'Port to listen on', '4321') + .option('--no-open', 'Do not open the browser automatically') + .action(safe(async (options) => { + if (!requireIndex()) return; + await startWeb(parseInt(String(options.port), 10), options.open !== false); + })); + // ── classify ──────────────────────────────────────────────────────────── program diff --git a/src/web.ts b/src/web.ts new file mode 100644 index 0000000..27ed19e --- /dev/null +++ b/src/web.ts @@ -0,0 +1,975 @@ +import { createServer, IncomingMessage, ServerResponse } from 'node:http'; +import { parse as parseUrl } from 'node:url'; +import { spawn } from 'node:child_process'; +import { buildVizData } from './bookmarks-viz.js'; +import { + listBookmarks, + countBookmarks, + getBookmarkById, +} from './bookmarks-db.js'; + +// ── Helpers ────────────────────────────────────────────────────────────────── + +function json(res: ServerResponse, data: unknown, status = 200): void { + const body = JSON.stringify(data); + res.writeHead(status, { + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(body), + 'Access-Control-Allow-Origin': '*', + }); + res.end(body); +} + +function html(res: ServerResponse, body: string): void { + res.writeHead(200, { + 'Content-Type': 'text/html; charset=utf-8', + 'Content-Length': Buffer.byteLength(body), + }); + res.end(body); +} + +function qs(req: IncomingMessage): Record { + const parsed = parseUrl(req.url ?? '', true); + const out: Record = {}; + for (const [k, v] of Object.entries(parsed.query)) { + if (typeof v === 'string') out[k] = v; + } + return out; +} + +function openInBrowser(url: string): void { + const platform = process.platform; + const cmd = platform === 'darwin' ? 'open' : platform === 'win32' ? 'cmd' : 'xdg-open'; + const args = platform === 'win32' ? ['/c', 'start', '', url] : [url]; + spawn(cmd, args, { detached: true, stdio: 'ignore' }).unref(); +} + +// ── HTML shell ──────────────────────────────────────────────────────────────── + +function buildHtml(): string { + return /* html */ ` + + + + + Field Theory · Bookmark Observatory + + + + + + + + + + + +
+ + +
+ + + + + Loading observatory data… +
+ + +
+ + +
+ + +
+
+ + + + + +
+
+ + + +
+
+ + +
+
+
+ + + +
+
+ + +
+ + + + +
+ + +
+ + +
+ + +
+ + +
+
+ + +
+ + +
+ + +
+
+ + + + + +
+ + + + +
+ + +
+
+
+ + + +`; +} + +// ── Request router ──────────────────────────────────────────────────────────── + +async function handleRequest(req: IncomingMessage, res: ServerResponse): Promise { + const parsed = parseUrl(req.url ?? '', true); + const pathname = parsed.pathname ?? '/'; + + // Static HTML shell + if (req.method === 'GET' && pathname === '/') { + html(res, buildHtml()); + return; + } + + if (req.method !== 'GET') { + json(res, { error: 'method not allowed' }, 405); + return; + } + + // /api/overview + if (pathname === '/api/overview') { + const data = await buildVizData(); + json(res, data); + return; + } + + // /api/count + if (pathname === '/api/count') { + const q = qs(req); + const count = await countBookmarks({ + query: q.q || undefined, + author: q.author || undefined, + category: q.category || undefined, + domain: q.domain || undefined, + after: q.after || undefined, + before: q.before || undefined, + }); + json(res, { count }); + return; + } + + // /api/bookmarks/:id + const detailMatch = pathname.match(/^\/api\/bookmarks\/(.+)$/); + if (detailMatch) { + const id = decodeURIComponent(detailMatch[1]); + const bookmark = await getBookmarkById(id); + if (!bookmark) { + json(res, { error: 'not found' }, 404); + return; + } + json(res, bookmark); + return; + } + + // /api/bookmarks + if (pathname === '/api/bookmarks') { + const q = qs(req); + const items = await listBookmarks({ + query: q.q || undefined, + author: q.author || undefined, + category: q.category || undefined, + domain: q.domain || undefined, + after: q.after || undefined, + before: q.before || undefined, + sort: q.sort === 'asc' ? 'asc' : 'desc', + limit: q.limit ? Math.min(200, Math.max(1, parseInt(q.limit, 10))) : 50, + offset: q.offset ? Math.max(0, parseInt(q.offset, 10)) : 0, + }); + json(res, items); + return; + } + + json(res, { error: 'not found' }, 404); +} + +// ── Server factory (exported for testing) ──────────────────────────────────── + +export async function createWebServer(port: number): Promise<{ port: number; close: () => Promise }> { + const server = createServer(async (req, res) => { + try { + await handleRequest(req, res); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + json(res, { error: message }, 500); + } + }); + + await new Promise((resolve, reject) => { + server.listen(port, '127.0.0.1', () => resolve()); + server.once('error', reject); + }); + + const addr = server.address(); + const actualPort = typeof addr === 'object' && addr !== null ? addr.port : port; + const close = (): Promise => + new Promise((resolve, reject) => server.close((err) => (err ? reject(err) : resolve()))); + + return { port: actualPort, close }; +} + +// ── Entry point ─────────────────────────────────────────────────────────────── + +export async function startWeb(port: number, openBrowser: boolean): Promise { + const { port: actualPort, close } = await createWebServer(port); + + const url = `http://localhost:${actualPort}`; + process.stdout.write(`\nField Theory web running at ${url}\nPress Ctrl+C to stop.\n\n`); + + if (openBrowser) { + openInBrowser(url); + } + + // Keep alive until interrupted + await new Promise((resolve) => { + process.once('SIGINT', () => { close().then(resolve).catch(resolve); }); + process.once('SIGTERM', () => { close().then(resolve).catch(resolve); }); + }); +} diff --git a/tests/bookmarks-viz.test.ts b/tests/bookmarks-viz.test.ts index 84435ee..eec4043 100644 --- a/tests/bookmarks-viz.test.ts +++ b/tests/bookmarks-viz.test.ts @@ -5,7 +5,7 @@ import { rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { buildIndex } from '../src/bookmarks-db.js'; -import { renderViz } from '../src/bookmarks-viz.js'; +import { renderViz, buildVizData } from '../src/bookmarks-viz.js'; async function withVizDataDir(records: any[], fn: () => Promise): Promise { const dir = await mkdtemp(path.join(tmpdir(), 'ft-viz-test-')); @@ -83,3 +83,68 @@ test('renderViz uses publication timing instead of fabricated bookmark timing', assert.doesNotMatch(output, /000Z/); }); }); + +test('buildVizData returns correctly shaped VizData', async () => { + const records = [ + { + id: '10', tweetId: '10', + url: 'https://x.com/alice/status/10', + text: 'AI is the future', + authorHandle: 'alice', authorName: 'Alice', + postedAt: 'Mon Jan 06 08:00:00 +0000 2025', + bookmarkedAt: '2025-01-07T09:00:00Z', + syncedAt: '2025-01-07T09:00:00Z', + mediaObjects: [], media: ['https://img.example.com/1.jpg'], + links: ['https://example.com/article'], + tags: [], ingestedVia: 'graphql', + }, + { + id: '11', tweetId: '11', + url: 'https://x.com/bob/status/11', + text: 'TypeScript is great for large codebases', + authorHandle: 'bob', authorName: 'Bob', + postedAt: 'Tue Feb 04 15:00:00 +0000 2025', + bookmarkedAt: '2025-02-05T12:00:00Z', + syncedAt: '2025-02-05T12:00:00Z', + mediaObjects: [], links: [], + tags: [], ingestedVia: 'graphql', + }, + ]; + + await withVizDataDir(records, async () => { + await buildIndex({ force: true }); + const data = await buildVizData(); + + // Aggregate counts + assert.equal(data.total, 2); + assert.equal(data.uniqueAuthors, 2); + + // Shape checks + assert.ok(typeof data.avgTextLength === 'number'); + assert.ok(Array.isArray(data.topAuthors)); + assert.ok(Array.isArray(data.monthlyActivity)); + assert.ok(Array.isArray(data.dayOfWeekActivity)); + assert.ok(Array.isArray(data.hourActivity)); + assert.ok(Array.isArray(data.topDomains)); + assert.ok(Array.isArray(data.languages)); + assert.ok(Array.isArray(data.categories)); + assert.ok(Array.isArray(data.domains)); + assert.ok(Array.isArray(data.timeCapsules)); + assert.ok(Array.isArray(data.hiddenGems)); + assert.ok(Array.isArray(data.risingVoices)); + assert.ok(Array.isArray(data.recentAuthors)); + assert.ok(typeof data.mediaStats === 'object'); + assert.ok(typeof data.dateRange === 'object'); + + // Media stats reflect fixture data (1 with media, 1 with links) + assert.equal(data.mediaStats.withMedia, 1); + assert.equal(data.mediaStats.withLinks, 1); + assert.equal(data.mediaStats.total, 2); + + // Top authors should contain both handles + const handles = data.topAuthors.map((a) => a.handle); + assert.ok(handles.includes('alice')); + assert.ok(handles.includes('bob')); + }); +}); + diff --git a/tests/web.test.ts b/tests/web.test.ts new file mode 100644 index 0000000..cbb9a68 --- /dev/null +++ b/tests/web.test.ts @@ -0,0 +1,255 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, writeFile } from 'node:fs/promises'; +import { rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { buildIndex } from '../src/bookmarks-db.js'; +import { createWebServer } from '../src/web.js'; + +// ── Fixtures ───────────────────────────────────────────────────────────────── + +const FIXTURES = [ + { + id: '1', tweetId: '1', + url: 'https://x.com/alice/status/1', + text: 'Machine learning is transforming healthcare and AI research', + authorHandle: 'alice', authorName: 'Alice Smith', + postedAt: 'Mon Jan 06 12:00:00 +0000 2025', + bookmarkedAt: '2025-01-07T08:00:00Z', + syncedAt: '2025-01-07T08:00:00Z', + language: 'en', mediaObjects: [], links: ['https://example.com'], + tags: [], ingestedVia: 'graphql', + }, + { + id: '2', tweetId: '2', + url: 'https://x.com/bob/status/2', + text: 'Rust is a great systems programming language for safety', + authorHandle: 'bob', authorName: 'Bob Jones', + postedAt: 'Tue Feb 04 14:30:00 +0000 2025', + bookmarkedAt: '2025-02-05T10:00:00Z', + syncedAt: '2025-02-05T10:00:00Z', + language: 'en', mediaObjects: [], links: [], + tags: [], ingestedVia: 'graphql', + }, + { + id: '3', tweetId: '3', + url: 'https://x.com/alice/status/3', + text: 'Deep learning models need massive compute infrastructure and optimization techniques', + authorHandle: 'alice', authorName: 'Alice Smith', + postedAt: 'Wed Mar 05 09:15:00 +0000 2025', + bookmarkedAt: '2025-03-06T07:00:00Z', + syncedAt: '2025-03-06T07:00:00Z', + language: 'en', + media: ['https://img.example.com/1.jpg'], + mediaObjects: [], + links: [], tags: [], ingestedVia: 'graphql', + }, +]; + +// ── Test helper ─────────────────────────────────────────────────────────────── + +async function withWebServer( + records: unknown[], + fn: (baseUrl: string) => Promise, +): Promise { + const dir = await mkdtemp(path.join(tmpdir(), 'ft-web-test-')); + const jsonl = records.map((r) => JSON.stringify(r)).join('\n') + '\n'; + await writeFile(path.join(dir, 'bookmarks.jsonl'), jsonl); + + const savedDir = process.env.FT_DATA_DIR; + process.env.FT_DATA_DIR = dir; + + let close: (() => Promise) | undefined; + try { + await buildIndex({ force: true }); + const server = await createWebServer(0); // port 0 = OS picks a free port + close = server.close; + await fn(`http://127.0.0.1:${server.port}`); + } finally { + await close?.(); + if (savedDir !== undefined) process.env.FT_DATA_DIR = savedDir; + else delete process.env.FT_DATA_DIR; + rmSync(dir, { recursive: true, force: true }); + } +} + +// ── HTML shell ──────────────────────────────────────────────────────────────── + +test('GET / returns HTML shell', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/`); + assert.equal(res.status, 200); + assert.match(res.headers.get('content-type') ?? '', /text\/html/); + const body = await res.text(); + assert.match(body, //i); + assert.match(body, /Field Theory/); + assert.match(body, /alpinejs/); + assert.match(body, /chart\.js/); + }); +}); + +// ── /api/overview ───────────────────────────────────────────────────────────── + +test('GET /api/overview returns VizData JSON', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/api/overview`); + assert.equal(res.status, 200); + assert.match(res.headers.get('content-type') ?? '', /application\/json/); + + const data = await res.json() as Record; + assert.equal(data.total, 3); + assert.equal(data.uniqueAuthors, 2); + assert.ok(Array.isArray(data.topAuthors), 'topAuthors should be an array'); + assert.ok(Array.isArray(data.monthlyActivity), 'monthlyActivity should be an array'); + assert.ok(Array.isArray(data.dayOfWeekActivity), 'dayOfWeekActivity should be an array'); + assert.ok(Array.isArray(data.hourActivity), 'hourActivity should be an array'); + assert.ok(Array.isArray(data.languages), 'languages should be an array'); + assert.ok(typeof (data.dateRange as Record)?.earliest === 'string'); + assert.ok(typeof (data.dateRange as Record)?.latest === 'string'); + assert.ok(typeof (data.mediaStats as Record)?.total === 'number'); + }); +}); + +test('GET /api/overview top authors reflect bookmark counts', async () => { + await withWebServer(FIXTURES, async (base) => { + const data = await fetch(`${base}/api/overview`).then((r) => r.json()) as Record; + const topAuthors = data.topAuthors as { handle: string; count: number }[]; + const alice = topAuthors.find((a) => a.handle === 'alice'); + const bob = topAuthors.find((a) => a.handle === 'bob'); + assert.ok(alice, 'alice should appear in topAuthors'); + assert.equal(alice!.count, 2); + assert.ok(bob, 'bob should appear in topAuthors'); + assert.equal(bob!.count, 1); + }); +}); + +// ── /api/bookmarks ──────────────────────────────────────────────────────────── + +test('GET /api/bookmarks returns all bookmarks', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/api/bookmarks`); + assert.equal(res.status, 200); + const items = await res.json() as unknown[]; + assert.equal(items.length, 3); + }); +}); + +test('GET /api/bookmarks?limit=1 returns exactly 1 item', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/api/bookmarks?limit=1`); + const items = await res.json() as unknown[]; + assert.equal(items.length, 1); + }); +}); + +test('GET /api/bookmarks?limit=999 is capped at the result set size', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/api/bookmarks?limit=999`); + const items = await res.json() as unknown[]; + assert.equal(items.length, 3); // only 3 in DB + }); +}); + +test('GET /api/bookmarks?author=alice returns only alice bookmarks', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/api/bookmarks?author=alice`); + const items = await res.json() as { authorHandle: string }[]; + assert.ok(items.length > 0); + for (const item of items) { + assert.equal(item.authorHandle, 'alice'); + } + }); +}); + +test('GET /api/bookmarks?q= full-text search filters results', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/api/bookmarks?q=Rust`); + const items = await res.json() as { authorHandle: string }[]; + assert.equal(items.length, 1); + assert.equal(items[0].authorHandle, 'bob'); + }); +}); + +test('GET /api/bookmarks?sort=asc returns oldest first', async () => { + await withWebServer(FIXTURES, async (base) => { + const items = await fetch(`${base}/api/bookmarks?sort=asc`).then((r) => r.json()) as { id: string }[]; + assert.equal(items[0].id, '1'); + }); +}); + +test('GET /api/bookmarks?sort=desc returns newest first', async () => { + await withWebServer(FIXTURES, async (base) => { + const items = await fetch(`${base}/api/bookmarks?sort=desc`).then((r) => r.json()) as { id: string }[]; + assert.equal(items[0].id, '3'); + }); +}); + +test('GET /api/bookmarks?offset=2 returns last bookmark', async () => { + await withWebServer(FIXTURES, async (base) => { + const items = await fetch(`${base}/api/bookmarks?sort=asc&offset=2`).then((r) => r.json()) as unknown[]; + assert.equal(items.length, 1); + }); +}); + +// ── /api/count ──────────────────────────────────────────────────────────────── + +test('GET /api/count returns total count', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/api/count`); + assert.equal(res.status, 200); + const data = await res.json() as { count: number }; + assert.equal(data.count, 3); + }); +}); + +test('GET /api/count?author=alice returns filtered count', async () => { + await withWebServer(FIXTURES, async (base) => { + const data = await fetch(`${base}/api/count?author=alice`).then((r) => r.json()) as { count: number }; + assert.equal(data.count, 2); + }); +}); + +// ── /api/bookmarks/:id ──────────────────────────────────────────────────────── + +test('GET /api/bookmarks/:id returns a single bookmark', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/api/bookmarks/1`); + assert.equal(res.status, 200); + const item = await res.json() as { id: string; authorHandle: string }; + assert.equal(item.id, '1'); + assert.equal(item.authorHandle, 'alice'); + }); +}); + +test('GET /api/bookmarks/:id returns 404 for unknown id', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/api/bookmarks/does-not-exist`); + assert.equal(res.status, 404); + const data = await res.json() as { error: string }; + assert.ok(data.error); + }); +}); + +// ── Error handling ──────────────────────────────────────────────────────────── + +test('POST / returns 405 method not allowed', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/`, { method: 'POST' }); + assert.equal(res.status, 405); + }); +}); + +test('GET /unknown-route returns 404', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/unknown-route`); + assert.equal(res.status, 404); + }); +}); + +test('GET /api/unknown returns 404', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/api/unknown`); + assert.equal(res.status, 404); + }); +}); From 89d9bad991bdd46575c9d45084755d4dcf07b930 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Me=CC=81de=CC=81de=CC=81=20Raymond=20KPATCHAA?= Date: Tue, 12 May 2026 11:49:31 +0100 Subject: [PATCH 2/9] feat(web): autocomplete for author, category, and domain filters Add /api/suggestions?field=author|category|domain&q=prefix endpoint backed by getFilterSuggestions() in bookmarks-db.ts (prefix-match, ordered by frequency). Wire Alpine.js custom dropdowns to each filter input so suggestions appear as the user types (debounced 300ms) and disappear on blur. Selecting a suggestion fills the input and triggers a search. Tests: 30/30 pass (11 new) --- src/bookmarks-db.ts | 50 +++++++++++++++++++++ src/web.ts | 106 ++++++++++++++++++++++++++++++++++++++++---- tests/web.test.ts | 104 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 251 insertions(+), 9 deletions(-) diff --git a/src/bookmarks-db.ts b/src/bookmarks-db.ts index cfc336b..c08520e 100644 --- a/src/bookmarks-db.ts +++ b/src/bookmarks-db.ts @@ -681,6 +681,56 @@ export async function listBookmarks( } } +export async function getFilterSuggestions( + field: 'author' | 'category' | 'domain', + prefix: string = '', + limit: number = 20, +): Promise { + const columnMap = { + author: 'author_handle', + category: 'primary_category', + domain: 'primary_domain', + } as const; + const col = columnMap[field]; + const dbPath = twitterBookmarksIndexPath(); + const db = await openDb(dbPath); + ensureMigrations(db); + + try { + let sql: string; + let params: Array; + + if (prefix) { + sql = ` + SELECT ${col}, COUNT(*) AS cnt + FROM bookmarks + WHERE ${col} IS NOT NULL AND ${col} != '' + AND ${col} LIKE ? COLLATE NOCASE + GROUP BY ${col} + ORDER BY cnt DESC + LIMIT ? + `; + params = [`${prefix}%`, limit]; + } else { + sql = ` + SELECT ${col}, COUNT(*) AS cnt + FROM bookmarks + WHERE ${col} IS NOT NULL AND ${col} != '' + GROUP BY ${col} + ORDER BY cnt DESC + LIMIT ? + `; + params = [limit]; + } + + const rows = db.exec(sql, params); + if (!rows.length) return []; + return rows[0].values.map((row) => row[0] as string); + } finally { + db.close(); + } +} + export async function countBookmarks( filters: BookmarkTimelineFilters = {}, ): Promise { diff --git a/src/web.ts b/src/web.ts index 27ed19e..bafca70 100644 --- a/src/web.ts +++ b/src/web.ts @@ -6,6 +6,7 @@ import { listBookmarks, countBookmarks, getBookmarkById, + getFilterSuggestions, } from './bookmarks-db.js'; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -294,15 +295,61 @@ function buildHtml(): string { - - - + + +
+ +
+ +
+
+ + +
+ +
+ +
+
+ + +
+ +
+ +
+
+ -
- + +
+ +
+
+ +
+
+ + +
No results
+
- -
- -
- + +
+ +
+
+ +
+
+ + +
No results
+
- -
- -
- + +
+ +
+
+ +
+
+ + +
No results
+
@@ -803,9 +842,9 @@ function app() { offset: 0, }, autocomplete: { - author: { open: false, items: [] }, - category: { open: false, items: [] }, - domain: { open: false, items: [] }, + author: { open: false, search: '', items: [] }, + category: { open: false, search: '', items: [] }, + domain: { open: false, search: '', items: [] }, }, chartsBuilt: false, @@ -876,22 +915,36 @@ function app() { clearFilters() { Object.assign(this.filters, { q: '', author: '', category: '', domain: '', after: '', before: '', sort: 'desc', offset: 0 }); - this.autocomplete.author.open = false; - this.autocomplete.category.open = false; - this.autocomplete.domain.open = false; + for (const f of ['author', 'category', 'domain']) { + Object.assign(this.autocomplete[f], { open: false, search: '', items: [] }); + } this.loadBookmarks(); }, - async fetchSuggestions(field, value) { - if (!value) { - this.autocomplete[field].open = false; - return; + async toggleDropdown(field) { + const isOpen = this.autocomplete[field].open; + // Close all others first + for (const f of ['author', 'category', 'domain']) { + this.autocomplete[f].open = false; } + if (!isOpen) { + this.autocomplete[field].open = true; + await this.fetchSuggestions(field, this.autocomplete[field].search); + // Focus the search input after opening + this.$nextTick(() => { + const ref = this.$refs[field + 'Search']; + if (ref) ref.focus(); + }); + } + }, + + async fetchSuggestions(field, value) { try { - const res = await fetch('/api/suggestions?field=' + field + '&q=' + encodeURIComponent(value)); - const items = await res.json(); - this.autocomplete[field].items = items; - this.autocomplete[field].open = items.length > 0; + const url = value + ? '/api/suggestions?field=' + field + '&q=' + encodeURIComponent(value) + : '/api/suggestions?field=' + field; + const res = await fetch(url); + this.autocomplete[field].items = await res.json(); } catch { /* silent */ } }, diff --git a/tests/web.test.ts b/tests/web.test.ts index 9e33934..588815d 100644 --- a/tests/web.test.ts +++ b/tests/web.test.ts @@ -309,29 +309,32 @@ test('GET /api/suggestions with no field returns 400', async () => { }); }); -// ── HTML autocomplete ───────────────────────────────────────────────────────── +// ── HTML searchable dropdowns ───────────────────────────────────────────────── -test('GET / HTML includes datalist-style autocomplete attributes for author', async () => { +test('GET / HTML includes searchable dropdown for author', async () => { await withWebServer(FIXTURES, async (base) => { const body = await fetch(`${base}/`).then((r) => r.text()); assert.match(body, /fetchSuggestions\('author'/); assert.match(body, /autocomplete\.author/); + assert.match(body, /toggleDropdown\('author'\)/); }); }); -test('GET / HTML includes datalist-style autocomplete attributes for category', async () => { +test('GET / HTML includes searchable dropdown for category', async () => { await withWebServer(FIXTURES, async (base) => { const body = await fetch(`${base}/`).then((r) => r.text()); assert.match(body, /fetchSuggestions\('category'/); assert.match(body, /autocomplete\.category/); + assert.match(body, /toggleDropdown\('category'\)/); }); }); -test('GET / HTML includes datalist-style autocomplete attributes for domain', async () => { +test('GET / HTML includes searchable dropdown for domain', async () => { await withWebServer(FIXTURES, async (base) => { const body = await fetch(`${base}/`).then((r) => r.text()); assert.match(body, /fetchSuggestions\('domain'/); assert.match(body, /autocomplete\.domain/); + assert.match(body, /toggleDropdown\('domain'\)/); }); }); From 9ca8e4cc61314bf0ee25b3929cf3063d9be0ed0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Me=CC=81de=CC=81de=CC=81=20Raymond=20KPATCHAA?= Date: Tue, 12 May 2026 12:48:05 +0100 Subject: [PATCH 4/9] feat(web): serve local media files and show thumbnails/gallery in visualizer --- src/web.ts | 130 +++++++++++++++++++++++++++++++++++++++---- tests/web.test.ts | 137 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 256 insertions(+), 11 deletions(-) diff --git a/src/web.ts b/src/web.ts index a6ef115..e50ba9b 100644 --- a/src/web.ts +++ b/src/web.ts @@ -1,6 +1,8 @@ import { createServer, IncomingMessage, ServerResponse } from 'node:http'; +import { readFile } from 'node:fs/promises'; import { parse as parseUrl } from 'node:url'; import { spawn } from 'node:child_process'; +import path from 'node:path'; import { buildVizData } from './bookmarks-viz.js'; import { listBookmarks, @@ -8,6 +10,46 @@ import { getBookmarkById, getFilterSuggestions, } from './bookmarks-db.js'; +import { pathExists, readJson } from './fs.js'; +import { bookmarkMediaDir, bookmarkMediaManifestPath } from './paths.js'; +import type { MediaFetchManifest } from './bookmark-media.js'; + +// ── Media index ─────────────────────────────────────────────────────────────── + +interface MediaEntry { filename: string; contentType: string; isProfileImage: boolean } +type MediaIndex = Map; // tweetId → entries + +const EXT_CONTENT_TYPE: Record = { + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.png': 'image/png', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.mp4': 'video/mp4', +}; + +async function buildMediaIndex(): Promise { + const manifestPath = bookmarkMediaManifestPath(); + if (!(await pathExists(manifestPath))) return new Map(); + try { + const manifest = await readJson(manifestPath); + const index = new Map(); + for (const entry of manifest.entries) { + if (entry.status !== 'downloaded' || !entry.localPath) continue; + const filename = path.basename(entry.localPath); + const arr = index.get(entry.tweetId) ?? []; + arr.push({ + filename, + contentType: entry.contentType ?? 'application/octet-stream', + isProfileImage: entry.sourceUrl.includes('/profile_images/'), + }); + index.set(entry.tweetId, arr); + } + return index; + } catch { + return new Map(); + } +} // ── Helpers ────────────────────────────────────────────────────────────────── @@ -474,9 +516,22 @@ function buildHtml(): string {
- +
+ + +
@@ -521,11 +576,11 @@ function buildHtml(): string {
- -
+
@@ -537,7 +592,20 @@ function buildHtml(): string {

- + +
@@ -986,7 +1054,7 @@ function app() { // ── Request router ──────────────────────────────────────────────────────────── -async function handleRequest(req: IncomingMessage, res: ServerResponse): Promise { +async function handleRequest(req: IncomingMessage, res: ServerResponse, mediaIndex: MediaIndex): Promise { const parsed = parseUrl(req.url ?? '', true); const pathname = parsed.pathname ?? '/'; @@ -1001,6 +1069,33 @@ async function handleRequest(req: IncomingMessage, res: ServerResponse): Promise return; } + // /media/:filename — serve locally cached media files + const mediaFileMatch = pathname.match(/^\/media\/([^/]+)$/); + if (mediaFileMatch) { + const filename = mediaFileMatch[1]; + const mediaDir = bookmarkMediaDir(); + const resolved = path.resolve(mediaDir, filename); + // Security: reject path traversal — resolved path must stay inside mediaDir + if (!resolved.startsWith(mediaDir + path.sep) && resolved !== mediaDir) { + json(res, { error: 'invalid filename' }, 400); + return; + } + try { + const buf = await readFile(resolved); + const ext = path.extname(filename).toLowerCase(); + const contentType = EXT_CONTENT_TYPE[ext] ?? 'application/octet-stream'; + res.writeHead(200, { + 'Content-Type': contentType, + 'Content-Length': buf.length, + 'Cache-Control': 'public, max-age=86400', + }); + res.end(buf); + } catch { + json(res, { error: 'not found' }, 404); + } + return; + } + // /api/overview if (pathname === '/api/overview') { const data = await buildVizData(); @@ -1046,7 +1141,14 @@ async function handleRequest(req: IncomingMessage, res: ServerResponse): Promise json(res, { error: 'not found' }, 404); return; } - json(res, bookmark); + const mediaEntries = mediaIndex.get(bookmark.tweetId) ?? []; + const localMediaUrls = mediaEntries + .filter((e) => !e.isProfileImage) + .map((e) => `/media/${e.filename}`); + const localProfileImageUrl = mediaEntries.find((e) => e.isProfileImage) + ? `/media/${mediaEntries.find((e) => e.isProfileImage)!.filename}` + : undefined; + json(res, { ...bookmark, localMediaUrls, localProfileImageUrl }); return; } @@ -1064,7 +1166,14 @@ async function handleRequest(req: IncomingMessage, res: ServerResponse): Promise limit: q.limit ? Math.min(200, Math.max(1, parseInt(q.limit, 10))) : 50, offset: q.offset ? Math.max(0, parseInt(q.offset, 10)) : 0, }); - json(res, items); + const enriched = items.map((b) => { + const entries = mediaIndex.get(b.tweetId) ?? []; + const localMediaUrls = entries + .filter((e) => !e.isProfileImage) + .map((e) => `/media/${e.filename}`); + return { ...b, localMediaUrls }; + }); + json(res, enriched); return; } @@ -1074,9 +1183,10 @@ async function handleRequest(req: IncomingMessage, res: ServerResponse): Promise // ── Server factory (exported for testing) ──────────────────────────────────── export async function createWebServer(port: number): Promise<{ port: number; close: () => Promise }> { + const mediaIndex = await buildMediaIndex(); const server = createServer(async (req, res) => { try { - await handleRequest(req, res); + await handleRequest(req, res, mediaIndex); } catch (err) { const message = err instanceof Error ? err.message : String(err); json(res, { error: message }, 500); diff --git a/tests/web.test.ts b/tests/web.test.ts index 588815d..986aa84 100644 --- a/tests/web.test.ts +++ b/tests/web.test.ts @@ -1,6 +1,6 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { mkdtemp, writeFile } from 'node:fs/promises'; +import { mkdtemp, mkdir, writeFile } from 'node:fs/promises'; import { rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; @@ -360,3 +360,138 @@ test('GET /api/unknown returns 404', async () => { assert.equal(res.status, 404); }); }); + +// ── /media/:filename ───────────────────────────────────────────────────────── + +test('GET /media/nonexistent.jpg returns 404 when no media downloaded', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/media/nonexistent.jpg`); + assert.equal(res.status, 404); + }); +}); + +test('GET /media/:filename serves a downloaded file from local disk', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'ft-web-media-test-')); + const jsonl = FIXTURES.map((r) => JSON.stringify(r)).join('\n') + '\n'; + await writeFile(path.join(dir, 'bookmarks.jsonl'), jsonl); + const mediaDir = path.join(dir, 'media'); + await mkdir(mediaDir, { recursive: true }); + const fakeBytes = Buffer.from('fake-image-bytes'); + await writeFile(path.join(mediaDir, 'abc123def456.jpg'), fakeBytes); + const manifest = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + limit: 100, maxBytes: 200 * 1024 * 1024, + processed: 1, downloaded: 1, skippedTooLarge: 0, failed: 0, + entries: [{ + bookmarkId: '3', tweetId: '3', + tweetUrl: 'https://x.com/alice/status/3', + authorHandle: 'alice', + sourceUrl: 'https://img.example.com/1.jpg', + localPath: path.join(mediaDir, 'abc123def456.jpg'), + contentType: 'image/jpeg', + bytes: fakeBytes.length, + status: 'downloaded', + fetchedAt: new Date().toISOString(), + }], + }; + await writeFile(path.join(dir, 'media-manifest.json'), JSON.stringify(manifest)); + const savedDir = process.env.FT_DATA_DIR; + process.env.FT_DATA_DIR = dir; + let close: (() => Promise) | undefined; + try { + await buildIndex({ force: true }); + const server = await createWebServer(0); + close = server.close; + const base = `http://127.0.0.1:${server.port}`; + const res = await fetch(`${base}/media/abc123def456.jpg`); + assert.equal(res.status, 200); + assert.match(res.headers.get('content-type') ?? '', /image\/jpeg/); + const body = Buffer.from(await res.arrayBuffer()); + assert.equal(body.toString(), 'fake-image-bytes'); + } finally { + await close?.(); + if (savedDir !== undefined) process.env.FT_DATA_DIR = savedDir; + else delete process.env.FT_DATA_DIR; + rmSync(dir, { recursive: true, force: true }); + } +}); + +// ── localMediaUrls in API responses ────────────────────────────────────────── + +test('GET /api/bookmarks/:id includes localMediaUrls field', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/api/bookmarks/3`); + assert.equal(res.status, 200); + const data = await res.json() as Record; + assert.ok(Array.isArray(data.localMediaUrls), 'localMediaUrls should be an array'); + }); +}); + +test('GET /api/bookmarks list includes localMediaUrls per item', async () => { + await withWebServer(FIXTURES, async (base) => { + const items = await fetch(`${base}/api/bookmarks`).then((r) => r.json()) as Record[]; + assert.ok(Array.isArray(items)); + for (const item of items) { + assert.ok(Array.isArray(item.localMediaUrls), 'each item should have localMediaUrls array'); + } + }); +}); + +test('GET /api/bookmarks/:id localMediaUrls map to /media/ paths', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'ft-web-media2-test-')); + const jsonl = FIXTURES.map((r) => JSON.stringify(r)).join('\n') + '\n'; + await writeFile(path.join(dir, 'bookmarks.jsonl'), jsonl); + const mediaDir = path.join(dir, 'media'); + await mkdir(mediaDir, { recursive: true }); + const fakeBytes = Buffer.from('img'); + await writeFile(path.join(mediaDir, '3-deadbeef.jpg'), fakeBytes); + const manifest = { + schemaVersion: 1, generatedAt: new Date().toISOString(), + limit: 100, maxBytes: 200 * 1024 * 1024, + processed: 1, downloaded: 1, skippedTooLarge: 0, failed: 0, + entries: [{ + bookmarkId: '3', tweetId: '3', + tweetUrl: 'https://x.com/alice/status/3', + authorHandle: 'alice', + sourceUrl: 'https://img.example.com/1.jpg', + localPath: path.join(mediaDir, '3-deadbeef.jpg'), + contentType: 'image/jpeg', bytes: 3, + status: 'downloaded', fetchedAt: new Date().toISOString(), + }], + }; + await writeFile(path.join(dir, 'media-manifest.json'), JSON.stringify(manifest)); + const savedDir = process.env.FT_DATA_DIR; + process.env.FT_DATA_DIR = dir; + let close: (() => Promise) | undefined; + try { + await buildIndex({ force: true }); + const server = await createWebServer(0); + close = server.close; + const base = `http://127.0.0.1:${server.port}`; + const data = await fetch(`${base}/api/bookmarks/3`).then((r) => r.json()) as Record; + const urls = data.localMediaUrls as string[]; + assert.ok(urls.includes('/media/3-deadbeef.jpg')); + } finally { + await close?.(); + if (savedDir !== undefined) process.env.FT_DATA_DIR = savedDir; + else delete process.env.FT_DATA_DIR; + rmSync(dir, { recursive: true, force: true }); + } +}); + +// ── HTML media UI ───────────────────────────────────────────────────────────── + +test('GET / HTML includes media gallery in detail panel', async () => { + await withWebServer(FIXTURES, async (base) => { + const body = await fetch(`${base}/`).then((r) => r.text()); + assert.match(body, /detail\.localMediaUrls/); + }); +}); + +test('GET / HTML includes media thumbnail strip in bookmark cards', async () => { + await withWebServer(FIXTURES, async (base) => { + const body = await fetch(`${base}/`).then((r) => r.text()); + assert.match(body, /b\.localMediaUrls/); + }); +}); From 3a2e9e65cea05319ec3ccb94e135e17e491fdcf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Me=CC=81de=CC=81de=CC=81=20Raymond=20KPATCHAA?= Date: Tue, 12 May 2026 13:05:15 +0100 Subject: [PATCH 5/9] feat(web): delete bookmark from local archive with one-click unbookmark on X --- src/bookmarks-db.ts | 38 +++++++++++++++++++++++++++++++++++++- src/web.ts | 39 +++++++++++++++++++++++++++++++++++++++ tests/web.test.ts | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) diff --git a/src/bookmarks-db.ts b/src/bookmarks-db.ts index c08520e..b9d8a0a 100644 --- a/src/bookmarks-db.ts +++ b/src/bookmarks-db.ts @@ -1,7 +1,7 @@ import type { Database } from 'sql.js'; import { openDb, saveDb } from './db.js'; import { parseTimestampMs, toIsoDate } from './date-utils.js'; -import { readJsonLines } from './fs.js'; +import { readJsonLines, writeJsonLines } from './fs.js'; import { twitterBookmarksCachePath, twitterBookmarksIndexPath } from './paths.js'; import type { BookmarkRecord, QuotedTweetSnapshot } from './types.js'; import { classifyCorpus, formatClassificationSummary } from './bookmark-classify.js'; @@ -875,6 +875,42 @@ export async function getBookmarkById(id: string): Promise { + const dbPath = twitterBookmarksIndexPath(); + const cachePath = twitterBookmarksCachePath(); + + const db = await openDb(dbPath); + ensureMigrations(db); + + let url: string | null = null; + try { + const rows = db.exec('SELECT url FROM bookmarks WHERE id = ? LIMIT 1', [id]); + url = (rows[0]?.values?.[0]?.[0] as string) ?? null; + if (!url) return null; + + db.run('DELETE FROM bookmarks WHERE id = ?', [id]); + // FTS5 content table is auto-updated via triggers set up in initSchema + db.run(`INSERT INTO bookmarks_fts(bookmarks_fts) VALUES('rebuild')`); + saveDb(db, dbPath); + } finally { + db.close(); + } + + // Remove from the JSONL cache so it won't re-appear on the next buildIndex + const records = await readJsonLines<{ id: string }>(cachePath); + const filtered = records.filter((r) => r.id !== id); + if (filtered.length !== records.length) { + await writeJsonLines(cachePath, filtered); + } + + return { url }; +} + export async function getStats(): Promise<{ totalBookmarks: number; uniqueAuthors: number; diff --git a/src/web.ts b/src/web.ts index e50ba9b..d027ec8 100644 --- a/src/web.ts +++ b/src/web.ts @@ -9,6 +9,7 @@ import { countBookmarks, getBookmarkById, getFilterSuggestions, + deleteBookmark, } from './bookmarks-db.js'; import { pathExists, readJson } from './fs.js'; import { bookmarkMediaDir, bookmarkMediaManifestPath } from './paths.js'; @@ -682,6 +683,12 @@ function buildHtml(): string { View on X ↗ + + +
@@ -1045,6 +1052,21 @@ function app() { this.detailLoading = false; } }, + + async deleteBookmark(id, tweetUrl) { + if (!confirm('Remove this bookmark from your local archive?')) return; + try { + const res = await fetch('/api/bookmarks/' + encodeURIComponent(id), { method: 'DELETE' }); + if (!res.ok) { console.error('Delete failed', await res.text()); return; } + } catch (e) { + console.error('Delete failed:', e); + return; + } + this.detailOpen = false; + this.detail = null; + if (tweetUrl) window.open(tweetUrl, '_blank', 'noopener,noreferrer'); + await this.loadBookmarks(); + }, }; } @@ -1064,6 +1086,23 @@ async function handleRequest(req: IncomingMessage, res: ServerResponse, mediaInd return; } + // DELETE /api/bookmarks/:id + if (req.method === 'DELETE') { + const deleteMatch = pathname.match(/^\/api\/bookmarks\/(.+)$/); + if (!deleteMatch) { + json(res, { error: 'not found' }, 404); + return; + } + const id = decodeURIComponent(deleteMatch[1]); + const deleted = await deleteBookmark(id); + if (!deleted) { + json(res, { error: 'not found' }, 404); + return; + } + json(res, { deleted: true, url: deleted.url }); + return; + } + if (req.method !== 'GET') { json(res, { error: 'method not allowed' }, 405); return; diff --git a/tests/web.test.ts b/tests/web.test.ts index 986aa84..dbc04ad 100644 --- a/tests/web.test.ts +++ b/tests/web.test.ts @@ -495,3 +495,46 @@ test('GET / HTML includes media thumbnail strip in bookmark cards', async () => assert.match(body, /b\.localMediaUrls/); }); }); + +// ── DELETE /api/bookmarks/:id ───────────────────────────────────────────────── + +test('DELETE /api/bookmarks/:id removes bookmark and returns url', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/api/bookmarks/2`, { method: 'DELETE' }); + assert.equal(res.status, 200); + const body = await res.json() as Record; + assert.equal(body.deleted, true); + assert.ok(typeof body.url === 'string' && body.url.length > 0); + }); +}); + +test('DELETE /api/bookmarks/:id makes record disappear from GET', async () => { + await withWebServer(FIXTURES, async (base) => { + await fetch(`${base}/api/bookmarks/2`, { method: 'DELETE' }); + const res = await fetch(`${base}/api/bookmarks/2`); + assert.equal(res.status, 404); + }); +}); + +test('DELETE /api/bookmarks/:id reduces bookmark count', async () => { + await withWebServer(FIXTURES, async (base) => { + const before = await fetch(`${base}/api/count`).then((r) => r.json()) as { count: number }; + await fetch(`${base}/api/bookmarks/2`, { method: 'DELETE' }); + const after = await fetch(`${base}/api/count`).then((r) => r.json()) as { count: number }; + assert.equal(after.count, before.count - 1); + }); +}); + +test('DELETE /api/bookmarks/:id returns 404 for unknown id', async () => { + await withWebServer(FIXTURES, async (base) => { + const res = await fetch(`${base}/api/bookmarks/nonexistent-id`, { method: 'DELETE' }); + assert.equal(res.status, 404); + }); +}); + +test('GET / HTML includes deleteBookmark function', async () => { + await withWebServer(FIXTURES, async (base) => { + const body = await fetch(`${base}/`).then((r) => r.text()); + assert.match(body, /deleteBookmark/); + }); +}); From ea6f9adf523ac7ad9c3ac35cd2c87b6ed21a8b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Me=CC=81de=CC=81de=CC=81=20Raymond=20KPATCHAA?= Date: Tue, 12 May 2026 13:12:19 +0100 Subject: [PATCH 6/9] fix(web): guard detail panel against null after delete, remove confirm dialog --- src/web.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/web.ts b/src/web.ts index d027ec8..dc1e3f0 100644 --- a/src/web.ts +++ b/src/web.ts @@ -549,7 +549,7 @@ function buildHtml(): string {
-
@@ -1054,7 +1054,6 @@ function app() { }, async deleteBookmark(id, tweetUrl) { - if (!confirm('Remove this bookmark from your local archive?')) return; try { const res = await fetch('/api/bookmarks/' + encodeURIComponent(id), { method: 'DELETE' }); if (!res.ok) { console.error('Delete failed', await res.text()); return; } From 6d5cf5a589e8df8edaa949be830385f830ac0aa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Me=CC=81de=CC=81de=CC=81=20Raymond=20KPATCHAA?= Date: Tue, 12 May 2026 13:24:53 +0100 Subject: [PATCH 7/9] fix(web): reload list before window.open to prevent tab-throttle stalling bookmarksLoading --- src/web.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web.ts b/src/web.ts index dc1e3f0..3eb1027 100644 --- a/src/web.ts +++ b/src/web.ts @@ -1062,9 +1062,9 @@ function app() { return; } this.detailOpen = false; + await this.loadBookmarks(); this.detail = null; if (tweetUrl) window.open(tweetUrl, '_blank', 'noopener,noreferrer'); - await this.loadBookmarks(); }, }; } From 554da0afc9546e8945b282fecaaf5397abfcff3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Me=CC=81de=CC=81de=CC=81=20Raymond=20KPATCHAA?= Date: Thu, 14 May 2026 09:46:34 +0100 Subject: [PATCH 8/9] feat(bookmarks-db): delete also removes associated media files and manifest entries --- src/bookmarks-db.ts | 20 ++++++- tests/bookmarks-db.test.ts | 108 +++++++++++++++++++++++++++++++++++-- 2 files changed, 123 insertions(+), 5 deletions(-) diff --git a/src/bookmarks-db.ts b/src/bookmarks-db.ts index b9d8a0a..ef61d72 100644 --- a/src/bookmarks-db.ts +++ b/src/bookmarks-db.ts @@ -1,8 +1,10 @@ import type { Database } from 'sql.js'; import { openDb, saveDb } from './db.js'; import { parseTimestampMs, toIsoDate } from './date-utils.js'; -import { readJsonLines, writeJsonLines } from './fs.js'; -import { twitterBookmarksCachePath, twitterBookmarksIndexPath } from './paths.js'; +import { unlink } from 'node:fs/promises'; +import { readJsonLines, writeJsonLines, readJson, writeJson, pathExists } from './fs.js'; +import { twitterBookmarksCachePath, twitterBookmarksIndexPath, bookmarkMediaManifestPath } from './paths.js'; +import type { MediaFetchManifest } from './bookmark-media.js'; import type { BookmarkRecord, QuotedTweetSnapshot } from './types.js'; import { classifyCorpus, formatClassificationSummary } from './bookmark-classify.js'; import type { ClassificationSummary } from './bookmark-classify.js'; @@ -908,6 +910,20 @@ export async function deleteBookmark(id: string): Promise<{ url: string } | null await writeJsonLines(cachePath, filtered); } + // Remove associated media files and manifest entries + const manifestPath = bookmarkMediaManifestPath(); + if (await pathExists(manifestPath)) { + const manifest = await readJson(manifestPath); + const toRemove = manifest.entries.filter((e) => e.bookmarkId === id); + for (const entry of toRemove) { + if (entry.localPath) { + await unlink(entry.localPath).catch(() => { /* already gone */ }); + } + } + manifest.entries = manifest.entries.filter((e) => e.bookmarkId !== id); + await writeJson(manifestPath, manifest); + } + return { url }; } diff --git a/tests/bookmarks-db.test.ts b/tests/bookmarks-db.test.ts index 7bea098..a3b277e 100644 --- a/tests/bookmarks-db.test.ts +++ b/tests/bookmarks-db.test.ts @@ -1,11 +1,12 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { mkdtemp, writeFile } from 'node:fs/promises'; +import { mkdtemp, writeFile, mkdir, access } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { buildIndex, searchBookmarks, getStats, formatSearchResults, getBookmarkById, listBookmarks, sanitizeFtsQuery, getCategoryCounts, sampleByCategory, getClassificationProgress } from '../src/bookmarks-db.js'; +import { buildIndex, searchBookmarks, getStats, formatSearchResults, getBookmarkById, listBookmarks, sanitizeFtsQuery, getCategoryCounts, sampleByCategory, getClassificationProgress, deleteBookmark } from '../src/bookmarks-db.js'; import { openDb, saveDb } from '../src/db.js'; -import { twitterBookmarksIndexPath } from '../src/paths.js'; +import { twitterBookmarksIndexPath, bookmarkMediaDir, bookmarkMediaManifestPath } from '../src/paths.js'; +import type { MediaFetchManifest } from '../src/bookmark-media.js'; const FIXTURES = [ { id: '1', tweetId: '1', url: 'https://x.com/alice/status/1', text: 'Machine learning is transforming healthcare', authorHandle: 'alice', authorName: 'Alice Smith', syncedAt: '2026-01-01T00:00:00Z', postedAt: '2026-01-01T12:00:00Z', language: 'en', engagement: { likeCount: 100, repostCount: 10 }, mediaObjects: [], links: ['https://example.com'], tags: [], ingestedVia: 'graphql' }, @@ -332,3 +333,104 @@ test('sanitizeFtsQuery: strips internal quotes to avoid double-escaping', () => // Internal quotes stripped; term wrapped once assert.ok(!result.includes('""')); }); + +// ── deleteBookmark: media cleanup ───────────────────────────────────────────── + +async function withMediaFixture(fn: (mediaDir: string) => Promise): Promise { + const dir = await mkdtemp(path.join(tmpdir(), 'ft-media-test-')); + const jsonl = FIXTURES.map((r) => JSON.stringify(r)).join('\n') + '\n'; + await writeFile(path.join(dir, 'bookmarks.jsonl'), jsonl); + + const mediaDir = path.join(dir, 'media'); + await mkdir(mediaDir, { recursive: true }); + + // Write two fake image files: one for bookmark 1, one for bookmark 2 + const file1 = path.join(mediaDir, 'tweet1-aabbcc.jpg'); + const file2 = path.join(mediaDir, 'tweet2-ddeeff.jpg'); + await writeFile(file1, 'fake-image-1'); + await writeFile(file2, 'fake-image-2'); + + const manifest: MediaFetchManifest = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + limit: 100, + maxBytes: 10_000_000, + processed: 2, + downloaded: 2, + skippedTooLarge: 0, + failed: 0, + entries: [ + { + bookmarkId: '1', + tweetId: '1', + tweetUrl: 'https://x.com/alice/status/1', + sourceUrl: 'https://img.com/a.jpg', + localPath: file1, + contentType: 'image/jpeg', + bytes: 12, + status: 'downloaded', + fetchedAt: new Date().toISOString(), + }, + { + bookmarkId: '2', + tweetId: '2', + tweetUrl: 'https://x.com/bob/status/2', + sourceUrl: 'https://img.com/b.jpg', + localPath: file2, + contentType: 'image/jpeg', + bytes: 12, + status: 'downloaded', + fetchedAt: new Date().toISOString(), + }, + ], + }; + await writeFile(path.join(dir, 'media-manifest.json'), JSON.stringify(manifest)); + + const saved = process.env.FT_DATA_DIR; + process.env.FT_DATA_DIR = dir; + try { + await fn(mediaDir); + } finally { + if (saved !== undefined) process.env.FT_DATA_DIR = saved; + else delete process.env.FT_DATA_DIR; + } +} + +async function fileExists(p: string): Promise { + try { await access(p); return true; } catch { return false; } +} + +test('deleteBookmark removes associated media file from disk', async () => { + await withMediaFixture(async (mediaDir) => { + await buildIndex(); + const file1 = path.join(mediaDir, 'tweet1-aabbcc.jpg'); + const file2 = path.join(mediaDir, 'tweet2-ddeeff.jpg'); + + await deleteBookmark('1'); + + assert.equal(await fileExists(file1), false, 'media file for deleted bookmark should be gone'); + assert.equal(await fileExists(file2), true, 'media file for unrelated bookmark should remain'); + }); +}); + +test('deleteBookmark removes manifest entries for deleted bookmark', async () => { + await withMediaFixture(async () => { + await buildIndex(); + + await deleteBookmark('1'); + + const { readJson } = await import('../src/fs.js'); + const manifest = await readJson(bookmarkMediaManifestPath()); + assert.equal(manifest.entries.every((e) => e.bookmarkId !== '1'), true); + assert.equal(manifest.entries.some((e) => e.bookmarkId === '2'), true); + }); +}); + +test('deleteBookmark leaves media intact when no manifest exists', async () => { + await withIsolatedDataDir(async () => { + await buildIndex(); + // No manifest → should not throw + const result = await deleteBookmark('1'); + assert.ok(result !== null); + }); +}); From ec6a90fa7e4264356ab74397a5392bbe0f9a26e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Me=CC=81de=CC=81de=CC=81=20Raymond=20KPATCHAA?= Date: Thu, 14 May 2026 09:47:21 +0100 Subject: [PATCH 9/9] fix(tests): correct formatting of updated fixture in buildIndex test --- tests/bookmarks-db.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/bookmarks-db.test.ts b/tests/bookmarks-db.test.ts index a3b277e..0a38ff3 100644 --- a/tests/bookmarks-db.test.ts +++ b/tests/bookmarks-db.test.ts @@ -58,10 +58,10 @@ test('buildIndex refreshes existing rows without dropping classifications', asyn const updatedFixtures = FIXTURES.map((fixture) => fixture.id === '1' ? { - ...fixture, - text: 'Machine learning note updated', - bookmarkedAt: '2026-04-02T00:00:00Z', - } + ...fixture, + text: 'Machine learning note updated', + bookmarkedAt: '2026-04-02T00:00:00Z', + } : fixture ); const jsonl = updatedFixtures.map((r) => JSON.stringify(r)).join('\n') + '\n';