extract-structured-data.mjs

#!/usr/bin/env node
/**
 * Extract structured data from a web page using Plasmate's SOM.
 *
 * Demonstrates how to walk the SOM and pull out specific data types:
 * headings, links, images, and text content organized by region.
 */
import { execSync } from "node:child_process";

const url = process.argv[2] || "https://news.ycombinator.com";

try {
  const output = execSync(`plasmate fetch "${url}"`, {
    encoding: "utf-8",
    stdio: ["pipe", "pipe", "pipe"],
  });

  const som = JSON.parse(output);
  console.log(`Structured data from: ${som.title ?? url}\n`);

  const data = {
    headings: [],
    links: [],
    images: [],
    textBlocks: [],
  };

  for (const region of som.regions ?? []) {
    for (const el of region.elements ?? []) {
      switch (el.role) {
        case "heading":
          data.headings.push({
            level: el.level,
            text: el.text,
            region: region.role,
          });
          break;
        case "link":
          if (el.href) {
            data.links.push({
              text: el.text?.trim(),
              href: el.href,
              region: region.role,
            });
          }
          break;
        case "image":
          if (el.src) {
            data.images.push({
              alt: el.alt ?? "",
              src: el.src,
              region: region.role,
            });
          }
          break;
        case "text":
          if (el.text?.trim()) {
            data.textBlocks.push({
              text: el.text.trim().slice(0, 200),
              region: region.role,
            });
          }
          break;
      }
    }
  }

  console.log(`📑 Headings (${data.headings.length}):`);
  for (const h of data.headings.slice(0, 10)) {
    console.log(`  ${"#".repeat(h.level)} ${h.text}`);
  }

  console.log(`\n🔗 Links (${data.links.length}):`);
  for (const l of data.links.slice(0, 10)) {
    console.log(`  ${l.text?.slice(0, 50) || "(no text)"} → ${l.href}`);
  }

  console.log(`\n🖼  Images (${data.images.length}):`);
  for (const img of data.images.slice(0, 5)) {
    console.log(`  ${img.alt || "(no alt)"} → ${img.src}`);
  }

  console.log(`\n📝 Text blocks (${data.textBlocks.length}):`);
  for (const t of data.textBlocks.slice(0, 5)) {
    console.log(`  [${t.region}] ${t.text.slice(0, 80)}...`);
  }
} catch (err) {
  console.error(`Error: ${err.message}`);
  process.exit(1);
}