scraper.py

#!/usr/bin/env python3
"""
PTCG Pocket Card Scraper
Scrapes pocket.limitlesstcg.com and outputs cards.json
Usage: python3 scraper.py
"""

import urllib.request
import urllib.parse
import json
import time
import re
import os
import sys

# All sets to scrape
SETS = [
    ("A1",  "Genetic Apex"),
    ("A1a", "Mythical Island"),
    ("A2",  "Space-Time Smackdown"),
    ("A2a", "Triumphant Light"),
    ("A2b", "Shining Revelry"),
    ("A3",  "Celestial Guardians"),
    ("A3a", "Extradimensional Crisis"),
    ("A3b", "Eevee Grove"),
    ("A4",  "Wisdom of Sea and Sky"),
    ("A4a", "Secluded Springs"),
    ("A4b", "Deluxe Pack: ex"),
    ("B1",  "Mega Rising"),
    ("B1a", "Crimson Blaze"),
    ("B2",  "Fantastical Parade"),
]

# Energy symbol mapping
SYMBOL_MAP = {
    "G": "Grass", "R": "Fire", "W": "Water", "L": "Lightning",
    "P": "Psychic", "F": "Fighting", "D": "Darkness", "M": "Metal",
    "N": "Dragon", "C": "Colorless", "Y": "Fairy",
}

def fetch(url, retries=3, delay=1.0):
    headers = {"User-Agent": "Mozilla/5.0 (compatible; PTCGPocketResearch/1.0)"}
    for attempt in range(retries):
        try:
            req = urllib.request.Request(url, headers=headers)
            with urllib.request.urlopen(req, timeout=15) as r:
                return r.read().decode("utf-8", errors="replace")
        except Exception as e:
            if attempt < retries - 1:
                time.sleep(delay * (attempt + 1))
            else:
                print(f"  ERROR fetching {url}: {e}")
                return None

def get_set_card_ids(set_code):
    """Scrape the set page to get all card numbers."""
    url = f"https://pocket.limitlesstcg.com/cards/{set_code}"
    html = fetch(url)
    if not html:
        return []
    # Find all card links like /cards/A1/6
    pattern = rf'/cards/{re.escape(set_code)}/(\d+)'
    nums = sorted(set(int(m) for m in re.findall(pattern, html)))
    return nums

def parse_symbols(sym_str):
    """Parse energy symbols like 'RRC' into list of types."""
    costs = []
    for ch in sym_str.strip():
        costs.append(SYMBOL_MAP.get(ch, "Colorless"))
    return costs

def parse_card_page(html, set_code, set_name, card_num):
    """Parse a single card page and return a card dict or None."""
    if not html:
        return None

    card = {
        "id": f"{set_code}-{card_num:03d}",
        "set_code": set_code,
        "set_name": set_name,
        "number": card_num,
    }

    # --- Name ---
    m = re.search(r'class="card-text-name"[^>]*><a[^>]*>([^<]+)</a>', html)
    if not m:
        return None
    card["name"] = m.group(1).strip()

    # --- Title line: type, HP ---
    m = re.search(r'class="card-text-title"[^>]*>(.*?)</p>', html, re.DOTALL)
    if m:
        title = m.group(1)
        hp_m = re.search(r'(\d+)\s*HP', title)
        card["hp"] = int(hp_m.group(1)) if hp_m else 0
        type_m = re.search(r'-\s*(Grass|Fire|Water|Lightning|Psychic|Fighting|Darkness|Metal|Dragon|Colorless|Fairy)\s*-', title)
        card["pokemon_type"] = type_m.group(1) if type_m else None
    else:
        card["hp"] = 0
        card["pokemon_type"] = None

    # --- Card type & stage ---
    m = re.search(r'class="card-text-type"[^>]*>(.*?)</p>', html, re.DOTALL)
    if m:
        tblock = m.group(1)
        if "Supporter" in tblock:
            card["card_type"] = "Supporter"
        elif "Item" in tblock:
            card["card_type"] = "Item"
        elif "Pokémon" in tblock or "Pokemon" in tblock:
            card["card_type"] = "Pokemon"
        else:
            card["card_type"] = "Trainer"

        if "Stage 2" in tblock:
            card["stage"] = "Stage2"
        elif "Stage 1" in tblock:
            card["stage"] = "Stage1"
        elif "Basic" in tblock:
            card["stage"] = "Basic"
        elif "ex" in card["name"].lower():
            card["stage"] = "ex"
        else:
            card["stage"] = "Basic"

        evo_m = re.search(r'Evolves from\s*<a[^>]*>([^<]+)</a>', tblock)
        card["evolves_from"] = evo_m.group(1).strip() if evo_m else None
    else:
        card["card_type"] = "Trainer"
        card["stage"] = None
        card["evolves_from"] = None

    # --- Attacks ---
    attacks = []
    for atk_block in re.finditer(r'class="card-text-attack"[^>]*>(.*?)(?=class="card-text-attack"|class="card-text-ability"|class="card-text-wrr"|</div>\s*</div>)', html, re.DOTALL):
        ab = atk_block.group(1)
        info_m = re.search(r'class="card-text-attack-info"[^>]*>(.*?)</p>', ab, re.DOTALL)
        if not info_m:
            continue
        info = info_m.group(1)
        # Extract symbols
        sym_m = re.search(r'class="ptcg-symbol"[^>]*>([^<]+)</span>', info)
        cost_str = sym_m.group(1).strip() if sym_m else ""
        cost = parse_symbols(cost_str)
        # Remove symbol span to get name + damage
        clean = re.sub(r'<[^>]+>', '', info).strip()
        parts = clean.split()
        # Last token might be damage number
        dmg = 0
        if parts and re.match(r'^\d+[+x]?$', parts[-1]):
            try:
                dmg = int(re.sub(r'[+x]', '', parts[-1]))
                name = " ".join(parts[:-1])
            except:
                name = " ".join(parts)
        else:
            name = " ".join(parts)
        # Effect
        eff_m = re.search(r'class="card-text-attack-effect"[^>]*>(.*?)</p>', ab, re.DOTALL)
        effect = re.sub(r'<[^>]+>', '', eff_m.group(1)).strip() if eff_m else ""
        attacks.append({
            "name": name.strip(),
            "cost": cost,
            "damage": dmg,
            "effect": effect,
        })
    card["attacks"] = attacks

    # --- Ability ---
    ab_m = re.search(r'class="card-text-ability"[^>]*>(.*?)</div>', html, re.DOTALL)
    if ab_m:
        ab_text = re.sub(r'<[^>]+>', ' ', ab_m.group(1)).strip()
        ab_text = re.sub(r'\s+', ' ', ab_text)
        card["ability"] = ab_text
    else:
        card["ability"] = None

    # --- Weakness & Retreat ---
    wrr_m = re.search(r'class="card-text-wrr"[^>]*>(.*?)</p>', html, re.DOTALL)
    if wrr_m:
        wrr = re.sub(r'<[^>]+>', '', wrr_m.group(1))
        wk_m = re.search(r'Weakness:\s*(\w+)', wrr)
        card["weakness"] = wk_m.group(1) if wk_m else None
        rt_m = re.search(r'Retreat:\s*(\d+)', wrr)
        card["retreat_cost"] = int(rt_m.group(1)) if rt_m else 1
    else:
        card["weakness"] = None
        card["retreat_cost"] = 1

    # --- Trainer effect (items/supporters) ---
    if card["card_type"] in ("Item", "Supporter", "Trainer"):
        eff_blocks = re.findall(r'class="card-text-section"[^>]*>(.*?)</div>', html, re.DOTALL)
        for blk in eff_blocks:
            clean = re.sub(r'<[^>]+>', '', blk).strip()
            if len(clean) > 15 and "Illustrated" not in clean and "Evolves" not in clean:
                card["trainer_effect"] = re.sub(r'\s+', ' ', clean)
                break
        else:
            card["trainer_effect"] = None
    else:
        card["trainer_effect"] = None

    return card

def scrape_all(sets=SETS, delay=0.3, out_path="cards.json"):
    all_cards = []
    total_sets = len(sets)

    # Load existing if resuming
    if os.path.exists(out_path):
        with open(out_path) as f:
            all_cards = json.load(f)
        existing_ids = {c["id"] for c in all_cards}
        print(f"Resuming — {len(all_cards)} cards already scraped.")
    else:
        existing_ids = set()

    for si, (set_code, set_name) in enumerate(sets):
        print(f"\n[{si+1}/{total_sets}] Scraping {set_name} ({set_code})...")
        nums = get_set_card_ids(set_code)
        if not nums:
            print(f"  No cards found for {set_code}, skipping.")
            continue
        print(f"  Found {len(nums)} cards.")
        scraped = 0
        for num in nums:
            cid = f"{set_code}-{num:03d}"
            if cid in existing_ids:
                continue
            url = f"https://pocket.limitlesstcg.com/cards/{set_code}/{num}"
            html = fetch(url)
            card = parse_card_page(html, set_code, set_name, num)
            if card:
                all_cards.append(card)
                existing_ids.add(cid)
                scraped += 1
                if scraped % 20 == 0:
                    with open(out_path, "w") as f:
                        json.dump(all_cards, f, indent=2)
                    print(f"  ...saved {scraped} new cards so far")
            time.sleep(delay)
        print(f"  Done. {scraped} new cards from {set_name}.")
        # Save after each set
        with open(out_path, "w") as f:
            json.dump(all_cards, f, indent=2)

    print(f"\n{'='*50}")
    print(f"COMPLETE: {len(all_cards)} total cards saved to {out_path}")

    # Print summary
    by_set = {}
    for c in all_cards:
        by_set[c["set_name"]] = by_set.get(c["set_name"], 0) + 1
    for s, cnt in by_set.items():
        print(f"  {s}: {cnt} cards")

    return all_cards

if __name__ == "__main__":
    out = os.path.expanduser("~/ptcgp_ai/cards.json")
    print("PTCG Pocket Card Scraper")
    print(f"Output: {out}")
    print("This will take ~10-20 minutes. Progress saves after every 20 cards.\n")
    scrape_all(out_path=out)
    print("\nDone! Now run: venv/bin/python3 ptcgp.py train --hours 8")