scraper.py

"""
Vinted scraper using ScrapingAnt API.
"""

import os
import time
from typing import List, Optional

import requests
from bs4 import BeautifulSoup

from config import (
    SCRAPINGANT_API_URL,
    SELECTORS,
    DEFAULT_DELAY,
    ITEMS_PER_PAGE
)
from models import VintedListing, ListingCollection
from utils import (
    build_search_url,
    parse_alt_text,
    extract_item_id,
    extract_favorites_count,
    make_absolute_url
)


class VintedScraper:
    """Scraper for Vinted marketplace listings."""

    def __init__(self, api_key: str, delay: float = DEFAULT_DELAY):
        """
        Initialize the scraper.

        Args:
            api_key: ScrapingAnt API key
            delay: Delay between requests in seconds
        """
        self.api_key = api_key
        self.delay = delay
        self.collection = ListingCollection()

    def _make_request(self, url: str) -> Optional[str]:
        """Make a request using ScrapingAnt API."""
        params = {
            "url": url,
            "x-api-key": self.api_key,
            "browser": "true",
            "proxy_country": "US"
        }

        try:
            response = requests.get(SCRAPINGANT_API_URL, params=params, timeout=120)
            response.raise_for_status()
            return response.text
        except requests.RequestException as e:
            print(f"Request failed for {url}: {e}")
            return None

    def _parse_listings(self, html: str, search_query: str = "", category: str = "") -> List[VintedListing]:
        """Parse listings from HTML content."""
        soup = BeautifulSoup(html, 'html.parser')
        listings = []

        items = soup.select(SELECTORS["listing"])

        for item in items:
            try:
                # Find the container with data-testid
                container = item.select_one(SELECTORS["container"])

                # Get link and URL
                link_el = item.select_one(SELECTORS["link"])
                if not link_el:
                    continue

                url = link_el.get('href', '')
                url = make_absolute_url(url)

                # Get image and alt text with structured data
                img_el = item.select_one(SELECTORS["image"])
                if not img_el:
                    continue

                alt_text = img_el.get('alt', '')
                img_src = img_el.get('src', '')

                # Parse structured data from alt text
                parsed_data = parse_alt_text(alt_text)

                # Extract item ID
                item_id = extract_item_id(url)
                if not item_id and container:
                    # Try from data-testid
                    test_id = container.get('data-testid', '')
                    if 'product-item-id-' in test_id:
                        item_id = test_id.replace('product-item-id-', '')

                # Get favorites count
                fav_btn = item.select_one(SELECTORS["favorites"])
                favorites = 0
                if fav_btn:
                    aria_label = fav_btn.get('aria-label', '')
                    favorites = extract_favorites_count(aria_label)

                listing = VintedListing(
                    title=parsed_data["title"],
                    price=parsed_data["price"],
                    price_with_protection=parsed_data["price_with_protection"],
                    brand=parsed_data["brand"],
                    condition=parsed_data["condition"],
                    size=parsed_data["size"],
                    item_id=item_id,
                    listing_url=url,
                    image_url=img_src,
                    favorites_count=favorites,
                    category=category,
                    search_query=search_query
                )

                listings.append(listing)

            except Exception as e:
                print(f"Error parsing listing: {e}")
                continue

        return listings

    def scrape_search(
        self,
        search_text: str = "",
        category: str = "",
        max_pages: int = 2,
        min_price: Optional[float] = None,
        max_price: Optional[float] = None
    ) -> int:
        """
        Scrape listings from Vinted search results.

        Args:
            search_text: Search query
            category: Category filter
            max_pages: Maximum number of pages to scrape
            min_price: Minimum price filter
            max_price: Maximum price filter

        Returns:
            Number of listings scraped
        """
        total_scraped = 0

        for page in range(1, max_pages + 1):
            url = build_search_url(
                search_text=search_text,
                category=category,
                page=page,
                min_price=min_price,
                max_price=max_price
            )

            print(f"Scraping page {page}: {url}")

            html = self._make_request(url)
            if not html:
                print(f"Failed to fetch page {page}")
                continue

            listings = self._parse_listings(html, search_query=search_text, category=category)
            added = self.collection.add_many(listings)

            print(f"Page {page}: scraped {len(listings)} listings, {added} new")
            total_scraped += added

            # Check if we got listings
            if len(listings) == 0:
                print("No more listings found, stopping pagination")
                break

            # Delay between requests
            if page < max_pages:
                time.sleep(self.delay)

        return total_scraped

    def scrape_categories(
        self,
        categories: List[str],
        max_pages: int = 2
    ) -> int:
        """
        Scrape multiple categories.

        Args:
            categories: List of category keys to scrape
            max_pages: Maximum pages per category

        Returns:
            Total number of listings scraped
        """
        total = 0

        for category in categories:
            print(f"\nScraping category: {category}")
            count = self.scrape_search(category=category, max_pages=max_pages)
            total += count
            print(f"Category {category}: {count} listings")

        return total

    def get_results(self) -> ListingCollection:
        """Get the collection of scraped listings."""
        return self.collection

    def export_csv(self, filepath: str) -> None:
        """Export results to CSV."""
        self.collection.to_csv(filepath)

    def export_json(self, filepath: str) -> None:
        """Export results to JSON."""
        self.collection.to_json(filepath)