diff --git a/src/polling/pollScraperResults.ts b/src/polling/pollScraperResults.ts index ed8af96..44b4cb4 100644 --- a/src/polling/pollScraperResults.ts +++ b/src/polling/pollScraperResults.ts @@ -37,14 +37,14 @@ export async function pollScraperResults( if (result.status === "SUCCEEDED") { const completedResult = result as { status: string; - datasetId: string; + dataset_id: string | null; data: unknown[]; }; return { run, pollResult: { runId: run.runId, - datasetId: completedResult.datasetId, + datasetId: completedResult.dataset_id ?? run.datasetId, status: completedResult.status, data: completedResult.data, }, @@ -54,7 +54,7 @@ export async function pollScraperResults( run, pollResult: { runId: run.runId, - datasetId: result.datasetId, + datasetId: result.dataset_id ?? run.datasetId, status: result.status, }, }; diff --git a/src/recoup/__tests__/getScraperResults.test.ts b/src/recoup/__tests__/getScraperResults.test.ts new file mode 100644 index 0000000..0574990 --- /dev/null +++ b/src/recoup/__tests__/getScraperResults.test.ts @@ -0,0 +1,93 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const mockFetch = vi.fn(); +vi.stubGlobal("fetch", mockFetch); + +vi.mock("@trigger.dev/sdk/v3", () => ({ + logger: { error: vi.fn(), log: vi.fn(), warn: vi.fn() }, +})); + +vi.mock("../../consts", () => ({ + NEW_API_BASE_URL: "https://recoup-api.vercel.app", + RECOUP_API_KEY: "test-key", +})); + +describe("getScraperResults", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("calls GET /api/apify/runs/{runId} with x-api-key header", async () => { + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ status: "RUNNING", dataset_id: "ds_1" }), + }); + + const { getScraperResults } = await import("../getScraperResults"); + await getScraperResults("run_123"); + + expect(mockFetch).toHaveBeenCalledWith( + "https://recoup-api.vercel.app/api/apify/runs/run_123", + expect.objectContaining({ + method: "GET", + headers: expect.objectContaining({ + "x-api-key": "test-key", + "Content-Type": "application/json", + }), + }) + ); + }); + + it("returns in-progress payload with nullable dataset_id", async () => { + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ status: "RUNNING", dataset_id: null }), + }); + const { getScraperResults } = await import("../getScraperResults"); + const result = await getScraperResults("run_1"); + expect(result).toEqual({ status: "RUNNING", dataset_id: null }); + }); + + it("returns completed payload with data", async () => { + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ + status: "SUCCEEDED", + dataset_id: "ds_1", + data: [{ foo: 1 }], + }), + }); + const { getScraperResults } = await import("../getScraperResults"); + const result = await getScraperResults("run_1"); + expect(result).toEqual({ + status: "SUCCEEDED", + dataset_id: "ds_1", + data: [{ foo: 1 }], + }); + }); + + it("returns undefined on non-ok response", async () => { + mockFetch.mockResolvedValue({ ok: false, status: 500, statusText: "err" }); + const { getScraperResults } = await import("../getScraperResults"); + expect(await getScraperResults("run_1")).toBeUndefined(); + }); + + it("returns undefined when runId is empty", async () => { + const { getScraperResults } = await import("../getScraperResults"); + expect(await getScraperResults("")).toBeUndefined(); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it("url-encodes runId", async () => { + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ status: "RUNNING", dataset_id: null }), + }); + const { getScraperResults } = await import("../getScraperResults"); + await getScraperResults("run/with?special"); + expect(mockFetch).toHaveBeenCalledWith( + "https://recoup-api.vercel.app/api/apify/runs/run%2Fwith%3Fspecial", + expect.anything() + ); + }); +}); diff --git a/src/recoup/getScraperResults.ts b/src/recoup/getScraperResults.ts index 7f8eba6..7437478 100644 --- a/src/recoup/getScraperResults.ts +++ b/src/recoup/getScraperResults.ts @@ -1,13 +1,12 @@ import { logger } from "@trigger.dev/sdk/v3"; import { z } from "zod"; +import { NEW_API_BASE_URL, RECOUP_API_KEY } from "../consts"; -// Base schema with shared fields const inProgressResponseSchema = z.object({ status: z.string(), - datasetId: z.string(), + dataset_id: z.string().nullable(), }); -// Completed response (base + data field) const completedResponseSchema = inProgressResponseSchema.extend({ data: z.array(z.unknown()), }); @@ -16,11 +15,8 @@ type ScraperResponse = | z.infer | z.infer; -const APIFY_SCRAPER_API_URL = "https://api.recoupable.com/api/apify/scraper"; - /** - * Checks the status and retrieves results from an Apify scraper run. - * Returns the response with status and data (if completed). + * Polls an Apify run's status and results via GET /api/apify/runs/{runId}. */ export async function getScraperResults( runId: string @@ -30,16 +26,22 @@ export async function getScraperResults( return undefined; } - try { - const url = new URL(APIFY_SCRAPER_API_URL); - url.searchParams.set("runId", runId); + if (!RECOUP_API_KEY) { + logger.error("RECOUP_API_KEY not configured"); + return undefined; + } - const response = await fetch(url.toString(), { - method: "GET", - headers: { - "Content-Type": "application/json", - }, - }); + try { + const response = await fetch( + `${NEW_API_BASE_URL}/api/apify/runs/${encodeURIComponent(runId)}`, + { + method: "GET", + headers: { + "Content-Type": "application/json", + "x-api-key": RECOUP_API_KEY, + }, + } + ); if (!response.ok) { logger.error("Recoup Apify Scraper API error", { @@ -52,13 +54,11 @@ export async function getScraperResults( const json = (await response.json()) as unknown; - // Try to parse as completed first (has data field) const completedValidation = completedResponseSchema.safeParse(json); if (completedValidation.success) { return completedValidation.data; } - // Otherwise parse as in-progress const inProgressValidation = inProgressResponseSchema.safeParse(json); if (inProgressValidation.success) { return inProgressValidation.data; @@ -80,4 +80,3 @@ export async function getScraperResults( return undefined; } } -