feat(tasks): migrate GET /api/apify/runs/{runId}

src/polling/pollScraperResults.ts

@@ -37,14 +37,14 @@ export async function pollScraperResults(
       if (result.status === "SUCCEEDED") {
         const completedResult = result as {
           status: string;
-          datasetId: string;
+          dataset_id: string | null;
           data: unknown[];
         };
         return {
           run,
           pollResult: {
             runId: run.runId,
-            datasetId: completedResult.datasetId,
+            datasetId: completedResult.dataset_id ?? run.datasetId,
             status: completedResult.status,
             data: completedResult.data,
           },
@@ -54,7 +54,7 @@ export async function pollScraperResults(
           run,
           pollResult: {
             runId: run.runId,
-            datasetId: result.datasetId,
+            datasetId: result.dataset_id ?? run.datasetId,
             status: result.status,
           },
         };

src/recoup/__tests__/getScraperResults.test.ts

@@ -0,0 +1,93 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+const mockFetch = vi.fn();
+vi.stubGlobal("fetch", mockFetch);
+
+vi.mock("@trigger.dev/sdk/v3", () => ({
+  logger: { error: vi.fn(), log: vi.fn(), warn: vi.fn() },
+}));
+
+vi.mock("../../consts", () => ({
+  NEW_API_BASE_URL: "https://recoup-api.vercel.app",
+  RECOUP_API_KEY: "test-key",
+}));
+
+describe("getScraperResults", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("calls GET /api/apify/runs/{runId} with x-api-key header", async () => {
+    mockFetch.mockResolvedValue({
+      ok: true,
+      json: async () => ({ status: "RUNNING", dataset_id: "ds_1" }),
+    });
+
+    const { getScraperResults } = await import("../getScraperResults");
+    await getScraperResults("run_123");
+
+    expect(mockFetch).toHaveBeenCalledWith(
+      "https://recoup-api.vercel.app/api/apify/runs/run_123",
+      expect.objectContaining({
+        method: "GET",
+        headers: expect.objectContaining({
+          "x-api-key": "test-key",
+          "Content-Type": "application/json",
+        }),
+      })
+    );
+  });
+
+  it("returns in-progress payload with nullable dataset_id", async () => {
+    mockFetch.mockResolvedValue({
+      ok: true,
+      json: async () => ({ status: "RUNNING", dataset_id: null }),
+    });
+    const { getScraperResults } = await import("../getScraperResults");
+    const result = await getScraperResults("run_1");
+    expect(result).toEqual({ status: "RUNNING", dataset_id: null });
+  });
+
+  it("returns completed payload with data", async () => {
+    mockFetch.mockResolvedValue({
+      ok: true,
+      json: async () => ({
+        status: "SUCCEEDED",
+        dataset_id: "ds_1",
+        data: [{ foo: 1 }],
+      }),
+    });
+    const { getScraperResults } = await import("../getScraperResults");
+    const result = await getScraperResults("run_1");
+    expect(result).toEqual({
+      status: "SUCCEEDED",
+      dataset_id: "ds_1",
+      data: [{ foo: 1 }],
+    });
+  });
+
+  it("returns undefined on non-ok response", async () => {
+    mockFetch.mockResolvedValue({ ok: false, status: 500, statusText: "err" });
+    const { getScraperResults } = await import("../getScraperResults");
+    expect(await getScraperResults("run_1")).toBeUndefined();
+  });
+
+  it("returns undefined when runId is empty", async () => {
+    const { getScraperResults } = await import("../getScraperResults");
+    expect(await getScraperResults("")).toBeUndefined();
+    expect(mockFetch).not.toHaveBeenCalled();
+  });
+
+  it("url-encodes runId", async () => {
+    mockFetch.mockResolvedValue({
+      ok: true,
+      json: async () => ({ status: "RUNNING", dataset_id: null }),
+    });
+    const { getScraperResults } = await import("../getScraperResults");
+    await getScraperResults("run/with?special");
+    expect(mockFetch).toHaveBeenCalledWith(
+      "https://recoup-api.vercel.app/api/apify/runs/run%2Fwith%3Fspecial",
+      expect.anything()
+    );
+  });
+});

src/recoup/getScraperResults.ts

@@ -1,13 +1,12 @@
 import { logger } from "@trigger.dev/sdk/v3";
 import { z } from "zod";
+import { NEW_API_BASE_URL, RECOUP_API_KEY } from "../consts";
 
-// Base schema with shared fields
 const inProgressResponseSchema = z.object({
   status: z.string(),
-  datasetId: z.string(),
+  dataset_id: z.string().nullable(),
 });
 
-// Completed response (base + data field)
 const completedResponseSchema = inProgressResponseSchema.extend({
   data: z.array(z.unknown()),
 });
@@ -16,11 +15,8 @@ type ScraperResponse =
   | z.infer<typeof inProgressResponseSchema>
   | z.infer<typeof completedResponseSchema>;
 
-const APIFY_SCRAPER_API_URL = "https://api.recoupable.com/api/apify/scraper";
-
 /**
- * Checks the status and retrieves results from an Apify scraper run.
- * Returns the response with status and data (if completed).
+ * Polls an Apify run's status and results via GET /api/apify/runs/{runId}.
  */
 export async function getScraperResults(
   runId: string
@@ -30,16 +26,22 @@ export async function getScraperResults(
     return undefined;
   }
 
-  try {
-    const url = new URL(APIFY_SCRAPER_API_URL);
-    url.searchParams.set("runId", runId);
+  if (!RECOUP_API_KEY) {
+    logger.error("RECOUP_API_KEY not configured");
+    return undefined;
+  }
 
-    const response = await fetch(url.toString(), {
-      method: "GET",
-      headers: {
-        "Content-Type": "application/json",
-      },
-    });
+  try {
+    const response = await fetch(
+      `${NEW_API_BASE_URL}/api/apify/runs/${encodeURIComponent(runId)}`,
+      {
+        method: "GET",
+        headers: {
+          "Content-Type": "application/json",
+          "x-api-key": RECOUP_API_KEY,
+        },
+      }
+    );
 
     if (!response.ok) {
       logger.error("Recoup Apify Scraper API error", {
@@ -52,13 +54,11 @@ export async function getScraperResults(
 
     const json = (await response.json()) as unknown;
 
-    // Try to parse as completed first (has data field)
     const completedValidation = completedResponseSchema.safeParse(json);
     if (completedValidation.success) {
       return completedValidation.data;
     }
 
-    // Otherwise parse as in-progress
     const inProgressValidation = inProgressResponseSchema.safeParse(json);
     if (inProgressValidation.success) {
       return inProgressValidation.data;
@@ -80,4 +80,3 @@ export async function getScraperResults(
     return undefined;
   }
 }
-