feat: EOL API pagination

e2e/scan/eol.test.ts

@@ -25,12 +25,27 @@ function mockReport(components: DeepPartial<EolScanComponent>[] = []) {
     eol: {
       createReport: {
         success: true,
-        report: {
-          id: 'test-123',
-          createdOn: new Date().toISOString(),
-          metadata: {},
-          components,
+        id: 'test-123',
+        totalRecords: components.length,
+      },
+    },
+  };
+}
+
+function mockGetReport(components: DeepPartial<EolScanComponent>[] = []) {
+  return {
+    eol: {
+      report: {
+        id: 'test-123',
+        createdOn: new Date().toISOString(),
+        metadata: {
+          totalComponentsCount: components.length,
+          unknownComponentsCount: 0,
+          totalUniqueComponentsCount: components.length,
         },
+        components,
+        page: 1,
+        totalRecords: components.length,
       },
     },
   };
@@ -58,16 +73,15 @@ describe('scan:eol e2e', () => {
 
   beforeEach(async () => {
     await mkdir(fixturesDir, { recursive: true });
-    fetchMock = new FetchMock().addGraphQL(
-      mockReport([
-        { purl: 'pkg:npm/bootstrap@3.1.1', metadata: { isEol: true } },
-        {
-          purl: 'pkg:npm/is-core-module@2.11.0',
-          metadata: {},
-          nesRemediation: { remediations: [{ urls: { main: 'https://example.com' } }] },
-        },
-      ]),
-    );
+    const components = [
+      { purl: 'pkg:npm/bootstrap@3.1.1', metadata: { isEol: true } },
+      {
+        purl: 'pkg:npm/is-core-module@2.11.0',
+        metadata: {},
+        nesRemediation: { remediations: [{ urls: { main: 'https://example.com' } }] },
+      },
+    ];
+    fetchMock = new FetchMock().addGraphQL(mockReport(components)).addGraphQL(mockGetReport(components));
   });
 
   afterEach(() => {
@@ -176,13 +190,12 @@ describe('scan:eol e2e', () => {
   });
 
   it('shows zero EOL components when scanning up-to-date packages', async () => {
+    const components = [
+      { purl: 'pkg:npm/bootstrap@5.3.5', metadata: {} },
+      { purl: 'pkg:npm/vue@3.5.13', metadata: {} },
+    ];
     fetchMock.restore();
-    fetchMock = new FetchMock().addGraphQL(
-      mockReport([
-        { purl: 'pkg:npm/bootstrap@5.3.5', metadata: {} },
-        { purl: 'pkg:npm/vue@3.5.13', metadata: {} },
-      ]),
-    );
+    fetchMock = new FetchMock().addGraphQL(mockReport(components)).addGraphQL(mockGetReport(components));
     const cmd = `scan:eol --file ${upToDateSbom}`;
     const { stdout } = await run(cmd);
     match(stdout, /Scan results:/, 'Should show results header');
@@ -192,7 +205,7 @@ describe('scan:eol e2e', () => {
 
   it('handles empty components array without errors', async () => {
     fetchMock.restore();
-    fetchMock = new FetchMock().addGraphQL(mockReport([]));
+    fetchMock = new FetchMock().addGraphQL(mockReport([])).addGraphQL(mockGetReport([]));
     const cmd = `scan:eol --file ${noComponentsSbom}`;
     const { stdout } = await run(cmd);
     match(stdout, /No components found in scan/, 'Should show no packages found in scan');
@@ -239,12 +252,11 @@ describe('scan:eol e2e', () => {
 
   it('scans up-to-date directory and shows modern packages', async () => {
     fetchMock.restore();
-    fetchMock = new FetchMock().addGraphQL(
-      mockReport([
-        { purl: 'pkg:npm/bootstrap@5.3.5', metadata: {} },
-        { purl: 'pkg:npm/vue@3.5.13', metadata: {} },
-      ]),
-    );
+    const components = [
+      { purl: 'pkg:npm/bootstrap@5.3.5', metadata: {} },
+      { purl: 'pkg:npm/vue@3.5.13', metadata: {} },
+    ];
+    fetchMock = new FetchMock().addGraphQL(mockReport(components)).addGraphQL(mockGetReport(components));
     const cmd = `scan:eol --dir ${upToDateDir}`;
     const { stdout } = await run(cmd);
     match(stdout, /Scan results:/, 'Should show results header');
@@ -361,7 +373,7 @@ describe('scan:eol e2e', () => {
     it('fails when NES returns unsuccessful result', async () => {
       // Override fetch mock to return unsuccessful mutation for this test
       fetchMock.restore();
-      fetchMock = new FetchMock().addGraphQL({ eol: { createReport: { success: false, report: null } } });
+      fetchMock = new FetchMock().addGraphQL({ eol: { createReport: { success: false, id: null, totalRecords: 0 } } });
       const out = await runExpectFail(`scan:eol --file ${simpleSbom}`);
       expectAny(out, [/Failed to submit scan to NES/i, /Scanning failed/i], 'Should indicate NES submission failure');
     });

package.json

@@ -39,7 +39,7 @@
     "@amplitude/analytics-node": "^1.5.0",
     "@apollo/client": "^3.13.8",
     "@cyclonedx/cdxgen": "^11.4.3",
-    "@herodevs/eol-shared": "github:herodevs/eol-shared#v0.1.4",
+    "@herodevs/eol-shared": "github:herodevs/eol-shared#v0.1.10",
     "@oclif/core": "^4.4.0",
     "@oclif/plugin-help": "^6.2.29",
     "@oclif/plugin-update": "^4.6.45",

src/api/gql-operations.ts

@@ -5,22 +5,33 @@ mutation createReport($input: CreateEolReportInput) {
   eol {
     createReport(input: $input) {
       success
-      report {
-        createdOn
-        id 
+      id
+      totalRecords
+    }
+  }
+}
+`;
+
+export const getEolReportQuery = gql`
+query GetEolReport($input: GetEolReportInput) {
+  eol {
+    report(input: $input) {
+      id
+      createdOn
+      metadata
+      components {
+        purl
         metadata
-        components {
-          purl
-          metadata
-          nesRemediation {
-            remediations {
-              urls {
-                main
-              }
+        nesRemediation {
+          remediations {
+            urls {
+              main
             }
           }
         }
       }
+      page
+      totalRecords
     }
   }
 }

src/api/nes.client.ts

@@ -1,16 +1,25 @@
-import { ApolloClient, ApolloLink, HttpLink, InMemoryCache } from '@apollo/client/core/index.js';
-import type { CreateEolReportInput, EolReport, EolReportMutationResponse } from '@herodevs/eol-shared';
+import { ApolloClient, InMemoryCache } from '@apollo/client/core/index.js';
+import type {
+  CreateEolReportInput,
+  EolReport,
+  EolReportMutationResponse,
+  EolReportQueryResponse,
+  GetEolReportInput,
+} from '@herodevs/eol-shared';
 import { config } from '../config/constants.ts';
 import { debugLogger } from '../service/log.svc.ts';
-import { createReportMutation } from './gql-operations.ts';
+import { createReportMutation, getEolReportQuery } from './gql-operations.ts';
 
 export const createApollo = (uri: string) =>
   new ApolloClient({
     cache: new InMemoryCache({ addTypename: false }),
+    defaultOptions: {
+      query: { fetchPolicy: 'no-cache' },
+    },
     headers: {
       'User-Agent': `hdcli/${process.env.npm_package_version ?? 'unknown'}`,
     },
-    link: ApolloLink.from([new HttpLink({ uri })]),
+    uri,
   });
 
 export const SbomScanner = (client: ReturnType<typeof createApollo>) => {
@@ -21,12 +30,45 @@ export const SbomScanner = (client: ReturnType<typeof createApollo>) => {
     });
 
     const result = res.data?.eol?.createReport;
-    if (!result?.success || !result.report) {
+    if (!result?.success || !result.id) {
       debugLogger('failed scan %o', result || {});
       throw new Error('Failed to create EOL report');
     }
 
-    return result.report;
+    const totalRecords = result.totalRecords || 0;
+    const totalPages = Math.ceil(totalRecords / config.pageSize);
+    const pages = Array.from({ length: totalPages }, (_, index) =>
+      client.query<EolReportQueryResponse, { input: GetEolReportInput }>({
+        query: getEolReportQuery,
+        variables: {
+          input: {
+            id: result.id,
+            page: index + 1,
+            size: config.pageSize,
+          },
+        },
+      }),
+    );
+
+    const components: EolReport['components'] = [];
+    let reportMetadata: EolReport | null = null;
+
+    for (let i = 0; i < pages.length; i += config.concurrentPageRequests) {
+      const batch = pages.slice(i, i + config.concurrentPageRequests);
+      const batchResponses = await Promise.all(batch);
+
+      for (const response of batchResponses) {
+        const report = response.data.eol.report;
+        reportMetadata ??= report;
+        components.push(...(report?.components ?? []));
+      }
+    }
+
+    if (!reportMetadata) {
+      throw new Error('Failed to fetch EOL report');
+    }
+
+    return { ...reportMetadata, components };
   };
 };
 

src/config/constants.ts

@@ -2,13 +2,29 @@ export const EOL_REPORT_URL = 'https://eol-report-card.apps.herodevs.com/reports
 export const GRAPHQL_HOST = 'https://api.nes.herodevs.com';
 export const GRAPHQL_PATH = '/graphql';
 export const ANALYTICS_URL = 'https://eol-api.herodevs.com/track';
+export const CONCURRENT_PAGE_REQUESTS = 3;
+export const PAGE_SIZE = 500;
+
+let concurrentPageRequests = CONCURRENT_PAGE_REQUESTS;
+const parsed = Number.parseInt(process.env.CONCURRENT_PAGE_REQUESTS ?? '0', 10);
+if (parsed > 0) {
+  concurrentPageRequests = parsed;
+}
+
+let pageSize = PAGE_SIZE;
+const parsedPageSize = Number.parseInt(process.env.PAGE_SIZE ?? '0', 10);
+if (parsedPageSize > 0) {
+  pageSize = parsedPageSize;
+}
 
 export const config = {
   eolReportUrl: process.env.EOL_REPORT_URL || EOL_REPORT_URL,
   graphqlHost: process.env.GRAPHQL_HOST || GRAPHQL_HOST,
   graphqlPath: process.env.GRAPHQL_PATH || GRAPHQL_PATH,
   analyticsUrl: process.env.ANALYTICS_URL || ANALYTICS_URL,
   showVulnCount: true,
+  concurrentPageRequests,
+  pageSize,
 };
 
 export const filenamePrefix = 'herodevs';

test/api/nes.client.test.ts

@@ -16,37 +16,45 @@ describe('nes.client', () => {
   });
 
   it('returns report on successful createReport mutation', async () => {
-    const report = {
-      id: 'test-123',
-      createdOn: new Date().toISOString(),
-      metadata: {},
-      components: [
-        { purl: 'pkg:npm/bootstrap@3.1.1', metadata: { isEol: true } },
-        {
-          purl: 'pkg:npm/is-core-module@2.11.0',
-          metadata: {},
-          nesRemediation: { remediations: [{ urls: { main: 'https://example.com' } }] },
-        },
-      ],
-    };
+    const components = [
+      { purl: 'pkg:npm/bootstrap@3.1.1', metadata: { isEol: true } },
+      {
+        purl: 'pkg:npm/is-core-module@2.11.0',
+        metadata: {},
+        nesRemediation: { remediations: [{ urls: { main: 'https://example.com' } }] },
+      },
+    ];
 
-    fetchMock.addGraphQL({
-      eol: { createReport: { success: true, report } },
-    });
+    fetchMock
+      .addGraphQL({
+        eol: { createReport: { success: true, id: 'test-123', totalRecords: components.length } },
+      })
+      .addGraphQL({
+        eol: {
+          report: {
+            id: 'test-123',
+            createdOn: new Date().toISOString(),
+            metadata: {},
+            components,
+            page: 1,
+            totalRecords: components.length,
+          },
+        },
+      });
 
     const input: CreateEolReportInput = {
       sbom: { bomFormat: 'CycloneDX', components: [], specVersion: '1.4', version: 1 },
     };
     const res = await submitScan(input);
 
-    assert.strictEqual(res.id, report.id);
+    assert.strictEqual(res.id, 'test-123');
     assert.strictEqual(Array.isArray(res.components), true);
-    assert.strictEqual(res.components.length, 2);
+    assert.strictEqual(res.components.length, components.length);
   });
 
   it('throws when mutation returns unsuccessful response or no report', async () => {
     fetchMock.addGraphQL({
-      eol: { createReport: { success: false, report: null } },
+      eol: { createReport: { success: false, id: null, totalRecords: 0 } },
     });
 
     const input: CreateEolReportInput = {

test/service/analytics.svc.test.ts

@@ -21,7 +21,7 @@ describe('analytics.svc', () => {
       namedExports: { config: { analyticsUrl: 'https://test-analytics.com' } },
     });
 
-    return import(import.meta.resolve(`../../src/service/analytics.svc.ts?${Math.random().toFixed(3)}`));
+    return import(import.meta.resolve(`../../src/service/analytics.svc.ts?${Math.random().toFixed(4)}`));
   }
 
   beforeEach(() => {