diff --git a/e2e/scan/eol.test.ts b/e2e/scan/eol.test.ts index 67468f14..152c003c 100644 --- a/e2e/scan/eol.test.ts +++ b/e2e/scan/eol.test.ts @@ -25,12 +25,27 @@ function mockReport(components: DeepPartial[] = []) { eol: { createReport: { success: true, - report: { - id: 'test-123', - createdOn: new Date().toISOString(), - metadata: {}, - components, + id: 'test-123', + totalRecords: components.length, + }, + }, + }; +} + +function mockGetReport(components: DeepPartial[] = []) { + return { + eol: { + report: { + id: 'test-123', + createdOn: new Date().toISOString(), + metadata: { + totalComponentsCount: components.length, + unknownComponentsCount: 0, + totalUniqueComponentsCount: components.length, }, + components, + page: 1, + totalRecords: components.length, }, }, }; @@ -58,16 +73,15 @@ describe('scan:eol e2e', () => { beforeEach(async () => { await mkdir(fixturesDir, { recursive: true }); - fetchMock = new FetchMock().addGraphQL( - mockReport([ - { purl: 'pkg:npm/bootstrap@3.1.1', metadata: { isEol: true } }, - { - purl: 'pkg:npm/is-core-module@2.11.0', - metadata: {}, - nesRemediation: { remediations: [{ urls: { main: 'https://example.com' } }] }, - }, - ]), - ); + const components = [ + { purl: 'pkg:npm/bootstrap@3.1.1', metadata: { isEol: true } }, + { + purl: 'pkg:npm/is-core-module@2.11.0', + metadata: {}, + nesRemediation: { remediations: [{ urls: { main: 'https://example.com' } }] }, + }, + ]; + fetchMock = new FetchMock().addGraphQL(mockReport(components)).addGraphQL(mockGetReport(components)); }); afterEach(() => { @@ -176,13 +190,12 @@ describe('scan:eol e2e', () => { }); it('shows zero EOL components when scanning up-to-date packages', async () => { + const components = [ + { purl: 'pkg:npm/bootstrap@5.3.5', metadata: {} }, + { purl: 'pkg:npm/vue@3.5.13', metadata: {} }, + ]; fetchMock.restore(); - fetchMock = new FetchMock().addGraphQL( - mockReport([ - { purl: 'pkg:npm/bootstrap@5.3.5', metadata: {} }, - { purl: 'pkg:npm/vue@3.5.13', metadata: {} }, - ]), - ); + fetchMock = new FetchMock().addGraphQL(mockReport(components)).addGraphQL(mockGetReport(components)); const cmd = `scan:eol --file ${upToDateSbom}`; const { stdout } = await run(cmd); match(stdout, /Scan results:/, 'Should show results header'); @@ -192,7 +205,7 @@ describe('scan:eol e2e', () => { it('handles empty components array without errors', async () => { fetchMock.restore(); - fetchMock = new FetchMock().addGraphQL(mockReport([])); + fetchMock = new FetchMock().addGraphQL(mockReport([])).addGraphQL(mockGetReport([])); const cmd = `scan:eol --file ${noComponentsSbom}`; const { stdout } = await run(cmd); match(stdout, /No components found in scan/, 'Should show no packages found in scan'); @@ -239,12 +252,11 @@ describe('scan:eol e2e', () => { it('scans up-to-date directory and shows modern packages', async () => { fetchMock.restore(); - fetchMock = new FetchMock().addGraphQL( - mockReport([ - { purl: 'pkg:npm/bootstrap@5.3.5', metadata: {} }, - { purl: 'pkg:npm/vue@3.5.13', metadata: {} }, - ]), - ); + const components = [ + { purl: 'pkg:npm/bootstrap@5.3.5', metadata: {} }, + { purl: 'pkg:npm/vue@3.5.13', metadata: {} }, + ]; + fetchMock = new FetchMock().addGraphQL(mockReport(components)).addGraphQL(mockGetReport(components)); const cmd = `scan:eol --dir ${upToDateDir}`; const { stdout } = await run(cmd); match(stdout, /Scan results:/, 'Should show results header'); @@ -361,7 +373,7 @@ describe('scan:eol e2e', () => { it('fails when NES returns unsuccessful result', async () => { // Override fetch mock to return unsuccessful mutation for this test fetchMock.restore(); - fetchMock = new FetchMock().addGraphQL({ eol: { createReport: { success: false, report: null } } }); + fetchMock = new FetchMock().addGraphQL({ eol: { createReport: { success: false, id: null, totalRecords: 0 } } }); const out = await runExpectFail(`scan:eol --file ${simpleSbom}`); expectAny(out, [/Failed to submit scan to NES/i, /Scanning failed/i], 'Should indicate NES submission failure'); }); diff --git a/package-lock.json b/package-lock.json index f98a82b0..7591e5e4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@amplitude/analytics-node": "^1.5.0", "@apollo/client": "^3.13.8", "@cyclonedx/cdxgen": "^11.4.3", - "@herodevs/eol-shared": "github:herodevs/eol-shared#v0.1.4", + "@herodevs/eol-shared": "github:herodevs/eol-shared#v0.1.10", "@oclif/core": "^4.4.0", "@oclif/plugin-help": "^6.2.29", "@oclif/plugin-update": "^4.6.45", @@ -2172,7 +2172,7 @@ }, "node_modules/@herodevs/eol-shared": { "version": "1.0.0", - "resolved": "git+ssh://git@github.com/herodevs/eol-shared.git#1e8326d833217847c470bd890f00b67d67f07998", + "resolved": "git+ssh://git@github.com/herodevs/eol-shared.git#880ac001bf4c4589de553c65288eb38ed8c3ead3", "license": "ISC", "dependencies": { "@cyclonedx/cyclonedx-library": "^8.5.0", diff --git a/package.json b/package.json index f5262a3f..622e58d1 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "@amplitude/analytics-node": "^1.5.0", "@apollo/client": "^3.13.8", "@cyclonedx/cdxgen": "^11.4.3", - "@herodevs/eol-shared": "github:herodevs/eol-shared#v0.1.4", + "@herodevs/eol-shared": "github:herodevs/eol-shared#v0.1.10", "@oclif/core": "^4.4.0", "@oclif/plugin-help": "^6.2.29", "@oclif/plugin-update": "^4.6.45", diff --git a/src/api/gql-operations.ts b/src/api/gql-operations.ts index 58faf7a0..8b7bb8be 100644 --- a/src/api/gql-operations.ts +++ b/src/api/gql-operations.ts @@ -5,22 +5,33 @@ mutation createReport($input: CreateEolReportInput) { eol { createReport(input: $input) { success - report { - createdOn - id + id + totalRecords + } + } +} +`; + +export const getEolReportQuery = gql` +query GetEolReport($input: GetEolReportInput) { + eol { + report(input: $input) { + id + createdOn + metadata + components { + purl metadata - components { - purl - metadata - nesRemediation { - remediations { - urls { - main - } + nesRemediation { + remediations { + urls { + main } } } } + page + totalRecords } } } diff --git a/src/api/nes.client.ts b/src/api/nes.client.ts index b609913c..c81cf534 100644 --- a/src/api/nes.client.ts +++ b/src/api/nes.client.ts @@ -1,16 +1,25 @@ -import { ApolloClient, ApolloLink, HttpLink, InMemoryCache } from '@apollo/client/core/index.js'; -import type { CreateEolReportInput, EolReport, EolReportMutationResponse } from '@herodevs/eol-shared'; +import { ApolloClient, InMemoryCache } from '@apollo/client/core/index.js'; +import type { + CreateEolReportInput, + EolReport, + EolReportMutationResponse, + EolReportQueryResponse, + GetEolReportInput, +} from '@herodevs/eol-shared'; import { config } from '../config/constants.ts'; import { debugLogger } from '../service/log.svc.ts'; -import { createReportMutation } from './gql-operations.ts'; +import { createReportMutation, getEolReportQuery } from './gql-operations.ts'; export const createApollo = (uri: string) => new ApolloClient({ cache: new InMemoryCache({ addTypename: false }), + defaultOptions: { + query: { fetchPolicy: 'no-cache' }, + }, headers: { 'User-Agent': `hdcli/${process.env.npm_package_version ?? 'unknown'}`, }, - link: ApolloLink.from([new HttpLink({ uri })]), + uri, }); export const SbomScanner = (client: ReturnType) => { @@ -21,12 +30,45 @@ export const SbomScanner = (client: ReturnType) => { }); const result = res.data?.eol?.createReport; - if (!result?.success || !result.report) { + if (!result?.success || !result.id) { debugLogger('failed scan %o', result || {}); throw new Error('Failed to create EOL report'); } - return result.report; + const totalRecords = result.totalRecords || 0; + const totalPages = Math.ceil(totalRecords / config.pageSize); + const pages = Array.from({ length: totalPages }, (_, index) => + client.query({ + query: getEolReportQuery, + variables: { + input: { + id: result.id, + page: index + 1, + size: config.pageSize, + }, + }, + }), + ); + + const components: EolReport['components'] = []; + let reportMetadata: EolReport | null = null; + + for (let i = 0; i < pages.length; i += config.concurrentPageRequests) { + const batch = pages.slice(i, i + config.concurrentPageRequests); + const batchResponses = await Promise.all(batch); + + for (const response of batchResponses) { + const report = response.data.eol.report; + reportMetadata ??= report; + components.push(...(report?.components ?? [])); + } + } + + if (!reportMetadata) { + throw new Error('Failed to fetch EOL report'); + } + + return { ...reportMetadata, components }; }; }; diff --git a/src/config/constants.ts b/src/config/constants.ts index b7259ff2..8ba931ff 100644 --- a/src/config/constants.ts +++ b/src/config/constants.ts @@ -2,6 +2,20 @@ export const EOL_REPORT_URL = 'https://eol-report-card.apps.herodevs.com/reports export const GRAPHQL_HOST = 'https://api.nes.herodevs.com'; export const GRAPHQL_PATH = '/graphql'; export const ANALYTICS_URL = 'https://eol-api.herodevs.com/track'; +export const CONCURRENT_PAGE_REQUESTS = 3; +export const PAGE_SIZE = 500; + +let concurrentPageRequests = CONCURRENT_PAGE_REQUESTS; +const parsed = Number.parseInt(process.env.CONCURRENT_PAGE_REQUESTS ?? '0', 10); +if (parsed > 0) { + concurrentPageRequests = parsed; +} + +let pageSize = PAGE_SIZE; +const parsedPageSize = Number.parseInt(process.env.PAGE_SIZE ?? '0', 10); +if (parsedPageSize > 0) { + pageSize = parsedPageSize; +} export const config = { eolReportUrl: process.env.EOL_REPORT_URL || EOL_REPORT_URL, @@ -9,6 +23,8 @@ export const config = { graphqlPath: process.env.GRAPHQL_PATH || GRAPHQL_PATH, analyticsUrl: process.env.ANALYTICS_URL || ANALYTICS_URL, showVulnCount: true, + concurrentPageRequests, + pageSize, }; export const filenamePrefix = 'herodevs'; diff --git a/test/api/nes.client.test.ts b/test/api/nes.client.test.ts index 7262b5a8..93f41072 100644 --- a/test/api/nes.client.test.ts +++ b/test/api/nes.client.test.ts @@ -16,37 +16,45 @@ describe('nes.client', () => { }); it('returns report on successful createReport mutation', async () => { - const report = { - id: 'test-123', - createdOn: new Date().toISOString(), - metadata: {}, - components: [ - { purl: 'pkg:npm/bootstrap@3.1.1', metadata: { isEol: true } }, - { - purl: 'pkg:npm/is-core-module@2.11.0', - metadata: {}, - nesRemediation: { remediations: [{ urls: { main: 'https://example.com' } }] }, - }, - ], - }; + const components = [ + { purl: 'pkg:npm/bootstrap@3.1.1', metadata: { isEol: true } }, + { + purl: 'pkg:npm/is-core-module@2.11.0', + metadata: {}, + nesRemediation: { remediations: [{ urls: { main: 'https://example.com' } }] }, + }, + ]; - fetchMock.addGraphQL({ - eol: { createReport: { success: true, report } }, - }); + fetchMock + .addGraphQL({ + eol: { createReport: { success: true, id: 'test-123', totalRecords: components.length } }, + }) + .addGraphQL({ + eol: { + report: { + id: 'test-123', + createdOn: new Date().toISOString(), + metadata: {}, + components, + page: 1, + totalRecords: components.length, + }, + }, + }); const input: CreateEolReportInput = { sbom: { bomFormat: 'CycloneDX', components: [], specVersion: '1.4', version: 1 }, }; const res = await submitScan(input); - assert.strictEqual(res.id, report.id); + assert.strictEqual(res.id, 'test-123'); assert.strictEqual(Array.isArray(res.components), true); - assert.strictEqual(res.components.length, 2); + assert.strictEqual(res.components.length, components.length); }); it('throws when mutation returns unsuccessful response or no report', async () => { fetchMock.addGraphQL({ - eol: { createReport: { success: false, report: null } }, + eol: { createReport: { success: false, id: null, totalRecords: 0 } }, }); const input: CreateEolReportInput = { diff --git a/test/service/analytics.svc.test.ts b/test/service/analytics.svc.test.ts index 66d08a7e..3f9dd3c9 100644 --- a/test/service/analytics.svc.test.ts +++ b/test/service/analytics.svc.test.ts @@ -21,7 +21,7 @@ describe('analytics.svc', () => { namedExports: { config: { analyticsUrl: 'https://test-analytics.com' } }, }); - return import(import.meta.resolve(`../../src/service/analytics.svc.ts?${Math.random().toFixed(3)}`)); + return import(import.meta.resolve(`../../src/service/analytics.svc.ts?${Math.random().toFixed(4)}`)); } beforeEach(() => {