From 70697d3f17f42f09f1789d6645521a1c3752bef6 Mon Sep 17 00:00:00 2001 From: Edward Ezekiel Date: Tue, 27 May 2025 11:06:19 -0500 Subject: [PATCH 1/2] fix: ensure purls are deduped and encoded --- src/api/nes/nes.client.ts | 22 +++++++- test/api/nes.client.test.ts | 103 ++++++++++++++++++++++++++---------- 2 files changed, 96 insertions(+), 29 deletions(-) diff --git a/src/api/nes/nes.client.ts b/src/api/nes/nes.client.ts index 1b25ea39..8d8ab0c4 100644 --- a/src/api/nes/nes.client.ts +++ b/src/api/nes/nes.client.ts @@ -1,4 +1,5 @@ import type * as apollo from '@apollo/client/core/index.js'; +import { PackageURL } from 'packageurl-js'; import { ApolloClient } from '../../api/client.ts'; import type { @@ -59,7 +60,8 @@ export const batchSubmitPurls = async ( batchSize = DEFAULT_SCAN_BATCH_SIZE, ): Promise => { try { - const batches = createBatches(purls, batchSize); + const dedupedAndEncodedPurls = dedupeAndEncodePurls(purls) + const batches = createBatches(dedupedAndEncodedPurls, batchSize); debugLogger('Processing %d batches', batches.length); if (batches.length === 0) { @@ -80,6 +82,24 @@ export const batchSubmitPurls = async ( } }; +export const dedupeAndEncodePurls = (purls: string[]): string[] => { + const dedupedAndEncodedPurls = new Set(); + + for (const purl of purls) { + try { + // The PackageURL.fromString method encodes each part of the purl + const encodedPurl = PackageURL.fromString(purl).toString(); + if (!dedupedAndEncodedPurls.has(encodedPurl)) { + dedupedAndEncodedPurls.add(encodedPurl); + } + } catch (error) { + debugLogger('Error encoding purl: %s', error); + } + } + + return Array.from(dedupedAndEncodedPurls); +}; + export const createBatches = (items: string[], batchSize: number): string[][] => { const numberOfBatches = Math.ceil(items.length / batchSize); diff --git a/test/api/nes.client.test.ts b/test/api/nes.client.test.ts index 81b4e32a..884a32a9 100644 --- a/test/api/nes.client.test.ts +++ b/test/api/nes.client.test.ts @@ -1,35 +1,82 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { createBatches } from '../../src/api/nes/nes.client.ts'; +import { createBatches, dedupeAndEncodePurls } from '../../src/api/nes/nes.client.ts'; import { DEFAULT_SCAN_BATCH_SIZE } from '../../src/api/types/hd-cli.types.ts'; -describe('createBatches', () => { - it('should handle empty array', () => { - const result = createBatches([], DEFAULT_SCAN_BATCH_SIZE); - assert.deepStrictEqual(result, []); +describe('nes.client', () => { + describe('createBatches', () => { + it('should handle empty array', () => { + const result = createBatches([], DEFAULT_SCAN_BATCH_SIZE); + assert.deepStrictEqual(result, []); + }); + + it('should create single batch when items length is less than batch size', () => { + const items = ['a', 'b', 'c']; + const result = createBatches(items, 5); + assert.deepStrictEqual(result, [['a', 'b', 'c']]); + }); + + it('should create single batch when items length equals batch size', () => { + const items = ['a', 'b', 'c', 'd', 'e']; + const result = createBatches(items, 5); + assert.deepStrictEqual(result, [['a', 'b', 'c', 'd', 'e']]); + }); + + it('should create multiple batches when items length exceeds batch size', () => { + const items = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']; + const result = createBatches(items, 3); + assert.deepStrictEqual(result, [['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i'], ['j']]); + }); + + it('should handle batch size of 1', () => { + const items = ['a', 'b', 'c']; + const result = createBatches(items, 1); + assert.deepStrictEqual(result, [['a'], ['b'], ['c']]); + }); }); - - it('should create single batch when items length is less than batch size', () => { - const items = ['a', 'b', 'c']; - const result = createBatches(items, 5); - assert.deepStrictEqual(result, [['a', 'b', 'c']]); - }); - - it('should create single batch when items length equals batch size', () => { - const items = ['a', 'b', 'c', 'd', 'e']; - const result = createBatches(items, 5); - assert.deepStrictEqual(result, [['a', 'b', 'c', 'd', 'e']]); - }); - - it('should create multiple batches when items length exceeds batch size', () => { - const items = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']; - const result = createBatches(items, 3); - assert.deepStrictEqual(result, [['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i'], ['j']]); - }); - - it('should handle batch size of 1', () => { - const items = ['a', 'b', 'c']; - const result = createBatches(items, 1); - assert.deepStrictEqual(result, [['a'], ['b'], ['c']]); + + describe('dedupeAndEncodePurls', () => { + const inputs = [ + { + purls: [ + "pkg:npm/@angular/core@14.3.0", + 'pkg:npm/npm-bundled@2.0.1', + 'pkg:npm/%40angular/core@14.3.0'], + expected: [ + "pkg:npm/%40angular/core@14.3.0", + 'pkg:npm/npm-bundled@2.0.1' + ], + description: 'should dedupe angular core purls' + }, + { + purls: [ + "pkg:npm/@angular/core@14.3.0", + 'pkg:npm/npm-bundled@2.0.1', + 'pkg:npm/rxjs@6.6.7'], + expected: [ + "pkg:npm/%40angular/core@14.3.0", + 'pkg:npm/npm-bundled@2.0.1', + 'pkg:npm/rxjs@6.6.7' + ], + description: 'should not dedupe unique purls' + }, + { + purls: [ + "pkg:maven/org.apache.commons/commons-lang3@3.12.0", + "pkg:maven/org.apache.commons/commons-lang3@3.12.0", + "pkg:maven/org.apache.commons/commons-lang3@3.12.0" + ], + expected: [ + "pkg:maven/org.apache.commons/commons-lang3@3.12.0" + ], + description: 'should dedupe maven purls' + } + ] + for (const input of inputs) { + it(`should dedupe and encode purls: ${input.description}`, () => { + const result = dedupeAndEncodePurls(input.purls); + assert.deepStrictEqual(result, input.expected); + }); + } }); }); From b38e9e2e5cfb61b93b87fe290cf6e4b54e5c3404 Mon Sep 17 00:00:00 2001 From: Edward Ezekiel Date: Tue, 27 May 2025 11:09:10 -0500 Subject: [PATCH 2/2] chore: run linting --- src/api/nes/nes.client.ts | 4 +-- test/api/nes.client.test.ts | 51 +++++++++++++------------------------ 2 files changed, 20 insertions(+), 35 deletions(-) diff --git a/src/api/nes/nes.client.ts b/src/api/nes/nes.client.ts index 8d8ab0c4..ac15f7b8 100644 --- a/src/api/nes/nes.client.ts +++ b/src/api/nes/nes.client.ts @@ -60,7 +60,7 @@ export const batchSubmitPurls = async ( batchSize = DEFAULT_SCAN_BATCH_SIZE, ): Promise => { try { - const dedupedAndEncodedPurls = dedupeAndEncodePurls(purls) + const dedupedAndEncodedPurls = dedupeAndEncodePurls(purls); const batches = createBatches(dedupedAndEncodedPurls, batchSize); debugLogger('Processing %d batches', batches.length); @@ -91,7 +91,7 @@ export const dedupeAndEncodePurls = (purls: string[]): string[] => { const encodedPurl = PackageURL.fromString(purl).toString(); if (!dedupedAndEncodedPurls.has(encodedPurl)) { dedupedAndEncodedPurls.add(encodedPurl); - } + } } catch (error) { debugLogger('Error encoding purl: %s', error); } diff --git a/test/api/nes.client.test.ts b/test/api/nes.client.test.ts index 884a32a9..d6b3eca5 100644 --- a/test/api/nes.client.test.ts +++ b/test/api/nes.client.test.ts @@ -9,69 +9,54 @@ describe('nes.client', () => { const result = createBatches([], DEFAULT_SCAN_BATCH_SIZE); assert.deepStrictEqual(result, []); }); - + it('should create single batch when items length is less than batch size', () => { const items = ['a', 'b', 'c']; const result = createBatches(items, 5); assert.deepStrictEqual(result, [['a', 'b', 'c']]); }); - + it('should create single batch when items length equals batch size', () => { const items = ['a', 'b', 'c', 'd', 'e']; const result = createBatches(items, 5); assert.deepStrictEqual(result, [['a', 'b', 'c', 'd', 'e']]); }); - + it('should create multiple batches when items length exceeds batch size', () => { const items = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']; const result = createBatches(items, 3); assert.deepStrictEqual(result, [['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i'], ['j']]); }); - + it('should handle batch size of 1', () => { const items = ['a', 'b', 'c']; const result = createBatches(items, 1); assert.deepStrictEqual(result, [['a'], ['b'], ['c']]); }); }); - + describe('dedupeAndEncodePurls', () => { const inputs = [ { - purls: [ - "pkg:npm/@angular/core@14.3.0", - 'pkg:npm/npm-bundled@2.0.1', - 'pkg:npm/%40angular/core@14.3.0'], - expected: [ - "pkg:npm/%40angular/core@14.3.0", - 'pkg:npm/npm-bundled@2.0.1' - ], - description: 'should dedupe angular core purls' + purls: ['pkg:npm/@angular/core@14.3.0', 'pkg:npm/npm-bundled@2.0.1', 'pkg:npm/%40angular/core@14.3.0'], + expected: ['pkg:npm/%40angular/core@14.3.0', 'pkg:npm/npm-bundled@2.0.1'], + description: 'should dedupe angular core purls', }, { - purls: [ - "pkg:npm/@angular/core@14.3.0", - 'pkg:npm/npm-bundled@2.0.1', - 'pkg:npm/rxjs@6.6.7'], - expected: [ - "pkg:npm/%40angular/core@14.3.0", - 'pkg:npm/npm-bundled@2.0.1', - 'pkg:npm/rxjs@6.6.7' - ], - description: 'should not dedupe unique purls' + purls: ['pkg:npm/@angular/core@14.3.0', 'pkg:npm/npm-bundled@2.0.1', 'pkg:npm/rxjs@6.6.7'], + expected: ['pkg:npm/%40angular/core@14.3.0', 'pkg:npm/npm-bundled@2.0.1', 'pkg:npm/rxjs@6.6.7'], + description: 'should not dedupe unique purls', }, { purls: [ - "pkg:maven/org.apache.commons/commons-lang3@3.12.0", - "pkg:maven/org.apache.commons/commons-lang3@3.12.0", - "pkg:maven/org.apache.commons/commons-lang3@3.12.0" + 'pkg:maven/org.apache.commons/commons-lang3@3.12.0', + 'pkg:maven/org.apache.commons/commons-lang3@3.12.0', + 'pkg:maven/org.apache.commons/commons-lang3@3.12.0', ], - expected: [ - "pkg:maven/org.apache.commons/commons-lang3@3.12.0" - ], - description: 'should dedupe maven purls' - } - ] + expected: ['pkg:maven/org.apache.commons/commons-lang3@3.12.0'], + description: 'should dedupe maven purls', + }, + ]; for (const input of inputs) { it(`should dedupe and encode purls: ${input.description}`, () => { const result = dedupeAndEncodePurls(input.purls);