Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 24 additions & 11 deletions resources/indexes/HierarchicalNavigableSmallWorld.ts
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ export class HierarchicalNavigableSmallWorld {
if (existingVector) {
// If we are updating an existing entry, we need to update the entry point
// if the new entry is closer to the entry point than the old one
oldNode = { ...this.indexStore.getSync(nodeId, options) };
oldNode = { ...this.safeGetSync(nodeId, options) };
} else oldNode = {} as Node;
if (vector) {
// Pre-compute 1/|vector| for cosine distance so searchLayer can skip sqrt per neighbor
Expand All @@ -174,7 +174,7 @@ export class HierarchicalNavigableSmallWorld {
for (const v of vector) magSq += v * v;
invMag = 1 / (Math.sqrt(magSq) || 1);
}
let entryPoint = entryPointId && this.indexStore.getSync(entryPointId, options);
let entryPoint = entryPointId && this.safeGetSync(entryPointId, options);
if (entryPoint == null) {
const level = Math.floor(-Math.log(Math.random()) * this.mL);
const node = {
Expand Down Expand Up @@ -281,8 +281,11 @@ export class HierarchicalNavigableSmallWorld {

for (const { fromId, toId } of connectionsToBeReplaced) {
let from = updateNode(fromId);
if (!from) from = updateNode(fromId, this.indexStore.getSync(fromId, options));
for (let i = 0; i < from[l].length; i++) {
if (!from) from = updateNode(fromId, this.safeGetSync(fromId, options));
if (!from) continue;
const fromAtLevel = from[l];
if (!fromAtLevel) continue;
for (let i = 0; i < fromAtLevel.length; i++) {
if (from[l][i].id === toId) {
if (Object.isFrozen(from[l])) {
from[l] = from[l].slice();
Expand Down Expand Up @@ -383,7 +386,7 @@ export class HierarchicalNavigableSmallWorld {
const oldConnections = oldNode[l];
for (const { id: neighborId } of oldConnections) {
// get and copy the neighbor node so we can modify it
const neighborNode = updateNode(neighborId, this.indexStore.getSync(neighborId, options));
const neighborNode = updateNode(neighborId, this.safeGetSync(neighborId, options));
if (!neighborNode) continue;
for (let l2 = 0; l2 <= l; l2++) {
// remove the connection to this node from the neighbor node
Expand Down Expand Up @@ -414,14 +417,24 @@ export class HierarchicalNavigableSmallWorld {
for (const [key, vector] of needsReindexing) {
this.index(key, vector, vector, options);
}
this.checkSymmetry(nodeId, this.indexStore.getSync(nodeId, options), options);
this.checkSymmetry(nodeId, this.safeGetSync(nodeId, options), options);
}

private safeGetSync(key: any, options?: any): any {
try {
return this.indexStore.getSync(key, options);
} catch {
logger.warn?.('Failed to decode HNSW node, skipping', key);
return undefined;
}
}

private getEntryPoint(options: { transaction?: any } = {}) {
// Get entry point
const entryPointId = this.indexStore.getSync(ENTRY_POINT, options);
if (entryPointId === undefined) return;
const node = this.indexStore.getSync(entryPointId, options);
const node = this.safeGetSync(entryPointId, options);
if (!node) return;
return { id: entryPointId, ...node };
}

Expand Down Expand Up @@ -486,7 +499,7 @@ export class HierarchicalNavigableSmallWorld {
if (visited.has(neighborId) || neighborId === undefined) continue;
visited.add(neighborId);

const neighbor = this.indexStore.getSync(neighborId, options);
const neighbor = this.safeGetSync(neighborId, options);
if (!neighbor) continue;
this.nodesVisitedCount++;
const distance = computeDistance(neighbor.vector, neighbor.invMag);
Expand Down Expand Up @@ -590,9 +603,9 @@ export class HierarchicalNavigableSmallWorld {
// verify that the level is not empty, otherwise this means we have an orphaned node
if (connections.length === 0) break;
for (const { id: neighbor } of connections) {
const neighborNode = this.indexStore.getSync(neighbor, options);
const neighborNode = this.safeGetSync(neighbor, options);
if (!neighborNode) {
logger.info?.('could not find neighbor node', neighborNode);
logger.info?.('could not find neighbor node', neighbor);
continue;
}
// verify that the connection is symmetrical
Expand Down Expand Up @@ -638,7 +651,7 @@ export class HierarchicalNavigableSmallWorld {
node[level] = keptConnections;
// For removed connections, ensure there's still a path to them
for (const removed of removedConnections) {
let removedNode = updateNode(removed.id) ?? this.indexStore.getSync(removed.id, options);
let removedNode = updateNode(removed.id) ?? this.safeGetSync(removed.id, options);
if (removedNode) {
// Remove the reverse connection if it exists
if (removedNode[level]) {
Expand Down
52 changes: 52 additions & 0 deletions unitTests/resources/vectorIndex.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,58 @@ describe('HierarchicalNavigableSmallWorld indexing', () => {
assert.equal(euclidean[0].id, 1);
assert.equal(dot[0].id, 2);
});
it('does not crash when an index node decodes as corrupt', () => {
const nodes = new Map();
let entryPoint;
let neighborReadCount = 0;

// Minimal mock indexStore: corrupt reads on numeric neighbor-node keys after the first few
const mockStore = {
encoder: { useFloat32: false },
getSync(key) {
if (key === Symbol.for('entryPoint')) return entryPoint;
if (typeof key === 'number') {
neighborReadCount++;
// After the graph has a few nodes, simulate a corrupt node read
if (neighborReadCount > 3) {
throw new Error('Data read, but end of buffer not reached 0');
}
return nodes.get(key);
}
return nodes.get(JSON.stringify(key));
},
put(key, value) {
if (key === Symbol.for('entryPoint')) {
entryPoint = value;
} else if (typeof key === 'number') {
nodes.set(key, value);
} else {
nodes.set(JSON.stringify(key), value);
}
},
remove(key) {
nodes.delete(typeof key === 'number' ? key : JSON.stringify(key));
},
getKeys() {
return [];
},
getUserSharedBuffer(_name, buffer) {
return buffer;
},
};

const hnsw = new HierarchicalNavigableSmallWorld(mockStore, {});

// Build a small graph (neighbor reads stay under threshold here)
for (let i = 0; i < 5; i++) {
hnsw.index(i, [i, i + 1, i + 2], null, {});
}
neighborReadCount = 0; // reset so subsequent inserts hit the corrupt path

// Inserting new nodes must not throw even though neighbor reads now corrupt
assert.doesNotThrow(() => hnsw.index(100, [1, 2, 3], null, {}));
assert.doesNotThrow(() => hnsw.index(101, [4, 5, 6], null, {}));
});
after(() => {
HNSWTest.dropTable();
});
Expand Down
Loading