diff --git a/package-lock.json b/package-lock.json index 8d2f8bd2c..82e270e47 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,7 +34,7 @@ "intl-messageformat": "^10.3.3", "ip": "^1.1.9", "ipfs-css": "^1.4.0", - "ipfs-geoip": "^9.2.0", + "ipfs-geoip": "^9.3.0", "ipfs-provider": "^2.1.0", "ipld-explorer-components": "^8.1.3", "is-ipfs": "^8.0.1", @@ -37806,9 +37806,9 @@ "license": "MIT" }, "node_modules/ipfs-geoip": { - "version": "9.2.0", - "resolved": "https://registry.npmjs.org/ipfs-geoip/-/ipfs-geoip-9.2.0.tgz", - "integrity": "sha512-f7lS2YppDVxbyDrcIz1G8QmKBOuFdCxyIDpN03/DF13XjnMnyV2YooFswOv+DBlcS4EMyuHTHn6+UvRf6K2gcg==", + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/ipfs-geoip/-/ipfs-geoip-9.3.0.tgz", + "integrity": "sha512-OL9GJdxaSOzHK2DKEQZq/0MU9ggj/uVAV6UrNeKyHaPKST32cg0dkvgyNtScH5l4kix/H4yLaQDwzxVfkUOklQ==", "license": "MIT", "engines": { "node": ">=16.0.0", diff --git a/package.json b/package.json index 859b6169f..fcc71a76e 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "intl-messageformat": "^10.3.3", "ip": "^1.1.9", "ipfs-css": "^1.4.0", - "ipfs-geoip": "^9.2.0", + "ipfs-geoip": "^9.3.0", "ipfs-provider": "^2.1.0", "ipld-explorer-components": "^8.1.3", "is-ipfs": "^8.0.1", diff --git a/src/bundles/peer-locations.js b/src/bundles/peer-locations.js index adf01d343..7af704467 100644 --- a/src/bundles/peer-locations.js +++ b/src/bundles/peer-locations.js @@ -6,7 +6,6 @@ import HLRU from 'hashlru' import { multiaddr } from '@multiformats/multiaddr' import ms from 'milliseconds' import ip from 'ip' -import memoize from 'p-memoize' import { createContextSelector } from '../helpers/context-bridge' import pkgJson from '../../package.json' @@ -25,7 +24,7 @@ const selectIdentityData = () => { // After this time interval, we re-check the locations for each peer // once again through PeerLocationResolver. -const UPDATE_EVERY = ms.seconds(1) +const UPDATE_EVERY = ms.seconds(3) // We reuse cached geoip lookups as long geoipVersion is the same. const geoipVersion = dependencies['ipfs-geoip'] @@ -44,8 +43,68 @@ function createPeersLocations (opts) { const bundle = createAsyncResourceBundle({ name: 'peerLocations', actionBaseType: 'PEER_LOCATIONS', - getPromise: ({ store }) => peerLocResolver.findLocations( - store.selectAvailableGatewayUrl(), store.selectPeers()), + getPromise: ({ store }) => { + const peers = store.selectPeers() + if (!peers) { + // Peers bundle hasn't loaded yet. Return empty result but schedule + // a quick retry -- the reactor doesn't depend on selectPeers, so + // without this we'd wait the full staleAfter (3s) before re-fetching. + setTimeout(() => store.doMarkPeerLocationsAsOutdated(), 100) + return Promise.resolve({}) + } + + const promise = peerLocResolver.findLocations( + store.selectAvailableGatewayUrl(), peers) + + // While optimizedPeerSet is still ramping up (pass 0β10, 1β100, 2β200, 3βall), + // chain an immediate re-fetch after the current one resolves instead of + // waiting for staleAfter (3s) between each pass. + if (peerLocResolver.pass <= 3) { + promise.then(() => setTimeout(() => store.doMarkPeerLocationsAsOutdated(), 0)) + } + + if (!peerLocResolver._completedHandler && peerLocResolver.queue.size > 0) { + let throttleTimer = null + let pendingUpdate = false + + const throttledUpdate = () => { + if (throttleTimer) { + pendingUpdate = true + return + } + store.doMarkPeerLocationsAsOutdated() + throttleTimer = setTimeout(() => { + throttleTimer = null + if (pendingUpdate) { + pendingUpdate = false + throttledUpdate() + } + }, 500) + } + + peerLocResolver._completedHandler = throttledUpdate + peerLocResolver.queue.on('completed', throttledUpdate) + + peerLocResolver.queue.onIdle().then(() => { + peerLocResolver.queue.off('completed', throttledUpdate) + peerLocResolver._completedHandler = null + clearTimeout(throttleTimer) + store.doMarkPeerLocationsAsOutdated() + }) + } + + // Avoid unnecessary selector recomputation and re-renders: return + // the previous data reference when nothing actually changed. This is + // cheap because memoryCache (HLRU) returns the same value references + // for the same IPs, so a shallow comparison suffices. + return promise.then(newLocations => { + const prev = store.selectPeerLocations() + if (prev && shallowEqualObjects(prev, newLocations)) { + return prev + } + return newLocations + }) + }, staleAfter: UPDATE_EVERY, retryAfter: UPDATE_EVERY, persist: false, @@ -66,9 +125,8 @@ function createPeersLocations (opts) { bundle.selectPeerLocationsForSwarm = createSelector( 'selectPeers', 'selectPeerLocations', - 'selectBootstrapPeers', selectIdentityData, // ipfs.id info from identity context, used for detecting local peers - (peers, locations = {}, bootstrapPeers, identity) => peers && Promise.all(peers.map(async (peer) => { + (peers, locations = {}, identity) => peers && peers.map((peer) => { const peerId = peer.peer const locationObj = locations ? locations[peerId] : null const location = toLocationString(locationObj) @@ -81,7 +139,7 @@ function createPeersLocations (opts) { const address = peer.addr.toString() const latency = parseLatency(peer.latency) const direction = peer.direction - const { isPrivate, isNearby } = await isPrivateAndNearby(peer.addr, identity) + const { isPrivate, isNearby } = isPrivateAndNearby(peer.addr, identity) const protocols = (Array.isArray(peer.streams) ? Array.from(new Set(peer.streams @@ -107,18 +165,17 @@ function createPeersLocations (opts) { isNearby, agentVersion } - })) + }) ) const COORDINATES_RADIUS = 4 bundle.selectPeersCoordinates = createSelector( 'selectPeerLocationsForSwarm', - async (peers) => { + (peers) => { if (!peers) return [] - const fetchedPeers = await peers - return fetchedPeers.reduce((previous, { peerId, coordinates }) => { + return peers.reduce((previous, { peerId, coordinates }) => { if (!coordinates) return previous let hasFoundACloseCoordinate = false @@ -152,7 +209,15 @@ function createPeersLocations (opts) { return bundle } -const isNonHomeIPv4 = t => t[0] === 4 && t[1] !== '127.0.0.1' +const shallowEqualObjects = (a, b) => { + const keysA = Object.keys(a) + const keysB = Object.keys(b) + if (keysA.length !== keysB.length) return false + return keysA.every(key => a[key] === b[key]) +} + +const isPublicIP = t => + (t[0] === 4 || t[0] === 41) && !ip.isPrivate(t[1]) const toLocationString = loc => { if (!loc) return null @@ -178,27 +243,32 @@ const parseLatency = (latency) => { return value } -const getPublicIP = memoize((identity) => { +let _cachedPublicIP +let _lastIdentityRef + +const getPublicIP = (identity) => { if (!identity) return + if (identity === _lastIdentityRef) return _cachedPublicIP + + _lastIdentityRef = identity + _cachedPublicIP = undefined for (const maddr of identity.addresses) { try { const addr = multiaddr(maddr).nodeAddress() if ((ip.isV4Format(addr.address) || ip.isV6Format(addr.address)) && !ip.isPrivate(addr.address)) { - return addr.address + _cachedPublicIP = addr.address + return _cachedPublicIP } } catch (e) { - // TODO: We should provide a way to log these errors when debugging - // if (['development', 'test'].includes(process.env.REACT_APP_ENV)) { - // console.error(e) - // } + // Might fail for non-IP multiaddrs, safe to ignore. } } -}) +} -const isPrivateAndNearby = async (maddr, identity) => { - const publicIP = await getPublicIP(identity) +const isPrivateAndNearby = (maddr, identity) => { + const publicIP = getPublicIP(identity) let isPrivate = false let isNearby = false let addr @@ -246,8 +316,10 @@ class PeerLocationResolver { }) this.geoipLookupPromises = new Map() + this.memoryCache = HLRU(500) this.pass = 0 + this._completedHandler = null } async findLocations (gatewayUrls, peers) { @@ -260,37 +332,46 @@ class PeerLocationResolver { for (const p of this.optimizedPeerSet(peers)) { const peerId = p.peer - const ipv4Tuple = p.addr.stringTuples().find(isNonHomeIPv4) - if (!ipv4Tuple) { + const ipTuple = p.addr.stringTuples().find(isPublicIP) + if (!ipTuple) { + continue + } + + const ipAddr = ipTuple[1] + if (this.failedAddrs.has(ipAddr)) { continue } - const ipv4Addr = ipv4Tuple[1] - if (this.failedAddrs.has(ipv4Addr)) { + // check in-memory cache first (avoids IndexedDB reads for known IPs) + const memoryCached = this.memoryCache.get(ipAddr) + if (memoryCached) { + res[peerId] = memoryCached continue } - // maybe we have it cached by ipv4 address already, check that. - const location = await this.geoipCache.get(ipv4Addr) + // maybe we have it cached by IP address in IndexedDB + const location = await this.geoipCache.get(ipAddr) if (location) { + this.memoryCache.set(ipAddr, location) res[peerId] = location continue } // no ip address cached. are we looking it up already? - if (this.geoipLookupPromises.has(ipv4Addr)) { + if (this.geoipLookupPromises.has(ipAddr)) { continue } - this.geoipLookupPromises.set(ipv4Addr, this.queue.add(async () => { + this.geoipLookupPromises.set(ipAddr, this.queue.add(async () => { try { - const data = await lookup(gatewayUrls, ipv4Addr) - await this.geoipCache.set(ipv4Addr, data) + const data = await lookup(gatewayUrls, ipAddr) + this.memoryCache.set(ipAddr, data) + await this.geoipCache.set(ipAddr, data) } catch (e) { // mark this one as failed so we don't retry again - this.failedAddrs.set(ipv4Addr, true) + this.failedAddrs.set(ipAddr, true) } finally { - this.geoipLookupPromises.delete(ipv4Addr) + this.geoipLookupPromises.delete(ipAddr) } })) } diff --git a/src/bundles/peer-locations.test.js b/src/bundles/peer-locations.test.js index aaa416dbd..474ceeb1d 100644 --- a/src/bundles/peer-locations.test.js +++ b/src/bundles/peer-locations.test.js @@ -133,8 +133,7 @@ describe('selectPeerLocationsForSwarm', () => { expect(createSelector).toHaveBeenNthCalledWith(2, 'selectPeers', 'selectPeerLocations', - 'selectBootstrapPeers', - 'selectIdentity', + expect.any(Function), // selectIdentityData expect.any(Function) ) }) @@ -145,7 +144,7 @@ describe('selectPeerLocationsForSwarm', () => { expect(callSelectorMethod(selectPeerLocationsForSwarm)).toBeFalsy() }) - it('should map the peers with the location information', async () => { + it('should map the peers with the location information', () => { const { selectPeerLocationsForSwarm } = createPeersLocationBundle() const locations = { @@ -185,7 +184,7 @@ describe('selectPeerLocationsForSwarm', () => { latency: '1s' } - const result = await callSelectorMethod(selectPeerLocationsForSwarm, [peer1, peer2], locations, ['/p2p/1']) + const result = callSelectorMethod(selectPeerLocationsForSwarm, [peer1, peer2], locations) expect(result).toEqual([ { address: '1.test', @@ -219,7 +218,7 @@ describe('selectPeerLocationsForSwarm', () => { ]) }) - it('should also handle the public ip', async () => { + it('should also handle the public ip', () => { const { selectPeerLocationsForSwarm } = createPeersLocationBundle() const locations = { @@ -250,7 +249,7 @@ describe('selectPeerLocationsForSwarm', () => { ] } - const result = await callSelectorMethod(selectPeerLocationsForSwarm, [peer1], locations, ['/ipfs/1'], identity) + const result = callSelectorMethod(selectPeerLocationsForSwarm, [peer1], locations, identity) expect(result).toEqual([ { address: '1.test', @@ -269,7 +268,87 @@ describe('selectPeerLocationsForSwarm', () => { ]) }) - it('should also handle the near addresses', async () => { + it('should map IPv6 peer with location information', () => { + const { selectPeerLocationsForSwarm } = createPeersLocationBundle() + + const locations = { + ipv6peer: { + country_name: 'Republic of Mocks', + city: 'Mocky', + country_code: 'ROM', + longitude: 3.33, + latitude: 3.03 + } + } + + const peer = { + peer: 'ipv6peer', + addr: { + protoNames: () => ['ip6', 'tcp'], + toString: () => '/ip6/2001:db8::1/tcp/4001', + encapsulate: (arg) => ({ toString: () => arg }) + }, + latency: '5ms' + } + + const result = callSelectorMethod(selectPeerLocationsForSwarm, [peer], locations) + expect(result).toEqual([ + { + address: '/ip6/2001:db8::1/tcp/4001', + agentVersion: undefined, + connection: 'ip6/tcp', + coordinates: [3.33, 3.03], + direction: undefined, + flagCode: 'ROM', + isNearby: false, + isPrivate: false, + latency: 5, + location: 'Republic of Mocks, Mocky', + peerId: 'ipv6peer', + protocols: '' + } + ]) + }) + + it('should show private IPv6 peer with isPrivate flag', () => { + const { selectPeerLocationsForSwarm } = createPeersLocationBundle() + + const peer = { + peer: 'privateV6', + addr: { + protoNames: () => ['ip6', 'tcp'], + toString: () => '/ip6/fe80::1/tcp/4001', + encapsulate: (arg) => ({ toString: () => arg }), + nodeAddress: () => ({ address: 'fe80::1' }) + }, + latency: 'n/a' + } + + const result = callSelectorMethod(selectPeerLocationsForSwarm, [peer], {}) + expect(result[0].isPrivate).toBe(true) + expect(result[0].peerId).toBe('privateV6') + }) + + it('should show private IPv4 peer with isPrivate flag', () => { + const { selectPeerLocationsForSwarm } = createPeersLocationBundle() + + const peer = { + peer: 'privateV4', + addr: { + protoNames: () => ['ip4', 'tcp'], + toString: () => '/ip4/192.168.1.1/tcp/4001', + encapsulate: (arg) => ({ toString: () => arg }), + nodeAddress: () => ({ address: '192.168.1.1' }) + }, + latency: 'n/a' + } + + const result = callSelectorMethod(selectPeerLocationsForSwarm, [peer], {}) + expect(result[0].isPrivate).toBe(true) + expect(result[0].peerId).toBe('privateV4') + }) + + it('should also handle the near addresses', () => { const { selectPeerLocationsForSwarm } = createPeersLocationBundle() const peer1 = { @@ -292,7 +371,7 @@ describe('selectPeerLocationsForSwarm', () => { ] } - const result = await callSelectorMethod(selectPeerLocationsForSwarm, [peer1], null, ['/ipfs/1'], identity) + const result = callSelectorMethod(selectPeerLocationsForSwarm, [peer1], null, identity) expect(result).toEqual([ { address: '1.test', @@ -322,14 +401,14 @@ describe('selectPeersCoordinates', () => { ) }) - it('should do nothing when there are no peers', async () => { + it('should do nothing when there are no peers', () => { const { selectPeersCoordinates } = createPeersLocationBundle() - expect(await callSelectorMethod(selectPeersCoordinates)).toEqual([]) + expect(callSelectorMethod(selectPeersCoordinates)).toEqual([]) }) - it('should aggregate peers by close coordinates', async () => { + it('should aggregate peers by close coordinates', () => { const { selectPeersCoordinates } = createPeersLocationBundle() - const result = await callSelectorMethod(selectPeersCoordinates, [ + const result = callSelectorMethod(selectPeersCoordinates, [ { peerId: '1', coordinates: [1, 1] }, { peerId: '2' }, { peerId: '3', coordinates: [1000, 1000] }, @@ -432,11 +511,79 @@ describe('PeerLocationResolver', () => { }) }) + describe('findLocations with IPv6', () => { + it('should resolve IPv6 peer from cache', async () => { + await mockGeoIpCache('2001:db8::1') + + const fakePeers = [ + { + peer: 'v6peer', + latency: '1ms', + addr: { + stringTuples: () => [[41, '2001:db8::1']] + } + } + ] + + const store = await createStore({ + selectors: { + selectAvailableGatewayUrl: () => 'https://ipfs.io', + selectIsOnline: () => true, + selectBootstrapPeers: () => fakePeers, + selectPeers: () => fakePeers, + selectRouteInfo: _ => ({ url: '/peers' }), + selectIdentity: () => ({ + addresses: ['/ip4/4.4.4.4/udp/1234'] + }) + } + }) + + const result = await getPeerLocationsFromStore({ store }) + expect(result).toEqual({ v6peer: 'location-cached' }) + }) + + it('should skip private IPs for both IPv4 and IPv6', async () => { + const privatePeers = [ + { + peer: 'loopbackV6', + latency: '1ms', + addr: { stringTuples: () => [[41, '::1']] } + }, + { + peer: 'linkLocalV6', + latency: '1ms', + addr: { stringTuples: () => [[41, 'fe80::1']] } + }, + { + peer: 'privateV4', + latency: '1ms', + addr: { stringTuples: () => [[4, '192.168.1.1']] } + } + ] + + const store = await createStore({ + selectors: { + selectAvailableGatewayUrl: () => 'https://ipfs.io', + selectIsOnline: () => true, + selectBootstrapPeers: () => privatePeers, + selectPeers: () => privatePeers, + selectRouteInfo: _ => ({ url: '/peers' }), + selectIdentity: () => ({ + addresses: ['/ip4/4.4.4.4/udp/1234'] + }) + } + }) + + const result = await getPeerLocationsFromStore({ store }) + expect(result).toEqual({}) + }) + }) + describe('optimizedPeerSet', () => { it('should return sets of 10, 100, 200 peers and more according to the number of calls', async () => { const ipAddresses = [] const peers = new Array(1000).fill().map((_, index) => { - const ipAddress = `${index}.0.${index}.${index}` + const ipAddress = `33.1.${Math.floor(index / 256)}.${index % 256}` ipAddresses.push(ipAddress) return ({ peer: `${index}aa`, diff --git a/src/peers/PeersTable/PeersTable.js b/src/peers/PeersTable/PeersTable.js index bfa1b9b23..2c6789002 100644 --- a/src/peers/PeersTable/PeersTable.js +++ b/src/peers/PeersTable/PeersTable.js @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useMemo, useState } from 'react' +import React, { useCallback, useMemo, useState } from 'react' import classNames from 'classnames' import ms from 'milliseconds' import { connect } from 'redux-bundler-react' @@ -143,7 +143,7 @@ const FilterInput = ({ filter, setFilter, t, filteredCount }) => { export const PeersTable = ({ className, t, peerLocationsForSwarm, selectedPeers }) => { const tableHeight = 400 - const [awaitedPeerLocationsForSwarm, setAwaitedPeerLocationsForSwarm] = useState([]) + const peers = useMemo(() => peerLocationsForSwarm || [], [peerLocationsForSwarm]) const [sortBy, setSortBy] = useState('latency') const [sortDirection, setSortDirection] = useState(SortDirection.ASC) const [filter, setFilter] = useState('') @@ -156,17 +156,11 @@ export const PeersTable = ({ className, t, peerLocationsForSwarm, selectedPeers setFilter(value) }, []) - useEffect(() => { - peerLocationsForSwarm?.then?.((peerLocationsForSwarm) => { - setAwaitedPeerLocationsForSwarm(peerLocationsForSwarm) - }) - }, [peerLocationsForSwarm]) - const filteredPeerList = useMemo(() => { const filterLower = filter.toLowerCase() - if (filterLower === '') return awaitedPeerLocationsForSwarm + if (filterLower === '') return peers const peerFilter = filter.startsWith('/p2p/') ? filter.slice(5) : filter - return awaitedPeerLocationsForSwarm.filter(({ location, latency, peerId, connection, protocols, agentVersion }) => { + return peers.filter(({ location, latency, peerId, connection, protocols, agentVersion }) => { if (location != null && location.toLowerCase().includes(filterLower)) { return true } @@ -188,7 +182,7 @@ export const PeersTable = ({ className, t, peerLocationsForSwarm, selectedPeers return false }) - }, [awaitedPeerLocationsForSwarm, filter]) + }, [peers, filter]) const sortedList = useMemo( () => filteredPeerList.sort(sortByProperty(sortBy, sortDirection === SortDirection.ASC ? 1 : -1)), @@ -198,13 +192,13 @@ export const PeersTable = ({ className, t, peerLocationsForSwarm, selectedPeers return (