From 1eda7f0b2457ff8298fc9448fe668744e54edc0a Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Sat, 25 Dec 2021 16:18:45 +0400 Subject: [PATCH 01/17] test: add netaddress tests. --- lib/net/netaddress.js | 24 +- test/net-netaddress-test.js | 732 ++++++++++++++++++++++++++++++++++++ 2 files changed, 748 insertions(+), 8 deletions(-) create mode 100644 test/net-netaddress-test.js diff --git a/lib/net/netaddress.js b/lib/net/netaddress.js index 96b051ec..a48fee29 100644 --- a/lib/net/netaddress.js +++ b/lib/net/netaddress.js @@ -18,6 +18,8 @@ const common = require('./common'); * Constants */ +const IPV4_ZERO_IP = Buffer.from('00000000000000000000ffff00000000', 'hex'); + const ZERO_KEY = Buffer.alloc(33, 0x00); /** @@ -49,7 +51,7 @@ class NetAddress extends bio.Struct { this.services = 0; this.time = 0; this.hostname = '0.0.0.0:0'; - this.raw = IP.ZERO_IP; + this.raw = IPV4_ZERO_IP; this.key = ZERO_KEY; if (options) @@ -63,26 +65,32 @@ class NetAddress extends bio.Struct { */ fromOptions(options) { - assert(typeof options.host === 'string'); - assert(typeof options.port === 'number'); + assert(typeof options.host === 'string', + 'NetAddress requires host string.'); + assert(typeof options.port === 'number', + 'NetAddress requires port number.'); + assert(options.port >= 0 && options.port <= 0xffff, + 'port number is incorrect.'); this.raw = IP.toBuffer(options.host); this.host = IP.toString(this.raw); this.port = options.port; if (options.services) { - assert(typeof options.services === 'number'); + assert(typeof options.services === 'number', + 'services must be a number.'); this.services = options.services; } if (options.time) { - assert(typeof options.time === 'number'); + assert(typeof options.time === 'number', + 'time must be a number.'); this.time = options.time; } if (options.key) { - assert(Buffer.isBuffer(options.key)); - assert(options.key.length === 33); + assert(Buffer.isBuffer(options.key), 'key must be a buffer.'); + assert(options.key.length === 33, 'key length must be 33.'); this.key = options.key; } @@ -211,7 +219,7 @@ class NetAddress extends bio.Struct { */ setNull() { - this.raw = IP.ZERO_IP; + this.raw = IPV4_ZERO_IP; this.host = '0.0.0.0'; this.key = ZERO_KEY; this.hostname = IP.toHostname(this.host, this.port, this.key); diff --git a/test/net-netaddress-test.js b/test/net-netaddress-test.js new file mode 100644 index 00000000..40e78685 --- /dev/null +++ b/test/net-netaddress-test.js @@ -0,0 +1,732 @@ +'use strict'; + +const assert = require('bsert'); +const NetAddress = require('../lib/net/netaddress'); +const Network = require('../lib/protocol/network'); +const util = require('../lib/utils/util') + +// 16 bytes (ipv6) - 4 (ipv4) byte - 2 ff = 10 +const IPV4_PREFIX = Buffer.from(`${'00'.repeat(10)}ffff`, 'hex'); + +const main = Network.get('main'); +const regtest = Network.get('regtest'); + +describe('NetAddress', function() { + it('should parse options', () => { + const options = [ + [null, { + host: '0.0.0.0', + port: 0, + hostname: '0.0.0.0:0', + isNull: true, + isIPv6: false, + isLocal: true, + isValid: false, + isRoutable: false + }], + [{ + host: '0.0.0.0', + port: 0 + }, { + host: '0.0.0.0', + port: 0, + hostname: '0.0.0.0:0', + isNull: true, + isIPv6: false, + isLocal: true, + isValid: false, + isRoutable: false + }], + [{ + host: '2345:0425:2CA1:0000:0000:0567:5673:23b5', + port: 1000 + }, { + host: '2345:425:2ca1::567:5673:23b5', + port: 1000, + hostname: '[2345:425:2ca1::567:5673:23b5]:1000', + isIPv6: true, + isLocal: false, + isValid: true, + isRoutable: true + }], + [{ + host: '1.1.1.1', + port: 1, + services: 1, + key: Buffer.alloc(33, 1) + }, { + hostname: + 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc@1.1.1.1:1', + key: Buffer.alloc(33, 1), + services: 1, + isIPv6: false, + isLocal: false, + isValid: true, + isRoutable: true + }], + [{ + host: '2.2.2.2', + port: 2, + key: Buffer.alloc(33, 2), + services: 2 + }, { + hostname: + 'aibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibae@2.2.2.2:2', + key: Buffer.alloc(33, 2), + services: 2, + isIPv6: false, + isLocal: false, + isValid: true, + isRoutable: true + }], [{ + host: '127.0.0.1', + port: 1000, + services: 3 + }, { + hostname: '127.0.0.1:1000', + services: 3, + isIPv6: false, + isLocal: true, + isValid: true, + isRoutable: false + }], [{ + host: '127.1.1.1', + port: 1000 + }, { + hostname: '127.1.1.1:1000', + isIPv6: false, + isLocal: true, + isValid: true, + isRoutable: false + }], [{ + host: '::1', + port: 1000 + }, { + hostname: '[::1]:1000', + isIPv6: true, + isLocal: true, + isValid: true, + isRoutable: false + }], [{ + host: 'fd87:d87e:eb43::1', + port: 1000 + }, { + host: 'aaaaaaaaaaaaaaab.onion', + hostname: 'aaaaaaaaaaaaaaab.onion:1000', + isIPv6: false, + isIPV4: false, + isLocal: false, + isValid: true, + isRoutable: true, + isOnion: true + }] + ]; + + for (const [i, [opts, expected]] of options.entries()) { + const naddr = new NetAddress(opts); + + if (expected.host == null) + expected.host = opts.host; + + if (expected.port == null) + expected.port = opts.port; + + assert.strictEqual(naddr.host, expected.host, `Failed #${i}`); + assert.strictEqual(naddr.port, expected.port, `Failed #${i}`); + assert.strictEqual(naddr.hostname, expected.hostname, `Failed #${i}`); + + const expectedKey = opts && opts.key; + assert.strictEqual(naddr.hasKey(), Boolean(expectedKey), `Failed #${i}`); + + if (expectedKey) + assert.bufferEqual(naddr.key, expectedKey, `Failed #${i}`); + + if (expected.isIPv6 != null) { + const isIPV4 = expected.isIPV4 != null + ? expected.isIPV4 + : !expected.isIPv6; + + assert.strictEqual(naddr.isIPv4(), isIPV4, `Failed #${i}`); + assert.strictEqual(naddr.isIPv6(), expected.isIPv6, `Failed #${i}`); + } + + if (opts && opts.services != null) { + assert.strictEqual(true, naddr.hasServices(expected.services), + `Failed #${i}`); + } + + assert.strictEqual(naddr.isRoutable(), expected.isRoutable, `Failed #${i}`); + assert.strictEqual(naddr.isValid(), expected.isValid, `Failed #${i}`); + assert.strictEqual(naddr.isNull(), Boolean(expected.isNull), + `Failed #${i}`); + assert.strictEqual(naddr.isOnion(), Boolean(expected.isOnion), + `Failed #${i}`); + assert.strictEqual(true, naddr.equal(naddr), `Failed #${i}`); + + assert.strictEqual(naddr.isLocal(), expected.isLocal, `Failed #${i}`); + } + }); + + it('should fail parsing options', () => { + const goodOptions = { + host: '0.0.0.0', + port: 12038 + }; + + const badOptions = [ + [{ port: goodOptions.port }, 'NetAddress requires host string.'], + [{ host: 1234 }, 'NetAddress requires host string.'], + [{ host: goodOptions.host }, 'NetAddress requires port number.'], + [{ host: goodOptions.host, port: '32' }, + 'NetAddress requires port number.'], + [{ host: goodOptions.host, port: -1 }, 'port number is incorrect.'], + [{ host: goodOptions.host, port: 0xffff + 1 }, + 'port number is incorrect.'], + [{ ...goodOptions, services: '12' }, 'services must be a number.'], + [{ ...goodOptions, services: {} }, 'services must be a number.'], + [{ ...goodOptions, key: '12' }, 'key must be a buffer.'], + [{ ...goodOptions, key: 11 }, 'key must be a buffer.'] + ]; + + for (const [opts, msg] of badOptions) { + let err; + try { + new NetAddress(opts); + } catch (e) { + err = e; + } + + assert(err, 'Expected err'); + assert.strictEqual(err.message, msg); + } + }); + + it('should check services', async () => { + const naddr = new NetAddress(); + + const serviceList = []; + for (let i = 0; i < 10; i++) + serviceList.push(1 << i); + + naddr.services = serviceList[7] | serviceList[8] | serviceList[9]; + + for (let i = 7; i < 10; i++) + assert.strictEqual(true, naddr.hasServices(serviceList[i])); + + for (let i = 0; i < 7; i++) + assert.strictEqual(false, naddr.hasServices(serviceList[i])); + + assert.strictEqual(true, + naddr.hasServices(serviceList[7] | serviceList[8])); + assert.strictEqual(false, + naddr.hasServices(serviceList[1] | serviceList[8])); + }); + + it('should set null', async () => { + const oldRaw = Buffer.from('2d4f86e1', 'hex'); + const nullRaw = Buffer.alloc(4, 0); + const naddr = new NetAddress({ + host: '45.79.134.225', + port: 1 + }); + + assert.strictEqual(false, naddr.isNull()); + assert.bufferEqual(naddr.raw, Buffer.concat([IPV4_PREFIX, oldRaw])); + assert.strictEqual(naddr.hostname, '45.79.134.225:1'); + + naddr.setNull(); + assert.strictEqual(true, naddr.isNull()); + assert.bufferEqual(naddr.raw, Buffer.concat([IPV4_PREFIX, nullRaw])); + assert.strictEqual(naddr.hostname, '0.0.0.0:1'); + }); + + it('should set host', async () => { + const oldHost = '45.79.134.225'; + const oldRaw = Buffer.from('2d4f86e1', 'hex'); + const newHost = '15.152.112.161'; + const newRaw = Buffer.from('0f9870a1', 'hex'); + + const naddr = new NetAddress({ + host: oldHost, + port: 12038 + }); + + assert.strictEqual(naddr.host, oldHost); + assert.bufferEqual(naddr.raw, Buffer.concat([IPV4_PREFIX, oldRaw])); + naddr.setHost(newHost); + assert.strictEqual(naddr.host, newHost); + assert.bufferEqual(naddr.raw, Buffer.concat([IPV4_PREFIX, newRaw])); + }); + + it('should set port', async () => { + const naddr = new NetAddress({ + host: '45.79.134.225', + port: 1000 + }); + + const badPorts = [ + -1, + -0xffff, + 0xffff + 1, + 0xffffff + ]; + + for (const port of badPorts) { + let err; + try { + naddr.setPort(port); + } catch (e) { + err = e; + } + + assert(err, `Error not found for ${port}.`); + } + + const goodPorts = [ + 0, + 0xffff, + 12038, + 44806 + ]; + + for (const port of goodPorts) { + naddr.setPort(port); + assert.strictEqual(naddr.port, port); + assert.strictEqual(naddr.hostname, `${naddr.host}:${port}`); + } + }); + + it('should set/get key', async () => { + const testKey = Buffer.alloc(33, 1); + + const naddr = new NetAddress({ + host: '0.0.0.0', + port: 1000 + }); + + assert.strictEqual(naddr.getKey(), null); + + naddr.setKey(testKey); + assert.bufferEqual(naddr.getKey(), testKey); + assert.strictEqual(naddr.getKey('base32'), + 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc'); + assert.strictEqual(naddr.getKey('hex'), + '01'.repeat(33)); + assert.strictEqual(naddr.hostname, + `${naddr.getKey('base32')}@${naddr.host}:${naddr.port}`); + + naddr.setKey(); + assert.strictEqual(naddr.getKey(), null); + assert.strictEqual(naddr.getKey('base32'), null); + assert.strictEqual(naddr.getKey('hex'), null); + assert.strictEqual(naddr.hostname, `${naddr.host}:${naddr.port}`); + + const badKeys = [ + 'badkey', + Buffer.alloc(32, 0), + Buffer.alloc(34, 11) + ]; + + for (const key of badKeys) { + let err; + try { + naddr.setKey(key); + } catch (e) { + err = e; + } + + assert(err); + } + }); + + it('should create from host', () => { + const vector = [ + [ + ['172.104.214.189', 1000, Buffer.alloc(33, 1)], + { + host: '172.104.214.189', + port: 1000, + key: Buffer.alloc(33, 1), + hostname: 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc' + + '@172.104.214.189:1000' + } + ], + [ + ['15.152.162.66', 1001, Buffer.alloc(33, 2)], + { + host: '15.152.162.66', + port: 1001, + key: Buffer.alloc(33, 2), + hostname: 'aibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibae' + + '@15.152.162.66:1001' + } + ], + [ + ['2345:0425:2CA1:0000:0000:0567:5673:23b5', 0xffff], + { + host: '2345:425:2ca1::567:5673:23b5', + port: 0xffff, + key: Buffer.alloc(33, 0), + hostname: '[2345:425:2ca1::567:5673:23b5]:65535' + } + ] + ]; + + const naddr = new NetAddress(); + + for (const [params, expected] of vector) { + naddr.fromHost(...params); + const naddr2 = NetAddress.fromHost(...params); + + for (const addr of [naddr, naddr2]) { + assert.strictEqual(addr.host, expected.host); + assert.strictEqual(addr.port, expected.port); + assert.bufferEqual(addr.key, expected.key); + assert.strictEqual(addr.hostname, expected.hostname); + } + } + }); + + it('should create from hostname', () => { + const vector = [ + [['127.0.0.1:100', 'main'], { + hostname: '127.0.0.1:100', + host: '127.0.0.1', + port: 100, + key: Buffer.alloc(33, 0) + }], + [ + [ + 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc@127.0.0.1:100', + 'main' + ], { + hostname: 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc' + + '@127.0.0.1:100', + host: '127.0.0.1', + port: 100, + key: Buffer.alloc(33, 1) + } + ], + [['127.0.0.1', 'main'], { + hostname: `127.0.0.1:${main.port}`, + host: '127.0.0.1', + port: main.port, + key: Buffer.alloc(33, 0) + }], + [ + [ + 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc@127.0.0.1', + 'main' + ], { + hostname: 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc' + + `@127.0.0.1:${main.brontidePort}`, + host: '127.0.0.1', + port: main.brontidePort, + key: Buffer.alloc(33, 1) + } + ], + [['127.0.0.1', 'regtest'], { + hostname: `127.0.0.1:${regtest.port}`, + host: '127.0.0.1', + port: regtest.port, + key: Buffer.alloc(33, 0) + }], + [ + [ + 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc@127.0.0.1', + 'regtest' + ], { + hostname: 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc' + + `@127.0.0.1:${regtest.brontidePort}`, + host: '127.0.0.1', + port: regtest.brontidePort, + key: Buffer.alloc(33, 1) + } + ] + ]; + + for (const [args, expected] of vector) { + const addr1 = new NetAddress(); + addr1.fromHostname(...args); + + const addr2 = NetAddress.fromHostname(...args); + + for (const addr of [addr1, addr2]) { + assert.strictEqual(addr.hostname, expected.hostname); + assert.strictEqual(addr.host, expected.host); + assert.strictEqual(addr.port, expected.port); + assert.bufferEqual(addr.key, expected.key); + } + } + }); + + it('should create from socket', () => { + const vector = [ + [[{ + remoteAddress: '2001:4860:a005::68', + remotePort: 1000 + }, 'main'], { + hostname: '[2001:4860:a005::68]:1000', + host: '2001:4860:a005::68', + port: 1000 + }], + [[{ + remoteAddress: '74.125.127.100', + remotePort: 2000 + }, 'main'], { + hostname: '74.125.127.100:2000', + host: '74.125.127.100', + port: 2000 + }] + ]; + + for (const [args, expected] of vector) { + const addr = NetAddress.fromSocket(...args); + + assert.strictEqual(addr.hostname, expected.hostname); + assert.strictEqual(addr.host, expected.host); + assert.strictEqual(addr.port, expected.port); + } + }); + + it('should compare addresses', () => { + const vector = [ + [['127.0.0.1', 10], ['127.1.1.1', 9], -1], + [['0.0.0.0', 10], ['1.1.1.1', 9], -1], + [['0.0.0.1', 10], ['0.0.0.1', 9], 1], + // IPV4 has two 0xff in the buffer before last 4 bytes. + // So any IPV6 from ::1 to :ffff:0:0 will be lower than IPV4. + // And any IPV6 from :ffff:0:0 to :ffff:ffff:ffff will be IPV4. + [['::1', 1], ['0.0.0.1', 1], -1], + [['::ffff:ffff', 1], ['0.0.0.1', 1], -1], + [['::ffff:ffff:ffff', 1], ['0.0.0.1', 1], 1], + [['::ffff:0:1', 1], ['0.0.0.1', 1], 0], + [['::ffff:ffff:ffff', 1], ['255.255.255.255', 1], 0], + // If IPs are same, then we compare ports. + [['::1', 102], ['::1', 101], 1], + [['::1', 100], ['::1', 101], -1], + [['::1', 100], ['::1', 100], 0] + ]; + + for (const [[hosta, porta], [hostb, portb], expected] of vector) { + const addrA = new NetAddress({ + host: hosta, + port: porta + }); + + const addrB = new NetAddress({ + host: hostb, + port: portb + }); + + assert.strictEqual(addrA.compare(addrB), expected, + `Failed for ${hosta}:${portb} compare to ${hostb}:${portb}.`); + } + }); + + it('should serialize/deserialize raw', () => { + const options = { + host: '::1', + port: 1000, + services: 0xff, + time: main.now(), + key: Buffer.alloc(33, 1) + }; + + const check = (addr, incorrectHost) => { + if (incorrectHost) + assert.notStrictEqual(addr.host, options.host); + else + assert.strictEqual(addr.host, options.host); + assert.strictEqual(addr.port, options.port); + assert.strictEqual(addr.time, options.time); + assert.strictEqual(addr.services, options.services); + assert.bufferEqual(addr.key, options.key); + }; + + { + const addr = new NetAddress(options); + const encoded = addr.encode(); + const decoded = NetAddress.decode(encoded); + + assert.strictEqual(decoded.equal(addr), true); + assert.strictEqual(decoded.compare(addr), 0); + assert.bufferEqual(decoded.encode(), encoded); + check(decoded); + } + + { + // Do not decode IP. + const addr = new NetAddress(options); + const encoded = addr.encode(); + // time(8) + services(4) + service bits(4) + encoded[8 + 8] = 1; + + const decoded = NetAddress.decode(encoded); + + assert.strictEqual(decoded.equal(addr), false); + assert.strictEqual(decoded.compare(addr), -1); + assert.notBufferEqual(decoded.encode(), encoded); + + check(decoded, true); + } + }); + + it('should serialize/deserialize JSON', () => { + const options = { + host: '::1', + port: 1000, + services: 0xff, + time: main.now(), + key: Buffer.alloc(33, 1) + }; + + const check = (addr, hex) => { + assert.strictEqual(addr.host, options.host); + assert.strictEqual(addr.port, options.port); + assert.strictEqual(addr.time, options.time); + assert.strictEqual(addr.services, options.services); + + if (hex) + assert.strictEqual(addr.key, options.key.toString('hex')); + else + assert.bufferEqual(addr.key, options.key); + }; + + const addr = new NetAddress(options); + const json = addr.getJSON(); + const decoded = NetAddress.fromJSON(json); + + assert.strictEqual(decoded.equal(addr), true); + assert.strictEqual(decoded.compare(addr), 0); + assert.bufferEqual(decoded.encode(), addr.encode()); + check(decoded); + check(json, true); + }); + + it('should inspect/format', () => { + const options = { + host: '::1', + port: 1000, + services: 0xff, + time: main.now(), + key: Buffer.alloc(33, 1) + }; + + const addr = new NetAddress(options); + const formatted = addr.format(); + + assert.strictEqual(formatted.startsWith(''), true); + assert.strictEqual(formatted.indexOf(`hostname=${addr.hostname}`) > 0, + true); + assert.strictEqual( + formatted.indexOf(`services=${addr.services.toString(2)}`) > 0, + true + ); + assert.strictEqual(formatted.indexOf(`date=${util.date(addr.time)}`) > 0, + true); + }); + + it('should get reachability score', () => { + // see: binet.getReachability for details + const UNREACHABLE = 0; + const DEFAULT = 1; + // const TEREDO = 2; + // const IPV6_WEAK = 3; + const IPV4 = 4; + // const IPV6_STRONG = 5; + // const PRIVATE = 6; + + const ipv4src = '74.125.127.100'; + const ipv4dest = '45.79.134.225'; + + const ipv6src = 'ffff::1'; + const ipv6dest = 'ffff::ffff'; + + const teredosrc = '2001::1'; + const teredodest = '2001:ffff::1'; + + const onionsrc = 'aaaaaaaaaaaaaaaa.onion'; + const oniondest = 'bbbbbbbbbbbbbbbb.onion'; + + const unreachable = [ + // Source UNREACHABLE + ['0.0.0.0', ipv4dest, UNREACHABLE], + ['127.0.0.1', ipv4dest, UNREACHABLE], + ['::', ipv4dest, UNREACHABLE], + ['::1', ipv4dest, UNREACHABLE], + // broadcast + ['255.255.255.255', ipv4dest, UNREACHABLE], + // SHIFTED + ['::ff:ff00:0:0:0', ipv4dest, UNREACHABLE], + // RFC3849 - IPv6 Reserved prefix + ['2001:0db8::', ipv4dest, UNREACHABLE], + ['2001:db8::1:1', ipv4dest, UNREACHABLE], + // RFC 1918 - Private Internets + ['192.168.1.1', ipv4dest, UNREACHABLE], + ['10.10.10.10', ipv4dest, UNREACHABLE], + ['172.20.1.1', ipv4dest, UNREACHABLE], + // RFC 2544 - IPv4 inter-network communications (198.18.0.0/15) + ['198.18.1.1', ipv4dest, UNREACHABLE], + ['198.19.255.255', ipv4dest, UNREACHABLE], + // RFC 3927 - Link local addresses (179.254.0.0/16) + ['169.254.0.0', ipv4dest, UNREACHABLE], + ['169.254.255.255', ipv4dest, UNREACHABLE], + // RFC 4862 - IPv6 Stateless address autoconfiguration + ['fe80::', ipv4dest, UNREACHABLE], + ['fe80::1', ipv4dest, UNREACHABLE], + // RFC 6598 - IANA-Reserved IPv4 Prefix for Shared Address Space + ['100.64.0.0', ipv4dest, UNREACHABLE], + ['100.127.255.255', ipv4dest, UNREACHABLE], + // RFC 5737 - IPv4 Address Blocks Reserved for Documentation + ['192.0.2.0', ipv4dest, UNREACHABLE], + ['192.0.2.255', ipv4dest, UNREACHABLE], + ['198.51.100.0', ipv4dest, UNREACHABLE], + ['198.51.100.255', ipv4dest, UNREACHABLE], + ['203.0.113.0', ipv4dest, UNREACHABLE], + ['203.0.113.255', ipv4dest, UNREACHABLE], + // RFC 4193 - Unique Local IPv6 Unicast Addresses + // FC00::/7 + ['fd00::', ipv4dest, UNREACHABLE], + ['fc00::', ipv4dest, UNREACHABLE], + ['fcff::', ipv4dest, UNREACHABLE], + // RFC 4843 - Overlay Routable Cryptographic Hash Identifiers prefix. + // ORCHID + ['2001:0010::', ipv4dest, UNREACHABLE], + ['2001:0010:ffff::', ipv4dest, UNREACHABLE], + // RFC 7343 - ORCHIDv2 + ['2001:0020::', ipv4dest, UNREACHABLE], + ['2001:0020::ffff', ipv4dest, UNREACHABLE] + ]; + + const destIPV4 = [ + // We already made sure above, source is not invalid. + // only proper outputs left. + [ipv4src, ipv4dest, IPV4], + [ipv6src, ipv4dest, DEFAULT], + [onionsrc, ipv4dest, DEFAULT], + [teredosrc, ipv4dest, DEFAULT] + ]; + + // const + + const vector = [ + ...unreachable, + ]; + + for (const [source, destination, reachability] of vector) { + const src = new NetAddress({ + host: source, + port: 1000 + }); + + const dest = new NetAddress({ + host: destination, + port: 1000 + }); + + assert.strictEqual(src.getReachability(dest), reachability); + } + }); +}); From 5232b33d2a10f93f0e4f32d90da6cf2d03b50ed3 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Wed, 26 Jan 2022 15:10:42 +0400 Subject: [PATCH 02/17] test: add look/resolve tests. --- test/net-lookup.js | 56 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 test/net-lookup.js diff --git a/test/net-lookup.js b/test/net-lookup.js new file mode 100644 index 00000000..daa64edd --- /dev/null +++ b/test/net-lookup.js @@ -0,0 +1,56 @@ +'use strict'; + +const assert = require('bsert'); +const Network = require('../lib/protocol/network'); +const {lookup, resolve} = require('../lib/net/lookup'); + +const main = Network.get('main'); + +const notAHost = 'not-a-domain.not-a-domain'; + +describe('Lookup', function() { + it('should lookup seed', async () => { + for (const host of main.seeds) { + const addresses = await lookup(host); + assert(addresses.length > 0, 'addresses not found.'); + } + }); + + it('should fail lookup', async () => { + let err; + + try { + await lookup(notAHost); + } catch (e) { + err = e; + } + + assert(err); + assert.strictEqual(err.message, 'No DNS results.'); + }); + + it('should lookup seed', async () => { + this.timeout(10000); + + for (const host of main.seeds) { + const addresses = await resolve(host); + + assert(addresses.length > 0, 'addresses not found.'); + } + }); + + it('should fail resolve', async () => { + let err; + + try { + await resolve(notAHost); + } catch (e) { + err = e; + } + + assert(err); + assert.strictEqual(err.message, `Query error: NXDOMAIN (${notAHost} A).`); + + // TODO: Host that does not have A/AAAA records? + }); +}); From 7d55fc65c5712d40e55c655414073f96bb838999 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Wed, 26 Jan 2022 19:12:29 +0400 Subject: [PATCH 03/17] test: netaddress add more tests. --- package.json | 2 +- test/data/netaddress-data.js | 335 ++++++++++++++++++++++++++++++++ test/net-netaddress-test.js | 367 +++-------------------------------- 3 files changed, 358 insertions(+), 346 deletions(-) create mode 100644 test/data/netaddress-data.js diff --git a/package.json b/package.json index bf9739a7..fad16162 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "bfile": "~0.2.2", "bfilter": "~1.0.5", "bheep": "~0.1.5", - "binet": "~0.3.6", + "binet": "~0.3.7", "blgr": "~0.2.0", "blru": "~0.1.6", "blst": "~0.1.5", diff --git a/test/data/netaddress-data.js b/test/data/netaddress-data.js new file mode 100644 index 00000000..6e3c102d --- /dev/null +++ b/test/data/netaddress-data.js @@ -0,0 +1,335 @@ +'use strict'; + +const Network = require('../../lib/protocol/network'); +const netaddressVectors = exports; + +const main = Network.get('main'); +const regtest = Network.get('regtest'); + +// [passedOptions, expectedValues] +netaddressVectors.options = [ + [null, { + host: '0.0.0.0', + port: 0, + hostname: '0.0.0.0:0', + isNull: true, + isIPv6: false, + isLocal: true, + isValid: false, + isRoutable: false + }], + [{ + host: '0.0.0.0', + port: 0 + }, { + host: '0.0.0.0', + port: 0, + hostname: '0.0.0.0:0', + isNull: true, + isIPv6: false, + isLocal: true, + isValid: false, + isRoutable: false + }], + [{ + host: '2345:0425:2CA1:0000:0000:0567:5673:23b5', + port: 1000 + }, { + host: '2345:425:2ca1::567:5673:23b5', + port: 1000, + hostname: '[2345:425:2ca1::567:5673:23b5]:1000', + isIPv6: true, + isLocal: false, + isValid: true, + isRoutable: true + }], + [{ + host: '1.1.1.1', + port: 1, + services: 1, + key: Buffer.alloc(33, 1) + }, { + hostname: + 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc@1.1.1.1:1', + key: Buffer.alloc(33, 1), + services: 1, + isIPv6: false, + isLocal: false, + isValid: true, + isRoutable: true + }], + [{ + host: '2.2.2.2', + port: 2, + key: Buffer.alloc(33, 2), + services: 2 + }, { + hostname: + 'aibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibae@2.2.2.2:2', + key: Buffer.alloc(33, 2), + services: 2, + isIPv6: false, + isLocal: false, + isValid: true, + isRoutable: true + }], [{ + host: '127.0.0.1', + port: 1000, + services: 3 + }, { + hostname: '127.0.0.1:1000', + services: 3, + isIPv6: false, + isLocal: true, + isValid: true, + isRoutable: false + }], [{ + host: '127.1.1.1', + port: 1000 + }, { + hostname: '127.1.1.1:1000', + isIPv6: false, + isLocal: true, + isValid: true, + isRoutable: false + }], [{ + host: '::1', + port: 1000 + }, { + hostname: '[::1]:1000', + isIPv6: true, + isLocal: true, + isValid: true, + isRoutable: false + }], [{ + host: 'fd87:d87e:eb43::1', + port: 1000 + }, { + host: 'aaaaaaaaaaaaaaab.onion', + hostname: 'aaaaaaaaaaaaaaab.onion:1000', + isIPv6: false, + isIPV4: false, + isLocal: false, + isValid: true, + isRoutable: true, + isOnion: true + }] +]; + +const goodOptions = { + host: '0.0.0.0', + port: 12038 +}; + +// [passedOptions, message] +netaddressVectors.failOptions = [ + [{ port: goodOptions.port }, 'NetAddress requires host string.'], + [{ host: 1234 }, 'NetAddress requires host string.'], + [{ host: goodOptions.host }, 'NetAddress requires port number.'], + [{ host: goodOptions.host, port: '32' }, + 'NetAddress requires port number.'], + [{ host: goodOptions.host, port: -1 }, 'port number is incorrect.'], + [{ host: goodOptions.host, port: 0xffff + 1 }, + 'port number is incorrect.'], + [{ ...goodOptions, services: '12' }, 'services must be a number.'], + [{ ...goodOptions, services: {} }, 'services must be a number.'], + [{ ...goodOptions, key: '12' }, 'key must be a buffer.'], + [{ ...goodOptions, key: 11 }, 'key must be a buffer.'] +]; + +// [options, expected] +netaddressVectors.fromHost = [ + [ + ['172.104.214.189', 1000, Buffer.alloc(33, 1)], + { + host: '172.104.214.189', + port: 1000, + key: Buffer.alloc(33, 1), + hostname: 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc' + + '@172.104.214.189:1000' + } + ], + [ + ['15.152.162.66', 1001, Buffer.alloc(33, 2)], + { + host: '15.152.162.66', + port: 1001, + key: Buffer.alloc(33, 2), + hostname: 'aibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibae' + + '@15.152.162.66:1001' + } + ], + [ + ['2345:0425:2CA1:0000:0000:0567:5673:23b5', 0xffff], + { + host: '2345:425:2ca1::567:5673:23b5', + port: 0xffff, + key: Buffer.alloc(33, 0), + hostname: '[2345:425:2ca1::567:5673:23b5]:65535' + } + ] +]; + +// [options, expected] +netaddressVectors.fromHostname = [ + [['127.0.0.1:100', 'main'], { + hostname: '127.0.0.1:100', + host: '127.0.0.1', + port: 100, + key: Buffer.alloc(33, 0) + }], + [ + [ + 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc@127.0.0.1:100', + 'main' + ], { + hostname: 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc' + + '@127.0.0.1:100', + host: '127.0.0.1', + port: 100, + key: Buffer.alloc(33, 1) + } + ], + [['127.0.0.1', 'main'], { + hostname: `127.0.0.1:${main.port}`, + host: '127.0.0.1', + port: main.port, + key: Buffer.alloc(33, 0) + }], + [ + [ + 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc@127.0.0.1', + 'main' + ], { + hostname: 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc' + + `@127.0.0.1:${main.brontidePort}`, + host: '127.0.0.1', + port: main.brontidePort, + key: Buffer.alloc(33, 1) + } + ], + [['127.0.0.1', 'regtest'], { + hostname: `127.0.0.1:${regtest.port}`, + host: '127.0.0.1', + port: regtest.port, + key: Buffer.alloc(33, 0) + }], + [ + [ + 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc@127.0.0.1', + 'regtest' + ], { + hostname: 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc' + + `@127.0.0.1:${regtest.brontidePort}`, + host: '127.0.0.1', + port: regtest.brontidePort, + key: Buffer.alloc(33, 1) + } + ] +]; + +// [args, expected] +netaddressVectors.fromSocket = [ + [[{ + remoteAddress: '2001:4860:a005::68', + remotePort: 1000 + }, 'main'], { + hostname: '[2001:4860:a005::68]:1000', + host: '2001:4860:a005::68', + port: 1000 + }], + [[{ + remoteAddress: '74.125.127.100', + remotePort: 2000 + }, 'main'], { + hostname: '74.125.127.100:2000', + host: '74.125.127.100', + port: 2000 + }] +]; + +// [addrA, addrB, expectedCompareResults] +netaddressVectors.compare = [ + [['127.0.0.1', 10], ['127.1.1.1', 9], -1], + [['0.0.0.0', 10], ['1.1.1.1', 9], -1], + [['0.0.0.1', 10], ['0.0.0.1', 9], 1], + // IPV4 has two 0xff in the buffer before last 4 bytes. + // So any IPV6 from ::1 to :ffff:0:0 will be lower than IPV4. + // And any IPV6 from :ffff:0:0 to :ffff:ffff:ffff will be IPV4. + [['::1', 1], ['0.0.0.1', 1], -1], + [['::ffff:ffff', 1], ['0.0.0.1', 1], -1], + [['::ffff:ffff:ffff', 1], ['0.0.0.1', 1], 1], + [['::ffff:0:1', 1], ['0.0.0.1', 1], 0], + [['::ffff:ffff:ffff', 1], ['255.255.255.255', 1], 0], + // If IPs are same, then we compare ports. + [['::1', 102], ['::1', 101], 1], + [['::1', 100], ['::1', 101], -1], + [['::1', 100], ['::1', 100], 0] +]; + +// Reachability scores +const rscores = { + UNREACHABLE: 0, + DEFAULT: 1, + TEREDO: 2, + IPV6_WEAK: 3, + IPV4: 4, + IPV6_STRONG: 5, + PRIVATE: 6 +}; + +// reachability IPs +const rips = { + ipv4: { + src: '74.125.127.100', + dest: '45.79.134.225' + }, + ipv6: { + src: 'ffff::1', + dest: 'ffff::ffff' + }, + onion: { + src: 'aaaaaaaaaaaaaaaa.onion', + dest: 'bbbbbbbbbbbbbbbb.onion' + }, + teredo: { + src: '2001::1', + dest: '2001:ffff::1' + } +}; + +netaddressVectors.getReachability = [ + // unroutable, destination does not matter + ['127.0.0.1', rips.ipv4.dest, rscores.UNREACHABLE], + + // IPv4 dest - DEFAULT + [rips.ipv4.src, rips.ipv4.dest, rscores.IPV4], + [rips.ipv6.src, rips.ipv4.dest, rscores.DEFAULT], + [rips.onion.src, rips.ipv4.dest, rscores.DEFAULT], + [rips.teredo.src, rips.ipv4.dest, rscores.DEFAULT], + + // IPv6 dest + [rips.ipv4.src, rips.ipv6.dest, rscores.IPV4], + ['2002::1', rips.ipv6.dest, rscores.IPV6_WEAK], + [rips.ipv6.src, rips.ipv6.dest, rscores.IPV6_STRONG], + [rips.onion.src, rips.ipv6.dest, rscores.DEFAULT], + [rips.teredo.src, rips.ipv6.dest, rscores.TEREDO], + + // ONION Dest + [rips.ipv4.src, rips.onion.src, rscores.IPV4], + [rips.ipv6.src, rips.onion.src, rscores.DEFAULT], + [rips.onion.src, rips.onion.src, rscores.PRIVATE], + [rips.teredo.src, rips.onion.src, rscores.DEFAULT], + + // TEREDO Dest + [rips.ipv4.src, rips.teredo.src, rscores.IPV4], + [rips.ipv6.src, rips.teredo.src, rscores.IPV6_WEAK], + [rips.onion.src, rips.teredo.src, rscores.DEFAULT], + [rips.teredo.src, rips.teredo.src, rscores.TEREDO], + + // UNREACHABLE Dest + [rips.ipv4.src, '127.0.0.1', rscores.IPV4], + [rips.ipv6.src, '127.0.0.1', rscores.IPV6_WEAK], + [rips.onion.src, '127.0.0.1', rscores.PRIVATE], + [rips.teredo.src, '127.0.0.1', rscores.TEREDO] +]; diff --git a/test/net-netaddress-test.js b/test/net-netaddress-test.js index 40e78685..e500f578 100644 --- a/test/net-netaddress-test.js +++ b/test/net-netaddress-test.js @@ -3,124 +3,18 @@ const assert = require('bsert'); const NetAddress = require('../lib/net/netaddress'); const Network = require('../lib/protocol/network'); -const util = require('../lib/utils/util') +const util = require('../lib/utils/util'); + +const netaddressVectors = require('./data/netaddress-data'); // 16 bytes (ipv6) - 4 (ipv4) byte - 2 ff = 10 const IPV4_PREFIX = Buffer.from(`${'00'.repeat(10)}ffff`, 'hex'); const main = Network.get('main'); -const regtest = Network.get('regtest'); describe('NetAddress', function() { it('should parse options', () => { - const options = [ - [null, { - host: '0.0.0.0', - port: 0, - hostname: '0.0.0.0:0', - isNull: true, - isIPv6: false, - isLocal: true, - isValid: false, - isRoutable: false - }], - [{ - host: '0.0.0.0', - port: 0 - }, { - host: '0.0.0.0', - port: 0, - hostname: '0.0.0.0:0', - isNull: true, - isIPv6: false, - isLocal: true, - isValid: false, - isRoutable: false - }], - [{ - host: '2345:0425:2CA1:0000:0000:0567:5673:23b5', - port: 1000 - }, { - host: '2345:425:2ca1::567:5673:23b5', - port: 1000, - hostname: '[2345:425:2ca1::567:5673:23b5]:1000', - isIPv6: true, - isLocal: false, - isValid: true, - isRoutable: true - }], - [{ - host: '1.1.1.1', - port: 1, - services: 1, - key: Buffer.alloc(33, 1) - }, { - hostname: - 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc@1.1.1.1:1', - key: Buffer.alloc(33, 1), - services: 1, - isIPv6: false, - isLocal: false, - isValid: true, - isRoutable: true - }], - [{ - host: '2.2.2.2', - port: 2, - key: Buffer.alloc(33, 2), - services: 2 - }, { - hostname: - 'aibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibae@2.2.2.2:2', - key: Buffer.alloc(33, 2), - services: 2, - isIPv6: false, - isLocal: false, - isValid: true, - isRoutable: true - }], [{ - host: '127.0.0.1', - port: 1000, - services: 3 - }, { - hostname: '127.0.0.1:1000', - services: 3, - isIPv6: false, - isLocal: true, - isValid: true, - isRoutable: false - }], [{ - host: '127.1.1.1', - port: 1000 - }, { - hostname: '127.1.1.1:1000', - isIPv6: false, - isLocal: true, - isValid: true, - isRoutable: false - }], [{ - host: '::1', - port: 1000 - }, { - hostname: '[::1]:1000', - isIPv6: true, - isLocal: true, - isValid: true, - isRoutable: false - }], [{ - host: 'fd87:d87e:eb43::1', - port: 1000 - }, { - host: 'aaaaaaaaaaaaaaab.onion', - hostname: 'aaaaaaaaaaaaaaab.onion:1000', - isIPv6: false, - isIPV4: false, - isLocal: false, - isValid: true, - isRoutable: true, - isOnion: true - }] - ]; + const {options} = netaddressVectors; for (const [i, [opts, expected]] of options.entries()) { const naddr = new NetAddress(opts); @@ -168,27 +62,9 @@ describe('NetAddress', function() { }); it('should fail parsing options', () => { - const goodOptions = { - host: '0.0.0.0', - port: 12038 - }; + const {failOptions} = netaddressVectors; - const badOptions = [ - [{ port: goodOptions.port }, 'NetAddress requires host string.'], - [{ host: 1234 }, 'NetAddress requires host string.'], - [{ host: goodOptions.host }, 'NetAddress requires port number.'], - [{ host: goodOptions.host, port: '32' }, - 'NetAddress requires port number.'], - [{ host: goodOptions.host, port: -1 }, 'port number is incorrect.'], - [{ host: goodOptions.host, port: 0xffff + 1 }, - 'port number is incorrect.'], - [{ ...goodOptions, services: '12' }, 'services must be a number.'], - [{ ...goodOptions, services: {} }, 'services must be a number.'], - [{ ...goodOptions, key: '12' }, 'key must be a buffer.'], - [{ ...goodOptions, key: 11 }, 'key must be a buffer.'] - ]; - - for (const [opts, msg] of badOptions) { + for (const [opts, msg] of failOptions) { let err; try { new NetAddress(opts); @@ -340,41 +216,10 @@ describe('NetAddress', function() { }); it('should create from host', () => { - const vector = [ - [ - ['172.104.214.189', 1000, Buffer.alloc(33, 1)], - { - host: '172.104.214.189', - port: 1000, - key: Buffer.alloc(33, 1), - hostname: 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc' - + '@172.104.214.189:1000' - } - ], - [ - ['15.152.162.66', 1001, Buffer.alloc(33, 2)], - { - host: '15.152.162.66', - port: 1001, - key: Buffer.alloc(33, 2), - hostname: 'aibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibae' - + '@15.152.162.66:1001' - } - ], - [ - ['2345:0425:2CA1:0000:0000:0567:5673:23b5', 0xffff], - { - host: '2345:425:2ca1::567:5673:23b5', - port: 0xffff, - key: Buffer.alloc(33, 0), - hostname: '[2345:425:2ca1::567:5673:23b5]:65535' - } - ] - ]; - + const {fromHost} = netaddressVectors; const naddr = new NetAddress(); - for (const [params, expected] of vector) { + for (const [params, expected] of fromHost) { naddr.fromHost(...params); const naddr2 = NetAddress.fromHost(...params); @@ -388,64 +233,9 @@ describe('NetAddress', function() { }); it('should create from hostname', () => { - const vector = [ - [['127.0.0.1:100', 'main'], { - hostname: '127.0.0.1:100', - host: '127.0.0.1', - port: 100, - key: Buffer.alloc(33, 0) - }], - [ - [ - 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc@127.0.0.1:100', - 'main' - ], { - hostname: 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc' - + '@127.0.0.1:100', - host: '127.0.0.1', - port: 100, - key: Buffer.alloc(33, 1) - } - ], - [['127.0.0.1', 'main'], { - hostname: `127.0.0.1:${main.port}`, - host: '127.0.0.1', - port: main.port, - key: Buffer.alloc(33, 0) - }], - [ - [ - 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc@127.0.0.1', - 'main' - ], { - hostname: 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc' - + `@127.0.0.1:${main.brontidePort}`, - host: '127.0.0.1', - port: main.brontidePort, - key: Buffer.alloc(33, 1) - } - ], - [['127.0.0.1', 'regtest'], { - hostname: `127.0.0.1:${regtest.port}`, - host: '127.0.0.1', - port: regtest.port, - key: Buffer.alloc(33, 0) - }], - [ - [ - 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc@127.0.0.1', - 'regtest' - ], { - hostname: 'aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqc' - + `@127.0.0.1:${regtest.brontidePort}`, - host: '127.0.0.1', - port: regtest.brontidePort, - key: Buffer.alloc(33, 1) - } - ] - ]; + const {fromHostname} = netaddressVectors; - for (const [args, expected] of vector) { + for (const [args, expected] of fromHostname) { const addr1 = new NetAddress(); addr1.fromHostname(...args); @@ -461,26 +251,9 @@ describe('NetAddress', function() { }); it('should create from socket', () => { - const vector = [ - [[{ - remoteAddress: '2001:4860:a005::68', - remotePort: 1000 - }, 'main'], { - hostname: '[2001:4860:a005::68]:1000', - host: '2001:4860:a005::68', - port: 1000 - }], - [[{ - remoteAddress: '74.125.127.100', - remotePort: 2000 - }, 'main'], { - hostname: '74.125.127.100:2000', - host: '74.125.127.100', - port: 2000 - }] - ]; + const {fromSocket} = netaddressVectors; - for (const [args, expected] of vector) { + for (const [args, expected] of fromSocket) { const addr = NetAddress.fromSocket(...args); assert.strictEqual(addr.hostname, expected.hostname); @@ -490,25 +263,9 @@ describe('NetAddress', function() { }); it('should compare addresses', () => { - const vector = [ - [['127.0.0.1', 10], ['127.1.1.1', 9], -1], - [['0.0.0.0', 10], ['1.1.1.1', 9], -1], - [['0.0.0.1', 10], ['0.0.0.1', 9], 1], - // IPV4 has two 0xff in the buffer before last 4 bytes. - // So any IPV6 from ::1 to :ffff:0:0 will be lower than IPV4. - // And any IPV6 from :ffff:0:0 to :ffff:ffff:ffff will be IPV4. - [['::1', 1], ['0.0.0.1', 1], -1], - [['::ffff:ffff', 1], ['0.0.0.1', 1], -1], - [['::ffff:ffff:ffff', 1], ['0.0.0.1', 1], 1], - [['::ffff:0:1', 1], ['0.0.0.1', 1], 0], - [['::ffff:ffff:ffff', 1], ['255.255.255.255', 1], 0], - // If IPs are same, then we compare ports. - [['::1', 102], ['::1', 101], 1], - [['::1', 100], ['::1', 101], -1], - [['::1', 100], ['::1', 100], 0] - ]; + const {compare} = netaddressVectors; - for (const [[hosta, porta], [hostb, portb], expected] of vector) { + for (const [[hosta, porta], [hostb, portb], expected] of compare) { const addrA = new NetAddress({ host: hosta, port: porta @@ -629,93 +386,12 @@ describe('NetAddress', function() { }); it('should get reachability score', () => { - // see: binet.getReachability for details - const UNREACHABLE = 0; - const DEFAULT = 1; - // const TEREDO = 2; - // const IPV6_WEAK = 3; - const IPV4 = 4; - // const IPV6_STRONG = 5; - // const PRIVATE = 6; - - const ipv4src = '74.125.127.100'; - const ipv4dest = '45.79.134.225'; - - const ipv6src = 'ffff::1'; - const ipv6dest = 'ffff::ffff'; - - const teredosrc = '2001::1'; - const teredodest = '2001:ffff::1'; - - const onionsrc = 'aaaaaaaaaaaaaaaa.onion'; - const oniondest = 'bbbbbbbbbbbbbbbb.onion'; - - const unreachable = [ - // Source UNREACHABLE - ['0.0.0.0', ipv4dest, UNREACHABLE], - ['127.0.0.1', ipv4dest, UNREACHABLE], - ['::', ipv4dest, UNREACHABLE], - ['::1', ipv4dest, UNREACHABLE], - // broadcast - ['255.255.255.255', ipv4dest, UNREACHABLE], - // SHIFTED - ['::ff:ff00:0:0:0', ipv4dest, UNREACHABLE], - // RFC3849 - IPv6 Reserved prefix - ['2001:0db8::', ipv4dest, UNREACHABLE], - ['2001:db8::1:1', ipv4dest, UNREACHABLE], - // RFC 1918 - Private Internets - ['192.168.1.1', ipv4dest, UNREACHABLE], - ['10.10.10.10', ipv4dest, UNREACHABLE], - ['172.20.1.1', ipv4dest, UNREACHABLE], - // RFC 2544 - IPv4 inter-network communications (198.18.0.0/15) - ['198.18.1.1', ipv4dest, UNREACHABLE], - ['198.19.255.255', ipv4dest, UNREACHABLE], - // RFC 3927 - Link local addresses (179.254.0.0/16) - ['169.254.0.0', ipv4dest, UNREACHABLE], - ['169.254.255.255', ipv4dest, UNREACHABLE], - // RFC 4862 - IPv6 Stateless address autoconfiguration - ['fe80::', ipv4dest, UNREACHABLE], - ['fe80::1', ipv4dest, UNREACHABLE], - // RFC 6598 - IANA-Reserved IPv4 Prefix for Shared Address Space - ['100.64.0.0', ipv4dest, UNREACHABLE], - ['100.127.255.255', ipv4dest, UNREACHABLE], - // RFC 5737 - IPv4 Address Blocks Reserved for Documentation - ['192.0.2.0', ipv4dest, UNREACHABLE], - ['192.0.2.255', ipv4dest, UNREACHABLE], - ['198.51.100.0', ipv4dest, UNREACHABLE], - ['198.51.100.255', ipv4dest, UNREACHABLE], - ['203.0.113.0', ipv4dest, UNREACHABLE], - ['203.0.113.255', ipv4dest, UNREACHABLE], - // RFC 4193 - Unique Local IPv6 Unicast Addresses - // FC00::/7 - ['fd00::', ipv4dest, UNREACHABLE], - ['fc00::', ipv4dest, UNREACHABLE], - ['fcff::', ipv4dest, UNREACHABLE], - // RFC 4843 - Overlay Routable Cryptographic Hash Identifiers prefix. - // ORCHID - ['2001:0010::', ipv4dest, UNREACHABLE], - ['2001:0010:ffff::', ipv4dest, UNREACHABLE], - // RFC 7343 - ORCHIDv2 - ['2001:0020::', ipv4dest, UNREACHABLE], - ['2001:0020::ffff', ipv4dest, UNREACHABLE] - ]; - - const destIPV4 = [ - // We already made sure above, source is not invalid. - // only proper outputs left. - [ipv4src, ipv4dest, IPV4], - [ipv6src, ipv4dest, DEFAULT], - [onionsrc, ipv4dest, DEFAULT], - [teredosrc, ipv4dest, DEFAULT] - ]; - - // const - - const vector = [ - ...unreachable, - ]; - - for (const [source, destination, reachability] of vector) { + // see: binet.getReachability for details. + // tests for the getReachability are covered in binet. + // + // Here we only test single case for all. + const {getReachability} = netaddressVectors; + for (const [source, destination, reachability] of getReachability) { const src = new NetAddress({ host: source, port: 1000 @@ -726,7 +402,8 @@ describe('NetAddress', function() { port: 1000 }); - assert.strictEqual(src.getReachability(dest), reachability); + assert.strictEqual(src.getReachability(dest), reachability, + `${source}->${destination} - ${reachability}`); } }); }); From e1f435969b7e7dd442adf6aa643bad973c93b287 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Sat, 5 Feb 2022 18:31:51 +0400 Subject: [PATCH 04/17] test: init net hostlists test. --- test/net-hostlist-test.js | 559 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 559 insertions(+) create mode 100644 test/net-hostlist-test.js diff --git a/test/net-hostlist-test.js b/test/net-hostlist-test.js new file mode 100644 index 00000000..944462a3 --- /dev/null +++ b/test/net-hostlist-test.js @@ -0,0 +1,559 @@ +'use strict'; + +const assert = require('bsert'); +const path = require('path'); +const fs = require('bfile'); +const Logger = require('blgr'); +const IP = require('binet'); +const base32 = require('bcrypto/lib/encoding/base32'); +const secp256k1 = require('bcrypto/lib/secp256k1'); +const random = require('bcrypto/lib/random'); +const common = require('./util/common'); +const Network = require('../lib/protocol/network'); +const NetAddress = require('../lib/net/netaddress'); +const HostList = require('../lib/net/hostlist'); + +const regtest = Network.get('regtest'); +const mainnet = Network.get('main'); + +/* + * Some configurations for the tests + */ + +function getHostsFromLocals(addresses, opts) { + const hosts = new HostList(opts); + + for (const [addr, score] of addresses) + hosts.pushLocal(addr, score); + + return hosts; +} + +describe('Net HostList', function() { + let testdir; + + before(async () => { + testdir = common.testdir('hostlist'); + + assert(await fs.mkdirp(testdir)); + }); + + after(async () => { + await fs.rimraf(testdir); + testdir = null; + }); + + it('should parse options', () => { + const network = regtest; + const logger = Logger.global; + const resolve = () => {}; + const banTime = 100; + const seeds = ['127.1.1.1', 'example.com']; + const nodes = ['127.2.2.2']; + const host = '127.0.0.1'; + const port = regtest.port; + const publicHost = getRandomIPv4(); + const publicPort = regtest.port; + const publicBrontidePort = regtest.brontidePort; + const identityKey = secp256k1.privateKeyGenerate(); + const pubIdentityKey = secp256k1.publicKeyCreate(identityKey); + const services = 1001; + const onion = false; + const brontideOnly = false; + const memory = true; + const prefix = testdir; + const filename = path.join(prefix, 'custom.json'); + const flushInterval = 2000; + + const options = { + network, + logger, + resolve, + banTime, + seeds, + nodes, + host, + port, + publicHost, + publicPort, + publicBrontidePort, + identityKey, + services, + onion, + brontideOnly, + memory, + prefix, + filename, + flushInterval + }; + + const hosts = new HostList(options); + + assert.strictEqual(hosts.network, network); + + // Hostlist will use context('hostlist') instead. + // assert.strictEqual(hostlist.logger, logger); + assert.strictEqual(hosts.resolve, resolve); + assert.strictEqual(hosts.options.banTime, banTime); + + // seeds are still stored in options until initAdd. + assert.deepStrictEqual(hosts.options.seeds, seeds); + + // Nodes are still stored in options until initAdd. + assert.deepStrictEqual(hosts.options.nodes, nodes); + + // Host:port will become local node after initAdd. + assert.strictEqual(hosts.options.host, host); + assert.strictEqual(hosts.options.port, port); + + { + // public address + const address = new NetAddress({ + host: publicHost, + port: publicPort, + services + }); + + assert.strictEqual(hosts.address.equal(address), true); + assert.strictEqual(hosts.address.services, services); + } + + { + // brontide Address + const address = new NetAddress({ + host: publicHost, + port: publicBrontidePort, + key: pubIdentityKey, + services + }); + + assert.strictEqual(hosts.brontide.equal(address), true); + assert.strictEqual(hosts.brontide.services, services); + assert.bufferEqual(hosts.brontide.getKey(), pubIdentityKey); + } + + assert.strictEqual(hosts.options.onion, onion); + assert.strictEqual(hosts.options.brontideOnly, brontideOnly); + assert.strictEqual(hosts.options.memory, memory); + assert.strictEqual(hosts.options.prefix, prefix); + assert.strictEqual(hosts.options.filename, filename); + assert.strictEqual(hosts.options.flushInterval, flushInterval); + + // Prefix check + { + const hostlist = new HostList({ + prefix: testdir + }); + + assert.strictEqual(hostlist.options.filename, + path.join(testdir, 'hosts.json')); + } + }); + + it('should init add ips', () => { + const network = regtest; + const host = '127.0.0.1'; + const port = regtest.port; + const publicHost = '1.1.1.1'; + const publicPort = regtest.port; + const publicBrontidePort = regtest.brontidePort; + const identityKey = secp256k1.privateKeyGenerate(); + const pubIdentityKey = secp256k1.publicKeyCreate(identityKey); + const seeds = ['127.1.1.1', 'example.com', + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@127.3.3.3']; + const nodes = ['127.2.2.2']; + + const hosts = new HostList({ + network, + host, + port, + publicHost, + publicPort, + publicBrontidePort, + identityKey, + seeds, + nodes + }); + + const ipGetPublic = IP.getPublic; + + const interfaceIPs = [ + getRandomIPv4(), + getRandomIPv4() + ]; + + IP.getPublic = () => interfaceIPs; + + hosts.initAdd(); + assert.strictEqual(hosts.added, true); + + // It's behind a check. + hosts.initAdd(); + + IP.getPublic = ipGetPublic; + + { + // one for brontinde public and one plaintext public + // two from interfaces + assert.strictEqual(hosts.local.size, 4); + const plaintextHost = IP.toHost(publicHost, publicPort); + const brontideHost = IP.toHost( + publicHost, + publicBrontidePort, + pubIdentityKey + ); + + const interfaceHosts = [ + IP.toHost(interfaceIPs[0], publicPort), + IP.toHost(interfaceIPs[1], publicPort) + ]; + + { + assert(hosts.local.has(plaintextHost)); + const local = hosts.local.get(plaintextHost); + assert.strictEqual(local.score, HostList.scores.MANUAL); + } + + { + assert(hosts.local.has(brontideHost)); + const local = hosts.local.get(brontideHost); + assert.strictEqual(local.score, HostList.scores.MANUAL); + } + + for (const ihost of interfaceHosts) { + assert(hosts.local.has(ihost)); + const local = hosts.local.get(ihost); + + assert.strictEqual(local.score, HostList.scores.IF); + } + } + + // After initAdd(); + // 127.1.1.1 - becomes normal peer + // example.com - will become dnsSeed + assert.strictEqual(hosts.dnsSeeds.length, 1); + assert.deepStrictEqual(hosts.dnsSeeds[0], IP.fromHost(seeds[1])); + + // Check 127.1.1.1 and 127.3.3.3 + assert(hosts.map.has(`${seeds[0]}:${regtest.port}`)); + assert(hosts.map.has(`127.3.3.3:${regtest.brontidePort}`)); + + // Check nodes have been added (127.2.2.2) + assert(hosts.map.has(`${nodes[0]}:${regtest.port}`)); + assert.strictEqual(hosts.map.size, 3); + }); + + it('should add push/local addresses', () => { + const services = 1000; + const hosts = new HostList({ services }); + + const unroutable = [ + '127.0.0.1', + '127.1.1.1', + '192.168.1.1', + '10.10.10.10' + ]; + + const routable = [ + getRandomIPv4(), + getRandomIPv4() + ]; + + assert.strictEqual(hosts.local.size, 0); + + // unroutable local addresses must not get added. + for (const host of unroutable) { + const port = regtest.port; + const score = HostList.scores.NONE; + const res = hosts.addLocal(host, port, null, score); + assert.strictEqual(res, false); + } + + assert.strictEqual(hosts.local.size, 0); + + const SCORE = HostList.scores.MANUAL; + // these must get added. + for (const host of routable) { + const port = regtest.port; + const res1 = hosts.addLocal(host, port, SCORE); + assert.strictEqual(res1, true); + + // add brontide versions + const bport = regtest.brontidePort; + const res2 = hosts.addLocal(host, bport, SCORE); + assert.strictEqual(res2, true); + } + + // one brontide and one plaintext + assert.strictEqual(hosts.local.size, routable.length * 2); + + const added = hosts.addLocal(routable[0], regtest.port, SCORE); + assert.strictEqual(added, false); + + for (const laddr of hosts.local.values()) { + assert.strictEqual(laddr.addr.services, services); + assert.strictEqual(laddr.score, SCORE); + assert.strictEqual(laddr.type, SCORE); + } + }); + + // w/o src it will only take into account the score. + it('should get local w/o src', () => { + const services = 1000; + const port = regtest.port; + + const addrs = []; + + // w/o key + for (let i = HostList.scores.NONE; i < HostList.scores.MAX - 1; i++) { + const address = new NetAddress({ + host: getRandomIPv4(), + port: port, + services: services + }); + addrs.push([address, i]); + } + + // max but with key + addrs.push([new NetAddress({ + host: getRandomIPv4(), + port: port, + services: services, + key: Buffer.alloc(33, 1) + }), HostList.scores.MAX]); + + const max = addrs[HostList.scores.MAX - 2]; + + const hosts = getHostsFromLocals(addrs); + const local = hosts.getLocal(); + + assert.strictEqual(local, max[0]); + }); + + it('should get local (score)', () => { + // NOTE: main network ignores everything other than MANUAL type. + // - scores.IF - addresses from the network interfaces. + // - scores.BIND - Listening IP. (publicHost/publicPort is MANUAL) + // - scores.DNS - Domain that needs to be resolved. + // - scores.UPNP - UPNP discovered address. + + const scores = HostList.scores; + + // same NET type but different scores: + const hostsByScore = [ + [getRandomIPv4(), scores.NONE], + [getRandomIPv4(), scores.IF], + [getRandomIPv4(), scores.BIND], + [getRandomIPv4(), scores.DNS], + [getRandomIPv4(), scores.UPNP], + [getRandomIPv4(), scores.MANUAL] + ]; + + const naddrsByScore = hostsByScore.map(([h, s]) => { + return [NetAddress.fromHostname(h, regtest), s]; + }); + + // testnet/regtest + for (let type = scores.NONE; type < scores.MAX; type++) { + const addrs = naddrsByScore.slice(scores.NONE, type + 1); + const hosts = getHostsFromLocals(addrs, { network: regtest }); + const src = NetAddress.fromHostname(getRandomIPv4(), regtest); + const best = hosts.getLocal(src); + assert.strictEqual(best, naddrsByScore[type][0]); + } + + // mainnet + { + const hosts = getHostsFromLocals(naddrsByScore, { network: mainnet }); + const src = NetAddress.fromHostname(getRandomIPv4(), regtest); + const best = hosts.getLocal(src); + assert.strictEqual(best, naddrsByScore[scores.MANUAL][0]); + } + + { + // everything below MANUAL is skipped on main network. + const addrs = naddrsByScore.slice(scores.NONE, scores.UPNP); + const hosts = getHostsFromLocals(addrs, { network: mainnet }); + const src = NetAddress.fromHostname(getRandomIPv4(), regtest); + const best = hosts.getLocal(src); + assert.strictEqual(best, null); + } + }); + + it('should get local (reachability)', () => { + // If we have multiple public host/ports (e.g. IPv4 and Ipv6) + // depending who is connecting to us, we will choose different + // address to advertise. + + // with src it will take into account the reachability score. + // See: binet.getReachability + // TLDR: + // UNREACHABLE = 0 + // < DEFAULT = 1 - non-ipv4 -> ipv4, onion -> ipv6, ... + // < TEREDO = 2 - teredo -> teredo, teredo -> ipv6 + // < IPV6_WEAK = 3 - ipv4 -> ipv6 tunnels, ipv6 -> teredo + // < IPV4 = 4 - ipv4 -> ipv4, ipv4 -> others + // < IPV6_STRONG = 5 - ipv6 -> ipv6 + // < PRIVATE = 6 - ONION -> ONION + + const {MANUAL} = HostList.scores; + + // same score (MANUAL), different reachability: + // remote(src) => [ local(dest)... ] - sorted by reachability scores. + const reachabilityMap = { + // unreachable => anything - will be UNREACHABLE = 0. + [getRandomTEREDO()]: [ + getRandomIPv4(), // DEFAULT = 1 + getRandomOnion(), // DEFAULT = 1 + getRandomTEREDO(), // TEREDO = 2 + getRandomIPv6() // TEREDO = 2 + ], + [getRandomIPv4()]: [ + getRandomIPv4(), // IPV4 = 4 + getRandomOnion(), // IPV4 = 4 + getRandomTEREDO(), // IPV4 = 4 + getRandomIPv6() // IPV4 = 4 + ], + [getRandomIPv6()]: [ + getRandomOnion(), // DEFAULT = 1 + getRandomIPv4(), // DEFAULT = 1 + getRandomTEREDO(), // IPV6_WEAK = 3 + getRandomIPv6() // IPV6_STRONG = 5 + ], + [getRandomOnion()]: [ + getRandomIPv4(), // DEFAULT = 1 + getRandomTEREDO(), // DEFAULT = 1 + getRandomIPv6(), // DEFAULT = 1 + getRandomOnion() // PRIVATE = 6 + ] + }; + + for (const [rawSrc, rawDests] of Object.entries(reachabilityMap)) { + const dests = rawDests.map((dest) => { + return [NetAddress.fromHostname(dest, mainnet), MANUAL]; + }); + + for (let i = 0; i < dests.length; i++) { + const addrs = dests.slice(0, i + 1); + const expected = addrs[addrs.length - 1]; + + // Because getLocal will choose first with the same score, + // we make the "best" choice (because of the sorting) at first. + addrs[addrs.length - 1] = addrs[0]; + addrs[0] = expected; + + const hosts = getHostsFromLocals(addrs); + const src = NetAddress.fromHostname(rawSrc, mainnet); + const best = hosts.getLocal(src); + + assert.strictEqual(best, expected[0]); + } + } + }); + + it('should not get local (skip brontide)', () => { + const {MANUAL} = HostList.scores; + + // skip with key + const src = getRandomIPv4(); + const KEY = base32.encode(Buffer.alloc(33, 1)); + const rawDests = [ + `${KEY}@${getRandomIPv4()}:${mainnet.brontidePort}`, + `${KEY}@${getRandomIPv4()}:${mainnet.brontidePort}` + ]; + + const dests = rawDests.map((d) => { + return [NetAddress.fromHostname(d, mainnet), MANUAL]; + }); + + const hosts = getHostsFromLocals(dests); + const best = hosts.getLocal(src); + assert.strictEqual(best, null); + }); + + it('should mark local', () => { + const {scores} = HostList; + + const rawDests = [ + [getRandomIPv4(), scores.IF], + [getRandomIPv4(), scores.BIND] + ]; + + const dests = rawDests.map(([h, s]) => { + return [NetAddress.fromHostname(h, regtest), s]; + }); + + const hosts = getHostsFromLocals(dests, { network: regtest }); + + { + const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + const marked = hosts.markLocal(addr); + + assert.strictEqual(marked, false); + } + + { + // we should get BIND, because BIND > IF + const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + const local = hosts.getLocal(addr); + assert.strictEqual(local, dests[1][0]); + } + + { + // with markLocal IF should get the same score (type remains). + hosts.markLocal(dests[0][0]); + const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + const local = hosts.getLocal(addr); + assert.strictEqual(local, dests[0][0]); + } + }); +}); + +/* + * Helpers + */ + +function getRandomIPv4() { + const prefix = Buffer.from('00000000000000000000ffff', 'hex'); + const number = random.randomBytes(4); + const ipv4 = Buffer.concat([prefix, number]); + + if (IP.getNetwork(ipv4) === IP.networks.INET4) + return IP.toString(ipv4); + + return getRandomIPv4(); +} + +function getRandomTEREDO() { + const prefix = Buffer.from('20010000', 'hex'); + const number = random.randomBytes(12); + const raw = Buffer.concat([prefix, number]); + + if (IP.getNetwork(raw) === IP.networks.TEREDO) + return IP.toString(raw); + + return getRandomTEREDO(); +} + +function getRandomIPv6() { + const raw = random.randomBytes(16); + + if (IP.isRFC3964(raw) || IP.isRFC6052(raw) || IP.isRFC6145(raw)) + return getRandomIPv6(); + + if (IP.getNetwork(raw) === IP.networks.INET6) + return IP.toString(raw); + + return getRandomIPv6(); +} + +function getRandomOnion() { + const prefix = Buffer.from('fd87d87eeb43', 'hex'); + const number = random.randomBytes(10); + const raw = Buffer.concat([prefix, number]); + + if (IP.getNetwork(raw) === IP.networks.ONION) + return IP.toString(raw); + + return getRandomOnion(); +} From f893e248a85224477b31ba4fe06b1aed81debf48 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Sun, 13 Feb 2022 18:44:14 +0400 Subject: [PATCH 05/17] net-tests: add hostslist add tests. --- lib/net/hostlist.js | 53 +++++- lib/net/netaddress.js | 6 +- test/net-hostlist-test.js | 340 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 391 insertions(+), 8 deletions(-) diff --git a/lib/net/hostlist.js b/lib/net/hostlist.js index 4b9726a0..bdd206de 100644 --- a/lib/net/hostlist.js +++ b/lib/net/hostlist.js @@ -23,6 +23,37 @@ const NetAddress = require('./netaddress'); const common = require('./common'); const seeds = require('./seeds'); +/** + * Stochastic address manager based on bitcoin addrman. + * + * Design goals: + * * Keep the address tables in-memory, and asynchronously dump the entire + * table to hosts.json. + * * Make sure no (localized) attacker can fill the entire table with his + * nodes/addresses. + * + * To that end: + * * Addresses are organized into buckets that can each store up + * to 64 entries (maxEntries). + * * Addresses to which our node has not successfully connected go into + * 1024 "fresh" buckets (maxFreshBuckets). + * * Based on the address range of the source of information + * 64 buckets are selected at random. + * * The actual bucket is chosen from one of these, based on the range in + * which the address itself is located. + * * The position in the bucket is chosen based on the full address. + * * One single address can occur in up to 8 different buckets to increase + * selection chances for addresses that are seen frequently. The chance + * for increasing this multiplicity decreases exponentially. + * * When adding a new address to an occupied position of a bucket, it + * will not replace the existing entry unless that address is also stored + * in another bucket or it doesn't meet one of several quality criteria + * (see isStale for exact criteria). + * * Bucket selection is based on cryptographic hashing, + * using a randomly-generated 256-bit key, which should not be observable + * by adversaries (key). + */ + /** * Host List * @alias module:net.HostList @@ -427,12 +458,15 @@ class HostList { * Get fresh bucket for host. * @private * @param {HostEntry} entry + * @param {NetAddress?} src * @returns {Map} */ - freshBucket(entry) { + freshBucket(entry, src) { const addr = entry.addr; - const src = entry.src; + + if (!src) + src = entry.src; this.hash.init(); this.hash.update(this.key); @@ -563,7 +597,7 @@ class HostList { this.totalFresh += 1; } - const bucket = this.freshBucket(entry); + const bucket = this.freshBucket(entry, src); if (bucket.has(entry.key())) return false; @@ -1409,6 +1443,14 @@ HostList.scores = { /** * Host Entry * @alias module:net.HostEntry + * @property {NetAddress} addr - host address. + * @property {NetAddress} src - the first address we discovered this entry + * from. + * @property {Boolean} used - is it used set. + * @property {Number} refCount - Reference count in new buckets. + * @property {Number} attempts - connection attempts since last successful one. + * @property {Number} lastSuccess - last success timestamp. + * @property {Number} lastAttempt - last attempt timestamp. */ class HostEntry { @@ -1842,4 +1884,7 @@ function groupKey(raw) { * Expose */ -module.exports = HostList; +exports = HostList; +exports.HostEntry = HostEntry; + +module.exports = exports; diff --git a/lib/net/netaddress.js b/lib/net/netaddress.js index a48fee29..c23bb2a8 100644 --- a/lib/net/netaddress.js +++ b/lib/net/netaddress.js @@ -18,8 +18,6 @@ const common = require('./common'); * Constants */ -const IPV4_ZERO_IP = Buffer.from('00000000000000000000ffff00000000', 'hex'); - const ZERO_KEY = Buffer.alloc(33, 0x00); /** @@ -51,7 +49,7 @@ class NetAddress extends bio.Struct { this.services = 0; this.time = 0; this.hostname = '0.0.0.0:0'; - this.raw = IPV4_ZERO_IP; + this.raw = IP.ZERO_IPV4; this.key = ZERO_KEY; if (options) @@ -219,7 +217,7 @@ class NetAddress extends bio.Struct { */ setNull() { - this.raw = IPV4_ZERO_IP; + this.raw = IP.ZERO_IPV4; this.host = '0.0.0.0'; this.key = ZERO_KEY; this.hostname = IP.toHostname(this.host, this.port, this.key); diff --git a/test/net-hostlist-test.js b/test/net-hostlist-test.js index 944462a3..4d0dfcc7 100644 --- a/test/net-hostlist-test.js +++ b/test/net-hostlist-test.js @@ -9,6 +9,7 @@ const base32 = require('bcrypto/lib/encoding/base32'); const secp256k1 = require('bcrypto/lib/secp256k1'); const random = require('bcrypto/lib/random'); const common = require('./util/common'); +const util = require('../lib/utils/util'); const Network = require('../lib/protocol/network'); const NetAddress = require('../lib/net/netaddress'); const HostList = require('../lib/net/hostlist'); @@ -29,6 +30,18 @@ function getHostsFromLocals(addresses, opts) { return hosts; } +// flat +function getFreshEntries(hosts) { + const naddrs = []; + + for (const bucket of hosts.fresh) { + for (const naddr of bucket.values()) + naddrs.push(naddr); + } + + return naddrs; +}; + describe('Net HostList', function() { let testdir; @@ -507,6 +520,333 @@ describe('Net HostList', function() { assert.strictEqual(local, dests[0][0]); } }); + + it('should add fresh address', () => { + { + const hosts = new HostList({ + network: regtest, + publicHost: getRandomIPv4() + }); + + // fresh, w/o src, not in the buckets + const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + + assert.strictEqual(hosts.totalFresh, 0); + assert.strictEqual(hosts.needsFlush, false); + assert.strictEqual(hosts.map.size, 0); + assert.strictEqual(getFreshEntries(hosts).length, 0); + + hosts.add(addr); + + assert.strictEqual(hosts.totalFresh, 1); + assert.strictEqual(hosts.needsFlush, true); + assert.strictEqual(hosts.map.size, 1); + + const freshEntries = getFreshEntries(hosts); + assert.strictEqual(freshEntries.length, 1); + + const entry = freshEntries[0]; + assert.strictEqual(entry.addr, addr, 'Entry addr is not correct.'); + assert.strictEqual(entry.src, hosts.address, 'Entry src is not correct.'); + } + + { + const hosts = new HostList({ + network: regtest + }); + + const src = NetAddress.fromHostname(getRandomIPv4(), regtest); + const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + + hosts.add(addr, src); + const freshEntries = getFreshEntries(hosts); + assert.strictEqual(freshEntries.length, 1); + + const entry = freshEntries[0]; + assert.strictEqual(entry.addr, addr, 'Entry addr is not correct.'); + assert.strictEqual(entry.src, src, 'Entry src is not correct.'); + assert.strictEqual(entry.refCount, 1); + assert.strictEqual(hosts.map.size, 1); + } + }); + + it('should add address (limits)', () => { + // Full Bucket? + { + const hosts = new HostList({ + network: regtest + }); + + const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + const src = NetAddress.fromHostname(getRandomIPv4(), regtest); + + let evicted = false; + + // always return first bucket for this test. + hosts.freshBucket = function() { + return this.fresh[0]; + }; + + hosts.evictFresh = function() { + evicted = true; + }; + + // Fill first bucket. + for (let i = 0; i < hosts.maxEntries; i++) { + const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + const added = hosts.add(addr, src); + assert.strictEqual(added, true); + assert.strictEqual(evicted, false); + } + + const added = hosts.add(addr, src); + assert.strictEqual(added, true); + assert.strictEqual(evicted, true); + } + + // Don't insert if entry is in a bucket. + { + const hosts = new HostList({ + network: regtest + }); + + const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + const src = NetAddress.fromHostname(getRandomIPv4(), regtest); + const entry = new HostList.HostEntry(addr, src); + + // insert entry in every bucket for this test. + for (const bucket of hosts.fresh) + bucket.set(entry.key(), entry); + + const added = hosts.add(addr, src); + assert.strictEqual(added, false); + } + }); + + it('should add seen address', () => { + const hosts = new HostList({ + network: regtest + }); + + // get addr clone that can be added (Online requirements) + const cloneAddr = (addr) => { + const addr2 = addr.clone(); + // just make sure this is < 3 * 60 * 60 (Online requirements + // with & without penalty) + addr2.time = util.now() + 60 * 60; + return addr2; + }; + + const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + const src = NetAddress.fromHostname(getRandomIPv4(), regtest); + addr.services = 0x01; + const added = hosts.add(addr, src); + assert.strictEqual(added, true); + assert.strictEqual(hosts.needsFlush, true); + hosts.needsFlush = false; + + const entries = getFreshEntries(hosts); + const entry = entries[0]; + assert.strictEqual(entries.length, 1); + assert.strictEqual(entry.addr.services, 0x01); + + // don't update - no new info (service will always get updated.) + { + const addr2 = addr.clone(); + addr2.services = 0x02; + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + + const entries = getFreshEntries(hosts); + const entry = entries[0]; + assert.strictEqual(entries.length, 1); + assert.strictEqual(entry.addr.services, 0x03); + assert.strictEqual(hosts.needsFlush, false); + } + + // update refCount. + { + const srcs = [ + NetAddress.fromHostname(getRandomIPv4(), regtest), + NetAddress.fromHostname(getRandomIPv4(), regtest) + ]; + const addr2 = cloneAddr(addr); + + // when we have 1 ref, so probability should be 50%. + // then we have 2 refs, probability will be 25%. + // ... until we have 8 refs. + // Because we are reusing src, we will get same bucket + // so it will only get added once. + let added = 0; + for (let i = 0; i < 100; i++) { + const res = hosts.add(addr2, srcs[added]); + + if (res) + added++; + + if (added === 2) + break; + } + + // at this point address should be in another bucket as well. + assert.strictEqual(added, 2); + assert.strictEqual(entry.refCount, 3); + const entries = getFreshEntries(hosts); + assert.strictEqual(entries.length, 3); + assert.strictEqual(hosts.needsFlush, true); + hosts.needsFlush = false; + } + + // should fail with max ref + { + const _refCount = entry.refCount; + entry.refCount = HostList.MAX_REFS; + const addr2 = cloneAddr(addr); + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + entry.refCount = _refCount; + assert.strictEqual(hosts.needsFlush, false); + } + + // should fail if it's used + { + entry.used = true; + const addr2 = cloneAddr(addr); + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + assert.strictEqual(hosts.needsFlush, false); + entry.used = false; + } + }); + + it('should add address (update time)', () => { + const getHosts = (time) => { + const hosts = new HostList({ + network: regtest + }); + + const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + const src = NetAddress.fromHostname(getRandomIPv4(), regtest); + + if (time) + addr.time = time; + + const added = hosts.add(addr, src); + assert.strictEqual(added, true); + assert.strictEqual(hosts.needsFlush, true); + hosts.needsFlush = false; + + const entries = getFreshEntries(hosts); + assert.strictEqual(entries.length, 1); + + // make sure we stop after updating time. + entries[0].used = true; + + return [hosts, entries[0], addr, src]; + }; + + // Update time - Online? + { + // a week ago + const [hosts, entry, addr, src] = getHosts(util.now() - 7 * 24 * 60 * 60); + const addr2 = addr.clone(); + + // a day ago (interval is a day, + // so we update if a day and 2 hrs have passed). + addr2.time = util.now() - 24 * 60 * 60; + + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + assert.strictEqual(entry.addr.time, addr2.time); + assert.strictEqual(hosts.needsFlush, true); + } + + { + // a day ago + const [hosts, entry, addr, src] = getHosts(util.now() - 24 * 60 * 60); + + const addr2 = addr.clone(); + + // now (interval becomes an hour, so instead we update if 3 hrs passed. + addr2.time = util.now(); + + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + assert.strictEqual(entry.addr.time, addr2.time); + assert.strictEqual(hosts.needsFlush, true); + } + + // Don't update + { + // a week ago + const weekAgo = util.now() - 7 * 24 * 60 * 60; + const sixDaysAgo = util.now() - 6 * 24 * 60 * 60 + 1; // and a second + + const [hosts, entry, addr, src] = getHosts(weekAgo); + + const addr2 = addr.clone(); + // 6 days ago (exactly 24 hrs after) because 2 hrs is penalty, + // we don't update. + addr2.time = sixDaysAgo; + + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + assert.strictEqual(entry.addr.time, weekAgo); + assert.strictEqual(hosts.needsFlush, false); + } + + // Update, because we are the ones inserting. + { + // a week ago + const weekAgo = util.now() - 7 * 24 * 60 * 60; + const sixDaysAgo = util.now() - 6 * 24 * 60 * 60 + 1; // and a second + + const [hosts, entry, addr] = getHosts(weekAgo); + + const addr2 = addr.clone(); + // 6 days ago (exactly 24 hrs after) because 2 hrs is penalty, + // we don't update. + addr2.time = sixDaysAgo; + + const added = hosts.add(addr2); + assert.strictEqual(added, false); + assert.strictEqual(entry.addr.time, sixDaysAgo); + assert.strictEqual(hosts.needsFlush, true); + } + + // Online - but still not updating (less than 3 hrs) + { + // now vs 3 hrs ago (exactly) + const now = util.now(); + const threeHoursAgo = now - 3 * 60 * 60; + + const [hosts, entry, addr, src] = getHosts(threeHoursAgo); + + const addr2 = addr.clone(); + addr2.time = now; + + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + assert.strictEqual(entry.addr.time, threeHoursAgo); + assert.strictEqual(hosts.needsFlush, false); + } + + { + // now vs 3 hrs and a second ago + const now = util.now(); + const threeHoursAgo = now - 1 - 3 * 60 * 60; + + const [hosts, entry, addr, src] = getHosts(threeHoursAgo); + + const addr2 = addr.clone(); + addr2.time = now; + + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + assert.strictEqual(entry.addr.time, now); + assert.strictEqual(hosts.needsFlush, true); + } + }); }); /* From 2a07d3c044e84e7a3b120d833cc1b1153b8b5ce9 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Tue, 15 Feb 2022 19:23:24 +0400 Subject: [PATCH 06/17] hostlist: make random an option. --- lib/net/hostlist.js | 19 ++- test/net-hostlist-test.js | 307 ++++++++++++++++++++++++++++++-------- 2 files changed, 259 insertions(+), 67 deletions(-) diff --git a/lib/net/hostlist.js b/lib/net/hostlist.js index bdd206de..7f922a89 100644 --- a/lib/net/hostlist.js +++ b/lib/net/hostlist.js @@ -73,6 +73,7 @@ class HostList { this.address = this.options.address; this.brontide = this.options.brontide; this.resolve = this.options.resolve; + this.random = this.options.random; this.key = rng.randomBytes(32); this.hash = new Hash256(); @@ -412,7 +413,7 @@ class HostList { buckets = this.fresh; if (this.totalUsed > 0) { - if (this.totalFresh === 0 || random(2) === 0) + if (this.totalFresh === 0 || this.random(2) === 0) buckets = this.used; } @@ -424,13 +425,13 @@ class HostList { let factor = 1; for (;;) { - const i = random(buckets.length); + const i = this.random(buckets.length); const bucket = buckets[i]; if (bucket.size === 0) continue; - let index = random(bucket.size); + let index = this.random(bucket.size); let entry; if (buckets === this.used) { @@ -445,7 +446,7 @@ class HostList { } } - const num = random(1 << 30); + const num = this.random(1 << 30); if (num < factor * entry.chance(now) * (1 << 30)) return entry; @@ -586,7 +587,7 @@ class HostList { for (let i = 0; i < entry.refCount; i++) factor *= 2; - if (random(factor) !== 0) + if (this.random(factor) !== 0) return false; } else { if (!src) @@ -862,7 +863,7 @@ class HostList { items.push(entry); for (let i = 0; i < items.length && out.length < 2500; i++) { - const j = random(items.length - i); + const j = this.random(items.length - i); [items[i], items[i + j]] = [items[i + j], items[i]]; @@ -1672,6 +1673,7 @@ class HostListOptions { this.onion = false; this.brontideOnly = false; this.banTime = common.BAN_TIME; + this.random = random; this.address = new NetAddress(); this.address.services = this.services; @@ -1804,6 +1806,11 @@ class HostListOptions { this.flushInterval = options.flushInterval; } + if (options.random != null) { + assert(typeof options.random === 'function'); + this.random = options.random;; + } + this.address.time = this.network.now(); this.address.services = this.services; diff --git a/test/net-hostlist-test.js b/test/net-hostlist-test.js index 4d0dfcc7..1b001061 100644 --- a/test/net-hostlist-test.js +++ b/test/net-hostlist-test.js @@ -42,6 +42,10 @@ function getFreshEntries(hosts) { return naddrs; }; +function getRandomNetAddr(network = regtest) { + return NetAddress.fromHostname(getRandomIPv4(), network); +} + describe('Net HostList', function() { let testdir; @@ -256,6 +260,130 @@ describe('Net HostList', function() { assert.strictEqual(hosts.map.size, 3); }); + it('should add/set nodes/seeds', () => { + // we need 3. + const hosts = [ + getRandomIPv4(), + getRandomIPv4(), + getRandomIPv4() + ]; + + const key = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; + + const tests = [ + // DNS Nodes + { + host: 'example.com', + hostname: 'example.com', + expected: { + addr: null, + dnsNodes: 1, + nodes: 0, + map: 0 + } + }, + // HSD Node w/o port - plaintext + { + host: hosts[0], + hostname: hosts[0], + expected: { + addr: { port: mainnet.port, host: hosts[0] }, + dnsNodes: 0, + nodes: 1, + map: 1 + } + }, + // HSD Node w/o port - brontide + { + host: hosts[1], + hostname: `${key}@${hosts[1]}`, + expected: { + addr: { host: hosts[1], port: mainnet.brontidePort }, + dnsNodes: 0, + nodes: 1, + map: 1 + } + }, + // HSD Node with port + { + host: hosts[2], + hostname: `${hosts[2]}:${mainnet.port + 1}`, + expected: { + addr: { host: hosts[2], port: mainnet.port + 1 }, + dnsNodes: 0, + nodes: 1, + map: 1 + } + } + ]; + + const allHosts = tests.map(t => t.hostname); + const sumExpected = tests.reduce((p, c) => { + p.dnsNodes += c.expected.dnsNodes; + p.nodes += c.expected.nodes; + p.map += c.expected.map; + return p; + }, { + dnsNodes: 0, + nodes: 0, + map: 0 + }); + + for (const test of tests) { + const hosts = new HostList(); + const {expected} = test; + + const addr = hosts.addNode(test.hostname); + + if (expected.addr == null) + assert.strictEqual(addr, null); + + if (expected.addr != null) { + assert.strictEqual(addr.host, expected.addr.host); + assert.strictEqual(addr.port, expected.addr.port); + } + + assert.strictEqual(hosts.dnsNodes.length, expected.dnsNodes); + assert.strictEqual(hosts.nodes.length, expected.nodes); + assert.strictEqual(hosts.map.size, expected.map); + } + + // set all nodes + { + const hosts = new HostList(); + hosts.setNodes(allHosts); + assert.strictEqual(hosts.dnsNodes.length, sumExpected.dnsNodes); + assert.strictEqual(hosts.nodes.length, sumExpected.nodes); + assert.strictEqual(hosts.map.size, sumExpected.map); + } + + for (const test of tests) { + const hosts = new HostList(); + const {expected} = test; + + const addr = hosts.addSeed(test.hostname); + + if (expected.addr == null) + assert.strictEqual(addr, null); + + if (expected.addr != null) { + assert.strictEqual(addr.host, expected.addr.host); + assert.strictEqual(addr.port, expected.addr.port); + } + + assert.strictEqual(hosts.dnsSeeds.length, expected.dnsNodes); + assert.strictEqual(hosts.map.size, expected.map); + } + + { + const hosts = new HostList(); + + hosts.setSeeds(allHosts); + assert.strictEqual(hosts.dnsSeeds.length, sumExpected.dnsNodes); + assert.strictEqual(hosts.map.size, sumExpected.map); + } + }); + it('should add push/local addresses', () => { const services = 1000; const hosts = new HostList({ services }); @@ -363,14 +491,14 @@ describe('Net HostList', function() { ]; const naddrsByScore = hostsByScore.map(([h, s]) => { - return [NetAddress.fromHostname(h, regtest), s]; + return [getRandomNetAddr(), s]; }); // testnet/regtest for (let type = scores.NONE; type < scores.MAX; type++) { const addrs = naddrsByScore.slice(scores.NONE, type + 1); const hosts = getHostsFromLocals(addrs, { network: regtest }); - const src = NetAddress.fromHostname(getRandomIPv4(), regtest); + const src = getRandomNetAddr(); const best = hosts.getLocal(src); assert.strictEqual(best, naddrsByScore[type][0]); } @@ -378,7 +506,7 @@ describe('Net HostList', function() { // mainnet { const hosts = getHostsFromLocals(naddrsByScore, { network: mainnet }); - const src = NetAddress.fromHostname(getRandomIPv4(), regtest); + const src = getRandomNetAddr(mainnet); const best = hosts.getLocal(src); assert.strictEqual(best, naddrsByScore[scores.MANUAL][0]); } @@ -387,7 +515,7 @@ describe('Net HostList', function() { // everything below MANUAL is skipped on main network. const addrs = naddrsByScore.slice(scores.NONE, scores.UPNP); const hosts = getHostsFromLocals(addrs, { network: mainnet }); - const src = NetAddress.fromHostname(getRandomIPv4(), regtest); + const src = getRandomNetAddr(mainnet); const best = hosts.getLocal(src); assert.strictEqual(best, null); } @@ -499,7 +627,7 @@ describe('Net HostList', function() { const hosts = getHostsFromLocals(dests, { network: regtest }); { - const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + const addr = getRandomNetAddr(); const marked = hosts.markLocal(addr); assert.strictEqual(marked, false); @@ -507,7 +635,7 @@ describe('Net HostList', function() { { // we should get BIND, because BIND > IF - const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + const addr = getRandomNetAddr(); const local = hosts.getLocal(addr); assert.strictEqual(local, dests[1][0]); } @@ -515,7 +643,7 @@ describe('Net HostList', function() { { // with markLocal IF should get the same score (type remains). hosts.markLocal(dests[0][0]); - const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + const addr = getRandomNetAddr(); const local = hosts.getLocal(addr); assert.strictEqual(local, dests[0][0]); } @@ -523,13 +651,10 @@ describe('Net HostList', function() { it('should add fresh address', () => { { - const hosts = new HostList({ - network: regtest, - publicHost: getRandomIPv4() - }); + const hosts = new HostList(); // fresh, w/o src, not in the buckets - const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + const addr = getRandomNetAddr(); assert.strictEqual(hosts.totalFresh, 0); assert.strictEqual(hosts.needsFlush, false); @@ -551,12 +676,9 @@ describe('Net HostList', function() { } { - const hosts = new HostList({ - network: regtest - }); - - const src = NetAddress.fromHostname(getRandomIPv4(), regtest); - const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + const hosts = new HostList(); + const addr = getRandomNetAddr(); + const src = getRandomNetAddr(); hosts.add(addr, src); const freshEntries = getFreshEntries(hosts); @@ -573,12 +695,9 @@ describe('Net HostList', function() { it('should add address (limits)', () => { // Full Bucket? { - const hosts = new HostList({ - network: regtest - }); - - const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); - const src = NetAddress.fromHostname(getRandomIPv4(), regtest); + const hosts = new HostList(); + const addr = getRandomNetAddr(); + const src = getRandomNetAddr(); let evicted = false; @@ -593,7 +712,7 @@ describe('Net HostList', function() { // Fill first bucket. for (let i = 0; i < hosts.maxEntries; i++) { - const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); + const addr = getRandomNetAddr(); const added = hosts.add(addr, src); assert.strictEqual(added, true); assert.strictEqual(evicted, false); @@ -606,12 +725,9 @@ describe('Net HostList', function() { // Don't insert if entry is in a bucket. { - const hosts = new HostList({ - network: regtest - }); - - const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); - const src = NetAddress.fromHostname(getRandomIPv4(), regtest); + const hosts = new HostList(); + const addr = getRandomNetAddr(); + const src = getRandomNetAddr(); const entry = new HostList.HostEntry(addr, src); // insert entry in every bucket for this test. @@ -624,9 +740,7 @@ describe('Net HostList', function() { }); it('should add seen address', () => { - const hosts = new HostList({ - network: regtest - }); + const hosts = new HostList(); // get addr clone that can be added (Online requirements) const cloneAddr = (addr) => { @@ -637,8 +751,8 @@ describe('Net HostList', function() { return addr2; }; - const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); - const src = NetAddress.fromHostname(getRandomIPv4(), regtest); + const addr = getRandomNetAddr(); + const src = getRandomNetAddr(); addr.services = 0x01; const added = hosts.add(addr, src); assert.strictEqual(added, true); @@ -664,37 +778,51 @@ describe('Net HostList', function() { assert.strictEqual(hosts.needsFlush, false); } - // update refCount. + // update refCount. (we only have 1 refCount, increase up to 8) { - const srcs = [ - NetAddress.fromHostname(getRandomIPv4(), regtest), - NetAddress.fromHostname(getRandomIPv4(), regtest) - ]; + const srcs = []; + for (let i = 0; i < 7; i++) + srcs.push(getRandomNetAddr()); + + const factors = []; + const _random = hosts.random; + const random = function (factor) { + factors.push(factor); + return 0; + }; + hosts.random = random; + const addr2 = cloneAddr(addr); - // when we have 1 ref, so probability should be 50%. - // then we have 2 refs, probability will be 25%. - // ... until we have 8 refs. + // when we have 1 ref, so probability of adding second one + // is 50% (1/2). + // then we have 2 refs, probability of adding will be 25% (1/4). + // ... until we have 8 refs. (last one being 1/128) // Because we are reusing src, we will get same bucket // so it will only get added once. let added = 0; - for (let i = 0; i < 100; i++) { - const res = hosts.add(addr2, srcs[added]); + for (let i = 0; i < 7; i++) { + const res = hosts.add(addr2, srcs[i]); - if (res) - added++; - - if (added === 2) - break; + // our custom random method always returns 0. + assert.strictEqual(res, true); + added++; } + // make sure factors are calculated properly. + assert.strictEqual(factors.length, 7); + + for (let i = 0; i < 7; i++) + assert.strictEqual(factors[i], 1 << (i + 1)); + // at this point address should be in another bucket as well. - assert.strictEqual(added, 2); - assert.strictEqual(entry.refCount, 3); + assert.strictEqual(added, 7); + assert.strictEqual(entry.refCount, 8); const entries = getFreshEntries(hosts); - assert.strictEqual(entries.length, 3); + assert.strictEqual(entries.length, 8); assert.strictEqual(hosts.needsFlush, true); hosts.needsFlush = false; + hosts.random = _random; } // should fail with max ref @@ -721,12 +849,9 @@ describe('Net HostList', function() { it('should add address (update time)', () => { const getHosts = (time) => { - const hosts = new HostList({ - network: regtest - }); - - const addr = NetAddress.fromHostname(getRandomIPv4(), regtest); - const src = NetAddress.fromHostname(getRandomIPv4(), regtest); + const hosts = new HostList(); + const addr = getRandomNetAddr(); + const src = getRandomNetAddr(); if (time) addr.time = time; @@ -738,11 +863,12 @@ describe('Net HostList', function() { const entries = getFreshEntries(hosts); assert.strictEqual(entries.length, 1); + const entry = hosts.map.get(addr.hostname); // make sure we stop after updating time. entries[0].used = true; - return [hosts, entries[0], addr, src]; + return [hosts, entry, addr, src]; }; // Update time - Online? @@ -847,6 +973,65 @@ describe('Net HostList', function() { assert.strictEqual(hosts.needsFlush, true); } }); + + it('should mark attempt', () => { + const hosts = new HostList(); + + // if we don't have the entry. + { + const addr = getRandomIPv4(); + hosts.markAttempt(addr); + } + + const src = getRandomNetAddr(); + const addr = getRandomNetAddr(); + + hosts.add(addr, src); + + const entry = hosts.map.get(addr.hostname); + assert.strictEqual(entry.attempts, 0); + assert.strictEqual(entry.lastAttempt, 0); + + hosts.markAttempt(addr.hostname); + assert.strictEqual(entry.attempts, 1); + assert(entry.lastAttempt > util.now() - 10); + }); + + it('should mark success', () => { + const hosts = new HostList(); + + // we don't have entry. + { + const addr = getRandomIPv4(); + hosts.markSuccess(addr); + } + + // Don't update time, it's recent. + { + const src = getRandomNetAddr(); + const addr = getRandomNetAddr(); + const oldTime = util.now() - 10 * 60; // last connection 11 minutes ago. + addr.time = oldTime; + + hosts.add(addr, src); + hosts.markSuccess(addr.hostname); + + const entry = hosts.map.get(addr.hostname); + assert.strictEqual(entry.addr.time, oldTime); + } + + // we update time. + const src = getRandomNetAddr(); + const addr = getRandomNetAddr(); + const oldTime = util.now() - 21 * 60; // last connection 21 minutes ago. + addr.time = oldTime; + + hosts.add(addr, src); + hosts.markSuccess(addr.hostname); + + const entry = hosts.map.get(addr.hostname); + assert(entry.addr.time > oldTime); + }); }); /* From 4f094d9d0464f15e0b56ce5100a98692accadd64 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Sat, 19 Feb 2022 20:20:10 +0400 Subject: [PATCH 07/17] net-test: add more hostlist tests. --- lib/net/hostlist.js | 5 + test/net-hostlist-test.js | 454 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 459 insertions(+) diff --git a/lib/net/hostlist.js b/lib/net/hostlist.js index 7f922a89..88920acd 100644 --- a/lib/net/hostlist.js +++ b/lib/net/hostlist.js @@ -49,6 +49,11 @@ const seeds = require('./seeds'); * will not replace the existing entry unless that address is also stored * in another bucket or it doesn't meet one of several quality criteria * (see isStale for exact criteria). + * * Addresses of nodes that are known to be accessible go into 256 "tried" + * buckets. + * * Each address range selects at random 8 of these buckets. + * * The actual bucket is chosen from one of these, based on the full + * address. * * Bucket selection is based on cryptographic hashing, * using a randomly-generated 256-bit key, which should not be observable * by adversaries (key). diff --git a/test/net-hostlist-test.js b/test/net-hostlist-test.js index 1b001061..112aae1e 100644 --- a/test/net-hostlist-test.js +++ b/test/net-hostlist-test.js @@ -13,6 +13,7 @@ const util = require('../lib/utils/util'); const Network = require('../lib/protocol/network'); const NetAddress = require('../lib/net/netaddress'); const HostList = require('../lib/net/hostlist'); +const {HostEntry} = HostList; const regtest = Network.get('regtest'); const mainnet = Network.get('main'); @@ -384,6 +385,29 @@ describe('Net HostList', function() { } }); + it('should remove node', () => { + const hosts = new HostList(); + const key = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; + const ips = [getRandomIPv4(), getRandomIPv4(), getRandomIPv4()]; + const nodes = [ + ips[0], + `${key}@${ips[1]}`, + `${ips[2]}:1000`, + `${ips[2]}:2000` + ]; + + for (const node of nodes) + assert(hosts.addNode(node)); + + assert.strictEqual(hosts.nodes.length, nodes.length); + + for (const node of nodes.reverse()) + assert(hosts.removeNode(node)); + + assert.strictEqual(hosts.removeNode(nodes[0]), false); + assert.strictEqual(hosts.nodes.length, 0); + }); + it('should add push/local addresses', () => { const services = 1000; const hosts = new HostList({ services }); @@ -792,6 +816,12 @@ describe('Net HostList', function() { }; hosts.random = random; + let index = 0; + const _freshBucket = hosts.freshBucket; + hosts.freshBucket = function () { + return this.fresh[index++]; + }; + const addr2 = cloneAddr(addr); // when we have 1 ref, so probability of adding second one @@ -823,6 +853,7 @@ describe('Net HostList', function() { assert.strictEqual(hosts.needsFlush, true); hosts.needsFlush = false; hosts.random = _random; + hosts.freshBucket = _freshBucket; } // should fail with max ref @@ -1032,6 +1063,429 @@ describe('Net HostList', function() { const entry = hosts.map.get(addr.hostname); assert(entry.addr.time > oldTime); }); + + it('should remove host', () => { + const hosts = new HostList(); + + const src = getRandomNetAddr(); + const addrs = [ + getRandomNetAddr(), + getRandomNetAddr(), + getRandomNetAddr(), + getRandomNetAddr() + ]; + + const used = addrs.slice(2); + + for (const addr of addrs) + hosts.add(addr, src); + + for (const addr of used) + hosts.markAck(addr.hostname, 0); + + assert.strictEqual(hosts.map.size, addrs.length); + assert.strictEqual(hosts.totalUsed, 2); + assert.strictEqual(hosts.totalFresh, 2); + const fresh = getFreshEntries(hosts); + assert.strictEqual(fresh.length, 2); + + assert.strictEqual(hosts.remove(getRandomIPv6()), null); + for (const addr of addrs.reverse()) + assert.strictEqual(hosts.remove(addr.hostname), addr); + assert.strictEqual(hosts.totalUsed, 0); + assert.strictEqual(hosts.totalFresh, 0); + }); + + it('should mark ack', () => { + // we don't have the entry + { + const hosts = new HostList(); + const addr = getRandomIPv4(); + hosts.markAck(addr); + } + + // Should update services, lastSuccess, lastAttempt and attempts + // even if it's already in the used. + { + const hosts = new HostList(); + const naddr = getRandomNetAddr(); + const nsrc = getRandomNetAddr(); + + naddr.services = 0x01; + hosts.add(naddr, nsrc); + + const entry = hosts.map.get(naddr.hostname); + const oldLastAttempt = util.now() - 1000; + const oldLastSuccess = util.now() - 1000; + const oldAttempts = 2; + + entry.lastAttempt = oldLastAttempt; + entry.lastSuccess = oldLastSuccess; + entry.attempts = oldAttempts; + entry.used = true; + + hosts.markAck(naddr.hostname, 0x02); + + assert(entry.lastSuccess > oldLastSuccess); + assert(entry.lastAttempt > oldLastAttempt); + assert.strictEqual(entry.attempts, 0); + assert.strictEqual(entry.addr.services, 0x01 | 0x02); + } + + // Should remove from fresh + { + const hosts = new HostList(); + + // make sure we have all 8 refs. + let index = 0; + hosts.random = () => 0; + + // make sure we always get different bucket. + hosts.freshBucket = function () { + return this.fresh[index++]; + }; + + const addr = getRandomNetAddr(); + + for (let i = 0; i < 8; i++) { + const src = getRandomNetAddr(); + const addr2 = addr.clone(); + addr2.time = addr.time + i + 1; + + const added = hosts.add(addr2, src); + assert.strictEqual(added, true); + } + + assert.strictEqual(hosts.totalFresh, 1); + assert.strictEqual(hosts.totalUsed, 0); + assert.strictEqual(getFreshEntries(hosts).length, 8); + const entry = hosts.map.get(addr.hostname); + assert.strictEqual(entry.refCount, 8); + + hosts.markAck(addr.hostname); + + assert.strictEqual(getFreshEntries(hosts).length, 0); + assert.strictEqual(entry.refCount, 0); + assert.strictEqual(hosts.totalFresh, 0); + assert.strictEqual(hosts.totalUsed, 1); + assert.strictEqual(entry.used, true); + } + + // evict used + { + const hosts = new HostList(); + + const addr = getRandomNetAddr(); + const src = getRandomNetAddr(); + + hosts.add(addr, src); + const entry = hosts.map.get(addr.hostname); + const bucket = hosts.usedBucket(entry); + + assert.strictEqual(hosts.totalFresh, 1); + assert.strictEqual(hosts.totalUsed, 0); + + // add 64 entries to the bucket. + const entries = []; + let expectedEvicted = null; + for (let i = 0; i < hosts.maxEntries; i++) { + const addr = getRandomNetAddr(); + const src = getRandomNetAddr(); + addr.time = util.now() - (i); + + const entry = new HostEntry(addr, src); + entry.used = true; + bucket.push(entry); + entries.push(entry); + } + + expectedEvicted = entries[0]; + hosts.markAck(addr.hostname); + assert.strictEqual(bucket.tail, entry); + assert.strictEqual(expectedEvicted.used, true); + } + }); + + it('should ban/unban', () => { + const hosts = new HostList(); + + const banIPs = [ + getRandomIPv4(), + getRandomIPv4(), + getRandomIPv4() + ]; + + assert.strictEqual(hosts.banned.size, 0); + + { + for (const ip of banIPs) + hosts.ban(ip); + + assert.strictEqual(hosts.banned.size, banIPs.length); + } + + { + for (const ip of banIPs) + hosts.unban(ip); + + assert.strictEqual(hosts.banned.size, 0); + } + + { + assert.strictEqual(hosts.banned.size, 0); + for (const ip of banIPs) + hosts.ban(ip); + assert.strictEqual(hosts.banned.size, banIPs.length); + + for (const ip of banIPs) + assert(hosts.isBanned(ip)); + + hosts.clearBanned(); + + for (const ip of banIPs) + assert.strictEqual(hosts.isBanned(ip), false); + } + + // ban time + { + assert.strictEqual(hosts.banned.size, 0); + + for (const ip of banIPs) + hosts.ban(ip); + + // change ban time for the first IP. + const banExpired = banIPs[0]; + hosts.banned.set(banExpired, util.now() - hosts.options.banTime - 1); + + const [, ...stillBanned] = banIPs; + for (const ip of stillBanned) + assert(hosts.isBanned(ip)); + + assert.strictEqual(hosts.banned.size, banIPs.length); + assert.strictEqual(hosts.isBanned(banExpired), false); + assert.strictEqual(hosts.banned.size, banIPs.length - 1); + } + }); + + it('should check if entry is stale', () => { + const hosts = new HostList(); + + const src = getRandomNetAddr(); + const addrs = []; + const entries = []; + + for (let i = 0; i < 10; i++) { + const addr = getRandomNetAddr(); + hosts.add(addr, src); + const entry = hosts.map.get(addr.hostname); + entries.push(entry); + addrs.push(addr); + } + + const A_DAY = 24 * 60 * 60; + + // address from the future? + entries[0].addr.time = util.now() + 30 * 60; + + entries[1].addr.time = 0; + + // too old + entries[2].addr.time = util.now() - HostList.HORIZON_DAYS * A_DAY - 1; + + // many attempts, no success + entries[3].attempts = HostList.RETRIES; + + // last success got old. + // and we failed max times. + entries[4].lastSuccess = util.now() - HostList.MIN_FAIL_DAYS * A_DAY - 1; + entries[4].attempts = HostList.MAX_FAILURES; + + // last attempt in last minute + entries[5].lastAttempt = util.now(); + + entries[6].lastSuccess = entries[5].lastSuccess; + entries[7].lastSuccess = entries[5].lastSuccess + A_DAY; + + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]; + + if (i < 5) + assert.strictEqual(hosts.isStale(entry), true); + else + assert.strictEqual(hosts.isStale(entry), false); + } + }); + + it('should evict entry from fresh bucket', () => { + const hosts = new HostList(); + const bucket = hosts.fresh[0]; + + const src = getRandomNetAddr(); + const entries = []; + + // Sort them young -> old + for (let i = 0; i < 10; i++) { + const entry = new HostEntry(getRandomNetAddr(), src); + entry.addr.time = util.now() - i; + entry.refCount = 1; + bucket.set(entry.key(), entry); + hosts.map.set(entry.key(), entry); + entries.push(entry); + hosts.totalFresh++; + } + + { + const staleEntry = entries[0]; + const expectedEvicted = entries[entries.length - 1]; + + // stales are evicted anyway. + staleEntry.addr.time = 0; + + // so we evict 2. + assert.strictEqual(hosts.isStale(staleEntry), true); + assert.strictEqual(bucket.has(staleEntry.key()), true); + assert.strictEqual(bucket.has(expectedEvicted.key()), true); + assert.strictEqual(hosts.map.has(staleEntry.key()), true); + hosts.evictFresh(bucket); + assert.strictEqual(bucket.has(staleEntry.key()), false); + assert.strictEqual(bucket.has(expectedEvicted.key()), false); + assert.strictEqual(hosts.map.has(staleEntry.key()), false); + } + + { + // evict older even if it's stale but is in another bucket as well.? + const staleEntry = entries[1]; + const expectedEvicted = entries[entries.length - 2]; + + staleEntry.attempts = HostList.RETRIES; + staleEntry.refCount = 2; + + assert.strictEqual(hosts.isStale(staleEntry), true); + + assert.strictEqual(bucket.has(staleEntry.key()), true); + assert.strictEqual(bucket.has(expectedEvicted.key()), true); + assert.strictEqual(hosts.map.has(staleEntry.key()), true); + hosts.evictFresh(bucket); + assert.strictEqual(bucket.has(staleEntry.key()), false); + assert.strictEqual(bucket.has(expectedEvicted.key()), false); + assert.strictEqual(hosts.map.has(staleEntry.key()), true); + } + + { + const expectedEvicted = entries[entries.length - 3]; + expectedEvicted.refCount = 2; + + assert.strictEqual(bucket.has(expectedEvicted.key()), true); + assert.strictEqual(hosts.map.has(expectedEvicted.key()), true); + hosts.evictFresh(bucket); + assert.strictEqual(bucket.has(expectedEvicted.key()), false); + assert.strictEqual(hosts.map.has(expectedEvicted.key()), true); + } + + assert.strictEqual(bucket.size, 5); + for (let i = entries.length - 4; i > 1; i--) { + const entry = entries[i]; + assert.strictEqual(bucket.has(entry.key()), true); + assert.strictEqual(hosts.map.has(entry.key()), true); + hosts.evictFresh(bucket); + assert.strictEqual(bucket.has(entry.key()), false); + assert.strictEqual(hosts.map.has(entry.key()), false); + } + + assert.strictEqual(bucket.size, 0); + hosts.evictFresh(bucket); + assert.strictEqual(bucket.size, 0); + }); + + it('should return array of entries', () => { + const hosts = new HostList(); + + const src = getRandomNetAddr(); + const addrs = []; + const entries = []; + + for (let i = 0; i < 20; i++) { + const addr = getRandomNetAddr(); + addrs.push(addr); + hosts.add(addr, src); + entries.push(hosts.map.get(addr.hostname)); + } + + // have first 2 entries stale. + entries[0].addr.time = 0; + entries[1].addr.time = util.now() + 20 * 60; + + const arr = hosts.toArray(); + const set = new Set(arr); + + assert.strictEqual(arr.length, entries.length - 2); + assert.strictEqual(set.size, entries.length - 2); + + for (let i = 0; i < 2; i++) + assert.strictEqual(set.has(addrs[i]), false); + + for (let i = 2; i < entries.length; i++) + assert.strictEqual(set.has(addrs[i]), true); + }); + + it('should get host', () => { + const add2bucket = (hosts, bucketIndex, entry, fresh = true) => { + if (fresh) { + assert(bucketIndex < hosts.maxFreshBuckets); + entry.refCount++; + hosts.totalFresh++; + hosts.map.set(entry.key(), entry); + hosts.fresh[bucketIndex].set(entry.key(), entry); + return; + } + + assert(bucketIndex < hosts.maxUsedBuckets); + assert(entry.refCount === 0); + entry.used = true; + hosts.map.set(entry.key(), entry); + hosts.used[bucketIndex].push(entry); + hosts.totalUsed++; + }; + + { + // empty + const hosts = new HostList(); + const host = hosts.getHost(); + assert.strictEqual(host, null); + } + + { + // fresh buckets + const hosts = new HostList(); + + const freshEntries = []; + + for (let i = 0; i < 100; i++) { + const entry = new HostEntry(getRandomNetAddr(), getRandomNetAddr()); + freshEntries.push(entry); + add2bucket(hosts, 0, entry, true); + } + + const found = hosts.getHost(); + assert.strictEqual(new Set(freshEntries).has(found), true); + } + + { + // used bucket - this is random. + const hosts = new HostList(); + // put 10 entries in the used. + const usedEntries = []; + + for (let i = 0; i < 100; i++) { + const entry = new HostEntry(getRandomNetAddr(), getRandomNetAddr()); + usedEntries.push(entry); + add2bucket(hosts, 0, usedEntries[i], false); + } + + const foundEntry = hosts.getHost(); + assert.strictEqual(new Set(usedEntries).has(foundEntry), true); + } + }); }); /* From 732d6278c300683f34a6a5f8dc73fd08567ad848 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Sun, 20 Feb 2022 16:53:12 +0400 Subject: [PATCH 08/17] net-test: add more tests to the hostlist. --- test/chain-locktime-test.js | 11 + test/net-hostlist-test.js | 2573 +++++++++++++++++++---------------- 2 files changed, 1420 insertions(+), 1164 deletions(-) diff --git a/test/chain-locktime-test.js b/test/chain-locktime-test.js index 2a760723..7777df90 100644 --- a/test/chain-locktime-test.js +++ b/test/chain-locktime-test.js @@ -72,6 +72,17 @@ describe('Chain Timelocks', function() { }); describe('Relative (CSV)', function() { + let timeOffset; + + // make sure we recover proper regtest Network. + before(() => { + timeOffset = network.time.offset; + }); + + after(() => { + network.time.offset = timeOffset; + }); + // Relative timelock by height const csvHeightScript = new Script([ Opcode.fromInt(2), diff --git a/test/net-hostlist-test.js b/test/net-hostlist-test.js index 112aae1e..eff42fff 100644 --- a/test/net-hostlist-test.js +++ b/test/net-hostlist-test.js @@ -35,32 +35,44 @@ function getHostsFromLocals(addresses, opts) { function getFreshEntries(hosts) { const naddrs = []; - for (const bucket of hosts.fresh) { - for (const naddr of bucket.values()) - naddrs.push(naddr); - } + for (const bucket of hosts.fresh) + naddrs.push(...bucket.values()); return naddrs; }; +function getUsedEntries(hosts) { + const naddrs = []; + + for (const bucket of hosts.used) + naddrs.push(...bucket.toArray()); + + return naddrs; +} + function getRandomNetAddr(network = regtest) { return NetAddress.fromHostname(getRandomIPv4(), network); } +function add2bucket(hosts, bucketIndex, entry, fresh = true) { + if (fresh) { + assert(bucketIndex < hosts.maxFreshBuckets); + entry.refCount++; + hosts.totalFresh++; + hosts.map.set(entry.key(), entry); + hosts.fresh[bucketIndex].set(entry.key(), entry); + return; + } + + assert(bucketIndex < hosts.maxUsedBuckets); + assert(entry.refCount === 0); + entry.used = true; + hosts.map.set(entry.key(), entry); + hosts.used[bucketIndex].push(entry); + hosts.totalUsed++; +}; + describe('Net HostList', function() { - let testdir; - - before(async () => { - testdir = common.testdir('hostlist'); - - assert(await fs.mkdirp(testdir)); - }); - - after(async () => { - await fs.rimraf(testdir); - testdir = null; - }); - it('should parse options', () => { const network = regtest; const logger = Logger.global; @@ -79,7 +91,7 @@ describe('Net HostList', function() { const onion = false; const brontideOnly = false; const memory = true; - const prefix = testdir; + const prefix = '/tmp/directory'; const filename = path.join(prefix, 'custom.json'); const flushInterval = 2000; @@ -160,11 +172,11 @@ describe('Net HostList', function() { // Prefix check { const hostlist = new HostList({ - prefix: testdir + prefix }); assert.strictEqual(hostlist.options.filename, - path.join(testdir, 'hosts.json')); + path.join(prefix, 'hosts.json')); } }); @@ -261,951 +273,6 @@ describe('Net HostList', function() { assert.strictEqual(hosts.map.size, 3); }); - it('should add/set nodes/seeds', () => { - // we need 3. - const hosts = [ - getRandomIPv4(), - getRandomIPv4(), - getRandomIPv4() - ]; - - const key = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; - - const tests = [ - // DNS Nodes - { - host: 'example.com', - hostname: 'example.com', - expected: { - addr: null, - dnsNodes: 1, - nodes: 0, - map: 0 - } - }, - // HSD Node w/o port - plaintext - { - host: hosts[0], - hostname: hosts[0], - expected: { - addr: { port: mainnet.port, host: hosts[0] }, - dnsNodes: 0, - nodes: 1, - map: 1 - } - }, - // HSD Node w/o port - brontide - { - host: hosts[1], - hostname: `${key}@${hosts[1]}`, - expected: { - addr: { host: hosts[1], port: mainnet.brontidePort }, - dnsNodes: 0, - nodes: 1, - map: 1 - } - }, - // HSD Node with port - { - host: hosts[2], - hostname: `${hosts[2]}:${mainnet.port + 1}`, - expected: { - addr: { host: hosts[2], port: mainnet.port + 1 }, - dnsNodes: 0, - nodes: 1, - map: 1 - } - } - ]; - - const allHosts = tests.map(t => t.hostname); - const sumExpected = tests.reduce((p, c) => { - p.dnsNodes += c.expected.dnsNodes; - p.nodes += c.expected.nodes; - p.map += c.expected.map; - return p; - }, { - dnsNodes: 0, - nodes: 0, - map: 0 - }); - - for (const test of tests) { - const hosts = new HostList(); - const {expected} = test; - - const addr = hosts.addNode(test.hostname); - - if (expected.addr == null) - assert.strictEqual(addr, null); - - if (expected.addr != null) { - assert.strictEqual(addr.host, expected.addr.host); - assert.strictEqual(addr.port, expected.addr.port); - } - - assert.strictEqual(hosts.dnsNodes.length, expected.dnsNodes); - assert.strictEqual(hosts.nodes.length, expected.nodes); - assert.strictEqual(hosts.map.size, expected.map); - } - - // set all nodes - { - const hosts = new HostList(); - hosts.setNodes(allHosts); - assert.strictEqual(hosts.dnsNodes.length, sumExpected.dnsNodes); - assert.strictEqual(hosts.nodes.length, sumExpected.nodes); - assert.strictEqual(hosts.map.size, sumExpected.map); - } - - for (const test of tests) { - const hosts = new HostList(); - const {expected} = test; - - const addr = hosts.addSeed(test.hostname); - - if (expected.addr == null) - assert.strictEqual(addr, null); - - if (expected.addr != null) { - assert.strictEqual(addr.host, expected.addr.host); - assert.strictEqual(addr.port, expected.addr.port); - } - - assert.strictEqual(hosts.dnsSeeds.length, expected.dnsNodes); - assert.strictEqual(hosts.map.size, expected.map); - } - - { - const hosts = new HostList(); - - hosts.setSeeds(allHosts); - assert.strictEqual(hosts.dnsSeeds.length, sumExpected.dnsNodes); - assert.strictEqual(hosts.map.size, sumExpected.map); - } - }); - - it('should remove node', () => { - const hosts = new HostList(); - const key = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; - const ips = [getRandomIPv4(), getRandomIPv4(), getRandomIPv4()]; - const nodes = [ - ips[0], - `${key}@${ips[1]}`, - `${ips[2]}:1000`, - `${ips[2]}:2000` - ]; - - for (const node of nodes) - assert(hosts.addNode(node)); - - assert.strictEqual(hosts.nodes.length, nodes.length); - - for (const node of nodes.reverse()) - assert(hosts.removeNode(node)); - - assert.strictEqual(hosts.removeNode(nodes[0]), false); - assert.strictEqual(hosts.nodes.length, 0); - }); - - it('should add push/local addresses', () => { - const services = 1000; - const hosts = new HostList({ services }); - - const unroutable = [ - '127.0.0.1', - '127.1.1.1', - '192.168.1.1', - '10.10.10.10' - ]; - - const routable = [ - getRandomIPv4(), - getRandomIPv4() - ]; - - assert.strictEqual(hosts.local.size, 0); - - // unroutable local addresses must not get added. - for (const host of unroutable) { - const port = regtest.port; - const score = HostList.scores.NONE; - const res = hosts.addLocal(host, port, null, score); - assert.strictEqual(res, false); - } - - assert.strictEqual(hosts.local.size, 0); - - const SCORE = HostList.scores.MANUAL; - // these must get added. - for (const host of routable) { - const port = regtest.port; - const res1 = hosts.addLocal(host, port, SCORE); - assert.strictEqual(res1, true); - - // add brontide versions - const bport = regtest.brontidePort; - const res2 = hosts.addLocal(host, bport, SCORE); - assert.strictEqual(res2, true); - } - - // one brontide and one plaintext - assert.strictEqual(hosts.local.size, routable.length * 2); - - const added = hosts.addLocal(routable[0], regtest.port, SCORE); - assert.strictEqual(added, false); - - for (const laddr of hosts.local.values()) { - assert.strictEqual(laddr.addr.services, services); - assert.strictEqual(laddr.score, SCORE); - assert.strictEqual(laddr.type, SCORE); - } - }); - - // w/o src it will only take into account the score. - it('should get local w/o src', () => { - const services = 1000; - const port = regtest.port; - - const addrs = []; - - // w/o key - for (let i = HostList.scores.NONE; i < HostList.scores.MAX - 1; i++) { - const address = new NetAddress({ - host: getRandomIPv4(), - port: port, - services: services - }); - addrs.push([address, i]); - } - - // max but with key - addrs.push([new NetAddress({ - host: getRandomIPv4(), - port: port, - services: services, - key: Buffer.alloc(33, 1) - }), HostList.scores.MAX]); - - const max = addrs[HostList.scores.MAX - 2]; - - const hosts = getHostsFromLocals(addrs); - const local = hosts.getLocal(); - - assert.strictEqual(local, max[0]); - }); - - it('should get local (score)', () => { - // NOTE: main network ignores everything other than MANUAL type. - // - scores.IF - addresses from the network interfaces. - // - scores.BIND - Listening IP. (publicHost/publicPort is MANUAL) - // - scores.DNS - Domain that needs to be resolved. - // - scores.UPNP - UPNP discovered address. - - const scores = HostList.scores; - - // same NET type but different scores: - const hostsByScore = [ - [getRandomIPv4(), scores.NONE], - [getRandomIPv4(), scores.IF], - [getRandomIPv4(), scores.BIND], - [getRandomIPv4(), scores.DNS], - [getRandomIPv4(), scores.UPNP], - [getRandomIPv4(), scores.MANUAL] - ]; - - const naddrsByScore = hostsByScore.map(([h, s]) => { - return [getRandomNetAddr(), s]; - }); - - // testnet/regtest - for (let type = scores.NONE; type < scores.MAX; type++) { - const addrs = naddrsByScore.slice(scores.NONE, type + 1); - const hosts = getHostsFromLocals(addrs, { network: regtest }); - const src = getRandomNetAddr(); - const best = hosts.getLocal(src); - assert.strictEqual(best, naddrsByScore[type][0]); - } - - // mainnet - { - const hosts = getHostsFromLocals(naddrsByScore, { network: mainnet }); - const src = getRandomNetAddr(mainnet); - const best = hosts.getLocal(src); - assert.strictEqual(best, naddrsByScore[scores.MANUAL][0]); - } - - { - // everything below MANUAL is skipped on main network. - const addrs = naddrsByScore.slice(scores.NONE, scores.UPNP); - const hosts = getHostsFromLocals(addrs, { network: mainnet }); - const src = getRandomNetAddr(mainnet); - const best = hosts.getLocal(src); - assert.strictEqual(best, null); - } - }); - - it('should get local (reachability)', () => { - // If we have multiple public host/ports (e.g. IPv4 and Ipv6) - // depending who is connecting to us, we will choose different - // address to advertise. - - // with src it will take into account the reachability score. - // See: binet.getReachability - // TLDR: - // UNREACHABLE = 0 - // < DEFAULT = 1 - non-ipv4 -> ipv4, onion -> ipv6, ... - // < TEREDO = 2 - teredo -> teredo, teredo -> ipv6 - // < IPV6_WEAK = 3 - ipv4 -> ipv6 tunnels, ipv6 -> teredo - // < IPV4 = 4 - ipv4 -> ipv4, ipv4 -> others - // < IPV6_STRONG = 5 - ipv6 -> ipv6 - // < PRIVATE = 6 - ONION -> ONION - - const {MANUAL} = HostList.scores; - - // same score (MANUAL), different reachability: - // remote(src) => [ local(dest)... ] - sorted by reachability scores. - const reachabilityMap = { - // unreachable => anything - will be UNREACHABLE = 0. - [getRandomTEREDO()]: [ - getRandomIPv4(), // DEFAULT = 1 - getRandomOnion(), // DEFAULT = 1 - getRandomTEREDO(), // TEREDO = 2 - getRandomIPv6() // TEREDO = 2 - ], - [getRandomIPv4()]: [ - getRandomIPv4(), // IPV4 = 4 - getRandomOnion(), // IPV4 = 4 - getRandomTEREDO(), // IPV4 = 4 - getRandomIPv6() // IPV4 = 4 - ], - [getRandomIPv6()]: [ - getRandomOnion(), // DEFAULT = 1 - getRandomIPv4(), // DEFAULT = 1 - getRandomTEREDO(), // IPV6_WEAK = 3 - getRandomIPv6() // IPV6_STRONG = 5 - ], - [getRandomOnion()]: [ - getRandomIPv4(), // DEFAULT = 1 - getRandomTEREDO(), // DEFAULT = 1 - getRandomIPv6(), // DEFAULT = 1 - getRandomOnion() // PRIVATE = 6 - ] - }; - - for (const [rawSrc, rawDests] of Object.entries(reachabilityMap)) { - const dests = rawDests.map((dest) => { - return [NetAddress.fromHostname(dest, mainnet), MANUAL]; - }); - - for (let i = 0; i < dests.length; i++) { - const addrs = dests.slice(0, i + 1); - const expected = addrs[addrs.length - 1]; - - // Because getLocal will choose first with the same score, - // we make the "best" choice (because of the sorting) at first. - addrs[addrs.length - 1] = addrs[0]; - addrs[0] = expected; - - const hosts = getHostsFromLocals(addrs); - const src = NetAddress.fromHostname(rawSrc, mainnet); - const best = hosts.getLocal(src); - - assert.strictEqual(best, expected[0]); - } - } - }); - - it('should not get local (skip brontide)', () => { - const {MANUAL} = HostList.scores; - - // skip with key - const src = getRandomIPv4(); - const KEY = base32.encode(Buffer.alloc(33, 1)); - const rawDests = [ - `${KEY}@${getRandomIPv4()}:${mainnet.brontidePort}`, - `${KEY}@${getRandomIPv4()}:${mainnet.brontidePort}` - ]; - - const dests = rawDests.map((d) => { - return [NetAddress.fromHostname(d, mainnet), MANUAL]; - }); - - const hosts = getHostsFromLocals(dests); - const best = hosts.getLocal(src); - assert.strictEqual(best, null); - }); - - it('should mark local', () => { - const {scores} = HostList; - - const rawDests = [ - [getRandomIPv4(), scores.IF], - [getRandomIPv4(), scores.BIND] - ]; - - const dests = rawDests.map(([h, s]) => { - return [NetAddress.fromHostname(h, regtest), s]; - }); - - const hosts = getHostsFromLocals(dests, { network: regtest }); - - { - const addr = getRandomNetAddr(); - const marked = hosts.markLocal(addr); - - assert.strictEqual(marked, false); - } - - { - // we should get BIND, because BIND > IF - const addr = getRandomNetAddr(); - const local = hosts.getLocal(addr); - assert.strictEqual(local, dests[1][0]); - } - - { - // with markLocal IF should get the same score (type remains). - hosts.markLocal(dests[0][0]); - const addr = getRandomNetAddr(); - const local = hosts.getLocal(addr); - assert.strictEqual(local, dests[0][0]); - } - }); - - it('should add fresh address', () => { - { - const hosts = new HostList(); - - // fresh, w/o src, not in the buckets - const addr = getRandomNetAddr(); - - assert.strictEqual(hosts.totalFresh, 0); - assert.strictEqual(hosts.needsFlush, false); - assert.strictEqual(hosts.map.size, 0); - assert.strictEqual(getFreshEntries(hosts).length, 0); - - hosts.add(addr); - - assert.strictEqual(hosts.totalFresh, 1); - assert.strictEqual(hosts.needsFlush, true); - assert.strictEqual(hosts.map.size, 1); - - const freshEntries = getFreshEntries(hosts); - assert.strictEqual(freshEntries.length, 1); - - const entry = freshEntries[0]; - assert.strictEqual(entry.addr, addr, 'Entry addr is not correct.'); - assert.strictEqual(entry.src, hosts.address, 'Entry src is not correct.'); - } - - { - const hosts = new HostList(); - const addr = getRandomNetAddr(); - const src = getRandomNetAddr(); - - hosts.add(addr, src); - const freshEntries = getFreshEntries(hosts); - assert.strictEqual(freshEntries.length, 1); - - const entry = freshEntries[0]; - assert.strictEqual(entry.addr, addr, 'Entry addr is not correct.'); - assert.strictEqual(entry.src, src, 'Entry src is not correct.'); - assert.strictEqual(entry.refCount, 1); - assert.strictEqual(hosts.map.size, 1); - } - }); - - it('should add address (limits)', () => { - // Full Bucket? - { - const hosts = new HostList(); - const addr = getRandomNetAddr(); - const src = getRandomNetAddr(); - - let evicted = false; - - // always return first bucket for this test. - hosts.freshBucket = function() { - return this.fresh[0]; - }; - - hosts.evictFresh = function() { - evicted = true; - }; - - // Fill first bucket. - for (let i = 0; i < hosts.maxEntries; i++) { - const addr = getRandomNetAddr(); - const added = hosts.add(addr, src); - assert.strictEqual(added, true); - assert.strictEqual(evicted, false); - } - - const added = hosts.add(addr, src); - assert.strictEqual(added, true); - assert.strictEqual(evicted, true); - } - - // Don't insert if entry is in a bucket. - { - const hosts = new HostList(); - const addr = getRandomNetAddr(); - const src = getRandomNetAddr(); - const entry = new HostList.HostEntry(addr, src); - - // insert entry in every bucket for this test. - for (const bucket of hosts.fresh) - bucket.set(entry.key(), entry); - - const added = hosts.add(addr, src); - assert.strictEqual(added, false); - } - }); - - it('should add seen address', () => { - const hosts = new HostList(); - - // get addr clone that can be added (Online requirements) - const cloneAddr = (addr) => { - const addr2 = addr.clone(); - // just make sure this is < 3 * 60 * 60 (Online requirements - // with & without penalty) - addr2.time = util.now() + 60 * 60; - return addr2; - }; - - const addr = getRandomNetAddr(); - const src = getRandomNetAddr(); - addr.services = 0x01; - const added = hosts.add(addr, src); - assert.strictEqual(added, true); - assert.strictEqual(hosts.needsFlush, true); - hosts.needsFlush = false; - - const entries = getFreshEntries(hosts); - const entry = entries[0]; - assert.strictEqual(entries.length, 1); - assert.strictEqual(entry.addr.services, 0x01); - - // don't update - no new info (service will always get updated.) - { - const addr2 = addr.clone(); - addr2.services = 0x02; - const added = hosts.add(addr2, src); - assert.strictEqual(added, false); - - const entries = getFreshEntries(hosts); - const entry = entries[0]; - assert.strictEqual(entries.length, 1); - assert.strictEqual(entry.addr.services, 0x03); - assert.strictEqual(hosts.needsFlush, false); - } - - // update refCount. (we only have 1 refCount, increase up to 8) - { - const srcs = []; - for (let i = 0; i < 7; i++) - srcs.push(getRandomNetAddr()); - - const factors = []; - const _random = hosts.random; - const random = function (factor) { - factors.push(factor); - return 0; - }; - hosts.random = random; - - let index = 0; - const _freshBucket = hosts.freshBucket; - hosts.freshBucket = function () { - return this.fresh[index++]; - }; - - const addr2 = cloneAddr(addr); - - // when we have 1 ref, so probability of adding second one - // is 50% (1/2). - // then we have 2 refs, probability of adding will be 25% (1/4). - // ... until we have 8 refs. (last one being 1/128) - // Because we are reusing src, we will get same bucket - // so it will only get added once. - let added = 0; - for (let i = 0; i < 7; i++) { - const res = hosts.add(addr2, srcs[i]); - - // our custom random method always returns 0. - assert.strictEqual(res, true); - added++; - } - - // make sure factors are calculated properly. - assert.strictEqual(factors.length, 7); - - for (let i = 0; i < 7; i++) - assert.strictEqual(factors[i], 1 << (i + 1)); - - // at this point address should be in another bucket as well. - assert.strictEqual(added, 7); - assert.strictEqual(entry.refCount, 8); - const entries = getFreshEntries(hosts); - assert.strictEqual(entries.length, 8); - assert.strictEqual(hosts.needsFlush, true); - hosts.needsFlush = false; - hosts.random = _random; - hosts.freshBucket = _freshBucket; - } - - // should fail with max ref - { - const _refCount = entry.refCount; - entry.refCount = HostList.MAX_REFS; - const addr2 = cloneAddr(addr); - const added = hosts.add(addr2, src); - assert.strictEqual(added, false); - entry.refCount = _refCount; - assert.strictEqual(hosts.needsFlush, false); - } - - // should fail if it's used - { - entry.used = true; - const addr2 = cloneAddr(addr); - const added = hosts.add(addr2, src); - assert.strictEqual(added, false); - assert.strictEqual(hosts.needsFlush, false); - entry.used = false; - } - }); - - it('should add address (update time)', () => { - const getHosts = (time) => { - const hosts = new HostList(); - const addr = getRandomNetAddr(); - const src = getRandomNetAddr(); - - if (time) - addr.time = time; - - const added = hosts.add(addr, src); - assert.strictEqual(added, true); - assert.strictEqual(hosts.needsFlush, true); - hosts.needsFlush = false; - - const entries = getFreshEntries(hosts); - assert.strictEqual(entries.length, 1); - const entry = hosts.map.get(addr.hostname); - - // make sure we stop after updating time. - entries[0].used = true; - - return [hosts, entry, addr, src]; - }; - - // Update time - Online? - { - // a week ago - const [hosts, entry, addr, src] = getHosts(util.now() - 7 * 24 * 60 * 60); - const addr2 = addr.clone(); - - // a day ago (interval is a day, - // so we update if a day and 2 hrs have passed). - addr2.time = util.now() - 24 * 60 * 60; - - const added = hosts.add(addr2, src); - assert.strictEqual(added, false); - assert.strictEqual(entry.addr.time, addr2.time); - assert.strictEqual(hosts.needsFlush, true); - } - - { - // a day ago - const [hosts, entry, addr, src] = getHosts(util.now() - 24 * 60 * 60); - - const addr2 = addr.clone(); - - // now (interval becomes an hour, so instead we update if 3 hrs passed. - addr2.time = util.now(); - - const added = hosts.add(addr2, src); - assert.strictEqual(added, false); - assert.strictEqual(entry.addr.time, addr2.time); - assert.strictEqual(hosts.needsFlush, true); - } - - // Don't update - { - // a week ago - const weekAgo = util.now() - 7 * 24 * 60 * 60; - const sixDaysAgo = util.now() - 6 * 24 * 60 * 60 + 1; // and a second - - const [hosts, entry, addr, src] = getHosts(weekAgo); - - const addr2 = addr.clone(); - // 6 days ago (exactly 24 hrs after) because 2 hrs is penalty, - // we don't update. - addr2.time = sixDaysAgo; - - const added = hosts.add(addr2, src); - assert.strictEqual(added, false); - assert.strictEqual(entry.addr.time, weekAgo); - assert.strictEqual(hosts.needsFlush, false); - } - - // Update, because we are the ones inserting. - { - // a week ago - const weekAgo = util.now() - 7 * 24 * 60 * 60; - const sixDaysAgo = util.now() - 6 * 24 * 60 * 60 + 1; // and a second - - const [hosts, entry, addr] = getHosts(weekAgo); - - const addr2 = addr.clone(); - // 6 days ago (exactly 24 hrs after) because 2 hrs is penalty, - // we don't update. - addr2.time = sixDaysAgo; - - const added = hosts.add(addr2); - assert.strictEqual(added, false); - assert.strictEqual(entry.addr.time, sixDaysAgo); - assert.strictEqual(hosts.needsFlush, true); - } - - // Online - but still not updating (less than 3 hrs) - { - // now vs 3 hrs ago (exactly) - const now = util.now(); - const threeHoursAgo = now - 3 * 60 * 60; - - const [hosts, entry, addr, src] = getHosts(threeHoursAgo); - - const addr2 = addr.clone(); - addr2.time = now; - - const added = hosts.add(addr2, src); - assert.strictEqual(added, false); - assert.strictEqual(entry.addr.time, threeHoursAgo); - assert.strictEqual(hosts.needsFlush, false); - } - - { - // now vs 3 hrs and a second ago - const now = util.now(); - const threeHoursAgo = now - 1 - 3 * 60 * 60; - - const [hosts, entry, addr, src] = getHosts(threeHoursAgo); - - const addr2 = addr.clone(); - addr2.time = now; - - const added = hosts.add(addr2, src); - assert.strictEqual(added, false); - assert.strictEqual(entry.addr.time, now); - assert.strictEqual(hosts.needsFlush, true); - } - }); - - it('should mark attempt', () => { - const hosts = new HostList(); - - // if we don't have the entry. - { - const addr = getRandomIPv4(); - hosts.markAttempt(addr); - } - - const src = getRandomNetAddr(); - const addr = getRandomNetAddr(); - - hosts.add(addr, src); - - const entry = hosts.map.get(addr.hostname); - assert.strictEqual(entry.attempts, 0); - assert.strictEqual(entry.lastAttempt, 0); - - hosts.markAttempt(addr.hostname); - assert.strictEqual(entry.attempts, 1); - assert(entry.lastAttempt > util.now() - 10); - }); - - it('should mark success', () => { - const hosts = new HostList(); - - // we don't have entry. - { - const addr = getRandomIPv4(); - hosts.markSuccess(addr); - } - - // Don't update time, it's recent. - { - const src = getRandomNetAddr(); - const addr = getRandomNetAddr(); - const oldTime = util.now() - 10 * 60; // last connection 11 minutes ago. - addr.time = oldTime; - - hosts.add(addr, src); - hosts.markSuccess(addr.hostname); - - const entry = hosts.map.get(addr.hostname); - assert.strictEqual(entry.addr.time, oldTime); - } - - // we update time. - const src = getRandomNetAddr(); - const addr = getRandomNetAddr(); - const oldTime = util.now() - 21 * 60; // last connection 21 minutes ago. - addr.time = oldTime; - - hosts.add(addr, src); - hosts.markSuccess(addr.hostname); - - const entry = hosts.map.get(addr.hostname); - assert(entry.addr.time > oldTime); - }); - - it('should remove host', () => { - const hosts = new HostList(); - - const src = getRandomNetAddr(); - const addrs = [ - getRandomNetAddr(), - getRandomNetAddr(), - getRandomNetAddr(), - getRandomNetAddr() - ]; - - const used = addrs.slice(2); - - for (const addr of addrs) - hosts.add(addr, src); - - for (const addr of used) - hosts.markAck(addr.hostname, 0); - - assert.strictEqual(hosts.map.size, addrs.length); - assert.strictEqual(hosts.totalUsed, 2); - assert.strictEqual(hosts.totalFresh, 2); - const fresh = getFreshEntries(hosts); - assert.strictEqual(fresh.length, 2); - - assert.strictEqual(hosts.remove(getRandomIPv6()), null); - for (const addr of addrs.reverse()) - assert.strictEqual(hosts.remove(addr.hostname), addr); - assert.strictEqual(hosts.totalUsed, 0); - assert.strictEqual(hosts.totalFresh, 0); - }); - - it('should mark ack', () => { - // we don't have the entry - { - const hosts = new HostList(); - const addr = getRandomIPv4(); - hosts.markAck(addr); - } - - // Should update services, lastSuccess, lastAttempt and attempts - // even if it's already in the used. - { - const hosts = new HostList(); - const naddr = getRandomNetAddr(); - const nsrc = getRandomNetAddr(); - - naddr.services = 0x01; - hosts.add(naddr, nsrc); - - const entry = hosts.map.get(naddr.hostname); - const oldLastAttempt = util.now() - 1000; - const oldLastSuccess = util.now() - 1000; - const oldAttempts = 2; - - entry.lastAttempt = oldLastAttempt; - entry.lastSuccess = oldLastSuccess; - entry.attempts = oldAttempts; - entry.used = true; - - hosts.markAck(naddr.hostname, 0x02); - - assert(entry.lastSuccess > oldLastSuccess); - assert(entry.lastAttempt > oldLastAttempt); - assert.strictEqual(entry.attempts, 0); - assert.strictEqual(entry.addr.services, 0x01 | 0x02); - } - - // Should remove from fresh - { - const hosts = new HostList(); - - // make sure we have all 8 refs. - let index = 0; - hosts.random = () => 0; - - // make sure we always get different bucket. - hosts.freshBucket = function () { - return this.fresh[index++]; - }; - - const addr = getRandomNetAddr(); - - for (let i = 0; i < 8; i++) { - const src = getRandomNetAddr(); - const addr2 = addr.clone(); - addr2.time = addr.time + i + 1; - - const added = hosts.add(addr2, src); - assert.strictEqual(added, true); - } - - assert.strictEqual(hosts.totalFresh, 1); - assert.strictEqual(hosts.totalUsed, 0); - assert.strictEqual(getFreshEntries(hosts).length, 8); - const entry = hosts.map.get(addr.hostname); - assert.strictEqual(entry.refCount, 8); - - hosts.markAck(addr.hostname); - - assert.strictEqual(getFreshEntries(hosts).length, 0); - assert.strictEqual(entry.refCount, 0); - assert.strictEqual(hosts.totalFresh, 0); - assert.strictEqual(hosts.totalUsed, 1); - assert.strictEqual(entry.used, true); - } - - // evict used - { - const hosts = new HostList(); - - const addr = getRandomNetAddr(); - const src = getRandomNetAddr(); - - hosts.add(addr, src); - const entry = hosts.map.get(addr.hostname); - const bucket = hosts.usedBucket(entry); - - assert.strictEqual(hosts.totalFresh, 1); - assert.strictEqual(hosts.totalUsed, 0); - - // add 64 entries to the bucket. - const entries = []; - let expectedEvicted = null; - for (let i = 0; i < hosts.maxEntries; i++) { - const addr = getRandomNetAddr(); - const src = getRandomNetAddr(); - addr.time = util.now() - (i); - - const entry = new HostEntry(addr, src); - entry.used = true; - bucket.push(entry); - entries.push(entry); - } - - expectedEvicted = entries[0]; - hosts.markAck(addr.hostname); - assert.strictEqual(bucket.tail, entry); - assert.strictEqual(expectedEvicted.used, true); - } - }); - it('should ban/unban', () => { const hosts = new HostList(); @@ -1267,224 +334,1402 @@ describe('Net HostList', function() { } }); - it('should check if entry is stale', () => { - const hosts = new HostList(); + describe('nodes and seeds', function() { + it('should add/set nodes/seeds', () => { + // we need 3. + const hosts = [ + getRandomIPv4(), + getRandomIPv4(), + getRandomIPv4() + ]; - const src = getRandomNetAddr(); - const addrs = []; - const entries = []; + const key = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; - for (let i = 0; i < 10; i++) { - const addr = getRandomNetAddr(); - hosts.add(addr, src); - const entry = hosts.map.get(addr.hostname); - entries.push(entry); - addrs.push(addr); - } + const tests = [ + // DNS Nodes + { + host: 'example.com', + hostname: 'example.com', + expected: { + addr: null, + dnsNodes: 1, + nodes: 0, + map: 0 + } + }, + // HSD Node w/o port - plaintext + { + host: hosts[0], + hostname: hosts[0], + expected: { + addr: { port: mainnet.port, host: hosts[0] }, + dnsNodes: 0, + nodes: 1, + map: 1 + } + }, + // HSD Node w/o port - brontide + { + host: hosts[1], + hostname: `${key}@${hosts[1]}`, + expected: { + addr: { host: hosts[1], port: mainnet.brontidePort }, + dnsNodes: 0, + nodes: 1, + map: 1 + } + }, + // HSD Node with port + { + host: hosts[2], + hostname: `${hosts[2]}:${mainnet.port + 1}`, + expected: { + addr: { host: hosts[2], port: mainnet.port + 1 }, + dnsNodes: 0, + nodes: 1, + map: 1 + } + } + ]; - const A_DAY = 24 * 60 * 60; + const allHosts = tests.map(t => t.hostname); + const sumExpected = tests.reduce((p, c) => { + p.dnsNodes += c.expected.dnsNodes; + p.nodes += c.expected.nodes; + p.map += c.expected.map; + return p; + }, { + dnsNodes: 0, + nodes: 0, + map: 0 + }); - // address from the future? - entries[0].addr.time = util.now() + 30 * 60; + for (const test of tests) { + const hosts = new HostList(); + const {expected} = test; - entries[1].addr.time = 0; + const addr = hosts.addNode(test.hostname); - // too old - entries[2].addr.time = util.now() - HostList.HORIZON_DAYS * A_DAY - 1; + if (expected.addr == null) + assert.strictEqual(addr, null); - // many attempts, no success - entries[3].attempts = HostList.RETRIES; + if (expected.addr != null) { + assert.strictEqual(addr.host, expected.addr.host); + assert.strictEqual(addr.port, expected.addr.port); + } - // last success got old. - // and we failed max times. - entries[4].lastSuccess = util.now() - HostList.MIN_FAIL_DAYS * A_DAY - 1; - entries[4].attempts = HostList.MAX_FAILURES; - - // last attempt in last minute - entries[5].lastAttempt = util.now(); - - entries[6].lastSuccess = entries[5].lastSuccess; - entries[7].lastSuccess = entries[5].lastSuccess + A_DAY; - - for (let i = 0; i < entries.length; i++) { - const entry = entries[i]; - - if (i < 5) - assert.strictEqual(hosts.isStale(entry), true); - else - assert.strictEqual(hosts.isStale(entry), false); - } - }); - - it('should evict entry from fresh bucket', () => { - const hosts = new HostList(); - const bucket = hosts.fresh[0]; - - const src = getRandomNetAddr(); - const entries = []; - - // Sort them young -> old - for (let i = 0; i < 10; i++) { - const entry = new HostEntry(getRandomNetAddr(), src); - entry.addr.time = util.now() - i; - entry.refCount = 1; - bucket.set(entry.key(), entry); - hosts.map.set(entry.key(), entry); - entries.push(entry); - hosts.totalFresh++; - } - - { - const staleEntry = entries[0]; - const expectedEvicted = entries[entries.length - 1]; - - // stales are evicted anyway. - staleEntry.addr.time = 0; - - // so we evict 2. - assert.strictEqual(hosts.isStale(staleEntry), true); - assert.strictEqual(bucket.has(staleEntry.key()), true); - assert.strictEqual(bucket.has(expectedEvicted.key()), true); - assert.strictEqual(hosts.map.has(staleEntry.key()), true); - hosts.evictFresh(bucket); - assert.strictEqual(bucket.has(staleEntry.key()), false); - assert.strictEqual(bucket.has(expectedEvicted.key()), false); - assert.strictEqual(hosts.map.has(staleEntry.key()), false); - } - - { - // evict older even if it's stale but is in another bucket as well.? - const staleEntry = entries[1]; - const expectedEvicted = entries[entries.length - 2]; - - staleEntry.attempts = HostList.RETRIES; - staleEntry.refCount = 2; - - assert.strictEqual(hosts.isStale(staleEntry), true); - - assert.strictEqual(bucket.has(staleEntry.key()), true); - assert.strictEqual(bucket.has(expectedEvicted.key()), true); - assert.strictEqual(hosts.map.has(staleEntry.key()), true); - hosts.evictFresh(bucket); - assert.strictEqual(bucket.has(staleEntry.key()), false); - assert.strictEqual(bucket.has(expectedEvicted.key()), false); - assert.strictEqual(hosts.map.has(staleEntry.key()), true); - } - - { - const expectedEvicted = entries[entries.length - 3]; - expectedEvicted.refCount = 2; - - assert.strictEqual(bucket.has(expectedEvicted.key()), true); - assert.strictEqual(hosts.map.has(expectedEvicted.key()), true); - hosts.evictFresh(bucket); - assert.strictEqual(bucket.has(expectedEvicted.key()), false); - assert.strictEqual(hosts.map.has(expectedEvicted.key()), true); - } - - assert.strictEqual(bucket.size, 5); - for (let i = entries.length - 4; i > 1; i--) { - const entry = entries[i]; - assert.strictEqual(bucket.has(entry.key()), true); - assert.strictEqual(hosts.map.has(entry.key()), true); - hosts.evictFresh(bucket); - assert.strictEqual(bucket.has(entry.key()), false); - assert.strictEqual(hosts.map.has(entry.key()), false); - } - - assert.strictEqual(bucket.size, 0); - hosts.evictFresh(bucket); - assert.strictEqual(bucket.size, 0); - }); - - it('should return array of entries', () => { - const hosts = new HostList(); - - const src = getRandomNetAddr(); - const addrs = []; - const entries = []; - - for (let i = 0; i < 20; i++) { - const addr = getRandomNetAddr(); - addrs.push(addr); - hosts.add(addr, src); - entries.push(hosts.map.get(addr.hostname)); - } - - // have first 2 entries stale. - entries[0].addr.time = 0; - entries[1].addr.time = util.now() + 20 * 60; - - const arr = hosts.toArray(); - const set = new Set(arr); - - assert.strictEqual(arr.length, entries.length - 2); - assert.strictEqual(set.size, entries.length - 2); - - for (let i = 0; i < 2; i++) - assert.strictEqual(set.has(addrs[i]), false); - - for (let i = 2; i < entries.length; i++) - assert.strictEqual(set.has(addrs[i]), true); - }); - - it('should get host', () => { - const add2bucket = (hosts, bucketIndex, entry, fresh = true) => { - if (fresh) { - assert(bucketIndex < hosts.maxFreshBuckets); - entry.refCount++; - hosts.totalFresh++; - hosts.map.set(entry.key(), entry); - hosts.fresh[bucketIndex].set(entry.key(), entry); - return; + assert.strictEqual(hosts.dnsNodes.length, expected.dnsNodes); + assert.strictEqual(hosts.nodes.length, expected.nodes); + assert.strictEqual(hosts.map.size, expected.map); } - assert(bucketIndex < hosts.maxUsedBuckets); - assert(entry.refCount === 0); - entry.used = true; - hosts.map.set(entry.key(), entry); - hosts.used[bucketIndex].push(entry); - hosts.totalUsed++; + // set all nodes + { + const hosts = new HostList(); + hosts.setNodes(allHosts); + assert.strictEqual(hosts.dnsNodes.length, sumExpected.dnsNodes); + assert.strictEqual(hosts.nodes.length, sumExpected.nodes); + assert.strictEqual(hosts.map.size, sumExpected.map); + } + + for (const test of tests) { + const hosts = new HostList(); + const {expected} = test; + + const addr = hosts.addSeed(test.hostname); + + if (expected.addr == null) + assert.strictEqual(addr, null); + + if (expected.addr != null) { + assert.strictEqual(addr.host, expected.addr.host); + assert.strictEqual(addr.port, expected.addr.port); + } + + assert.strictEqual(hosts.dnsSeeds.length, expected.dnsNodes); + assert.strictEqual(hosts.map.size, expected.map); + } + + { + const hosts = new HostList(); + + hosts.setSeeds(allHosts); + assert.strictEqual(hosts.dnsSeeds.length, sumExpected.dnsNodes); + assert.strictEqual(hosts.map.size, sumExpected.map); + } + }); + + it('should remove node', () => { + const hosts = new HostList(); + const key = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; + const ips = [getRandomIPv4(), getRandomIPv4(), getRandomIPv4()]; + const nodes = [ + ips[0], + `${key}@${ips[1]}`, + `${ips[2]}:1000`, + `${ips[2]}:2000` + ]; + + for (const node of nodes) + assert(hosts.addNode(node)); + + assert.strictEqual(hosts.nodes.length, nodes.length); + + for (const node of nodes.reverse()) + assert(hosts.removeNode(node)); + + assert.strictEqual(hosts.removeNode(nodes[0]), false); + assert.strictEqual(hosts.nodes.length, 0); + }); + }); + + describe('Local addresses', function() { + it('should add push/local addresses', () => { + const services = 1000; + const hosts = new HostList({ services }); + + const unroutable = [ + '127.0.0.1', + '127.1.1.1', + '192.168.1.1', + '10.10.10.10' + ]; + + const routable = [ + getRandomIPv4(), + getRandomIPv4() + ]; + + assert.strictEqual(hosts.local.size, 0); + + // unroutable local addresses must not get added. + for (const host of unroutable) { + const port = regtest.port; + const score = HostList.scores.NONE; + const res = hosts.addLocal(host, port, null, score); + assert.strictEqual(res, false); + } + + assert.strictEqual(hosts.local.size, 0); + + const SCORE = HostList.scores.MANUAL; + // these must get added. + for (const host of routable) { + const port = regtest.port; + const res1 = hosts.addLocal(host, port, SCORE); + assert.strictEqual(res1, true); + + // add brontide versions + const bport = regtest.brontidePort; + const res2 = hosts.addLocal(host, bport, SCORE); + assert.strictEqual(res2, true); + } + + // one brontide and one plaintext + assert.strictEqual(hosts.local.size, routable.length * 2); + + const added = hosts.addLocal(routable[0], regtest.port, SCORE); + assert.strictEqual(added, false); + + for (const laddr of hosts.local.values()) { + assert.strictEqual(laddr.addr.services, services); + assert.strictEqual(laddr.score, SCORE); + assert.strictEqual(laddr.type, SCORE); + } + }); + + // w/o src it will only take into account the score. + it('should get local w/o src', () => { + const services = 1000; + const port = regtest.port; + + const addrs = []; + + // w/o key + for (let i = HostList.scores.NONE; i < HostList.scores.MAX - 1; i++) { + const address = new NetAddress({ + host: getRandomIPv4(), + port: port, + services: services + }); + addrs.push([address, i]); + } + + // max but with key + addrs.push([new NetAddress({ + host: getRandomIPv4(), + port: port, + services: services, + key: Buffer.alloc(33, 1) + }), HostList.scores.MAX]); + + const max = addrs[HostList.scores.MAX - 2]; + + const hosts = getHostsFromLocals(addrs); + const local = hosts.getLocal(); + + assert.strictEqual(local, max[0]); + }); + + it('should get local (score)', () => { + // NOTE: main network ignores everything other than MANUAL type. + // - scores.IF - addresses from the network interfaces. + // - scores.BIND - Listening IP. (publicHost/publicPort is MANUAL) + // - scores.DNS - Domain that needs to be resolved. + // - scores.UPNP - UPNP discovered address. + + const scores = HostList.scores; + + // same NET type but different scores: + const hostsByScore = [ + [getRandomIPv4(), scores.NONE], + [getRandomIPv4(), scores.IF], + [getRandomIPv4(), scores.BIND], + [getRandomIPv4(), scores.DNS], + [getRandomIPv4(), scores.UPNP], + [getRandomIPv4(), scores.MANUAL] + ]; + + const naddrsByScore = hostsByScore.map(([h, s]) => { + return [getRandomNetAddr(), s]; + }); + + // testnet/regtest + for (let type = scores.NONE; type < scores.MAX; type++) { + const addrs = naddrsByScore.slice(scores.NONE, type + 1); + const hosts = getHostsFromLocals(addrs, { network: regtest }); + const src = getRandomNetAddr(); + const best = hosts.getLocal(src); + assert.strictEqual(best, naddrsByScore[type][0]); + } + + // mainnet + { + const hosts = getHostsFromLocals(naddrsByScore, { network: mainnet }); + const src = getRandomNetAddr(mainnet); + const best = hosts.getLocal(src); + assert.strictEqual(best, naddrsByScore[scores.MANUAL][0]); + } + + { + // everything below MANUAL is skipped on main network. + const addrs = naddrsByScore.slice(scores.NONE, scores.UPNP); + const hosts = getHostsFromLocals(addrs, { network: mainnet }); + const src = getRandomNetAddr(mainnet); + const best = hosts.getLocal(src); + assert.strictEqual(best, null); + } + }); + + it('should get local (reachability)', () => { + // If we have multiple public host/ports (e.g. IPv4 and Ipv6) + // depending who is connecting to us, we will choose different + // address to advertise. + + // with src it will take into account the reachability score. + // See: binet.getReachability + // TLDR: + // UNREACHABLE = 0 + // < DEFAULT = 1 - non-ipv4 -> ipv4, onion -> ipv6, ... + // < TEREDO = 2 - teredo -> teredo, teredo -> ipv6 + // < IPV6_WEAK = 3 - ipv4 -> ipv6 tunnels, ipv6 -> teredo + // < IPV4 = 4 - ipv4 -> ipv4, ipv4 -> others + // < IPV6_STRONG = 5 - ipv6 -> ipv6 + // < PRIVATE = 6 - ONION -> ONION + + const {MANUAL} = HostList.scores; + + // same score (MANUAL), different reachability: + // remote(src) => [ local(dest)... ] - sorted by reachability scores. + const reachabilityMap = { + // unreachable => anything - will be UNREACHABLE = 0. + [getRandomTEREDO()]: [ + getRandomIPv4(), // DEFAULT = 1 + getRandomOnion(), // DEFAULT = 1 + getRandomTEREDO(), // TEREDO = 2 + getRandomIPv6() // TEREDO = 2 + ], + [getRandomIPv4()]: [ + getRandomIPv4(), // IPV4 = 4 + getRandomOnion(), // IPV4 = 4 + getRandomTEREDO(), // IPV4 = 4 + getRandomIPv6() // IPV4 = 4 + ], + [getRandomIPv6()]: [ + getRandomOnion(), // DEFAULT = 1 + getRandomIPv4(), // DEFAULT = 1 + getRandomTEREDO(), // IPV6_WEAK = 3 + getRandomIPv6() // IPV6_STRONG = 5 + ], + [getRandomOnion()]: [ + getRandomIPv4(), // DEFAULT = 1 + getRandomTEREDO(), // DEFAULT = 1 + getRandomIPv6(), // DEFAULT = 1 + getRandomOnion() // PRIVATE = 6 + ] + }; + + for (const [rawSrc, rawDests] of Object.entries(reachabilityMap)) { + const dests = rawDests.map((dest) => { + return [NetAddress.fromHostname(dest, mainnet), MANUAL]; + }); + + for (let i = 0; i < dests.length; i++) { + const addrs = dests.slice(0, i + 1); + const expected = addrs[addrs.length - 1]; + + // Because getLocal will choose first with the same score, + // we make the "best" choice (because of the sorting) at first. + addrs[addrs.length - 1] = addrs[0]; + addrs[0] = expected; + + const hosts = getHostsFromLocals(addrs); + const src = NetAddress.fromHostname(rawSrc, mainnet); + const best = hosts.getLocal(src); + + assert.strictEqual(best, expected[0]); + } + } + }); + + it('should not get local (skip brontide)', () => { + const {MANUAL} = HostList.scores; + + // skip with key + const src = getRandomIPv4(); + const KEY = base32.encode(Buffer.alloc(33, 1)); + const rawDests = [ + `${KEY}@${getRandomIPv4()}:${mainnet.brontidePort}`, + `${KEY}@${getRandomIPv4()}:${mainnet.brontidePort}` + ]; + + const dests = rawDests.map((d) => { + return [NetAddress.fromHostname(d, mainnet), MANUAL]; + }); + + const hosts = getHostsFromLocals(dests); + const best = hosts.getLocal(src); + assert.strictEqual(best, null); + }); + + it('should mark local', () => { + const {scores} = HostList; + + const rawDests = [ + [getRandomIPv4(), scores.IF], + [getRandomIPv4(), scores.BIND] + ]; + + const dests = rawDests.map(([h, s]) => { + return [NetAddress.fromHostname(h, regtest), s]; + }); + + const hosts = getHostsFromLocals(dests, { network: regtest }); + + { + const addr = getRandomNetAddr(); + const marked = hosts.markLocal(addr); + + assert.strictEqual(marked, false); + } + + { + // we should get BIND, because BIND > IF + const addr = getRandomNetAddr(); + const local = hosts.getLocal(addr); + assert.strictEqual(local, dests[1][0]); + } + + { + // with markLocal IF should get the same score (type remains). + hosts.markLocal(dests[0][0]); + const addr = getRandomNetAddr(); + const local = hosts.getLocal(addr); + assert.strictEqual(local, dests[0][0]); + } + }); + }); + + describe('Fresh bucket', function() { + it('should add fresh address', () => { + { + const hosts = new HostList(); + + // fresh, w/o src, not in the buckets + const addr = getRandomNetAddr(); + + assert.strictEqual(hosts.totalFresh, 0); + assert.strictEqual(hosts.needsFlush, false); + assert.strictEqual(hosts.map.size, 0); + assert.strictEqual(getFreshEntries(hosts).length, 0); + + hosts.add(addr); + + assert.strictEqual(hosts.totalFresh, 1); + assert.strictEqual(hosts.needsFlush, true); + assert.strictEqual(hosts.map.size, 1); + + const freshEntries = getFreshEntries(hosts); + assert.strictEqual(freshEntries.length, 1); + + const entry = freshEntries[0]; + assert.strictEqual(entry.addr, addr, 'Entry addr is not correct.'); + assert.strictEqual(entry.src, hosts.address, 'Entry src is not correct.'); + } + + { + const hosts = new HostList(); + const addr = getRandomNetAddr(); + const src = getRandomNetAddr(); + + hosts.add(addr, src); + const freshEntries = getFreshEntries(hosts); + assert.strictEqual(freshEntries.length, 1); + + const entry = freshEntries[0]; + assert.strictEqual(entry.addr, addr, 'Entry addr is not correct.'); + assert.strictEqual(entry.src, src, 'Entry src is not correct.'); + assert.strictEqual(entry.refCount, 1); + assert.strictEqual(hosts.map.size, 1); + } + }); + + it('should add address (limits)', () => { + // Full Bucket? + { + const hosts = new HostList(); + const addr = getRandomNetAddr(); + const src = getRandomNetAddr(); + + let evicted = false; + + // always return first bucket for this test. + hosts.freshBucket = function() { + return this.fresh[0]; + }; + + hosts.evictFresh = function() { + evicted = true; + }; + + // Fill first bucket. + for (let i = 0; i < hosts.maxEntries; i++) { + const addr = getRandomNetAddr(); + const added = hosts.add(addr, src); + assert.strictEqual(added, true); + assert.strictEqual(evicted, false); + } + + const added = hosts.add(addr, src); + assert.strictEqual(added, true); + assert.strictEqual(evicted, true); + } + + // Don't insert if entry is in a bucket. + { + const hosts = new HostList(); + const addr = getRandomNetAddr(); + const src = getRandomNetAddr(); + const entry = new HostList.HostEntry(addr, src); + + // insert entry in every bucket for this test. + for (const bucket of hosts.fresh) + bucket.set(entry.key(), entry); + + const added = hosts.add(addr, src); + assert.strictEqual(added, false); + } + }); + + it('should add seen address', () => { + const hosts = new HostList(); + + // make sure we don't insert into the same bucket twice. (refcount test) + let index = 0; + hosts.freshBucket = function () { + return this.fresh[index++]; + }; + + // get addr clone that can be added (Online requirements) + const cloneAddr = (addr) => { + const addr2 = addr.clone(); + // just make sure this is < 3 * 60 * 60 (Online requirements + // with & without penalty) + addr2.time = util.now() + 60 * 60; + return addr2; + }; + + const addr = getRandomNetAddr(); + const src = getRandomNetAddr(); + addr.services = 0x01; + const added = hosts.add(addr, src); + assert.strictEqual(added, true); + assert.strictEqual(hosts.needsFlush, true); + hosts.needsFlush = false; + + const entries = getFreshEntries(hosts); + const entry = entries[0]; + assert.strictEqual(entries.length, 1); + assert.strictEqual(entry.addr.services, 0x01); + + // don't update - no new info (service will always get updated.) + { + const addr2 = addr.clone(); + addr2.services = 0x02; + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + + const entries = getFreshEntries(hosts); + const entry = entries[0]; + assert.strictEqual(entries.length, 1); + assert.strictEqual(entry.addr.services, 0x03); + assert.strictEqual(hosts.needsFlush, false); + } + + // update refCount. (we only have 1 refCount, increase up to 8) + { + const srcs = []; + for (let i = 0; i < 7; i++) + srcs.push(getRandomNetAddr()); + + const factors = []; + const _random = hosts.random; + const random = function (factor) { + factors.push(factor); + return 0; + }; + hosts.random = random; + + const addr2 = cloneAddr(addr); + + // when we have 1 ref, so probability of adding second one + // is 50% (1/2). + // then we have 2 refs, probability of adding will be 25% (1/4). + // ... until we have 8 refs. (last one being 1/128) + // Because we are reusing src, we will get same bucket + // so it will only get added once. + let added = 0; + for (let i = 0; i < 7; i++) { + const res = hosts.add(addr2, srcs[i]); + + // our custom random method always returns 0. + assert.strictEqual(res, true); + added++; + } + + // make sure factors are calculated properly. + assert.strictEqual(factors.length, 7); + + for (let i = 0; i < 7; i++) + assert.strictEqual(factors[i], 1 << (i + 1)); + + // at this point address should be in another bucket as well. + assert.strictEqual(added, 7); + assert.strictEqual(entry.refCount, 8); + const entries = getFreshEntries(hosts); + assert.strictEqual(entries.length, 8); + assert.strictEqual(hosts.needsFlush, true); + hosts.needsFlush = false; + hosts.random = _random; + } + + // should fail with max ref + { + const _refCount = entry.refCount; + entry.refCount = HostList.MAX_REFS; + const addr2 = cloneAddr(addr); + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + entry.refCount = _refCount; + assert.strictEqual(hosts.needsFlush, false); + } + + // should fail if it's used + { + entry.used = true; + const addr2 = cloneAddr(addr); + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + assert.strictEqual(hosts.needsFlush, false); + entry.used = false; + } + }); + + it('should add address (update time)', () => { + const getHosts = (time) => { + const hosts = new HostList(); + const addr = getRandomNetAddr(); + const src = getRandomNetAddr(); + + if (time) + addr.time = time; + + const added = hosts.add(addr, src); + assert.strictEqual(added, true); + assert.strictEqual(hosts.needsFlush, true); + hosts.needsFlush = false; + + const entries = getFreshEntries(hosts); + assert.strictEqual(entries.length, 1); + const entry = hosts.map.get(addr.hostname); + + // make sure we stop after updating time. + entries[0].used = true; + + return [hosts, entry, addr, src]; + }; + + // Update time - Online? + { + // a week ago + const [hosts, entry, addr, src] = getHosts(util.now() - 7 * 24 * 60 * 60); + const addr2 = addr.clone(); + + // a day ago (interval is a day, + // so we update if a day and 2 hrs have passed). + addr2.time = util.now() - 24 * 60 * 60; + + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + assert.strictEqual(entry.addr.time, addr2.time); + assert.strictEqual(hosts.needsFlush, true); + } + + { + // a day ago + const [hosts, entry, addr, src] = getHosts(util.now() - 24 * 60 * 60); + + const addr2 = addr.clone(); + + // now (interval becomes an hour, so instead we update if 3 hrs passed. + addr2.time = util.now(); + + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + assert.strictEqual(entry.addr.time, addr2.time); + assert.strictEqual(hosts.needsFlush, true); + } + + // Don't update + { + // a week ago + const weekAgo = util.now() - 7 * 24 * 60 * 60; + const sixDaysAgo = util.now() - 6 * 24 * 60 * 60 + 1; // and a second + + const [hosts, entry, addr, src] = getHosts(weekAgo); + + const addr2 = addr.clone(); + // 6 days ago (exactly 24 hrs after) because 2 hrs is penalty, + // we don't update. + addr2.time = sixDaysAgo; + + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + assert.strictEqual(entry.addr.time, weekAgo); + assert.strictEqual(hosts.needsFlush, false); + } + + // Update, because we are the ones inserting. + { + // a week ago + const weekAgo = util.now() - 7 * 24 * 60 * 60; + const sixDaysAgo = util.now() - 6 * 24 * 60 * 60 + 1; // and a second + + const [hosts, entry, addr] = getHosts(weekAgo); + + const addr2 = addr.clone(); + // 6 days ago (exactly 24 hrs after) because 2 hrs is penalty, + // we don't update. + addr2.time = sixDaysAgo; + + const added = hosts.add(addr2); + assert.strictEqual(added, false); + assert.strictEqual(entry.addr.time, sixDaysAgo); + assert.strictEqual(hosts.needsFlush, true); + } + + // Online - but still not updating (less than 3 hrs) + { + // now vs 3 hrs ago (exactly) + const now = util.now(); + const threeHoursAgo = now - 3 * 60 * 60; + + const [hosts, entry, addr, src] = getHosts(threeHoursAgo); + + const addr2 = addr.clone(); + addr2.time = now; + + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + assert.strictEqual(entry.addr.time, threeHoursAgo); + assert.strictEqual(hosts.needsFlush, false); + } + + { + // now vs 3 hrs and a second ago + const now = util.now(); + const threeHoursAgo = now - 1 - 3 * 60 * 60; + + const [hosts, entry, addr, src] = getHosts(threeHoursAgo); + + const addr2 = addr.clone(); + addr2.time = now; + + const added = hosts.add(addr2, src); + assert.strictEqual(added, false); + assert.strictEqual(entry.addr.time, now); + assert.strictEqual(hosts.needsFlush, true); + } + }); + + it('should evict entry from fresh bucket', () => { + const hosts = new HostList(); + const bucket = hosts.fresh[0]; + + const src = getRandomNetAddr(); + const entries = []; + + // Sort them young -> old + for (let i = 0; i < 10; i++) { + const entry = new HostEntry(getRandomNetAddr(), src); + entry.addr.time = util.now() - i; + entry.refCount = 1; + bucket.set(entry.key(), entry); + hosts.map.set(entry.key(), entry); + entries.push(entry); + hosts.totalFresh++; + } + + { + const staleEntry = entries[0]; + const expectedEvicted = entries[entries.length - 1]; + + // stales are evicted anyway. + staleEntry.addr.time = 0; + + // so we evict 2. + assert.strictEqual(hosts.isStale(staleEntry), true); + assert.strictEqual(bucket.has(staleEntry.key()), true); + assert.strictEqual(bucket.has(expectedEvicted.key()), true); + assert.strictEqual(hosts.map.has(staleEntry.key()), true); + hosts.evictFresh(bucket); + assert.strictEqual(bucket.has(staleEntry.key()), false); + assert.strictEqual(bucket.has(expectedEvicted.key()), false); + assert.strictEqual(hosts.map.has(staleEntry.key()), false); + } + + { + // evict older even if it's stale but is in another bucket as well.? + const staleEntry = entries[1]; + const expectedEvicted = entries[entries.length - 2]; + + staleEntry.attempts = HostList.RETRIES; + staleEntry.refCount = 2; + + assert.strictEqual(hosts.isStale(staleEntry), true); + + assert.strictEqual(bucket.has(staleEntry.key()), true); + assert.strictEqual(bucket.has(expectedEvicted.key()), true); + assert.strictEqual(hosts.map.has(staleEntry.key()), true); + hosts.evictFresh(bucket); + assert.strictEqual(bucket.has(staleEntry.key()), false); + assert.strictEqual(bucket.has(expectedEvicted.key()), false); + assert.strictEqual(hosts.map.has(staleEntry.key()), true); + } + + { + const expectedEvicted = entries[entries.length - 3]; + expectedEvicted.refCount = 2; + + assert.strictEqual(bucket.has(expectedEvicted.key()), true); + assert.strictEqual(hosts.map.has(expectedEvicted.key()), true); + hosts.evictFresh(bucket); + assert.strictEqual(bucket.has(expectedEvicted.key()), false); + assert.strictEqual(hosts.map.has(expectedEvicted.key()), true); + } + + assert.strictEqual(bucket.size, 5); + for (let i = entries.length - 4; i > 1; i--) { + const entry = entries[i]; + assert.strictEqual(bucket.has(entry.key()), true); + assert.strictEqual(hosts.map.has(entry.key()), true); + hosts.evictFresh(bucket); + assert.strictEqual(bucket.has(entry.key()), false); + assert.strictEqual(hosts.map.has(entry.key()), false); + } + + assert.strictEqual(bucket.size, 0); + hosts.evictFresh(bucket); + assert.strictEqual(bucket.size, 0); + }); + }); + + describe('Host manipulation (used/fresh)', function() { + it('should mark attempt', () => { + const hosts = new HostList(); + + // if we don't have the entry. + { + const addr = getRandomIPv4(); + hosts.markAttempt(addr); + } + + const src = getRandomNetAddr(); + const addr = getRandomNetAddr(); + + hosts.add(addr, src); + + const entry = hosts.map.get(addr.hostname); + assert.strictEqual(entry.attempts, 0); + assert.strictEqual(entry.lastAttempt, 0); + + hosts.markAttempt(addr.hostname); + assert.strictEqual(entry.attempts, 1); + assert(entry.lastAttempt > util.now() - 10); + }); + + it('should mark success', () => { + const hosts = new HostList(); + + // we don't have entry. + { + const addr = getRandomIPv4(); + hosts.markSuccess(addr); + } + + // Don't update time, it's recent. + { + const src = getRandomNetAddr(); + const addr = getRandomNetAddr(); + const oldTime = util.now() - 10 * 60; // last connection 11 minutes ago. + addr.time = oldTime; + + hosts.add(addr, src); + hosts.markSuccess(addr.hostname); + + const entry = hosts.map.get(addr.hostname); + assert.strictEqual(entry.addr.time, oldTime); + } + + // we update time. + const src = getRandomNetAddr(); + const addr = getRandomNetAddr(); + const oldTime = util.now() - 21 * 60; // last connection 21 minutes ago. + addr.time = oldTime; + + hosts.add(addr, src); + hosts.markSuccess(addr.hostname); + + const entry = hosts.map.get(addr.hostname); + assert(entry.addr.time > oldTime); + }); + + it('should remove host', () => { + const hosts = new HostList(); + + const src = getRandomNetAddr(); + const addrs = [ + getRandomNetAddr(), + getRandomNetAddr(), + getRandomNetAddr(), + getRandomNetAddr() + ]; + + const used = addrs.slice(2); + + for (const addr of addrs) + hosts.add(addr, src); + + for (const addr of used) + hosts.markAck(addr.hostname, 0); + + assert.strictEqual(hosts.map.size, addrs.length); + assert.strictEqual(hosts.totalUsed, 2); + assert.strictEqual(hosts.totalFresh, 2); + const fresh = getFreshEntries(hosts); + assert.strictEqual(fresh.length, 2); + + assert.strictEqual(hosts.remove(getRandomIPv6()), null); + for (const addr of addrs.reverse()) + assert.strictEqual(hosts.remove(addr.hostname), addr); + assert.strictEqual(hosts.totalUsed, 0); + assert.strictEqual(hosts.totalFresh, 0); + }); + + it('should mark ack', () => { + // we don't have the entry + { + const hosts = new HostList(); + const addr = getRandomIPv4(); + hosts.markAck(addr); + } + + // Should update services, lastSuccess, lastAttempt and attempts + // even if it's already in the used. + { + const hosts = new HostList(); + const naddr = getRandomNetAddr(); + const nsrc = getRandomNetAddr(); + + naddr.services = 0x01; + hosts.add(naddr, nsrc); + + const entry = hosts.map.get(naddr.hostname); + const oldLastAttempt = util.now() - 1000; + const oldLastSuccess = util.now() - 1000; + const oldAttempts = 2; + + entry.lastAttempt = oldLastAttempt; + entry.lastSuccess = oldLastSuccess; + entry.attempts = oldAttempts; + entry.used = true; + + hosts.markAck(naddr.hostname, 0x02); + + assert(entry.lastSuccess > oldLastSuccess); + assert(entry.lastAttempt > oldLastAttempt); + assert.strictEqual(entry.attempts, 0); + assert.strictEqual(entry.addr.services, 0x01 | 0x02); + } + + // Should remove from fresh + { + const hosts = new HostList(); + + // make sure we have all 8 refs. + let index = 0; + hosts.random = () => 0; + + // make sure we always get different bucket. + hosts.freshBucket = function () { + return this.fresh[index++]; + }; + + const addr = getRandomNetAddr(); + + for (let i = 0; i < 8; i++) { + const src = getRandomNetAddr(); + const addr2 = addr.clone(); + addr2.time = addr.time + i + 1; + + const added = hosts.add(addr2, src); + assert.strictEqual(added, true); + } + + assert.strictEqual(hosts.totalFresh, 1); + assert.strictEqual(hosts.totalUsed, 0); + assert.strictEqual(getFreshEntries(hosts).length, 8); + const entry = hosts.map.get(addr.hostname); + assert.strictEqual(entry.refCount, 8); + + hosts.markAck(addr.hostname); + + assert.strictEqual(getFreshEntries(hosts).length, 0); + assert.strictEqual(entry.refCount, 0); + assert.strictEqual(hosts.totalFresh, 0); + assert.strictEqual(hosts.totalUsed, 1); + assert.strictEqual(entry.used, true); + } + + // evict used + { + const hosts = new HostList(); + + const addr = getRandomNetAddr(); + const src = getRandomNetAddr(); + + hosts.add(addr, src); + const entry = hosts.map.get(addr.hostname); + const bucket = hosts.usedBucket(entry); + + assert.strictEqual(hosts.totalFresh, 1); + assert.strictEqual(hosts.totalUsed, 0); + + // add 64 entries to the bucket. + const entries = []; + let expectedEvicted = null; + for (let i = 0; i < hosts.maxEntries; i++) { + const addr = getRandomNetAddr(); + const src = getRandomNetAddr(); + addr.time = util.now() - (i); + + const entry = new HostEntry(addr, src); + entry.used = true; + bucket.push(entry); + entries.push(entry); + } + + expectedEvicted = entries[0]; + hosts.markAck(addr.hostname); + assert.strictEqual(bucket.tail, entry); + assert.strictEqual(expectedEvicted.used, true); + } + }); + + it('should check if entry is stale', () => { + const hosts = new HostList(); + + const src = getRandomNetAddr(); + const addrs = []; + const entries = []; + + for (let i = 0; i < 10; i++) { + const addr = getRandomNetAddr(); + hosts.add(addr, src); + const entry = hosts.map.get(addr.hostname); + entries.push(entry); + addrs.push(addr); + } + + const A_DAY = 24 * 60 * 60; + + // address from the future? + entries[0].addr.time = util.now() + 30 * 60; + + entries[1].addr.time = 0; + + // too old + entries[2].addr.time = util.now() - HostList.HORIZON_DAYS * A_DAY - 1; + + // many attempts, no success + entries[3].attempts = HostList.RETRIES; + + // last success got old. + // and we failed max times. + entries[4].lastSuccess = util.now() - HostList.MIN_FAIL_DAYS * A_DAY - 1; + entries[4].attempts = HostList.MAX_FAILURES; + + // last attempt in last minute + entries[5].lastAttempt = util.now(); + + entries[6].lastSuccess = entries[5].lastSuccess; + entries[7].lastSuccess = entries[5].lastSuccess + A_DAY; + + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]; + + if (i < 5) + assert.strictEqual(hosts.isStale(entry), true); + else + assert.strictEqual(hosts.isStale(entry), false); + } + }); + + it('should return array of entries', () => { + const hosts = new HostList(); + + const src = getRandomNetAddr(); + const addrs = []; + const entries = []; + + for (let i = 0; i < 20; i++) { + const addr = getRandomNetAddr(); + addrs.push(addr); + hosts.add(addr, src); + entries.push(hosts.map.get(addr.hostname)); + } + + // have first 2 entries stale. + entries[0].addr.time = 0; + entries[1].addr.time = util.now() + 20 * 60; + + const arr = hosts.toArray(); + const set = new Set(arr); + + assert.strictEqual(arr.length, entries.length - 2); + assert.strictEqual(set.size, entries.length - 2); + + for (let i = 0; i < 2; i++) + assert.strictEqual(set.has(addrs[i]), false); + + for (let i = 2; i < entries.length; i++) + assert.strictEqual(set.has(addrs[i]), true); + }); + + it('should get host', () => { + { + // empty + const hosts = new HostList(); + const host = hosts.getHost(); + assert.strictEqual(host, null); + } + + { + // fresh buckets + const hosts = new HostList(); + + const freshEntries = []; + + for (let i = 0; i < 100; i++) { + const entry = new HostEntry(getRandomNetAddr(), getRandomNetAddr()); + freshEntries.push(entry); + add2bucket(hosts, 0, entry, true); + } + + const found = hosts.getHost(); + assert.strictEqual(new Set(freshEntries).has(found), true); + } + + { + // used bucket - this is random. + const hosts = new HostList(); + // put 10 entries in the used. + const usedEntries = []; + + for (let i = 0; i < 100; i++) { + const entry = new HostEntry(getRandomNetAddr(), getRandomNetAddr()); + usedEntries.push(entry); + add2bucket(hosts, 0, usedEntries[i], false); + } + + const foundEntry = hosts.getHost(); + assert.strictEqual(new Set(usedEntries).has(foundEntry), true); + } + }); + }); + + describe('populate', function() { + const DNS_SEED = IP.fromHost('example.com'); + + let hosts; + beforeEach(() => { + hosts = new HostList(); + }); + + it('should populate', async () => { + let err; + try { + // only DNS name is a valid seed. + await hosts.populate(getRandomIPv4()); + } catch (e) { + err = e; + } + + assert(err); + assert.strictEqual(err.message, 'Resolved host passed.'); + + hosts.resolve = () => { + throw new Error('pop error'); + }; + + const failedAddrs = await hosts.populate(DNS_SEED); + assert.strictEqual(failedAddrs.length, 0); + + const addrs = [ + getRandomIPv4(), + getRandomIPv4(), + getRandomIPv4() + ]; + + hosts.resolve = () => addrs; + const resAddrs = await hosts.populate(DNS_SEED); + assert.strictEqual(resAddrs.length, addrs.length); + + for (const [i, resAddr] of resAddrs.entries()) + assert.strictEqual(resAddr.host, addrs[i]); + }); + + it('should populate seed', async () => { + const addrs = [ + getRandomIPv4(), + getRandomIPv4(), + getRandomIPv4() + ]; + + hosts.resolve = () => addrs; + + await hosts.populateSeed(DNS_SEED); + + assert.strictEqual(hosts.map.size, addrs.length); + for (const addr of addrs) + assert(hosts.map.has(`${addr}:${mainnet.port}`)); + + const fresh = getFreshEntries(hosts); + assert.strictEqual(fresh.length, addrs.length); + }); + + it('should populate node', async () => { + // Populate empty. + hosts.resolve = () => []; + await hosts.populateNode(DNS_SEED); + + assert.strictEqual(hosts.nodes.length, 0); + assert.strictEqual(hosts.map.size, 0); + + const addrs = [ + getRandomIPv4(), + getRandomIPv4(), + getRandomIPv4() + ]; + + // we just take first resolved IP as a Node. + hosts.resolve = () => addrs; + await hosts.populateNode(DNS_SEED); + assert.strictEqual(hosts.nodes.length, 1); + assert.strictEqual(hosts.map.size, 1); + + assert.strictEqual(hosts.nodes[0].host, addrs[0]); + assert(hosts.map.has(`${addrs[0]}:${mainnet.port}`)); + }); + + it('should discover seeds', async () => { + const seeds = { + 'example.com': [ + getRandomIPv4(), + getRandomIPv4() + ], + 'example.org': [ + getRandomIPv4(), + getRandomIPv4() + ] + }; + + hosts.resolve = host => seeds[host]; + + for (const seed of Object.keys(seeds)) + hosts.addSeed(seed); + + await hosts.discoverSeeds(); + + assert.strictEqual(hosts.map.size, 4); + + for (const ips of Object.values(seeds)) { + for (const ip of ips) + assert(hosts.map.has(`${ip}:${mainnet.port}`)); + } + }); + + it('should discover nodes', async () => { + const seeds = { + 'example.com': [ + getRandomIPv4(), + getRandomIPv4() + ], + 'example.org': [ + getRandomIPv4(), + getRandomIPv4() + ] + }; + + hosts.resolve = host => seeds[host]; + + for (const seed of Object.keys(seeds)) + hosts.addNode(seed); + + await hosts.discoverNodes(); + + assert.strictEqual(hosts.map.size, 2); + + for (const [i, ips] of Object.values(seeds).entries()) { + assert(hosts.map.has(`${ips[0]}:${mainnet.port}`)); + assert(hosts.nodes[i].host, ips[0]); + } + }); + }); + + describe('File', function() { + let testdir; + + beforeEach(async () => { + testdir = common.testdir('hostlist'); + + assert(await fs.mkdirp(testdir)); + }); + + afterEach(async () => { + await fs.rimraf(testdir); + testdir = null; + }); + + // Generate 1 entry per bucket from 0 to nFresh or nUsed. + const genEntries = (hosts, nFresh, nUsed) => { + const fresh = []; + const used = []; + + assert(nFresh < hosts.maxFreshBuckets); + assert(nUsed < hosts.maxUsedBuckets); + + for (let i = 0; i < nFresh; i++) { + const entry = new HostEntry(getRandomNetAddr(), getRandomNetAddr()); + add2bucket(hosts, i, entry, true); + fresh.push(entry); + } + + for (let i = 0; i < nUsed; i++) { + const entry = new HostEntry(getRandomNetAddr(), getRandomNetAddr()); + add2bucket(hosts, i, entry, false); + used.push(entry); + } + + return [fresh, used]; }; - { - // empty - const hosts = new HostList(); - const host = hosts.getHost(); - assert.strictEqual(host, null); - } + it('should reserialize JSON', () => { + const hosts = new HostList({ + // network: regtest + }); - { - // fresh buckets - const hosts = new HostList(); + genEntries(hosts, 10, 10); - const freshEntries = []; + const json = hosts.toJSON(); + const hosts2 = HostList.fromJSON({ + // network: regtest + }, json); - for (let i = 0; i < 100; i++) { - const entry = new HostEntry(getRandomNetAddr(), getRandomNetAddr()); - freshEntries.push(entry); - add2bucket(hosts, 0, entry, true); - } + assert.deepStrictEqual(hosts2.fresh, hosts.fresh); + assert.deepStrictEqual(hosts2.used, hosts.used); + assert.deepStrictEqual(hosts2.map, hosts.map); + assert.strictEqual(hosts2.map.size, 20); + assert.strictEqual(getFreshEntries(hosts2).length, 10); + assert.strictEqual(getUsedEntries(hosts2).length, 10); + }); - const found = hosts.getHost(); - assert.strictEqual(new Set(freshEntries).has(found), true); - } + it('should open empty', async () => { + const hosts = new HostList({ + network: regtest, + prefix: testdir + }); - { - // used bucket - this is random. - const hosts = new HostList(); - // put 10 entries in the used. - const usedEntries = []; + await hosts.open(); + assert.strictEqual(hosts.map.size, 0); + assert.strictEqual(getFreshEntries(hosts).length, 0); + assert.strictEqual(getUsedEntries(hosts).length, 0); + await hosts.close(); - for (let i = 0; i < 100; i++) { - const entry = new HostEntry(getRandomNetAddr(), getRandomNetAddr()); - usedEntries.push(entry); - add2bucket(hosts, 0, usedEntries[i], false); - } + // it does not need flushing. + assert(!await fs.exists(path.join(testdir, 'hosts.json'))); + }); - const foundEntry = hosts.getHost(); - assert.strictEqual(new Set(usedEntries).has(foundEntry), true); - } + it('should create hosts.json', async () => { + const hosts = new HostList({ + network: regtest, + prefix: testdir, + memory: false + }); + + await hosts.open(); + + genEntries(hosts, 10, 10); + hosts.needsFlush = true; + assert.strictEqual(hosts.map.size, 20); + assert.strictEqual(getFreshEntries(hosts).length, 10); + assert.strictEqual(getUsedEntries(hosts).length, 10); + await hosts.close(); + + assert(await fs.exists(path.join(testdir, 'hosts.json'))); + + const hosts2 = new HostList({ + network: regtest, + prefix: testdir, + memory: false + }); + + await hosts2.open(); + assert.strictEqual(hosts2.map.size, 20); + assert.strictEqual(getFreshEntries(hosts2).length, 10); + assert.strictEqual(getUsedEntries(hosts2).length, 10); + await hosts2.close(); + }); }); }); From 721dd9a314819c7199015525f8d50e6a1d90dce5 Mon Sep 17 00:00:00 2001 From: Roy Marples Date: Thu, 11 Jul 2019 17:41:49 +0100 Subject: [PATCH 09/17] pool: Use a shorter loop to get hosts for refilling peers Currently, pool::getHost() uses a fixed loop of 0-99 regardless of how many hosts it has in it's lists so it can use a percentage to factor in whether it should use an entry or not. Using network regtest and a single host of 127.0.0.1:14038, hsd burns a lot of CPU when refilling peers and this blocks incoming requests so much it times out. This is less noticeable the more hosts you have. Instead, use shorter loop of 10. This brings CPU utlisation on my NetBSD Xen DOMU on regtest with one host from 100% to 6% and connecting 4 to peers on testnet from 85% to 8%, allowing RPC commands to now work. Fixes #220. --- lib/net/pool.js | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/net/pool.js b/lib/net/pool.js index a280b3c3..f4531a2a 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -1717,8 +1717,6 @@ class Pool extends EventEmitter { } } } - - this.fillOutbound(); } /** @@ -3395,12 +3393,27 @@ class Pool extends EventEmitter { const services = this.options.getRequiredServices(); const now = this.network.now(); - for (let i = 0; i < 100; i++) { + // Calculate maximum number of hosts we can get. + let max = this.hosts.totalFresh + this.hosts.totalUsed; + + // We don't want to loop a lot here as it's expensive on CPU. + // If this gets high, such as 100, it could cause a local DoS + // for incoming RPC calls. + if (max > 10) + max = 10; + + // Work out a percentage based hit rate outside of the + // loop to save CPU. + // Subtract 1 because we're zero based. + const pc1 = max / 100; + const pc30 = (pc1 * 30) - 1; + const pc50 = (pc1 * 50) - 1; + + for (let i = 0; i < max; i++) { const entry = this.hosts.getHost(); if (!entry) break; - const addr = entry.addr; if (this.peers.has(addr.hostname)) @@ -3424,10 +3437,10 @@ class Pool extends EventEmitter { if (this.options.brontideOnly && !addr.hasKey()) continue; - if (i < 30 && now - entry.lastAttempt < 600) + if (i < pc30 && now - entry.lastAttempt < 600) continue; - if (i < 50 && addr.port !== this.network.port) + if (i < pc50 && addr.port !== this.network.port) continue; return entry.addr; From 03b962eb261da945239b6cb55c7d2355db3502c1 Mon Sep 17 00:00:00 2001 From: kilpatty Date: Tue, 2 Jul 2019 11:44:57 -0500 Subject: [PATCH 10/17] net: remove discoverExternal() from the pool This commit removes the discover external check in the pool. Currently we already store all local addresses that are sent to us from peers in the hostlist, so there is no reason to rely on an external service to check for our local/external address. This was removed from bitcoin in commit 845c86d. --- lib/net/pool.js | 47 ----------------------------------------------- 1 file changed, 47 deletions(-) diff --git a/lib/net/pool.js b/lib/net/pool.js index f4531a2a..90bb6270 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -342,7 +342,6 @@ class Pool extends EventEmitter { await this.hosts.open(); await this.discoverGateway(); - await this.discoverExternal(); await this.discoverSeeds(); await this.listen(); @@ -595,52 +594,6 @@ class Pool extends EventEmitter { } } - /** - * Attempt to discover external IP via DNS. - * @returns {Promise} - */ - - async discoverExternal() { - const port = this.options.publicPort; - - // Pointless if we're not listening. - if (!this.options.listen) - return; - - // Never hit a DNS server if - // we're using an outbound proxy. - if (this.options.proxy) - return; - - // Try not to hit this if we can avoid it. - if (this.hosts.local.size > 0) - return; - - let host4 = null; - - try { - host4 = await dns.getIPv4(2000); - } catch (e) { - this.logger.debug('Could not find external IPv4 (dns).'); - this.logger.debug(e); - } - - if (host4 && this.hosts.addLocal(host4, port, scores.DNS)) - this.logger.info('External IPv4 found (dns): %s.', host4); - - let host6 = null; - - try { - host6 = await dns.getIPv6(2000); - } catch (e) { - this.logger.debug('Could not find external IPv6 (dns).'); - this.logger.debug(e); - } - - if (host6 && this.hosts.addLocal(host6, port, scores.DNS)) - this.logger.info('External IPv6 found (dns): %s.', host6); - } - /** * Handle incoming connection. * @private From 4ee4a425edf7f9834c7789d5ea1fe11e427347fb Mon Sep 17 00:00:00 2001 From: kilpatty Date: Sat, 6 Jul 2019 01:01:16 -0500 Subject: [PATCH 11/17] net-address: Add get group function. This commit adds the getGroup function to net addresses. It allows for network addresses to be grouped into buckets, such that we can limit the number of outgoing connections per group to a certain amount. This will allow for increased connection diversity and reduced attack surface area. Co-authored-by: Nodari Chkuaselidze --- lib/net/hostlist.js | 65 ++-------------------- lib/net/netaddress.js | 106 ++++++++++++++++++++++++++++++++++++ test/net-netaddress-test.js | 97 +++++++++++++++++++++++++++++++++ 3 files changed, 207 insertions(+), 61 deletions(-) diff --git a/lib/net/hostlist.js b/lib/net/hostlist.js index 88920acd..d33debe6 100644 --- a/lib/net/hostlist.js +++ b/lib/net/hostlist.js @@ -476,8 +476,8 @@ class HostList { this.hash.init(); this.hash.update(this.key); - this.hash.update(groupKey(addr.raw)); - this.hash.update(groupKey(src.raw)); + this.hash.update(addr.getGroup()); + this.hash.update(src.getGroup()); const hash1 = this.hash.final(); const hash32 = bio.readU32(hash1, 0) % 64; @@ -486,7 +486,7 @@ class HostList { this.hash.init(); this.hash.update(this.key); - this.hash.update(groupKey(src.raw)); + this.hash.update(src.getGroup()); this.hash.update(this.hashbuf); const hash2 = this.hash.final(); @@ -521,7 +521,7 @@ class HostList { this.hash.init(); this.hash.update(this.key); - this.hash.update(groupKey(addr.raw)); + this.hash.update(addr.getGroup()); this.hash.update(this.hashbuf); const hash2 = this.hash.final(); @@ -1835,63 +1835,6 @@ function random(max) { return Math.floor(Math.random() * max); } -function groupKey(raw) { - // See: https://github.com/bitcoin/bitcoin/blob/e258ce7/src/netaddress.cpp#L413 - // Todo: Use IP->ASN mapping, see: - // https://github.com/bitcoin/bitcoin/blob/adea5e1/src/addrman.h#L274 - let type = 6; // NET_IPV6 - let start = 0; - let bits = 16; - let i = 0; - - if (IP.isLocal(raw)) { - type = 255; // NET_LOCAL - bits = 0; - } else if (!IP.isRoutable(raw)) { - type = 0; // NET_UNROUTABLE - bits = 0; - } else if (IP.isIPv4(raw) || IP.isRFC6145(raw) || IP.isRFC6052(raw)) { - type = 4; // NET_IPV4 - start = 12; - } else if (IP.isRFC3964(raw)) { - type = 4; // NET_IPV4 - start = 2; - } else if (IP.isRFC4380(raw)) { - const buf = Buffer.alloc(3); - buf[0] = 4; // NET_IPV4 - buf[1] = raw[12] ^ 0xff; - buf[2] = raw[13] ^ 0xff; - return buf; - } else if (IP.isOnion(raw)) { - type = 8; // NET_ONION - start = 6; - bits = 4; - } else if (raw[0] === 0x20 - && raw[1] === 0x01 - && raw[2] === 0x04 - && raw[3] === 0x70) { - bits = 36; - } else { - bits = 32; - } - - const out = Buffer.alloc(1 + ((bits + 7) >>> 3)); - - out[i++] = type; - - while (bits >= 8) { - out[i++] = raw[start++]; - bits -= 8; - } - - if (bits > 0) - out[i++] = raw[start] | ((1 << (8 - bits)) - 1); - - assert(i === out.length); - - return out; -} - /* * Expose */ diff --git a/lib/net/netaddress.js b/lib/net/netaddress.js index c23bb2a8..7f59d47f 100644 --- a/lib/net/netaddress.js +++ b/lib/net/netaddress.js @@ -125,6 +125,42 @@ class NetAddress extends bio.Struct { return IP.isIPv6(this.raw); } + /** + * Test whether the address is RFC3964. + * @returns {Boolean} + */ + + isRFC3964() { + return IP.isRFC3964(this.raw); + } + + /** + * Test whether the address is RFC4380. + * @returns {Boolean} + */ + + isRFC4380() { + return IP.isRFC4380(this.raw); + } + + /** + * Test whether the address is RFC6052. + * @returns {Boolean} + */ + + isRFC6052() { + return IP.isRFC6052(this.raw); + } + + /** + * Test whether the address is RFC6145. + * @returns {Boolean} + */ + + isRFC6145() { + return IP.isRFC6145(this.raw); + } + /** * Test whether the host is null. * @returns {Boolean} @@ -212,6 +248,15 @@ class NetAddress extends bio.Struct { return IP.getReachability(this.raw, dest.raw); } + /** + * Get the canonical identifier of our network group + * @returns {Buffer} + */ + + getGroup() { + return groupKey(this.raw); + } + /** * Set null host. */ @@ -493,6 +538,67 @@ NetAddress.DEFAULT_SERVICES = 0 | common.services.NETWORK | common.services.BLOOM; +/* + * Helpers + */ + +function groupKey(raw) { + // See: https://github.com/bitcoin/bitcoin/blob/e258ce7/src/netaddress.cpp#L413 + // Todo: Use IP->ASN mapping, see: + // https://github.com/bitcoin/bitcoin/blob/adea5e1/src/addrman.h#L274 + let type = IP.networks.INET6; // NET_IPV6 + let start = 0; + let bits = 16; + let i = 0; + + if (IP.isLocal(raw)) { + type = 255; // NET_LOCAL + bits = 0; + } else if (!IP.isRoutable(raw)) { + type = IP.networks.NONE; // NET_UNROUTABLE + bits = 0; + } else if (IP.isIPv4(raw) || IP.isRFC6145(raw) || IP.isRFC6052(raw)) { + type = IP.networks.INET4; // NET_IPV4 + start = 12; + } else if (IP.isRFC3964(raw)) { + type = IP.networks.INET4; // NET_IPV4 + start = 2; + } else if (IP.isRFC4380(raw)) { + const buf = Buffer.alloc(3); + buf[0] = IP.networks.INET4; // NET_IPV4 + buf[1] = raw[12] ^ 0xff; + buf[2] = raw[13] ^ 0xff; + return buf; + } else if (IP.isOnion(raw)) { + type = IP.networks.ONION; // NET_ONION + start = 6; + bits = 4; + } else if (raw[0] === 0x20 + && raw[1] === 0x01 + && raw[2] === 0x04 + && raw[3] === 0x70) { + bits = 36; + } else { + bits = 32; + } + + const out = Buffer.alloc(1 + ((bits + 7) >>> 3)); + + out[i++] = type; + + while (bits >= 8) { + out[i++] = raw[start++]; + bits -= 8; + } + + if (bits > 0) + out[i++] = raw[start] | ((1 << (8 - bits)) - 1); + + assert(i === out.length); + + return out; +} + /* * Expose */ diff --git a/test/net-netaddress-test.js b/test/net-netaddress-test.js index e500f578..222d2087 100644 --- a/test/net-netaddress-test.js +++ b/test/net-netaddress-test.js @@ -406,4 +406,101 @@ describe('NetAddress', function() { `${source}->${destination} - ${reachability}`); } }); + + it('should return the correct group', () => { + // Local -> !Routable() + assert.bufferEqual( + NetAddress.fromHost('127.0.0.1', 13038, null, 'testnet').getGroup(), + Buffer.from([0xff]) + ); + + // RFC1918 -> !Routable() + assert.bufferEqual( + NetAddress.fromHost('169.254.1.1', 13038, null, 'testnet').getGroup(), + Buffer.from([0]) + ); + + // IPv4 + assert.bufferEqual( + NetAddress.fromHost('1.2.3.4', 13038, null, 'testnet').getGroup(), + Buffer.from([1, 1, 2]) + ); + + // RFC6145 + assert.bufferEqual( + NetAddress.fromHost( + '::FFFF:0:102:304', + 13038, + null, + 'testnet' + ).getGroup(), + Buffer.from([1, 1, 2]) + ); + + // RFC6052 + assert.bufferEqual( + NetAddress.fromHost( + '64:FF9B::102:304', + 13038, + null, + 'testnet' + ).getGroup(), + Buffer.from([1, 1, 2]) + ); + + // RFC3964 + assert.bufferEqual( + NetAddress.fromHost( + '2002:102:304:9999:9999:9999:9999:9999', + 13038, + null, + 'testnet' + ).getGroup(), + Buffer.from([1, 1, 2]) + ); + + // RFC4380 + assert.bufferEqual( + NetAddress.fromHost( + '2001:0:9999:9999:9999:9999:FEFD:FCFB', + 13038, + null, + 'testnet' + ).getGroup(), + Buffer.from([1, 1, 2]) + ); + + // Tor + assert.bufferEqual( + NetAddress.fromHost( + 'FD87:D87E:EB43:edb1:8e4:3588:e546:35ca', + 13038, + null, + 'testnet' + ).getGroup(), + Buffer.from([3, 239]) + ); + + // he.net + assert.bufferEqual( + NetAddress.fromHost( + '2001:470:abcd:9999:9999:9999:9999:9999', + 13038, + null, + 'testnet' + ).getGroup(), + Buffer.from([2, 32, 1, 4, 112, 175]) + ); + + // IPv6 + assert.bufferEqual( + NetAddress.fromHost( + '2001:2001:9999:9999:9999:9999:9999:9999', + 13038, + null, + 'testnet' + ).getGroup(), + Buffer.from([2, 32, 1, 32, 1]) + ); + }); }); From f52e924cda59173ef63dd2635034fd954d09ce7e Mon Sep 17 00:00:00 2001 From: kilpatty Date: Sat, 6 Jul 2019 13:43:11 -0500 Subject: [PATCH 12/17] net-pool: Check outbound peers for same group before connecting This commit checks the outbound peers to see which group their network addresses lives in before connecting. By doing this, we prevent a node from connecting to all outbound addresses in a very close network group bunch. This helps prevent someone spinning up multiple nodes on a similar network e.g. AWS, GCP and using those to fill other nodes' outbounds. This commit just adds 2 operations in both addOutbound and addLoader. It first checks if the new address pull from hostlist belongs in the same group as an already connected peer. If it does not, it precedes as usual, and if the connection succeeds then it adds that new outbound group to the list of groups currently connected to. Co-authored-by: Nodari Chkuaselidze --- lib/net/pool.js | 12 +++++++++++- test/net-netaddress-test.js | 9 +++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/net/pool.js b/lib/net/pool.js index 90bb6270..2c4d2f32 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -10,7 +10,6 @@ const assert = require('bsert'); const EventEmitter = require('events'); const {Lock} = require('bmutex'); const IP = require('binet'); -const dns = require('bdns'); const tcp = require('btcp'); const UPNP = require('bupnp'); const socks = require('bsocks'); @@ -85,6 +84,7 @@ class Pool extends EventEmitter { this.pendingFilter = null; this.refillTimer = null; this.discoverTimer = null; + this.connectedGroups = new Set(); this.checkpoints = false; this.headerChain = new List(); @@ -663,6 +663,7 @@ class Pool extends EventEmitter { this.logger.info('Adding loader peer (%s).', peer.hostname()); this.peers.add(peer); + this.connectedGroups.add(addr.getGroup()); this.setLoader(peer); } @@ -3367,6 +3368,7 @@ class Pool extends EventEmitter { if (!entry) break; + const addr = entry.addr; if (this.peers.has(addr.hostname)) @@ -3390,6 +3392,10 @@ class Pool extends EventEmitter { if (this.options.brontideOnly && !addr.hasKey()) continue; + // Don't connect to outbound peers in the same group. + if (this.connectedGroups.has(addr.getGroup())) + continue; + if (i < pc30 && now - entry.lastAttempt < 600) continue; @@ -3428,6 +3434,7 @@ class Pool extends EventEmitter { const peer = this.createOutbound(addr); this.peers.add(peer); + this.connectedGroups.add(addr.getGroup()); this.emit('peer', peer); } @@ -3482,6 +3489,9 @@ class Pool extends EventEmitter { removePeer(peer) { this.peers.remove(peer); + if (peer.outbound) + this.connectedGroups.delete(peer.address.getGroup()); + for (const hash of peer.blockMap.keys()) this.resolveBlock(peer, hash); diff --git a/test/net-netaddress-test.js b/test/net-netaddress-test.js index 222d2087..fa4a275b 100644 --- a/test/net-netaddress-test.js +++ b/test/net-netaddress-test.js @@ -1,5 +1,14 @@ 'use strict'; +/* Parts of this software are based on bitcoin/bitcoin: + * Copyright (c) 2009-2019, The Bitcoin Core Developers (MIT License). + * Copyright (c) 2009-2019, The Bitcoin Developers (MIT License). + * https://github.com/bitcoin/bitcoin + * + * Resources: + * https://github.com/bitcoin/bitcoin/blob/46fc4d1a24c88e797d6080336e3828e45e39c3fd/src/test/netbase_tests.cpp + */ + const assert = require('bsert'); const NetAddress = require('../lib/net/netaddress'); const Network = require('../lib/protocol/network'); From a7d214ce569c3c3e16071ee6c637ec91554b819d Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Thu, 24 Feb 2022 19:52:19 +0400 Subject: [PATCH 13/17] pkg: update package-lock. --- package-lock.json | 479 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 475 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1787ba10..58d7903b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,8 +1,479 @@ { "name": "hsd", "version": "3.0.1", - "lockfileVersion": 1, + "lockfileVersion": 2, "requires": true, + "packages": { + "": { + "name": "hsd", + "version": "3.0.1", + "license": "MIT", + "dependencies": { + "bcfg": "~0.1.7", + "bcrypto": "~5.4.0", + "bdb": "~1.3.0", + "bdns": "~0.1.5", + "bevent": "~0.1.5", + "bfile": "~0.2.2", + "bfilter": "~1.0.5", + "bheep": "~0.1.5", + "binet": "~0.3.7", + "blgr": "~0.2.0", + "blru": "~0.1.6", + "blst": "~0.1.5", + "bmutex": "~0.1.6", + "bns": "~0.15.0", + "bsert": "~0.0.10", + "bsock": "~0.1.9", + "bsocks": "~0.2.6", + "btcp": "~0.1.5", + "buffer-map": "~0.0.7", + "bufio": "~1.0.7", + "bupnp": "~0.2.6", + "bval": "~0.1.6", + "bweb": "~0.1.10", + "goosig": "~0.10.0", + "hs-client": "~0.0.10", + "n64": "~0.2.10", + "urkel": "~0.7.0" + }, + "bin": { + "hs-seeder": "bin/hs-seeder", + "hs-wallet": "bin/hsw", + "hsd": "bin/hsd", + "hsd-cli": "bin/hsd-cli", + "hsd-node": "bin/node", + "hsd-spvnode": "bin/spvnode", + "hsw-cli": "bin/hsw-cli" + }, + "devDependencies": { + "bmocha": "^2.1.5" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bcfg": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/bcfg/-/bcfg-0.1.7.tgz", + "integrity": "sha512-+4beq5bXwfmxdcEoHYQsaXawh1qFzjLcRvPe5k5ww/NEWzZTm56Jk8LuPmfeGB7X584jZ8xGq6UgMaZnNDa5Ww==", + "dependencies": { + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bcrypto": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/bcrypto/-/bcrypto-5.4.0.tgz", + "integrity": "sha512-KDX2CR29o6ZoqpQndcCxFZAtYA1jDMnXU3jmCfzP44g++Cu7AHHtZN/JbrN/MXAg9SLvtQ8XISG+eVD9zH1+Jg==", + "hasInstallScript": true, + "dependencies": { + "bufio": "~1.0.7", + "loady": "~0.0.5" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bcurl": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/bcurl/-/bcurl-0.1.9.tgz", + "integrity": "sha512-WV9LKCqFPtmGwIOqHexJx3Mm/9H/G5bwSCZxJXq9WRrnVQmd58L+Ltxgp/2QicveDG6AgTfepP6JtNiYWbbeHQ==", + "dependencies": { + "brq": "~0.1.8", + "bsert": "~0.0.10", + "bsock": "~0.1.9" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bdb": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/bdb/-/bdb-1.3.0.tgz", + "integrity": "sha512-oJnWnHOTcnJhazwpEzQvPFtSR1IdHtS3PczuLY3klgZTTtRUbARX7tdphQS8iNUUwEVMfuO93eHDWwTICoeJlg==", + "hasInstallScript": true, + "dependencies": { + "bsert": "~0.0.10", + "loady": "~0.0.5" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/bdns": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/bdns/-/bdns-0.1.5.tgz", + "integrity": "sha512-LNVkfM7ynlAD0CvPvO9cKxW8YXt1KOCRQZlRsGZWeMyymUWVdHQpZudAzH9chaFAz6HiwAnQxwDemCKDPy6Mag==", + "dependencies": { + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bevent": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/bevent/-/bevent-0.1.5.tgz", + "integrity": "sha512-hs6T3BjndibrAmPSoKTHmKa3tz/c6Qgjv9iZw+tAoxuP6izfTCkzfltBQrW7SuK5xnY22gv9jCEf51+mRH+Qvg==", + "dependencies": { + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bfile": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/bfile/-/bfile-0.2.2.tgz", + "integrity": "sha512-X205SsJ7zFAnjeJ/pBLqDqF10x/4Su3pBy8UdVKw4hdGJk7t5pLoRi+uG4rPaDAClGbrEfT/06PGUbYiMYKzTg==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bfilter": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/bfilter/-/bfilter-1.0.5.tgz", + "integrity": "sha512-GupIidtCvLbKhXnA1sxvrwa+gh95qbjafy7P1U1x/2DHxNabXq4nGW0x3rmgzlJMYlVl+c8fMxoMRIwpKYlgcQ==", + "dependencies": { + "bsert": "~0.0.10", + "bufio": "~1.0.6", + "mrmr": "~0.1.6" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bheep": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/bheep/-/bheep-0.1.5.tgz", + "integrity": "sha512-0KR5Zi8hgJBKL35+aYzndCTtgSGakOMxrYw2uszd5UmXTIfx3+drPGoETlVbQ6arTdAzSoQYA1j35vbaWpQXBg==", + "dependencies": { + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/binet": { + "version": "0.3.7", + "resolved": "https://registry.npmjs.org/binet/-/binet-0.3.7.tgz", + "integrity": "sha512-GF+QD4ajs3GWabaVzso7Kn9aZEbwI0e54FKU2ID8bM/7rIk7BpSJytB1KS7SMpix+fWAi9MAGkOgSFljl0aaKg==", + "dependencies": { + "bs32": "~0.1.5", + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/blgr": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/blgr/-/blgr-0.2.0.tgz", + "integrity": "sha512-2jZdqajYCGD5rwGdOooQpxgjKsiAAV2g8LapwSnbTjAYTZAqmqBAS+GsVGFi+/y7t1Pspidv/5HsWBbJrsEuFw==", + "dependencies": { + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/blru": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/blru/-/blru-0.1.6.tgz", + "integrity": "sha512-34+xZ2u4ys/aUzWCU9m6Eee4nVuN1ywdxbi8b3Z2WULU6qvnfeHvCWEdGzlVfRbbhimG2xxJX6R77GD2cuVO6w==", + "dependencies": { + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/blst": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/blst/-/blst-0.1.5.tgz", + "integrity": "sha512-TPl04Cx3CHdPFAJ2x9Xx1Z1FOfpAzmNPfHkfo+pGAaNH4uLhS58ExvamVkZh3jadF+B7V5sMtqvrqdf9mHINYA==", + "dependencies": { + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bmocha": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/bmocha/-/bmocha-2.1.5.tgz", + "integrity": "sha512-hEO+jQC+6CMxdxSqKPjqAdIDvRWHfdGgsMh4fUmatkMewbYr2O6qMIbW7Lhcmkcnz8bwRHZuEdDaBt/16NofoA==", + "dev": true, + "bin": { + "_bmocha": "bin/_bmocha", + "bmocha": "bin/bmocha" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bmutex": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/bmutex/-/bmutex-0.1.6.tgz", + "integrity": "sha512-nXWOXtQHbfPaMl6jyEF/rmRMrcemj2qn+OCAI/uZYurjfx7Dg3baoXdPzHOL0U8Cfvn8CWxKcnM/rgxL7DR4zw==", + "dependencies": { + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bns": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/bns/-/bns-0.15.0.tgz", + "integrity": "sha512-iJWQVE399vQzPfhalFMJGEQ7k5Ot2D6Mz8dkoPeLO8huWAMOiJNJ1tHzOu5j+ZyNNew6ITgG/LsSyaRPxvkXuw==", + "dependencies": { + "bcrypto": "~5.4.0", + "bfile": "~0.2.2", + "bheep": "~0.1.5", + "binet": "~0.3.6", + "bs32": "~0.1.6", + "bsert": "~0.0.10", + "btcp": "~0.1.5", + "budp": "~0.1.6", + "bufio": "~1.0.7" + }, + "bin": { + "bns-keygen": "bin/bns-keygen", + "bns-prove": "bin/bns-prove", + "dig.js": "bin/dig.js", + "dig2json": "bin/dig2json", + "json2dig": "bin/json2dig", + "json2rr": "bin/json2rr", + "json2zone": "bin/json2zone", + "named.js": "bin/named.js", + "rr2json": "bin/rr2json", + "whois.js": "bin/whois.js", + "zone2json": "bin/zone2json" + }, + "engines": { + "node": ">=8.0.0" + }, + "optionalDependencies": { + "unbound": "~0.4.3" + } + }, + "node_modules/brq": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/brq/-/brq-0.1.8.tgz", + "integrity": "sha512-6SDY1lJMKXgt5TZ6voJQMH2zV1XPWWtm203PSkx3DSg9AYNYuRfOPFSBDkNemabzgpzFW9/neR4YhTvyJml8rQ==", + "dependencies": { + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bs32": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/bs32/-/bs32-0.1.6.tgz", + "integrity": "sha512-usjDesQqZ8ihHXOnOEQuAdymBHnJEfSd+aELFSg1jN/V3iAf12HrylHlRJwIt6DTMmXpBDQ+YBg3Q3DIYdhRgQ==", + "dependencies": { + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bsert": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/bsert/-/bsert-0.0.10.tgz", + "integrity": "sha512-NHNwlac+WPy4t2LoNh8pXk8uaIGH3NSaIUbTTRXGpE2WEbq0te/tDykYHkFK57YKLPjv/aGHmbqvnGeVWDz57Q==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bsock": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/bsock/-/bsock-0.1.9.tgz", + "integrity": "sha512-/l9Kg/c5o+n/0AqreMxh2jpzDMl1ikl4gUxT7RFNe3A3YRIyZkiREhwcjmqxiymJSRI/Qhew357xGn1SLw/xEw==", + "dependencies": { + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bsocks": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/bsocks/-/bsocks-0.2.6.tgz", + "integrity": "sha512-66UkjoB9f7lhT+WKgYq8MQa6nkr96mlX64JYMlIsXe/X4VeqNwvsx7UOE3ZqD6lkwg8GvBhapRTWj0qWO3Pw8w==", + "dependencies": { + "binet": "~0.3.5", + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/btcp": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/btcp/-/btcp-0.1.5.tgz", + "integrity": "sha512-tkrtMDxeJorn5p0KxaLXELneT8AbfZMpOFeoKYZ5qCCMMSluNuwut7pGccLC5YOJqmuk0DR774vNVQLC9sNq/A==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/budp": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/budp/-/budp-0.1.6.tgz", + "integrity": "sha512-o+a8NPq3DhV91j4nInjht2md6mbU1XL+7ciPltP66rw5uD3KP1m5r8lA94LZVaPKcFdJ0l2HVVzRNxnY26Pefg==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/buffer-map": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buffer-map/-/buffer-map-0.0.7.tgz", + "integrity": "sha512-95try3p/vMRkIAAnJDaGkFhGpT/65NoeW6XelEPjAomWYR58RQtW4khn0SwKj34kZoE7uxL7w2koZSwbnszvQQ==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bufio": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/bufio/-/bufio-1.0.7.tgz", + "integrity": "sha512-bd1dDQhiC+bEbEfg56IdBv7faWa6OipMs/AFFFvtFnB3wAYjlwQpQRZ0pm6ZkgtfL0pILRXhKxOiQj6UzoMR7A==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bupnp": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/bupnp/-/bupnp-0.2.6.tgz", + "integrity": "sha512-J6ykzJhZMxXKN78K+1NzFi3v/51X2Mvzp2hW42BWwmxIVfau6PaN99gyABZ8x05e8MObWbsAis23gShhj9qpbw==", + "dependencies": { + "binet": "~0.3.5", + "brq": "~0.1.7", + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bval": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/bval/-/bval-0.1.6.tgz", + "integrity": "sha512-jxNH9gSx7g749hQtS+nTxXYz/bLxwr4We1RHFkCYalNYcj12RfbW6qYWsKu0RYiKAdFcbNoZRHmWrIuXIyhiQQ==", + "dependencies": { + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bweb": { + "version": "0.1.10", + "resolved": "https://registry.npmjs.org/bweb/-/bweb-0.1.10.tgz", + "integrity": "sha512-3Kkz/rfsyAWUS+8DV5XYhwcgVN4DfDewrP+iFTcpQfdZzcF6+OypAq7dHOtXV0sW7U/3msA/sEEqz0MHZ9ERWg==", + "dependencies": { + "bsert": "~0.0.10", + "bsock": "~0.1.8" + }, + "bin": { + "bweb": "bin/bweb" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/goosig": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/goosig/-/goosig-0.10.0.tgz", + "integrity": "sha512-+BVVLfxmawAmGVjjJpXzu5LNcFIOfgXgP7kWEyc3qu/xn9RMqbPbNfYDdHBZKfZkDMIO7Q4vD790iNYQAXhoFA==", + "hasInstallScript": true, + "dependencies": { + "bcrypto": "~5.4.0", + "bsert": "~0.0.10", + "loady": "~0.0.5" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/hs-client": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/hs-client/-/hs-client-0.0.10.tgz", + "integrity": "sha512-15tfeQEMRS1FZA0q9gFbQ1jYs8v4z9oKw9xFwVEyRuckn72hoVAglN4IrFxkOCDMYV7TWCY/nO/yNZp5njYFBw==", + "dependencies": { + "bcfg": "~0.1.7", + "bcurl": "~0.1.9", + "bsert": "~0.0.10" + }, + "bin": { + "hsd-cli": "bin/hsd-cli", + "hsd-rpc": "bin/hsd-rpc", + "hsw-cli": "bin/hsw-cli", + "hsw-rpc": "bin/hsw-rpc" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/loady": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/loady/-/loady-0.0.5.tgz", + "integrity": "sha512-uxKD2HIj042/HBx77NBcmEPsD+hxCgAtjEWlYNScuUjIsh/62Uyu39GOR68TBR68v+jqDL9zfftCWoUo4y03sQ==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/mrmr": { + "version": "0.1.10", + "resolved": "https://registry.npmjs.org/mrmr/-/mrmr-0.1.10.tgz", + "integrity": "sha512-NJRJs+yJyRWwcTqLRf7O32n56UP1+UQoTrGVEoB3LMj0h2jlon790drDbxKvi5mK5k4HfC0cpNkxqHcrJK/evg==", + "hasInstallScript": true, + "dependencies": { + "bsert": "~0.0.10", + "loady": "~0.0.5" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/n64": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/n64/-/n64-0.2.10.tgz", + "integrity": "sha512-uH9geV4+roR1tohsrrqSOLCJ9Mh1iFcDI+9vUuydDlDxUS1UCAWUfuGb06p3dj3flzywquJNrGsQ7lHP8+4RVQ==", + "engines": { + "node": ">=2.0.0" + } + }, + "node_modules/unbound": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/unbound/-/unbound-0.4.3.tgz", + "integrity": "sha512-2ISqZLXtzp1l9f1V8Yr6S+zuhXxEwE1CjKHjXULFDHJcfhc9Gm3mn19hdPp4rlNGEdCivKYGKjYe3WRGnafYdA==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "loady": "~0.0.5" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/urkel": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/urkel/-/urkel-0.7.0.tgz", + "integrity": "sha512-7Z3Gor4DkKKi0Ehp6H9xehWXqyL12+PA4JM41dcVc1LWks4zI4PGWv6DWgxaLCC+otpEuGdq3Vh5ayD/Mvzfbg==", + "dependencies": { + "bfile": "~0.2.1", + "bmutex": "~0.1.6", + "bsert": "~0.0.10" + }, + "engines": { + "node": ">=8.0.0" + } + } + }, "dependencies": { "bcfg": { "version": "0.1.7", @@ -80,9 +551,9 @@ } }, "binet": { - "version": "0.3.6", - "resolved": "https://registry.npmjs.org/binet/-/binet-0.3.6.tgz", - "integrity": "sha512-6pm+Gc3uNiiJZEv0k8JDWqQlo9ki/o9UNAkLmr0EGm7hI5MboOJVIOlO1nw3YuDkLHWN78OPsaC4JhRkn2jMLw==", + "version": "0.3.7", + "resolved": "https://registry.npmjs.org/binet/-/binet-0.3.7.tgz", + "integrity": "sha512-GF+QD4ajs3GWabaVzso7Kn9aZEbwI0e54FKU2ID8bM/7rIk7BpSJytB1KS7SMpix+fWAi9MAGkOgSFljl0aaKg==", "requires": { "bs32": "~0.1.5", "bsert": "~0.0.10" From 43f5c2c025c7475c6312003dfcf385274a023d72 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Thu, 24 Feb 2022 20:02:30 +0400 Subject: [PATCH 14/17] test: minor. --- test/net-hostlist-test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/net-hostlist-test.js b/test/net-hostlist-test.js index eff42fff..236eaeb2 100644 --- a/test/net-hostlist-test.js +++ b/test/net-hostlist-test.js @@ -1633,7 +1633,7 @@ describe('Net HostList', function() { beforeEach(async () => { testdir = common.testdir('hostlist'); - assert(await fs.mkdirp(testdir)); + await fs.mkdirp(testdir); }); afterEach(async () => { From 81f3736d5da93c19b364b2e70d65caf236943d46 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Thu, 3 Mar 2022 13:10:24 +0400 Subject: [PATCH 15/17] net: use PeerOptions for the Peer. --- lib/net/peer.js | 14 ++------------ lib/net/pool.js | 15 ++++++++++----- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/lib/net/peer.js b/lib/net/peer.js index c63eed5e..a42bb99c 100644 --- a/lib/net/peer.js +++ b/lib/net/peer.js @@ -51,7 +51,7 @@ class Peer extends EventEmitter { constructor(options) { super(); - this.options = options; + this.options = new PeerOptions(options); this.network = this.options.network; this.logger = this.options.logger.context('peer'); this.locker = new Lock(); @@ -164,16 +164,6 @@ class Peer extends EventEmitter { return peer; } - /** - * Create a peer from options. - * @param {Object} options - * @returns {Peer} - */ - - static fromOptions(options) { - return new this(new PeerOptions(options)); - } - /** * Begin peer initialization. * @private @@ -2161,7 +2151,7 @@ class PeerOptions { this.compact = false; this.headers = false; this.banScore = common.BAN_SCORE; - this.proofPRS = 100; + this.maxProofRPS = 100; this.getHeight = PeerOptions.getHeight; this.isFull = PeerOptions.isFull; diff --git a/lib/net/pool.js b/lib/net/pool.js index 2c4d2f32..af2cff7a 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -4354,6 +4354,11 @@ class PoolOptions { this.createSocket = this._createSocket.bind(this); this.createServer = tcp.createServer; this.resolve = this._resolve.bind(this); + this.createNonce = this._createNonce.bind(this); + this.hasNonce = this._hasNonce.bind(this); + this.getHeight = this._getHeight.bind(this); + this.isFull = this._isFull.bind(this); + this.getRate = this._getRate.bind(this); this.proxy = null; this.onion = false; this.brontideOnly = false; @@ -4646,7 +4651,7 @@ class PoolOptions { * @returns {Number} */ - getHeight() { + _getHeight() { return this.chain.height; } @@ -4656,7 +4661,7 @@ class PoolOptions { * @returns {Boolean} */ - isFull() { + _isFull() { return this.chain.synced; } @@ -4687,7 +4692,7 @@ class PoolOptions { * @returns {Buffer} */ - createNonce(hostname) { + _createNonce(hostname) { return this.nonces.alloc(hostname); } @@ -4698,7 +4703,7 @@ class PoolOptions { * @returns {Boolean} */ - hasNonce(nonce) { + _hasNonce(nonce) { return this.nonces.has(nonce); } @@ -4709,7 +4714,7 @@ class PoolOptions { * @returns {Rate} */ - getRate(hash) { + _getRate(hash) { if (!this.mempool) return -1; From 5fc3122307c4e7d55e62dc25d63d28f7cd774b93 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Thu, 24 Mar 2022 14:28:16 +0400 Subject: [PATCH 16/17] net: nits. --- lib/net/hostlist.js | 2 +- test/net-hostlist-test.js | 21 ++++++++++----------- test/{net-lookup.js => net-lookup-test.js} | 0 3 files changed, 11 insertions(+), 12 deletions(-) rename test/{net-lookup.js => net-lookup-test.js} (100%) diff --git a/lib/net/hostlist.js b/lib/net/hostlist.js index d33debe6..69a8f037 100644 --- a/lib/net/hostlist.js +++ b/lib/net/hostlist.js @@ -1452,7 +1452,7 @@ HostList.scores = { * @property {NetAddress} addr - host address. * @property {NetAddress} src - the first address we discovered this entry * from. - * @property {Boolean} used - is it used set. + * @property {Boolean} used - is it in the used set. * @property {Number} refCount - Reference count in new buckets. * @property {Number} attempts - connection attempts since last successful one. * @property {Number} lastSuccess - last success timestamp. diff --git a/test/net-hostlist-test.js b/test/net-hostlist-test.js index 236eaeb2..8b666cc3 100644 --- a/test/net-hostlist-test.js +++ b/test/net-hostlist-test.js @@ -630,12 +630,12 @@ describe('Net HostList', function() { // See: binet.getReachability // TLDR: // UNREACHABLE = 0 - // < DEFAULT = 1 - non-ipv4 -> ipv4, onion -> ipv6, ... - // < TEREDO = 2 - teredo -> teredo, teredo -> ipv6 - // < IPV6_WEAK = 3 - ipv4 -> ipv6 tunnels, ipv6 -> teredo - // < IPV4 = 4 - ipv4 -> ipv4, ipv4 -> others - // < IPV6_STRONG = 5 - ipv6 -> ipv6 - // < PRIVATE = 6 - ONION -> ONION + // < DEFAULT = 1 -- non-ipv4 -> ipv4, onion -> ipv6, ... + // < TEREDO = 2 -- teredo -> teredo, teredo -> ipv6 + // < IPV6_WEAK = 3 -- ipv4 -> ipv6 tunnels, ipv6 -> teredo + // < IPV4 = 4 -- ipv4 -> ipv4, ipv4 -> others + // < IPV6_STRONG = 5 -- ipv6 -> ipv6 + // < PRIVATE = 6 -- ONION -> ONION const {MANUAL} = HostList.scores; @@ -647,7 +647,7 @@ describe('Net HostList', function() { getRandomIPv4(), // DEFAULT = 1 getRandomOnion(), // DEFAULT = 1 getRandomTEREDO(), // TEREDO = 2 - getRandomIPv6() // TEREDO = 2 + getRandomIPv6() // TEREDO = 2 ], [getRandomIPv4()]: [ getRandomIPv4(), // IPV4 = 4 @@ -906,8 +906,8 @@ describe('Net HostList', function() { // is 50% (1/2). // then we have 2 refs, probability of adding will be 25% (1/4). // ... until we have 8 refs. (last one being 1/128) - // Because we are reusing src, we will get same bucket - // so it will only get added once. + // We use different SRC for the same host, so we don't get the + // same bucket. let added = 0; for (let i = 0; i < 7; i++) { const res = hosts.add(addr2, srcs[i]); @@ -1347,7 +1347,6 @@ describe('Net HostList', function() { // add 64 entries to the bucket. const entries = []; - let expectedEvicted = null; for (let i = 0; i < hosts.maxEntries; i++) { const addr = getRandomNetAddr(); const src = getRandomNetAddr(); @@ -1359,7 +1358,7 @@ describe('Net HostList', function() { entries.push(entry); } - expectedEvicted = entries[0]; + const expectedEvicted = entries[0]; hosts.markAck(addr.hostname); assert.strictEqual(bucket.tail, entry); assert.strictEqual(expectedEvicted.used, true); diff --git a/test/net-lookup.js b/test/net-lookup-test.js similarity index 100% rename from test/net-lookup.js rename to test/net-lookup-test.js From c96b6ecda9f96608aeb7ba010ae8f6ab975267c5 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Thu, 24 Mar 2022 14:28:54 +0400 Subject: [PATCH 17/17] net: Fix connectedGroup filter. --- lib/net/netaddress.js | 29 ++++++++++++++++++----------- lib/net/pool.js | 2 +- test/net-lookup-test.js | 3 +-- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/lib/net/netaddress.js b/lib/net/netaddress.js index 7f59d47f..f949df67 100644 --- a/lib/net/netaddress.js +++ b/lib/net/netaddress.js @@ -249,12 +249,12 @@ class NetAddress extends bio.Struct { } /** - * Get the canonical identifier of our network group - * @returns {Buffer} - */ + * Get the canonical identifier of our network group + * @returns {Buffer} + */ getGroup() { - return groupKey(this.raw); + return groupKey(this); } /** @@ -542,7 +542,14 @@ NetAddress.DEFAULT_SERVICES = 0 * Helpers */ -function groupKey(raw) { +/** + * @param {NetAddress} addr + * @returns {Number} + */ + +function groupKey(addr) { + const raw = addr.raw; + // See: https://github.com/bitcoin/bitcoin/blob/e258ce7/src/netaddress.cpp#L413 // Todo: Use IP->ASN mapping, see: // https://github.com/bitcoin/bitcoin/blob/adea5e1/src/addrman.h#L274 @@ -551,25 +558,25 @@ function groupKey(raw) { let bits = 16; let i = 0; - if (IP.isLocal(raw)) { + if (addr.isLocal()) { type = 255; // NET_LOCAL bits = 0; - } else if (!IP.isRoutable(raw)) { + } else if (!addr.isRoutable()) { type = IP.networks.NONE; // NET_UNROUTABLE bits = 0; - } else if (IP.isIPv4(raw) || IP.isRFC6145(raw) || IP.isRFC6052(raw)) { + } else if (addr.isIPv4() || addr.isRFC6145() || addr.isRFC6052()) { type = IP.networks.INET4; // NET_IPV4 start = 12; - } else if (IP.isRFC3964(raw)) { + } else if (addr.isRFC3964()) { type = IP.networks.INET4; // NET_IPV4 start = 2; - } else if (IP.isRFC4380(raw)) { + } else if (addr.isRFC4380()) { const buf = Buffer.alloc(3); buf[0] = IP.networks.INET4; // NET_IPV4 buf[1] = raw[12] ^ 0xff; buf[2] = raw[13] ^ 0xff; return buf; - } else if (IP.isOnion(raw)) { + } else if (addr.isOnion()) { type = IP.networks.ONION; // NET_ONION start = 6; bits = 4; diff --git a/lib/net/pool.js b/lib/net/pool.js index af2cff7a..c7773db3 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -84,7 +84,7 @@ class Pool extends EventEmitter { this.pendingFilter = null; this.refillTimer = null; this.discoverTimer = null; - this.connectedGroups = new Set(); + this.connectedGroups = new BufferSet(); this.checkpoints = false; this.headerChain = new List(); diff --git a/test/net-lookup-test.js b/test/net-lookup-test.js index daa64edd..28ed6561 100644 --- a/test/net-lookup-test.js +++ b/test/net-lookup-test.js @@ -9,6 +9,7 @@ const main = Network.get('main'); const notAHost = 'not-a-domain.not-a-domain'; describe('Lookup', function() { + this.timeout(10000); it('should lookup seed', async () => { for (const host of main.seeds) { const addresses = await lookup(host); @@ -30,8 +31,6 @@ describe('Lookup', function() { }); it('should lookup seed', async () => { - this.timeout(10000); - for (const host of main.seeds) { const addresses = await resolve(host);