fix: refactor query logic

.github/workflows/test.yml

@@ -0,0 +1,39 @@
+name: ci
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - run: npm install
+    - run: npx aegir lint
+    - uses: gozala/typescript-error-reporter-action@v1.0.8
+    - run: npx aegir build
+    - run: npx aegir dep-check
+    - uses: ipfs/aegir/actions/bundle-size@master
+      name: size
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+  test-node:
+    needs: check
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest, ubuntu-latest, macos-latest]
+        node: [14, 16]
+      fail-fast: true
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node }}
+      - run: npm install
+      - run: npx nyc --reporter=lcov aegir test -t node -- --bail
+      - uses: codecov/codecov-action@v1

.gitignore

@@ -41,3 +41,4 @@ test/test-data/go-ipfs-repo/LOG.old
 # while testing npm5
 package-lock.json
 yarn.lock
+.nyc_output

README.md

@@ -1,10 +1,10 @@
-# js-libp2p-kad-dht
+# js-libp2p-kad-dht <!-- omit in toc -->
 
 [![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io)
 [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/)
 [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs)
 [![Discourse posts](https://img.shields.io/discourse/https/discuss.libp2p.io/posts.svg)](https://discuss.libp2p.io)
-[![Build Status](https://travis-ci.org/libp2p/js-libp2p-kad-dht.svg?style=flat-square)](https://travis-ci.org/libp2p/js-libp2p-kad-dht)
+[![Build status](https://github.com/libp2p/js-libp2p-kad-dht/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/libp2p/js-libp2p-kad-dht/actions/workflows/test.yml)
 [![Coverage Status](https://coveralls.io/repos/github/libp2p/js-libp2p-kad-dht/badge.svg?branch=master)](https://coveralls.io/github/libp2p/js-libp2p-kad-dht?branch=master)
 [![Dependency Status](https://david-dm.org/libp2p/js-libp2p-kad-dht.svg?style=flat-square)](https://david-dm.org/libp2p/js-libp2p-kad-dht)
 [![Bundle Size](https://flat.badgen.net/bundlephobia/minzip/libp2p-kad-dht)](https://bundlephobia.com/result?p=libp2p-kad-dht)
@@ -15,19 +15,23 @@
 
 > JavaScript implementation of the Kademlia DHT for libp2p, based on [go-libp2p-kad-dht](https://github.com/libp2p/go-libp2p-kad-dht).
 
-## Lead Maintainer
+## Lead Maintainer  <!-- omit in toc -->
 
 [Vasco Santos](https://github.com/vasco-santos).
 
-## Table of Contents
+## Table of Contents  <!-- omit in toc -->
 
 - [Install](#install)
   - [npm](#npm)
   - [Use in Node.js](#use-in-nodejs)
 - [API](#api)
+  - [Custom secondary DHT in libp2p](#custom-secondary-dht-in-libp2p)
+  - [Peer Routing](#peer-routing)
+  - [Content Routing](#content-routing)
+  - [Peer Discovery](#peer-discovery)
+  - [Implementation Summary](#implementation-summary)
 - [Contribute](#contribute)
 - [License](#license)
-
 ## Install
 
 ### npm
@@ -39,7 +43,7 @@
 ### Use in Node.js
 
 ```js
-const KadDHT = require('libp2p-kad-dht')
+import { create } from 'libp2p-kad-dht'
 ```
 
 ## API
@@ -51,20 +55,18 @@ The libp2p-kad-dht module offers 3 APIs: Peer Routing, Content Routing and Peer
 ### Custom secondary DHT in libp2p
 
 ```js
+import { create } from 'libp2p-kad-dht'
+
 /**
  * @param {Libp2p} libp2p
  */
 function addDHT(libp2p) {
-    const customDHT = new KadDHT({
+    const customDHT = create({
         libp2p,
-        dialer: libp2p.dialer,
-        peerId: libp2p.peerId,
-        peerStore: libp2p.peerStore,
-        registrar: libp2p.registrar,
         protocolPrefix: '/custom'
     })
     customDHT.start()
-    customDHT.on('peer', libp2p._onDiscoveryPeer)
+
     return customDHT
 }
 ```

docs/IMPL_SUMMARY.MD

@@ -1,67 +1,26 @@
-# js-libp2p-kad-dht
+# js-libp2p-kad-dht <!-- omit in toc -->
 
-js-libp2p-kad-dht is a JavaScript implementation of the [Kademlia DHT](http://www.scs.stanford.edu/%7Edm/home/papers/kpos.pdf) with some features of S/Kademlia. A "provider" node uses the DHT to advertise that it has a particular piece of content, and "querying" nodes will search the DHT for peers that have a particular piece of content. Content is modeled as a value that is identified by a key, where the key and value are Buffers.
+js-libp2p-kad-dht is a JavaScript implementation of the [Kademlia DHT][] with some features of [S/Kademlia][]. A "provider" node uses the DHT to advertise that it has a particular piece of content, and "querying" nodes will search the DHT for peers that have a particular piece of content. Content is modelled as a value that is identified by a key, where the key and value are Uint8Arrays.
 
-#### DHT Identifiers
+## Table of contents <!-- omit in toc -->
 
-The DHT uses a sha2-256 hash for identifiers:
-- For peers the DHT identifier is the hash of the [PeerId][PeerId]
-- For content the DHT identifier is the hash of the key (eg a Block CID)
+- [Spec](#spec)
+- [Extensions](#extensions)
+  - [Disjoint paths](#disjoint-paths)
 
-#### FIND_NODE
+## Spec
 
-`findPeer (PeerId):` [PeerInfo][PeerInfo]
+js-libp2p-kad-dht follows the [libp2p/kad-dht spec](https://github.com/libp2p/specs/tree/master/kad-dht) and implements the algorithms described in the [IPFS DHT documentation](https://docs.ipfs.io/concepts/dht/).
 
-The address space is so large (256 bits) that there are big gaps between DHT ids, and nodes frequently join and leave the DHT.
+## Extensions
 
-To find a particular node
-- the `querying node` converts the [PeerId][PeerId] to a DHT id
-- the `querying node` sends a request to the nearest peers to that DHT id that it knows about
-- those peers respond with the nearest peers to the DHT id that they know about
-- the `querying node` sorts the responses and recursively queries the closest peers to the DHT id, continuing until it finds the node or it has queried all the closest peers.
+js-libp2p-kad-dht implements some additional functionality not described in the libp2p KAD-DHT spec.
 
-#### PUT
+### Disjoint paths
 
-`put (Key, Value)`
-
-To store a value in the DHT, the `provider node`
-- converts the key to a DHT id
-- follows the "closest peers" algorithm as above to find the nearest peers to the DHT id
-- sends the value to those nearest peers
-
-Note that DHT nodes will only store values that are accepted by its "validators", configurable functions that validate the key/value to ensure the node can control what kind of content it stores (eg IPNS records).
-
-#### GET
-
-`get (Key): [Value]`
-
-To retrieve a value from the DHT
-- the `querying node` converts the key to a DHT id
-- the `querying node` follows the "closest peers" algorithm to find the nearest peers to the DHT id
-- at each iteration of the algorithm, if the peer has the value it responds with the value itself in addition to closer peers.
-
-Note that the value for a particular key is stored by many nodes, and these nodes receive `PUT` requests asynchronously, so it's possible that nodes may have distinct values for the same key. For example if node A `PUT`s the value `hello` to key `greeting` and node B concurrently `PUT`s the value `bonjour` to key `greeting`, some nodes close to the key `greeting` may receive `hello` first and others may receive `bonjour` first.
-
-Therefore a `GET` request to the DHT may collect distinct values (eg `hello` and `bonjour`) for a particular key from the nodes close to the key. The DHT has "selectors", configurable functions that choose the "best" value (for example IPNS records include a sequence number, and the "best" value is the record with the highest sequence number).
-
-#### PROVIDE
-
-`provide (Key)`
-
-To advertise that it has the content for a particular key
-- the `provider node` converts the key to a DHT id
-- the `provider node` follows the "closest peers" algorithm to find the nearest peers to the DHT id
-- the `provider node` sends a "provide" message to each of the nearest peers
-- each of the nearest peers saves the association between the "provider" peer and the key
-
-#### FIND_PROVIDERS
-
-`findProviders (Key):` [[PeerInfo][PeerInfo]]
-
-To find providers for a particular key
-- the `querying node` converts the key to a DHT id
-- the `querying node` follows the "closest peers" algorithm to find the nearest peers to the DHT id
-- at each iteration of the algorithm, if the peer knows which nodes are providing the value it responds with the provider nodes in addition to closer peers.
+js-libp2p-kad-dht uses disjoint paths when performing DHT queries. These are described in the [S/Kademlia][] paper.
 
+[Kademlia DHT]: (http://www.scs.stanford.edu/%7Edm/home/papers/kpos.pdf)
+[S/Kademlia]: (https://git.gnunet.org/bibliography.git/plain/docs/SKademlia2007.pdf)
 [PeerId]: https://github.com/libp2p/js-peer-id
 [PeerInfo]: https://github.com/libp2p/js-peer-info

package.json

@@ -5,8 +5,8 @@
   "leadMaintainer": "Vasco Santos <vasco.santos@moxy.studio>",
   "main": "src/index.js",
   "scripts": {
-    "lint": "aegir ts -p check && aegir lint",
     "prepare": "npm run build",
+    "lint": "aegir ts -p check && aegir lint",
     "build": "npm run build:proto && npm run build:proto-types && aegir build",
     "build:proto": "pbjs -t static-module -w commonjs -r libp2p-dht-message --force-number --no-verify --no-delimited --no-create --no-beautify --no-defaults --lint eslint-disable -o src/message/dht.js ./src/message/dht.proto",
     "build:proto-types": "pbts -o src/message/dht.d.ts src/message/dht.js",
@@ -18,7 +18,8 @@
     "release-major": "aegir release --type major --docs -t node",
     "coverage": "aegir coverage",
     "coverage-publish": "aegir-coverage publish",
-    "sim": "node test/simulation/index.js"
+    "sim": "node test/simulation/index.js",
+    "dep-check": "aegir dep-check"
   },
   "files": [
     "src",
@@ -47,47 +48,53 @@
   "homepage": "https://github.com/libp2p/js-libp2p-kad-dht",
   "types": "dist/src/index.d.ts",
   "dependencies": {
+    "any-signal": "^2.1.2",
     "datastore-core": "^6.0.7",
     "debug": "^4.3.1",
     "err-code": "^3.0.0",
     "hashlru": "^2.3.0",
-    "heap": "~0.2.6",
     "interface-datastore": "^6.0.2",
+    "it-all": "^1.0.5",
+    "it-drain": "^1.0.4",
     "it-first": "^1.0.4",
     "it-length": "^1.0.3",
     "it-length-prefixed": "^5.0.2",
+    "it-map": "^1.0.5",
+    "it-merge": "^1.0.3",
+    "it-parallel": "^2.0.1",
     "it-pipe": "^1.1.0",
+    "it-take": "^1.0.2",
     "k-bucket": "^5.1.0",
     "libp2p-crypto": "^0.19.5",
     "libp2p-interfaces": "^1.0.0",
     "libp2p-record": "^0.10.4",
     "multiaddr": "^10.0.0",
     "multiformats": "^9.4.5",
+    "native-abort-controller": "^1.0.4",
+    "p-defer": "^3.0.0",
     "p-map": "^4.0.0",
     "p-queue": "^6.6.2",
-    "p-timeout": "^4.1.0",
     "peer-id": "^0.15.0",
+    "private-ip": "^2.3.3",
     "protobufjs": "^6.10.2",
     "streaming-iterables": "^6.0.0",
-    "timeout-abort-controller": "^1.1.1",
+    "timeout-abort-controller": "^2.0.0",
     "uint8arrays": "^3.0.0",
     "varint": "^6.0.0"
   },
   "devDependencies": {
     "@types/debug": "^4.1.7",
     "aegir": "^35.0.1",
     "async-iterator-all": "^1.0.0",
-    "crypto-browserify": "^3.12.0",
     "datastore-level": "^7.0.1",
     "delay": "^5.0.0",
     "execa": "^5.1.1",
+    "it-filter": "^1.0.3",
+    "it-last": "^1.0.6",
     "it-pair": "^1.0.0",
-    "libp2p": "^0.32.3",
+    "libp2p": "^0.33.0",
     "lodash.random": "^3.2.0",
     "lodash.range": "^3.2.0",
-    "p-defer": "^3.0.0",
-    "p-each-series": "^2.1.0",
-    "p-map-series": "^2.1.0",
     "p-retry": "^4.2.0",
     "sinon": "^11.1.1",
     "which": "^2.0.2"

src/constants.js

@@ -33,3 +33,6 @@ exports.K = 20
 
 // Alpha is the concurrency for asynchronous requests
 exports.ALPHA = 3
+
+// How often we look for our closest DHT neighbours
+exports.QUERY_SELF_INTERVAL = Number(5 * minute)