Merge pull request #400 from ipfs/feat/go-libp2p-0.16-and-sharding-co…

…nfig-option

feat: go-libp2p 0.16 and autosharding

.github/workflows/test.yml

@@ -7,8 +7,8 @@ on:
 # To run CI against unrelased go-ipfs or js-ipfs-* code (eg. wip PR),
 # uncomment env below and define git revisions in ./scripts/custom-runtime.sh
 
-#env:
-  #IPFS_GO_EXEC: /tmp/go-ipfs/cmd/ipfs/ipfs
+env:
+  IPFS_GO_EXEC: /tmp/go-ipfs/cmd/ipfs/ipfs
   #IPFS_JS_EXEC: /tmp/js-ipfs/packages/ipfs/src/cli.js
   #IPFS_JS_MODULE: /tmp/js-ipfs/packages/ipfs/dist/cjs/src/index.js
   #IPFS_JS_HTTP_MODULE: /tmp/js-ipfs/packages/ipfs-http-client/dist/cjs/src/index.js
@@ -31,6 +31,7 @@ jobs:
         with:
           path: |
             /tmp/*-ipfs/**
+            ./go-libp2p-relay-daemon
             ~/.npm
             ./node_modules
           key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
@@ -56,6 +57,7 @@ jobs:
         with:
           path: |
             /tmp/*-ipfs/**
+            ./go-libp2p-relay-daemon
             ~/.npm
             ./node_modules
           key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
@@ -84,6 +86,7 @@ jobs:
         with:
           path: |
             /tmp/*-ipfs/**
+            ./go-libp2p-relay-daemon
             ~/.npm
             ./node_modules
           key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
@@ -106,6 +109,7 @@ jobs:
         type:
           - browser
           - webworker
+      fail-fast: false
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
@@ -118,6 +122,7 @@ jobs:
         with:
           path: |
             /tmp/*-ipfs/**
+            ./go-libp2p-relay-daemon
             ~/.npm
             ./node_modules
           key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
@@ -137,6 +142,7 @@ jobs:
         type:
           - electron-main
           - electron-renderer
+      fail-fast: false
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
@@ -149,6 +155,7 @@ jobs:
         with:
           path: |
             /tmp/*-ipfs/**
+            ./go-libp2p-relay-daemon
             ~/.npm
             ./node_modules
           key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}

.gitignore

@@ -37,4 +37,6 @@ dist
 test/test-data/go-ipfs-repo/LOCK
 test/test-data/go-ipfs-repo/LOG
 test/test-data/go-ipfs-repo/LOG.old
-types
+types
+go-libp2p-relay-daemon
+*.identity

package.json

@@ -23,6 +23,8 @@
     "npm": ">6.0.0"
   },
   "scripts": {
+    "postinstall": "./scripts/setup-relayd.sh",
+    "clean": "rimraf /tmp/js-ipfs /tmp/go-ipfs ./go-libp2p-relay-daemon",
     "lint": "aegir lint",
     "build": "aegir build",
     "pretest": "aegir build --esm-tests",
@@ -45,7 +47,9 @@
   },
   "homepage": "https://github.com/ipfs/interop#readme",
   "browser": {
-    "go-ipfs": false
+    "go-ipfs": false,
+    "fs": false,
+    "execa": false
   },
   "dependencies": {
     "aegir": "^35.0.2",
@@ -55,8 +59,8 @@
     "delay": "^5.0.0",
     "detect-node": "^2.0.4",
     "ipfs-unixfs": "^6.0.3",
-    "ipfsd-ctl": "^10.0.3",
     "ipfs-utils": "^9.0.1",
+    "ipfsd-ctl": "^10.0.3",
     "ipns": "^0.15.0",
     "is-ci": "^3.0.0",
     "is-os": "^1.0.1",
@@ -75,7 +79,7 @@
     "promisify-es6": "^1.0.3",
     "random-fs": "^1.0.3",
     "readable-stream-buffer-stream": "^1.0.0",
-    "rimraf": "^3.0.0",
+    "rimraf": "^3.0.2",
     "uint8arrays": "^3.0.0",
     "wherearewe": "^1.0.0"
   },
@@ -97,6 +101,7 @@
     "Richard Littauer <richard.littauer@gmail.com>"
   ],
   "devDependencies": {
+    "execa": "^5.1.1",
     "go-ipfs": "^0.9.1",
     "ipfs": "^0.59.0",
     "ipfs-http-client": "^53.0.0"

scripts/custom-runtime.sh

@@ -25,7 +25,7 @@ if [ ! -d /tmp/go-ipfs ]; then
         git clone https://github.com/ipfs/go-ipfs.git
         cd go-ipfs
             # set implementation to specific commit
-            git checkout CHANGEME_GO
+            git checkout 8c902b156b1c35d17ce45de2525c80bf71c77b5b # https://github.com/ipfs/go-ipfs/pull/8563
             make build
 fi
 fi

scripts/relayd_v1.config.json

@@ -0,0 +1,15 @@
+{
+    "RelayV1": {
+        "Enabled": true
+    },
+    "RelayV2": {
+        "Enabled": false
+    },
+    "Network": {
+        "ListenAddrs": ["/ip4/127.0.0.1/tcp/14111/ws"],
+        "AnnounceAddrs": ["/ip4/127.0.0.1/tcp/14111/ws"]
+    },
+    "Daemon": {
+        "PprofPort": -1
+    }
+}

scripts/relayd_v2.config.json

@@ -0,0 +1,15 @@
+{
+    "RelayV1": {
+        "Enabled": false
+    },
+    "RelayV2": {
+        "Enabled": true
+    },
+    "Network": {
+        "ListenAddrs": ["/ip4/127.0.0.1/tcp/24222/ws"],
+        "AnnounceAddrs": ["/ip4/127.0.0.1/tcp/24222/ws"]
+    },
+    "Daemon": {
+        "PprofPort": -1
+    }
+}

scripts/setup-relayd.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# This script ensures go-libp2p-relay-daemon is available
+# for use in  circuit v1 and v2 tests.
+
+set -eo pipefail
+
+if ! test -e ./go-libp2p-relay-daemon/relayd; then
+    echo "Building ./go-libp2p-relay-daemon/relayd binary.."
+    rm -rf ./go-libp2p-relay-daemon
+    git clone https://github.com/libp2p/go-libp2p-relay-daemon.git
+    cd go-libp2p-relay-daemon
+    # no releases atm, so we pin implementation to specific commit
+    git checkout 65211a0b6d881086feb7c386d780f55c37dff101 # 2021-11-19
+    go build ./...
+    echo "./go-libp2p-relay-daemon/relayd is ready"
+fi

test/circuit.js

@@ -1,10 +1,13 @@
 /* eslint max-nested-callbacks: ["error", 8] */
 /* eslint-env mocha */
 
-import all from './circuit/all.js'
-import browser from './circuit/browser.js'
+import allV1 from './circuit/v1/all.js'
+import allV2 from './circuit/v2/all.js'
+import browserV1 from './circuit/v1/browser.js'
+import browserV2 from './circuit/v2/browser.js'
 import isNode from 'detect-node'
 import { connect, send, clean } from './utils/circuit.js'
+import { closeRelays } from './utils/relayd.js'
 import { daemonFactory } from './utils/daemon-factory.js'
 
 const timeout = 80 * 1000
@@ -15,40 +18,84 @@ const baseTest = {
 }
 
 describe('circuit', () => {
-  let factory
+  after(closeRelays)
 
-  before(async () => {
-    factory = await daemonFactory()
-  })
+  // Legacy v1 (unlimited relay)
+  describe('v1', () => {
+    let factory
 
-  const tests = isNode ? all : browser
+    before(async () => {
+      factory = await daemonFactory()
+    })
 
-  Object.keys(tests).forEach((test) => {
-    let nodeA
-    let relay
-    let nodeB
+    const tests = isNode ? allV1 : browserV1
 
-    tests[test] = Object.assign({}, baseTest, tests[test])
+    Object.keys(tests).forEach((test) => {
+      let nodeA
+      let relay
+      let nodeB
 
-    const dsc = tests[test].skip && tests[test].skip()
-      ? describe.skip
-      : describe
+      tests[test] = Object.assign({}, baseTest, tests[test])
 
-    dsc(test, function () {
-      this.timeout(tests[test].timeout)
+      const dsc = tests[test].skip && tests[test].skip()
+        ? describe.skip
+        : describe
 
-      before(async () => {
-        [nodeA, relay, nodeB] = await tests[test].create(factory)
-      })
+      dsc(test, function () {
+        this.timeout(tests[test].timeout)
+
+        before(async () => {
+          [nodeA, relay, nodeB] = await tests[test].create(factory)
+        })
+
+        after(() => clean(factory))
 
-      after(() => clean(factory))
+        it('connect', () => {
+          return tests[test].connect(nodeA, nodeB, relay)
+        })
 
-      it('connect', () => {
-        return tests[test].connect(nodeA, nodeB, relay)
+        it('send', () => {
+          return tests[test].send(nodeA, nodeB)
+        })
       })
+    })
+  })
+
+  // Modern v2 (limited relay)
+  // https://github.com/libp2p/specs/blob/master/relay/circuit-v2.md
+  describe('v2', () => {
+    let factory
+
+    before(async () => {
+      factory = await daemonFactory()
+    })
+
+    const tests = isNode ? allV2 : browserV2
+
+    Object.keys(tests).forEach((test) => {
+      let nodeA
+      let relay
+      let nodeB
+
+      tests[test] = Object.assign({}, baseTest, tests[test])
+
+      const dsc = tests[test].skip && tests[test].skip()
+        ? describe.skip
+        : describe
+
+      dsc(test, function () {
+        this.timeout(tests[test].timeout)
+
+        before(async () => {
+          [nodeA, relay, nodeB] = await tests[test].create(factory)
+        })
+
+        after(() => clean(factory))
 
-      it('send', () => {
-        return tests[test].send(nodeA, nodeB)
+        it('connect', () => {
+          return tests[test].connect(nodeA, nodeB, relay)
+        })
+        // Note: v2 provides a limited relay for things like hole punching – no send test
       })
     })
   })