fix: disable provide over HTTP with Routing.Type=auto (#9511)

Closes #9504
ipfs · Jan 4, 2023 · a7d6dca · a7d6dca
1 parent 94f8f89
commit a7d6dca
Show file tree

Hide file tree

Showing 8 changed files with 111 additions and 13 deletions.
diff --git a/core/node/libp2p/routingopt.go b/core/node/libp2p/routingopt.go
@@ -2,6 +2,8 @@ package libp2p
 
 import (
 	"context"
+	"os"
+	"strings"
 	"time"
 
 	"github.com/ipfs/go-datastore"
@@ -30,6 +32,13 @@ var defaultHTTPRouters = []string{
 	// TODO: add an independent router from Cloudflare
 }
 
+func init() {
+	// Override HTTP routers if custom ones were passed via env
+	if routers := os.Getenv("IPFS_HTTP_ROUTERS"); routers != "" {
+		defaultHTTPRouters = strings.Split(routers, " ")
+	}
+}
+
 // ConstructDefaultRouting returns routers used when Routing.Type is unset or set to "auto"
 func ConstructDefaultRouting(peerID string, addrs []string, privKey string) func(
 	ctx context.Context,
@@ -67,8 +76,17 @@ func ConstructDefaultRouting(peerID string, addrs []string, privKey string) func
 			if err != nil {
 				return nil, err
 			}
+
+			r := &irouting.Composer{
+				GetValueRouter:      routinghelpers.Null{},
+				PutValueRouter:      routinghelpers.Null{},
+				ProvideRouter:       routinghelpers.Null{}, // modify this when indexers supports provide
+				FindPeersRouter:     routinghelpers.Null{},
+				FindProvidersRouter: httpRouter,
+			}
+
 			routers = append(routers, &routinghelpers.ParallelRouter{
-				Router:       httpRouter,
+				Router:       r,
 				IgnoreError:  true,             // https://github.com/ipfs/kubo/pull/9475#discussion_r1042507387
 				Timeout:      15 * time.Second, // 5x server value from https://github.com/ipfs/kubo/pull/9475#discussion_r1042428529
 				ExecuteAfter: 0,

diff --git a/docs/environment-variables.md b/docs/environment-variables.md
@@ -114,6 +114,23 @@ $ ipfs resolve -r /ipns/dnslink-test2.example.com
 /ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am
 ```
 
+## `IPFS_HTTP_ROUTERS`
+
+Overrides all implicit HTTP routers enabled when `Routing.Type=auto` with
+the space-separated list of URLs provided in this variable.
+Useful for testing and debugging in offline contexts.
+
+Example:
+
+```console
+$ ipfs config Routing.Type auto
+$ IPFS_HTTP_ROUTERS="http://127.0.0.1:7423" ipfs daemon
+```
+
+The above will replace implicit HTTP routers with single one, allowing for
+inspection/debug of HTTP requests sent by Kubo via `while true ; do nc -l 7423; done`
+or more advanced tools like [mitmproxy](https://docs.mitmproxy.org/stable/#mitmproxy).
+
 ## `LIBP2P_TCP_REUSEPORT`
 
 Kubo tries to reuse the same source port for all connections to improve NAT

diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod
@@ -127,7 +127,7 @@ require (
 	github.com/libp2p/go-libp2p-pubsub v0.8.2 // indirect
 	github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect
 	github.com/libp2p/go-libp2p-record v0.2.0 // indirect
-	github.com/libp2p/go-libp2p-routing-helpers v0.4.1 // indirect
+	github.com/libp2p/go-libp2p-routing-helpers v0.5.0 // indirect
 	github.com/libp2p/go-libp2p-xor v0.1.0 // indirect
 	github.com/libp2p/go-mplex v0.7.0 // indirect
 	github.com/libp2p/go-msgio v0.2.0 // indirect
@@ -198,7 +198,7 @@ require (
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/dig v1.15.0 // indirect
 	go.uber.org/fx v1.18.2 // indirect
-	go.uber.org/multierr v1.8.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
 	go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
 	golang.org/x/crypto v0.3.0 // indirect

diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum
@@ -793,8 +793,8 @@ github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqU
 github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q=
 github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0=
 github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk=
-github.com/libp2p/go-libp2p-routing-helpers v0.4.1 h1:rOlZiFpUt7SgHm4w62MBvWaQ4UHh7bVJnSnor6RN7j8=
-github.com/libp2p/go-libp2p-routing-helpers v0.4.1/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E=
+github.com/libp2p/go-libp2p-routing-helpers v0.5.0 h1:Byujua1X9MeTzbF54i5OwjUNopeg7PYBykuNow/w3p4=
+github.com/libp2p/go-libp2p-routing-helpers v0.5.0/go.mod h1:wwK/XSLt6njjO7sRbjhf8w7PGBOfdntMQ2mOQPZ5s/Q=
 github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
 github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g=
 github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8=
@@ -1377,8 +1377,8 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
-go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
-go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
+go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=

diff --git a/go.mod b/go.mod
@@ -79,7 +79,7 @@ require (
 	github.com/libp2p/go-libp2p-pubsub v0.8.2
 	github.com/libp2p/go-libp2p-pubsub-router v0.6.0
 	github.com/libp2p/go-libp2p-record v0.2.0
-	github.com/libp2p/go-libp2p-routing-helpers v0.4.0
+	github.com/libp2p/go-libp2p-routing-helpers v0.5.0
 	github.com/libp2p/go-libp2p-testing v0.12.0
 	github.com/libp2p/go-socket-activation v0.1.0
 	github.com/miekg/dns v1.1.50
@@ -231,7 +231,7 @@ require (
 	go.opentelemetry.io/otel/metric v0.30.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.16.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
-	go.uber.org/multierr v1.8.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
 	go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
 	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
 	golang.org/x/net v0.3.0 // indirect

diff --git a/go.sum b/go.sum
@@ -828,8 +828,8 @@ github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqU
 github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q=
 github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0=
 github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk=
-github.com/libp2p/go-libp2p-routing-helpers v0.4.0 h1:b7y4aixQ7AwbqYfcOQ6wTw8DQvuRZeTAA0Od3YYN5yc=
-github.com/libp2p/go-libp2p-routing-helpers v0.4.0/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E=
+github.com/libp2p/go-libp2p-routing-helpers v0.5.0 h1:Byujua1X9MeTzbF54i5OwjUNopeg7PYBykuNow/w3p4=
+github.com/libp2p/go-libp2p-routing-helpers v0.5.0/go.mod h1:wwK/XSLt6njjO7sRbjhf8w7PGBOfdntMQ2mOQPZ5s/Q=
 github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
 github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g=
 github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8=
@@ -1441,8 +1441,8 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
-go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
-go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
+go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=

diff --git a/routing/composer.go b/routing/composer.go
@@ -100,9 +100,18 @@ func (c *Composer) GetValue(ctx context.Context, key string, opts ...routing.Opt
 func (c *Composer) SearchValue(ctx context.Context, key string, opts ...routing.Option) (<-chan []byte, error) {
 	log.Debug("composer: calling searchValue: ", key)
 	ch, err := c.GetValueRouter.SearchValue(ctx, key, opts...)
+
+	// avoid nil channels on implementations not supporting SearchValue method.
+	if err == routing.ErrNotFound && ch == nil {
+		out := make(chan []byte)
+		close(out)
+		return out, err
+	}
+
 	if err != nil {
 		log.Debug("composer: calling searchValue error: ", key, err)
 	}
+
 	return ch, err
 }
 

diff --git a/test/sharness/t0172-content-routing-over-http.sh b/test/sharness/t0172-content-routing-over-http.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+test_description="Test content routing over HTTP"
+
+. lib/test-lib.sh
+
+
+if ! test_have_prereq SOCAT; then
+  skip_all="skipping '$test_description': socat is not available"
+  test_done
+fi
+
+test_init_ipfs
+
+# Run listener on a free port to log HTTP requests sent by Kubo in Routing.Type=auto mode
+export ROUTER_PORT=$(comm -23 <(seq 49152 65535 | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2) | head -n 1)
+export IPFS_HTTP_ROUTERS="http://127.0.0.1:$ROUTER_PORT"
+
+test_launch_ipfs_daemon
+
+test_expect_success "start HTTP router proxy" '
+  socat TCP-LISTEN:$ROUTER_PORT,reuseaddr,fork,bind=127.0.0.1 STDOUT > http_requests &
+  NCPID=$!
+'
+
+## HTTP GETs
+
+test_expect_success 'create unique CID without adding it to the local datastore' '
+  WANT_CID=$(date +"%FT%T.%N%z" | ipfs add -qn)
+'
+
+test_expect_success 'expect HTTP request for unknown CID' '
+  ipfs routing findprovs --timeout 3s "$WANT_CID" &&
+  test_should_contain "GET /routing/v1/providers/$WANT_CID" http_requests
+'
+
+## HTTP PUTs
+
+test_expect_success 'add new CID to the local datastore' '
+  ADD_CID=$(date +"%FT%T.%N%z" | ipfs add -q)
+'
+
+# cid.contact supports GET-only: https://github.com/ipfs/kubo/issues/9504
+# which means no announcements over HTTP should be made.
+test_expect_success 'expect no HTTP requests to be sent with locally added CID' '
+  test_should_not_contain "$ADD_CID" http_requests
+'
+
+test_expect_success "stop nc" '
+  kill "$NCPID" && wait "$NCPID" || true
+'
+
+test_kill_ipfs_daemon
+test_done