From 3aafda4e6738519bad2203fcdf11199ad4a772c8 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Mon, 19 Jul 2021 03:58:53 +0200 Subject: [PATCH 1/6] feat: append annouces Closes #7791 --- core/node/groups.go | 2 +- core/node/libp2p/addrs.go | 24 +++++++++++++++++------- docs/config.md | 1 + go.mod | 2 ++ go.sum | 4 ++-- 5 files changed, 23 insertions(+), 10 deletions(-) diff --git a/core/node/groups.go b/core/node/groups.go index 930a2151bef..0cc32934ad9 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -128,7 +128,7 @@ func LibP2P(bcfg *BuildCfg, cfg *config.Config) fx.Option { BaseLibP2P, fx.Provide(libp2p.AddrFilters(cfg.Swarm.AddrFilters)), - fx.Provide(libp2p.AddrsFactory(cfg.Addresses.Announce, cfg.Addresses.NoAnnounce)), + fx.Provide(libp2p.AddrsFactory(cfg.Addresses.Announce, cfg.Addresses.AppendAnnounce, cfg.Addresses.NoAnnounce)), fx.Provide(libp2p.SmuxTransport(cfg.Swarm.Transports)), fx.Provide(libp2p.Relay(enableRelay, cfg.Swarm.EnableRelayHop)), fx.Provide(libp2p.Transports(cfg.Swarm.Transports)), diff --git a/core/node/libp2p/addrs.go b/core/node/libp2p/addrs.go index 818385194a7..06aa059a4ea 100644 --- a/core/node/libp2p/addrs.go +++ b/core/node/libp2p/addrs.go @@ -25,14 +25,23 @@ func AddrFilters(filters []string) func() (*ma.Filters, Libp2pOpts, error) { } } -func makeAddrsFactory(announce []string, noAnnounce []string) (p2pbhost.AddrsFactory, error) { - var annAddrs []ma.Multiaddr - for _, addr := range announce { - maddr, err := ma.NewMultiaddr(addr) +func makeAddrsFactory(announce []string, appendAnnouce []string, noAnnounce []string) (p2pbhost.AddrsFactory, error) { + var err error // To assign to the slice in the for loop + + annAddrs := make([]ma.Multiaddr, len(announce)) + for i, addr := range announce { + annAddrs[i], err = ma.NewMultiaddr(addr) + if err != nil { + return nil, err + } + } + + appendAnnAddrs := make([]ma.Multiaddr, len(appendAnnouce)) + for i, addr := range appendAnnouce { + appendAnnAddrs[i], err = ma.NewMultiaddr(addr) if err != nil { return nil, err } - annAddrs = append(annAddrs, maddr) } filters := ma.NewFilters() @@ -57,6 +66,7 @@ func makeAddrsFactory(announce []string, noAnnounce []string) (p2pbhost.AddrsFac } else { addrs = allAddrs } + addrs = append(addrs, appendAnnAddrs...) var out []ma.Multiaddr for _, maddr := range addrs { @@ -71,9 +81,9 @@ func makeAddrsFactory(announce []string, noAnnounce []string) (p2pbhost.AddrsFac }, nil } -func AddrsFactory(announce []string, noAnnounce []string) func() (opts Libp2pOpts, err error) { +func AddrsFactory(announce []string, appendAnnouce []string, noAnnounce []string) func() (opts Libp2pOpts, err error) { return func() (opts Libp2pOpts, err error) { - addrsFactory, err := makeAddrsFactory(announce, noAnnounce) + addrsFactory, err := makeAddrsFactory(announce, appendAnnouce, noAnnounce) if err != nil { return opts, err } diff --git a/docs/config.md b/docs/config.md index ae39a3f88ca..c45f7ad7a11 100644 --- a/docs/config.md +++ b/docs/config.md @@ -21,6 +21,7 @@ config file at runtime. - [`Addresses.Gateway`](#addressesgateway) - [`Addresses.Swarm`](#addressesswarm) - [`Addresses.Announce`](#addressesannounce) + - [`Addresses.AppendAnnounce`](#addressesappendannounce) - [`Addresses.NoAnnounce`](#addressesnoannounce) - [`API`](#api) - [`API.HTTPHeaders`](#apihttpheaders) diff --git a/go.mod b/go.mod index 739d85be399..d405f843b62 100644 --- a/go.mod +++ b/go.mod @@ -114,3 +114,5 @@ require ( ) go 1.16 + +replace github.com/ipfs/go-ipfs-config => github.com/Jorropo/go-ipfs-config v0.14.1-0.20211115134851-f197db464bd5 // Temporary while waiting for https://github.com/ipfs/go-ipfs-config/pull/135/ diff --git a/go.sum b/go.sum index 643d4206182..fdbc1a46b82 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Jorropo/go-ipfs-config v0.14.1-0.20211115134851-f197db464bd5 h1:frQtt2vWTOYbpKPBbraLaTfxlKdwmTx7Qsz6iBnlAuU= +github.com/Jorropo/go-ipfs-config v0.14.1-0.20211115134851-f197db464bd5/go.mod h1:wz2lKzOjgJeYJa6zx8W9VT7mz+iSd0laBMqS/9wmX6A= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMGXfDk= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= @@ -465,8 +467,6 @@ github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7Na github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-cmds v0.6.0 h1:yAxdowQZzoFKjcLI08sXVNnqVj3jnABbf9smrPQmBsw= github.com/ipfs/go-ipfs-cmds v0.6.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= -github.com/ipfs/go-ipfs-config v0.16.0 h1:CBtIYyp/iWIczCv83bmfge8EA2KqxOOfqmETs3tUnnU= -github.com/ipfs/go-ipfs-config v0.16.0/go.mod h1:wz2lKzOjgJeYJa6zx8W9VT7mz+iSd0laBMqS/9wmX6A= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= From aec3c206d9f3abfaea038a5b6d2999dbcc826260 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 15 Nov 2021 14:59:12 +0100 Subject: [PATCH 2/6] refactor: go-ipfs-config - nat.go needs to use OptionalDuration - libp2p config no longer use relay v1 setting --- core/node/groups.go | 1 - core/node/libp2p/nat.go | 14 ++++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/core/node/groups.go b/core/node/groups.go index 0cc32934ad9..d80c5fc268a 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -130,7 +130,6 @@ func LibP2P(bcfg *BuildCfg, cfg *config.Config) fx.Option { fx.Provide(libp2p.AddrFilters(cfg.Swarm.AddrFilters)), fx.Provide(libp2p.AddrsFactory(cfg.Addresses.Announce, cfg.Addresses.AppendAnnounce, cfg.Addresses.NoAnnounce)), fx.Provide(libp2p.SmuxTransport(cfg.Swarm.Transports)), - fx.Provide(libp2p.Relay(enableRelay, cfg.Swarm.EnableRelayHop)), fx.Provide(libp2p.Transports(cfg.Swarm.Transports)), fx.Invoke(libp2p.StartListening(cfg.Addresses.Swarm)), fx.Invoke(libp2p.SetupDiscovery(cfg.Discovery.MDNS.Enabled, cfg.Discovery.MDNS.Interval)), diff --git a/core/node/libp2p/nat.go b/core/node/libp2p/nat.go index 9fe74d91cb3..ce0ca345f70 100644 --- a/core/node/libp2p/nat.go +++ b/core/node/libp2p/nat.go @@ -3,7 +3,7 @@ package libp2p import ( "time" - "github.com/ipfs/go-ipfs-config" + config "github.com/ipfs/go-ipfs-config" "github.com/libp2p/go-libp2p" ) @@ -13,14 +13,12 @@ func AutoNATService(throttle *config.AutoNATThrottleConfig) func() Libp2pOpts { return func() (opts Libp2pOpts) { opts.Opts = append(opts.Opts, libp2p.EnableNATService()) if throttle != nil { - global := throttle.GlobalLimit - peer := throttle.PeerLimit - interval := time.Duration(throttle.Interval) - if interval == 0 { - interval = time.Minute - } opts.Opts = append(opts.Opts, - libp2p.AutoNATServiceRateLimit(global, peer, interval), + libp2p.AutoNATServiceRateLimit( + throttle.GlobalLimit, + throttle.PeerLimit, + throttle.Interval.WithDefault(time.Minute), + ), ) } return opts From 8d6a5e575b6d8eee064d4fc6647af011cceec053 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 15 Nov 2021 15:10:23 +0100 Subject: [PATCH 3/6] docs: Addresses.AppendAnnounce --- docs/config.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/config.md b/docs/config.md index c45f7ad7a11..37de0d1a125 100644 --- a/docs/config.md +++ b/docs/config.md @@ -326,8 +326,19 @@ Default: `[]` Type: `array[string]` (multiaddrs) +### `Addresses.AppendAnnounce` + +Similar to [`Addresses.Announce`](#addressesannounce) except this doesn't +override inferred swarm addresses if non-empty. + +Default: `[]` + +Type: `array[string]` (multiaddrs) + ### `Addresses.NoAnnounce` + An array of swarm addresses not to announce to the network. +Takes precedence over `Addresses.Announce` and `Addresses.AppendAnnounce`. Default: `[]` From aacf2fcaaad83f50b8eefd5e98a866f76aac216c Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 15 Nov 2021 15:28:24 +0100 Subject: [PATCH 4/6] test: Addresses.AppendAnnounce --- test/sharness/t0140-swarm.sh | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/test/sharness/t0140-swarm.sh b/test/sharness/t0140-swarm.sh index 20994c13e7c..04a4c8f3a93 100755 --- a/test/sharness/t0140-swarm.sh +++ b/test/sharness/t0140-swarm.sh @@ -65,6 +65,26 @@ test_expect_success 'Addresses.Announce affects addresses' ' test_kill_ipfs_daemon +announceCfg='["/dnsaddr/dynamic.example.com", "/ip4/10.20.30.40/tcp/4321"]' +test_expect_success "test_config_set AppendAnnounce succeeds" " + ipfs config --json Addresses.AppendAnnounce '$announceCfg' +" + +test_launch_ipfs_daemon + +test_expect_success 'Addresses.AppendAnnounce is applied on top of Announce' ' + ipfs swarm addrs local >actual && + grep "/ip4/1.2.3.4/tcp/1234" actual && + grep "/dnsaddr/dynamic.example.com" actual && + grep "/ip4/10.20.30.40/tcp/4321" actual && + ipfs id -f"" | xargs -n1 echo | tee actual && + grep "/ip4/1.2.3.4/tcp/1234/p2p" actual && + grep "/dnsaddr/dynamic.example.com/p2p/" actual && + grep "/ip4/10.20.30.40/tcp/4321/p2p/" actual +' + +test_kill_ipfs_daemon + noAnnounceCfg='["/ip4/1.2.3.4/tcp/1234"]' test_expect_success "test_config_set succeeds" " ipfs config --json Addresses.NoAnnounce '$noAnnounceCfg' @@ -72,11 +92,13 @@ test_expect_success "test_config_set succeeds" " test_launch_ipfs_daemon -test_expect_success "Addresses.NoAnnounce affects addresses" ' +test_expect_success "Addresses.NoAnnounce affects addresses from Announce and AppendAnnounce" ' ipfs swarm addrs local >actual && grep -v "/ip4/1.2.3.4/tcp/1234" actual && + grep -v "/ip4/10.20.30.40/tcp/4321" actual && ipfs id -f"" | xargs -n1 echo >actual && - grep -v "/ip4/1.2.3.4/tcp/1234" actual + grep -v "/ip4/1.2.3.4/tcp/1234" actual && + grep -v "//ip4/10.20.30.40/tcp/4321" actual ' test_kill_ipfs_daemon From 23c028b22bd138c562ef23a533785630705e67af Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 23 Nov 2021 19:33:21 +0100 Subject: [PATCH 5/6] chore: go-ipfs-config v0.18.0 --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index d405f843b62..df6de570381 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/ipfs/go-ipfs-blockstore v0.1.6 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-cmds v0.6.0 - github.com/ipfs/go-ipfs-config v0.16.0 + github.com/ipfs/go-ipfs-config v0.18.0 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 github.com/ipfs/go-ipfs-files v0.0.9 @@ -114,5 +114,3 @@ require ( ) go 1.16 - -replace github.com/ipfs/go-ipfs-config => github.com/Jorropo/go-ipfs-config v0.14.1-0.20211115134851-f197db464bd5 // Temporary while waiting for https://github.com/ipfs/go-ipfs-config/pull/135/ diff --git a/go.sum b/go.sum index fdbc1a46b82..1f44f594b30 100644 --- a/go.sum +++ b/go.sum @@ -48,8 +48,6 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Jorropo/go-ipfs-config v0.14.1-0.20211115134851-f197db464bd5 h1:frQtt2vWTOYbpKPBbraLaTfxlKdwmTx7Qsz6iBnlAuU= -github.com/Jorropo/go-ipfs-config v0.14.1-0.20211115134851-f197db464bd5/go.mod h1:wz2lKzOjgJeYJa6zx8W9VT7mz+iSd0laBMqS/9wmX6A= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMGXfDk= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= @@ -467,6 +465,8 @@ github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7Na github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-cmds v0.6.0 h1:yAxdowQZzoFKjcLI08sXVNnqVj3jnABbf9smrPQmBsw= github.com/ipfs/go-ipfs-cmds v0.6.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= +github.com/ipfs/go-ipfs-config v0.18.0 h1:Ta1aNGNEq6RIvzbw7dqzCVZJKb7j+Dd35JFnAOCpT8g= +github.com/ipfs/go-ipfs-config v0.18.0/go.mod h1:wz2lKzOjgJeYJa6zx8W9VT7mz+iSd0laBMqS/9wmX6A= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= From 9673571a201bfe5a781015a2b9003aa3d506f46e Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 30 Nov 2021 17:07:14 +0100 Subject: [PATCH 6/6] fix: deduplicate Swarm.Announce and AppendAnnounce https://github.com/ipfs/go-ipfs/pull/8177#discussion_r759317772 https://github.com/ipfs/go-ipfs/pull/8177#discussion_r759320332 --- core/node/libp2p/addrs.go | 15 +++++++++++---- test/sharness/t0140-swarm.sh | 13 ++++++++++--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/core/node/libp2p/addrs.go b/core/node/libp2p/addrs.go index 06aa059a4ea..827c6cc4c9d 100644 --- a/core/node/libp2p/addrs.go +++ b/core/node/libp2p/addrs.go @@ -26,7 +26,8 @@ func AddrFilters(filters []string) func() (*ma.Filters, Libp2pOpts, error) { } func makeAddrsFactory(announce []string, appendAnnouce []string, noAnnounce []string) (p2pbhost.AddrsFactory, error) { - var err error // To assign to the slice in the for loop + var err error // To assign to the slice in the for loop + existing := make(map[string]bool) // To avoid duplicates annAddrs := make([]ma.Multiaddr, len(announce)) for i, addr := range announce { @@ -34,14 +35,20 @@ func makeAddrsFactory(announce []string, appendAnnouce []string, noAnnounce []st if err != nil { return nil, err } + existing[addr] = true } - appendAnnAddrs := make([]ma.Multiaddr, len(appendAnnouce)) - for i, addr := range appendAnnouce { - appendAnnAddrs[i], err = ma.NewMultiaddr(addr) + var appendAnnAddrs []ma.Multiaddr + for _, addr := range appendAnnouce { + if existing[addr] { + // skip AppendAnnounce that is on the Announce list already + continue + } + appendAddr, err := ma.NewMultiaddr(addr) if err != nil { return nil, err } + appendAnnAddrs = append(appendAnnAddrs, appendAddr) } filters := ma.NewFilters() diff --git a/test/sharness/t0140-swarm.sh b/test/sharness/t0140-swarm.sh index 04a4c8f3a93..d65831d3e22 100755 --- a/test/sharness/t0140-swarm.sh +++ b/test/sharness/t0140-swarm.sh @@ -65,9 +65,16 @@ test_expect_success 'Addresses.Announce affects addresses' ' test_kill_ipfs_daemon -announceCfg='["/dnsaddr/dynamic.example.com", "/ip4/10.20.30.40/tcp/4321"]' -test_expect_success "test_config_set AppendAnnounce succeeds" " - ipfs config --json Addresses.AppendAnnounce '$announceCfg' + +announceCfg='["/ip4/127.0.0.1/tcp/4001", "/ip4/1.2.3.4/tcp/1234"]' +test_expect_success "test_config_set succeeds" " + ipfs config --json Addresses.Announce '$announceCfg' +" +# Include "/ip4/1.2.3.4/tcp/1234" to ensure we deduplicate addrs already present in Swarm.Announce +appendAnnounceCfg='["/dnsaddr/dynamic.example.com", "/ip4/10.20.30.40/tcp/4321", "/ip4/1.2.3.4/tcp/1234"]' +test_expect_success "test_config_set Announce and AppendAnnounce succeeds" " + ipfs config --json Addresses.Announce '$announceCfg' && + ipfs config --json Addresses.AppendAnnounce '$appendAnnounceCfg' " test_launch_ipfs_daemon