diff --git a/README.md b/README.md index 6337cdb..817f564 100644 --- a/README.md +++ b/README.md @@ -71,10 +71,38 @@ Note that the `multiaddr` can be: ### Check results -The server performs several checks given a CID. The results of the check are expressed by the `output` type: +The server performs several checks depending on whether you also pass a **multiaddr** or just a **cid**. + +#### Results when only a `cid` is passed + +The results of the check are expressed by the `cidCheckOutput` type: + +```go +type cidCheckOutput *[]providerOutput + +type providerOutput struct { + ID string + ConnectionError string + Addrs []string + ConnectionMaddrs []string + DataAvailableOverBitswap BitswapCheckOutput +} +``` + +The `providerOutput` type contains the following fields: + +- `ID`: The peer ID of the provider. +- `ConnectionError`: An error message if the connection to the provider failed. +- `Addrs`: The multiaddrs of the provider from the DHT. +- `ConnectionMaddrs`: The multiaddrs that were used to connect to the provider. +- `DataAvailableOverBitswap`: The result of the Bitswap check. + +#### Results when a `multiaddr` and a `cid` are passed + +The results of the check are expressed by the `peerCheckOutput` type: ```go -type output struct { +type peerCheckOutput struct { ConnectionError string PeerFoundInDHT map[string]int CidInDHT bool diff --git a/daemon.go b/daemon.go index e16ab36..fb2f385 100644 --- a/daemon.go +++ b/daemon.go @@ -2,10 +2,8 @@ package main import ( "context" - "errors" "fmt" "log" - "net/url" "sync" "time" @@ -20,10 +18,12 @@ import ( record "github.com/libp2p/go-libp2p-record" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/peerstore" "github.com/libp2p/go-libp2p/core/routing" "github.com/libp2p/go-libp2p/p2p/net/connmgr" + "github.com/libp2p/go-libp2p/p2p/protocol/identify" "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" + "github.com/prometheus/client_golang/prometheus" ) type kademlia interface { @@ -36,8 +36,13 @@ type daemon struct { dht kademlia dhtMessenger *dhtpb.ProtocolMessenger createTestHost func() (host.Host, error) + promRegistry *prometheus.Registry } +// number of providers at which to stop looking for providers in the DHT +// When doing a check only with a CID +var MaxProvidersCount = 10 + func newDaemon(ctx context.Context, acceleratedDHT bool) (*daemon, error) { rm, err := NewResourceManager() if err != nil { @@ -49,6 +54,9 @@ func newDaemon(ctx context.Context, acceleratedDHT bool) (*daemon, error) { return nil, err } + // Create a custom registry for all prometheus metrics + promRegistry := prometheus.NewRegistry() + h, err := libp2p.New( libp2p.DefaultMuxers, libp2p.Muxer(mplex.ID, mplex.DefaultTransport), @@ -56,6 +64,7 @@ func newDaemon(ctx context.Context, acceleratedDHT bool) (*daemon, error) { libp2p.ConnectionGater(&privateAddrFilterConnectionGater{}), libp2p.ResourceManager(rm), libp2p.EnableHolePunching(), + libp2p.PrometheusRegisterer(promRegistry), libp2p.UserAgent(userAgent), ) if err != nil { @@ -88,15 +97,20 @@ func newDaemon(ctx context.Context, acceleratedDHT bool) (*daemon, error) { return nil, err } - return &daemon{h: h, dht: d, dhtMessenger: pm, createTestHost: func() (host.Host, error) { - return libp2p.New( - libp2p.ConnectionGater(&privateAddrFilterConnectionGater{}), - libp2p.DefaultMuxers, - libp2p.Muxer("/mplex/6.7.0", mplex.DefaultTransport), - libp2p.EnableHolePunching(), - libp2p.UserAgent(userAgent), - ) - }}, nil + return &daemon{ + h: h, + dht: d, + dhtMessenger: pm, + promRegistry: promRegistry, + createTestHost: func() (host.Host, error) { + return libp2p.New( + libp2p.ConnectionGater(&privateAddrFilterConnectionGater{}), + libp2p.DefaultMuxers, + libp2p.Muxer("/mplex/6.7.0", mplex.DefaultTransport), + libp2p.EnableHolePunching(), + libp2p.UserAgent(userAgent), + ) + }}, nil } func (d *daemon) mustStart() { @@ -109,18 +123,101 @@ func (d *daemon) mustStart() { } -func (d *daemon) runCheck(query url.Values) (*output, error) { - maStr := query.Get("multiaddr") - cidStr := query.Get("cid") +type cidCheckOutput *[]providerOutput - if maStr == "" { - return nil, errors.New("missing 'multiaddr' argument") +type providerOutput struct { + ID string + ConnectionError string + Addrs []string + ConnectionMaddrs []string + DataAvailableOverBitswap BitswapCheckOutput +} + +// runCidCheck looks up the DHT for providers of a given CID and then checks their connectivity and Bitswap availability +func (d *daemon) runCidCheck(ctx context.Context, cidStr string) (cidCheckOutput, error) { + cid, err := cid.Decode(cidStr) + if err != nil { + return nil, err } - if cidStr == "" { - return nil, errors.New("missing 'cid' argument") + out := make([]providerOutput, 0, MaxProvidersCount) + + queryCtx, cancel := context.WithCancel(ctx) + defer cancel() + provsCh := d.dht.FindProvidersAsync(queryCtx, cid, MaxProvidersCount) + + var wg sync.WaitGroup + var mu sync.Mutex + + for provider := range provsCh { + wg.Add(1) + go func(provider peer.AddrInfo) { + defer wg.Done() + + addrs := []string{} + if len(provider.Addrs) > 0 { + for _, addr := range provider.Addrs { + if manet.IsPublicAddr(addr) { // only return public addrs + addrs = append(addrs, addr.String()) + } + } + } + + provOutput := providerOutput{ + ID: provider.ID.String(), + Addrs: addrs, + DataAvailableOverBitswap: BitswapCheckOutput{}, + } + + testHost, err := d.createTestHost() + if err != nil { + log.Printf("Error creating test host: %v\n", err) + return + } + defer testHost.Close() + + // Test Is the target connectable + dialCtx, dialCancel := context.WithTimeout(ctx, time.Second*15) + defer dialCancel() + + testHost.Connect(dialCtx, provider) + // Call NewStream to force NAT hole punching. see https://github.com/libp2p/go-libp2p/issues/2714 + _, connErr := testHost.NewStream(dialCtx, provider.ID, "/ipfs/bitswap/1.2.0", "/ipfs/bitswap/1.1.0", "/ipfs/bitswap/1.0.0", "/ipfs/bitswap") + + if connErr != nil { + provOutput.ConnectionError = connErr.Error() + } else { + // since we pass a libp2p host that's already connected to the peer the actual connection maddr we pass in doesn't matter + p2pAddr, _ := multiaddr.NewMultiaddr("/p2p/" + provider.ID.String()) + provOutput.DataAvailableOverBitswap = checkBitswapCID(ctx, testHost, cid, p2pAddr) + + for _, c := range testHost.Network().ConnsToPeer(provider.ID) { + provOutput.ConnectionMaddrs = append(provOutput.ConnectionMaddrs, c.RemoteMultiaddr().String()) + } + } + + mu.Lock() + out = append(out, provOutput) + mu.Unlock() + }(provider) } + // Wait for all goroutines to finish + wg.Wait() + + return &out, nil +} + +type peerCheckOutput struct { + ConnectionError string + PeerFoundInDHT map[string]int + CidInDHT bool + ConnectionMaddrs []string + DataAvailableOverBitswap BitswapCheckOutput +} + +// runPeerCheck checks the connectivity and Bitswap availability of a CID from a given peer (either with just peer ID or specific multiaddr) +func (d *daemon) runPeerCheck(ctx context.Context, maStr, cidStr string) (*peerCheckOutput, error) { ma, err := multiaddr.NewMultiaddr(maStr) if err != nil { return nil, err @@ -139,12 +236,11 @@ func (d *daemon) runCheck(query url.Values) (*output, error) { return nil, err } - ctx := context.Background() - out := &output{} + out := &peerCheckOutput{} connectionFailed := false - out.CidInDHT = providerRecordInDHT(ctx, d.dht, c, ai.ID) + out.CidInDHT = providerRecordForPeerInDHT(ctx, d.dht, c, ai.ID) addrMap, peerAddrDHTErr := peerAddrsInDHT(ctx, d.dht, d.dhtMessenger, ai.ID) out.PeerFoundInDHT = addrMap @@ -174,15 +270,28 @@ func (d *daemon) runCheck(query url.Values) (*output, error) { if !connectionFailed { // Test Is the target connectable - dialCtx, dialCancel := context.WithTimeout(ctx, time.Second*15) + dialCtx, dialCancel := context.WithTimeout(ctx, time.Second*120) - // we call NewStream instead of Connect to force NAT hole punching - // See https://github.com/libp2p/go-libp2p/issues/2714 - testHost.Peerstore().AddAddrs(ai.ID, ai.Addrs, peerstore.RecentlyConnectedAddrTTL) + testHost.Connect(dialCtx, *ai) + // Call NewStream to force NAT hole punching. see https://github.com/libp2p/go-libp2p/issues/2714 _, connErr := testHost.NewStream(dialCtx, ai.ID, "/ipfs/bitswap/1.2.0", "/ipfs/bitswap/1.1.0", "/ipfs/bitswap/1.0.0", "/ipfs/bitswap") dialCancel() if connErr != nil { - out.ConnectionError = fmt.Sprintf("error dialing to peer: %s", connErr.Error()) + log.Printf("Error connecting to peer %s: %v", ai.ID, connErr) + ids, ok := testHost.(interface{ IDService() identify.IDService }) + if ok { + log.Printf("Own observed addrs: %v", ids.IDService().OwnObservedAddrs()) + } + + // Log all open connections + for _, conn := range testHost.Network().Conns() { + log.Printf("Open connection: Peer ID: %s, Remote Addr: %s, Local Addr: %s", + conn.RemotePeer(), + conn.RemoteMultiaddr(), + conn.LocalMultiaddr(), + ) + } + out.ConnectionError = connErr.Error() connectionFailed = true } } @@ -203,8 +312,15 @@ func (d *daemon) runCheck(query url.Values) (*output, error) { return out, nil } +type BitswapCheckOutput struct { + Duration time.Duration + Found bool + Responded bool + Error string +} + func checkBitswapCID(ctx context.Context, host host.Host, c cid.Cid, ma multiaddr.Multiaddr) BitswapCheckOutput { - log.Printf("Start of Bitswap check for cid %s by attempting to connect to ma: %v with the temporary peer: %s", c, ma, host.ID()) + log.Printf("Start of Bitswap check for cid %s by attempting to connect to ma: %v with the peer: %s", c, ma, host.ID()) out := BitswapCheckOutput{} start := time.Now() @@ -224,21 +340,6 @@ func checkBitswapCID(ctx context.Context, host host.Host, c cid.Cid, ma multiadd return out } -type BitswapCheckOutput struct { - Duration time.Duration - Found bool - Responded bool - Error string -} - -type output struct { - ConnectionError string - PeerFoundInDHT map[string]int - CidInDHT bool - ConnectionMaddrs []string - DataAvailableOverBitswap BitswapCheckOutput -} - func peerAddrsInDHT(ctx context.Context, d kademlia, messenger *dhtpb.ProtocolMessenger, p peer.ID) (map[string]int, error) { closestPeers, err := d.GetClosestPeers(ctx, string(p)) if err != nil { @@ -282,7 +383,7 @@ func peerAddrsInDHT(ctx context.Context, d kademlia, messenger *dhtpb.ProtocolMe return addrMap, nil } -func providerRecordInDHT(ctx context.Context, d kademlia, c cid.Cid, p peer.ID) bool { +func providerRecordForPeerInDHT(ctx context.Context, d kademlia, c cid.Cid, p peer.ID) bool { queryCtx, cancel := context.WithCancel(ctx) defer cancel() provsCh := d.FindProvidersAsync(queryCtx, c, 0) diff --git a/integration_test.go b/integration_test.go index 6dd2da0..f2ed060 100644 --- a/integration_test.go +++ b/integration_test.go @@ -21,7 +21,9 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" "github.com/libp2p/go-libp2p/p2p/net/connmgr" + manet "github.com/multiformats/go-multiaddr/net" "github.com/multiformats/go-multihash" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" ) @@ -61,6 +63,7 @@ func TestBasicIntegration(t *testing.T) { require.NoError(t, err) d := &daemon{ + promRegistry: prometheus.NewRegistry(), h: queryHost, dht: queryDHT, dhtMessenger: pm, @@ -157,4 +160,34 @@ func TestBasicIntegration(t *testing.T) { obj.Value("DataAvailableOverBitswap").Object().Value("Found").Boolean().IsFalse() obj.Value("DataAvailableOverBitswap").Object().Value("Responded").Boolean().IsTrue() }) + + t.Run("Data found on reachable peer with just cid", func(t *testing.T) { + testData := []byte(t.Name()) + mh, err := multihash.Sum(testData, multihash.SHA2_256, -1) + require.NoError(t, err) + testCid := cid.NewCidV1(cid.Raw, mh) + testBlock, err := blocks.NewBlockWithCid(testData, testCid) + require.NoError(t, err) + err = bstore.Put(ctx, testBlock) + require.NoError(t, err) + err = dhtClient.Provide(ctx, testCid, true) + require.NoError(t, err) + + res := test.QueryCid(t, "http://localhost:1234", testCid.String()) + + res.Length().IsEqual(1) + res.Value(0).Object().Value("ID").String().IsEqual(h.ID().String()) + res.Value(0).Object().Value("ConnectionError").String().IsEmpty() + testHostAddrs := h.Addrs() + for _, addr := range testHostAddrs { + if manet.IsPublicAddr(addr) { + res.Value(0).Object().Value("Addrs").Array().ContainsAny(addr.String()) + } + } + + res.Value(0).Object().Value("ConnectionMaddrs").Array() + res.Value(0).Object().Value("DataAvailableOverBitswap").Object().Value("Error").String().IsEmpty() + res.Value(0).Object().Value("DataAvailableOverBitswap").Object().Value("Found").Boolean().IsTrue() + res.Value(0).Object().Value("DataAvailableOverBitswap").Object().Value("Responded").Boolean().IsTrue() + }) } diff --git a/main.go b/main.go index 36cfb27..889a921 100644 --- a/main.go +++ b/main.go @@ -4,13 +4,14 @@ import ( "context" "crypto/subtle" "encoding/json" - "fmt" + "errors" "log" "net" "net/http" "os" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/urfave/cli/v2" @@ -48,6 +49,7 @@ func main() { } app.Action = func(cctx *cli.Context) error { ctx := cctx.Context + d, err := newDaemon(ctx, cctx.Bool("accelerated-dht")) if err != nil { return err @@ -62,13 +64,12 @@ func main() { } func startServer(ctx context.Context, d *daemon, tcpListener, metricsUsername, metricPassword string) error { - fmt.Printf("Starting %s %s\n", name, version) + log.Printf("Starting %s %s\n", name, version) l, err := net.Listen("tcp", tcpListener) if err != nil { return err } - log.Printf("listening on %v\n", l.Addr()) log.Printf("Libp2p host peer id %s\n", d.h.ID()) log.Printf("Libp2p host listening on %v\n", d.h.Addrs()) log.Printf("listening on %v\n", l.Addr()) @@ -79,7 +80,23 @@ func startServer(ctx context.Context, d *daemon, tcpListener, metricsUsername, m checkHandler := func(w http.ResponseWriter, r *http.Request) { w.Header().Add("Access-Control-Allow-Origin", "*") - data, err := d.runCheck(r.URL.Query()) + + maStr := r.URL.Query().Get("multiaddr") + cidStr := r.URL.Query().Get("cid") + + if cidStr == "" { + err = errors.New("missing 'cid' argument") + } + + var err error + var data interface{} + + if maStr == "" { + data, err = d.runCidCheck(r.Context(), cidStr) + } else { + data, err = d.runPeerCheck(r.Context(), maStr, cidStr) + } + if err == nil { w.Header().Add("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(data) @@ -89,13 +106,16 @@ func startServer(ctx context.Context, d *daemon, tcpListener, metricsUsername, m } } - // Create a custom registry - reg := prometheus.NewRegistry() + // Register the default Go collector + d.promRegistry.MustRegister(collectors.NewGoCollector()) + + // Register the process collector + d.promRegistry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) requestsTotal := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "http_requests_total", - Help: "Total number of slow requests", + Help: "Total number of HTTP requests", }, []string{"code"}, ) @@ -103,22 +123,23 @@ func startServer(ctx context.Context, d *daemon, tcpListener, metricsUsername, m requestDuration := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "http_request_duration_seconds", - Help: "Duration of slow requests", + Help: "Duration of HTTP requests", Buckets: prometheus.DefBuckets, }, []string{"code"}, ) requestsInFlight := prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "slow_requests_in_flight", - Help: "Number of slow requests currently being served", + Name: "http_requests_in_flight", + Help: "Number of HTTP requests currently being served", }) // Register metrics with our custom registry - reg.MustRegister(requestsTotal) - reg.MustRegister(requestDuration) - reg.MustRegister(requestsInFlight) - // Instrument the slowHandler + d.promRegistry.MustRegister(requestsTotal) + d.promRegistry.MustRegister(requestDuration) + d.promRegistry.MustRegister(requestsInFlight) + + // Instrument the checkHandler instrumentedHandler := promhttp.InstrumentHandlerCounter( requestsTotal, promhttp.InstrumentHandlerDuration( @@ -130,14 +151,10 @@ func startServer(ctx context.Context, d *daemon, tcpListener, metricsUsername, m ), ) - // 1. Is the peer findable in the DHT? - // 2. Does the multiaddr work? If not, what's the error? - // 3. Is the CID in the DHT? - // 4. Does the peer respond that it has the given data over Bitswap? http.Handle("/check", instrumentedHandler) - http.Handle("/metrics/libp2p", BasicAuth(promhttp.Handler(), metricsUsername, metricPassword)) - http.Handle("/metrics/http", BasicAuth(promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), metricsUsername, metricPassword)) + // Use a single metrics endpoint for all Prometheus metrics + http.Handle("/metrics", BasicAuth(promhttp.HandlerFor(d.promRegistry, promhttp.HandlerOpts{}), metricsUsername, metricPassword)) done := make(chan error, 1) go func() { diff --git a/test/tools.go b/test/tools.go index 39cc275..8ab1e9b 100644 --- a/test/tools.go +++ b/test/tools.go @@ -51,6 +51,26 @@ func Query( JSON(opts).Object() } +func QueryCid( + t *testing.T, + url string, + cid string, +) *httpexpect.Array { + expectedContentType := "application/json" + + opts := httpexpect.ContentOpts{ + MediaType: expectedContentType, + } + + e := httpexpect.Default(t, url) + + return e.GET("/check"). + WithQuery("cid", cid). + Expect(). + Status(http.StatusOK). + JSON(opts).Array() +} + func GetEnv(key string, fallback string) string { if value, ok := os.LookupEnv(key); ok { return value diff --git a/web/index.html b/web/index.html index a5e92fe..f295e78 100644 --- a/web/index.html +++ b/web/index.html @@ -17,20 +17,20 @@

- Check the retrievability of CID from an IPFS peer + Check the retrievability of CID

- Paste in a Content ID and the multiaddr of a host to check if it is expected to be retrievable + Paste in a Content ID and the multiaddr (optional) of a host to check if it is expected to be retrievable

- - + +
- Optional fields + Backend Config @@ -46,7 +46,11 @@

Run Test -
+
+
+ Raw Output +
+

@@ -122,6 +126,7 @@

What does it mean if I get an error?

e.preventDefault() // dont do a browser form post showOutput('') // clear out previous results + showRawOutput('') // clear out previous results const formData = new FormData(document.getElementById('queryForm')) const backendURL = getBackendUrl(formData) @@ -133,13 +138,21 @@

What does it mean if I get an error?

if (res.ok) { const respObj = await res.json() - const output = formatOutput(formData, respObj) - showOutput(output) + showRawOutput(JSON.stringify(respObj, null, 2)) + + if(formData.get('multiaddr') == '') { + const output = formatJustCidOutput(respObj) + showOutput(output) + } else { + const output = formatMaddrOutput(formData.get('multiaddr'), respObj) + showOutput(output) + } } else { const resText = await res.text() showOutput(`⚠️ backend returned an error: ${res.status} ${resText}`) } } catch (e) { + console.log(e) showOutput(`⚠️ backend error: ${e}`) } finally { toggleSubmitButton() @@ -177,6 +190,11 @@

What does it mean if I get an error?

outObj.textContent = output } + function showRawOutput (output) { + const outObj = document.getElementById('raw-output') + outObj.textContent = output + } + function toggleSubmitButton() { const button = document.getElementById('submit') button.toggleAttribute('disabled') @@ -184,11 +202,10 @@

What does it mean if I get an error?

spinner.classList.toggle('dn') } - function formatOutput (formData, respObj) { - const ma = formData.get('multiaddr') - const peerIDStartIndex = ma.lastIndexOf("/p2p/") - const peerID = ma.slice(peerIDStartIndex + 5); - const addrPart = ma.slice(0, peerIDStartIndex); + function formatMaddrOutput (multiaddr, respObj) { + const peerIDStartIndex = multiaddr.lastIndexOf("/p2p/") + const peerID = multiaddr.slice(peerIDStartIndex + 5); + const addrPart = multiaddr.slice(0, peerIDStartIndex); let outText = "" if (respObj.ConnectionError !== "") { @@ -198,7 +215,7 @@

What does it mean if I get an error?

outText += `✅ Successfully connected to multiaddr${madrs?.length > 1 ? 's' : '' }: \n\t${madrs.join('\n\t')}\n` } - if (ma.indexOf("/p2p/") === 0 && ma.lastIndexOf("/") === 4) { + if (multiaddr.indexOf("/p2p/") === 0 && multiaddr.lastIndexOf("/") === 4) { // only peer id passed with /p2p/PeerID if (Object.keys(respObj.PeerFoundInDHT).length === 0) { outText += "❌ Could not find any multiaddrs in the dht\n" @@ -243,6 +260,53 @@

What does it mean if I get an error?

} return outText } + + function formatJustCidOutput (resp) { + let outText = "" + if (resp.length === 0) { + outText += "❌ No providers found for the given CID" + return outText + } + + const successfulProviders = resp.reduce((acc, provider) => { + if(provider.ConnectionError === '' && provider.DataAvailableOverBitswap?.Found === true) { + acc++ + } + return acc + }, 0) + + const failedProviders = resp.length - successfulProviders + + // Show providers without connection errors first + resp.sort((a, b) => { + if (a.ConnectionError === '' && b.ConnectionError !== '') { + return -1; + } else if (a.ConnectionError !== '' && b.ConnectionError === '') { + return 1; + } + + // If both have a connection error, list the one with addresses first + if(a.Addrs.length > 0 && b.Addrs.length === 0) { + return -1 + } else if(a.Addrs.length === 0 && b.Addrs.length > 0) { + return 1 + } else { + return 0 + } + }) + + outText += `${successfulProviders > 0 ? '✅' : '❌'} Found ${successfulProviders} working providers (out of ${resp.length} provider records sampled from Amino DHT) that could be connected to and had the CID available over Bitswap:` + for (const provider of resp) { + const couldConnect = provider.ConnectionError === '' + + outText += `\n\t${provider.ID}\n\t\tConnected: ${couldConnect ? "✅" : `❌ ${provider.ConnectionError.replaceAll('\n', '\n\t\t')}` }` + outText += couldConnect ? `\n\t\tBitswap Check: ${provider.DataAvailableOverBitswap.Found ? `✅` : "❌"} ${provider.DataAvailableOverBitswap.Error || ''}` : '' + outText += (couldConnect && provider.ConnectionMaddrs) ? `\n\t\tSuccessful Connection Multiaddr${provider.ConnectionMaddrs.length > 1 ? 's' : ''}:\n\t\t\t${provider.ConnectionMaddrs?.join('\n\t\t\t') || ''}` : '' + outText += (provider.Addrs.length > 0) ? `\n\t\tPeer Multiaddrs:\n\t\t\t${provider.Addrs.join('\n\t\t\t')}` : '' + } + + return outText + }