chore: update ipni components

.github/actions/go-test-setup/action.yml

@@ -0,0 +1,10 @@
+name: extend-tests-timeout
+description: add -timeout 10m flag to GOFLAGS to extend timeout for tests
+
+runs:
+  using: "composite"
+  steps:
+    - name: Extend timeout for tests
+      shell: bash
+      run: |
+        echo "GOFLAGS=$GOFLAGS -timeout=10m" >> $GITHUB_ENV

cmd/frisbii/flags.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"errors"
+	"net/url"
 	"path/filepath"
 	"strconv"
 	"time"
@@ -26,6 +27,16 @@ var Flags = []cli.Flag{
 		Usage: "content to announce to the indexer, one of [none,roots]",
 		Value: "none",
 	},
+	&cli.StringFlag{
+		Name:  "announce-url",
+		Usage: "announcement endpoint url for the indexer",
+		Value: IndexerAnnounceUrl,
+	},
+	&cli.StringFlag{
+		Name:  "ipni-path",
+		Usage: "the local path to serve IPNI content from, requests will have /ipni/v1/ad/ automatically appended to it",
+		Value: IndexerHandlerPath,
+	},
 	&cli.StringFlag{
 		Name:  "public-addr",
 		Usage: "multiaddr or URL of this server as seen by the indexer and other peers if it is different to the listen address",
@@ -61,6 +72,8 @@ type Config struct {
 	Cars                []string
 	Listen              string
 	Announce            AnnounceType
+	AnnounceUrl         *url.URL
+	IpniPath            string
 	PublicAddr          string
 	LogFile             string
 	MaxResponseDuration time.Duration
@@ -90,7 +103,12 @@ func ToConfig(c *cli.Context) (Config, error) {
 	default:
 		return Config{}, errors.New("invalid announce parameter, must be of value [none,roots]")
 	}
+	announceUrl, err := url.Parse(c.String("announce-url"))
+	if err != nil {
+		return Config{}, err
+	}
 
+	ipniPath := c.String("ipni-path")
 	listen := c.String("listen")
 	publicAddr := c.String("public-addr")
 	logFile := c.String("log-file")
@@ -110,6 +128,8 @@ func ToConfig(c *cli.Context) (Config, error) {
 		Cars:                carPaths,
 		Listen:              listen,
 		Announce:            announceType,
+		AnnounceUrl:         announceUrl,
+		IpniPath:            ipniPath,
 		PublicAddr:          publicAddr,
 		LogFile:             logFile,
 		MaxResponseDuration: maxResponseDuration,

cmd/frisbii/main.go

@@ -6,11 +6,13 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 
 	"github.com/ipfs/go-log/v2"
 	"github.com/ipfs/go-unixfsnode"
 	"github.com/ipld/frisbii"
+	util "github.com/ipld/frisbii/internal/util"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
 	"github.com/ipni/go-libipni/maurl"
 	"github.com/ipni/index-provider/engine"
@@ -21,31 +23,49 @@ import (
 )
 
 const (
-	ConfigDir          = ".frisbii"
-	IndexerHandlerPath = "/_ipni/"
+	IndexerHandlerPath = "/ipni/"
 	IndexerAnnounceUrl = "https://cid.contact/ingest/announce"
 	DefaultHttpPort    = 3747
 )
 
 var logger = log.Logger("frisbii")
 
 func main() {
+	// Set up a context that is canceled when the command is interrupted
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	app := &cli.App{
 		Name:   "frisbii",
 		Usage:  "A minimal IPLD data provider for IPFS",
 		Flags:  Flags,
 		Action: action,
 	}
 
-	err := app.Run(os.Args)
+	// Set up a signal handler to cancel the context
+	go func() {
+		interrupt := make(chan os.Signal, 1)
+		signal.Notify(interrupt, syscall.SIGTERM, syscall.SIGINT)
+		select {
+		case <-interrupt:
+			cancel()
+			fmt.Println("Received interrupt signal, shutting down...")
+			fmt.Println("(Hit ctrl-c again to force-shutdown the daemon.)")
+		case <-ctx.Done():
+		}
+		// Allow any further SIGTERM or SIGINT to kill process
+		signal.Stop(interrupt)
+	}()
+
+	err := app.RunContext(ctx, os.Args)
 	if err != nil {
 		logger.Error(err)
 		os.Exit(1)
 	}
 }
 
 func action(c *cli.Context) error {
-	ctx := context.Background()
+	ctx := c.Context
 
 	config, err := ToConfig(c)
 	if err != nil {
@@ -71,20 +91,20 @@ func action(c *cli.Context) error {
 		}()
 	}
 
-	confDir, err := configDir()
+	confDir, err := util.ConfigDir()
 	if err != nil {
 		return err
 	}
 
-	privKey, id, err := loadPrivKey(confDir)
+	privKey, id, err := util.LoadPrivKey(confDir)
 	if err != nil {
 		return err
 	}
 
 	multicar := frisbii.NewMultiReadableStorage()
 	for ii, carPath := range config.Cars {
 		loader.SetStatus(fmt.Sprintf("Loading CARs (%d / %d) ...", ii+1, len(config.Cars)))
-		loadCar(multicar, carPath)
+		util.LoadCar(multicar, carPath)
 	}
 
 	loader.SetStatus("Loaded CARs, starting server ...")
@@ -117,7 +137,7 @@ func action(c *cli.Context) error {
 		errCh <- server.Serve()
 	}()
 
-	frisbiiListenAddr, err := getListenAddr(server.Addr().String(), config.PublicAddr)
+	frisbiiListenAddr, err := util.GetListenAddr(server.Addr().String(), config.PublicAddr)
 	if err != nil {
 		return err
 	}
@@ -140,19 +160,29 @@ func action(c *cli.Context) error {
 			return err
 		}
 
-		httpath, err := multiaddr.NewComponent("httpath", url.PathEscape(IndexerHandlerPath))
-		if err != nil {
-			return err
+		announceAddr := frisbiiListenAddr.Maddr
+		ipniPath := config.IpniPath
+		if strings.HasPrefix("/ipni/v1/ad/", config.IpniPath) {
+			// if it's some subset of /ipni/v1/ad/ then we won't need to add it,
+			// ipnisync's ServeHTTP is agnostic and only cares about the path.Base()
+			// of the path.
+			ipniPath = ""
+		}
+		if ipniPath != "" {
+			httpath, err := multiaddr.NewComponent("httpath", url.PathEscape(ipniPath))
+			if err != nil {
+				return err
+			}
+			announceAddr = multiaddr.Join(announceAddr, httpath)
 		}
-		announceAddr := multiaddr.Join(frisbiiListenAddr.Maddr, httpath)
 
 		engine, err := engine.New(
 			engine.WithPrivateKey(privKey),
 			engine.WithProvider(peer.AddrInfo{ID: id, Addrs: []multiaddr.Multiaddr{frisbiiListenAddr.Maddr}}),
-			engine.WithDirectAnnounce(IndexerAnnounceUrl),
+			engine.WithDirectAnnounce(config.AnnounceUrl.String()),
 			engine.WithPublisherKind(engine.HttpPublisher),
 			engine.WithHttpPublisherWithoutServer(),
-			engine.WithHttpPublisherHandlerPath(IndexerHandlerPath),
+			engine.WithHttpPublisherHandlerPath(ipniPath),
 			engine.WithHttpPublisherListenAddr(listenUrl.Host),
 			engine.WithHttpPublisherAnnounceAddr(announceAddr.String()),
 		)
@@ -168,7 +198,10 @@ func action(c *cli.Context) error {
 			return err
 		}
 
-		server.SetIndexerProvider(IndexerHandlerPath, engine)
+		// use config.IpniPath here, but the adjusted ipniPath above for setting up
+		// the engine; here we set our local mount expectations and it can't be
+		// ""
+		server.SetIndexerProvider(config.IpniPath, engine)
 
 		if err := server.Announce(); err != nil {
 			return err

frisbii.go

@@ -83,7 +83,7 @@ func (fs *FrisbiiServer) SetIndexerProvider(handlerPath string, indexerProvider
 	if err != nil {
 		return err
 	}
-	if handlerPath[len(handlerPath)-1] != '/' {
+	if handlerPath == "" || handlerPath[len(handlerPath)-1] != '/' {
 		handlerPath += "/"
 	}
 	fs.mux.HandleFunc(handlerPath, handlerFunc)

go.mod

@@ -15,9 +15,10 @@ require (
 	github.com/ipld/go-ipld-prime v0.21.0
 	github.com/ipld/go-trustless-utils v0.2.0
 	github.com/ipld/ipld/specs v0.0.0-20230907004443-0e4ff95ff474
-	github.com/ipni/go-libipni v0.3.4
-	github.com/ipni/index-provider v0.13.5
-	github.com/libp2p/go-libp2p v0.30.0
+	github.com/ipni/go-libipni v0.5.0
+	github.com/ipni/index-provider v0.14.1
+	github.com/ipni/storetheindex v0.7.9-0.20230914224616-e6658bcf60c5
+	github.com/libp2p/go-libp2p v0.31.0
 	github.com/multiformats/go-multiaddr v0.11.0
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/stretchr/testify v1.8.4
@@ -57,13 +58,15 @@ require (
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gopacket v1.1.19 // indirect
-	github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect
+	github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c // indirect
 	github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
 	github.com/huin/goupnp v1.2.0 // indirect
 	github.com/ipfs/go-bitfield v1.1.0 // indirect
@@ -88,7 +91,6 @@ require (
 	github.com/libp2p/go-cidranger v1.1.0 // indirect
 	github.com/libp2p/go-flow-metrics v0.1.0 // indirect
 	github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect
-	github.com/libp2p/go-libp2p-gostream v0.6.0 // indirect
 	github.com/libp2p/go-libp2p-pubsub v0.9.3 // indirect
 	github.com/libp2p/go-msgio v0.3.0 // indirect
 	github.com/libp2p/go-nat v0.2.0 // indirect
@@ -119,13 +121,13 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polydawn/refmt v0.89.0 // indirect
-	github.com/prometheus/client_golang v1.15.1 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.1 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
-	github.com/quic-go/qtls-go1-20 v0.3.2 // indirect
-	github.com/quic-go/quic-go v0.37.6 // indirect
+	github.com/quic-go/qtls-go1-20 v0.3.3 // indirect
+	github.com/quic-go/quic-go v0.38.1 // indirect
 	github.com/quic-go/webtransport-go v0.5.3 // indirect
 	github.com/raulk/go-watchdog v1.3.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
@@ -145,6 +147,7 @@ require (
 	go.uber.org/fx v1.20.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.25.0 // indirect
+	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
 	golang.org/x/crypto v0.12.0 // indirect
 	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
 	golang.org/x/mod v0.12.0 // indirect
@@ -154,7 +157,7 @@ require (
 	golang.org/x/text v0.12.0 // indirect
 	golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/blake3 v1.2.1 // indirect
 )