From 78965ec837b970ae2c03cbc21ac5f5edb686aff9 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Tue, 12 Sep 2023 19:51:15 +1000 Subject: [PATCH] feat: add --ipni-path, remove /_ipni default, test variations --- cmd/frisbii/flags.go | 8 ++ cmd/frisbii/main.go | 28 +++- frisbii.go | 2 +- internal/integration/ipni_test.go | 205 ++++++++++++++++-------------- 4 files changed, 142 insertions(+), 101 deletions(-) diff --git a/cmd/frisbii/flags.go b/cmd/frisbii/flags.go index 5397d88..4986403 100644 --- a/cmd/frisbii/flags.go +++ b/cmd/frisbii/flags.go @@ -32,6 +32,11 @@ var Flags = []cli.Flag{ Usage: "announcement endpoint url for the indexer", Value: IndexerAnnounceUrl, }, + &cli.StringFlag{ + Name: "ipni-path", + Usage: "the local path to serve IPNI content from, requests will have /ipni/v1/ad/ automatically appended to it", + Value: IndexerHandlerPath, + }, &cli.StringFlag{ Name: "public-addr", Usage: "multiaddr or URL of this server as seen by the indexer and other peers if it is different to the listen address", @@ -68,6 +73,7 @@ type Config struct { Listen string Announce AnnounceType AnnounceUrl *url.URL + IpniPath string PublicAddr string LogFile string MaxResponseDuration time.Duration @@ -102,6 +108,7 @@ func ToConfig(c *cli.Context) (Config, error) { return Config{}, err } + ipniPath := c.String("ipni-path") listen := c.String("listen") publicAddr := c.String("public-addr") logFile := c.String("log-file") @@ -122,6 +129,7 @@ func ToConfig(c *cli.Context) (Config, error) { Listen: listen, Announce: announceType, AnnounceUrl: announceUrl, + IpniPath: ipniPath, PublicAddr: publicAddr, LogFile: logFile, MaxResponseDuration: maxResponseDuration, diff --git a/cmd/frisbii/main.go b/cmd/frisbii/main.go index c103f00..88aaf14 100644 --- a/cmd/frisbii/main.go +++ b/cmd/frisbii/main.go @@ -6,6 +6,7 @@ import ( "net/url" "os" "os/signal" + "strings" "syscall" "github.com/ipfs/go-log/v2" @@ -22,7 +23,7 @@ import ( ) const ( - IndexerHandlerPath = "/_ipni/" + IndexerHandlerPath = "/ipni/" IndexerAnnounceUrl = "https://cid.contact/ingest/announce" DefaultHttpPort = 3747 ) @@ -159,11 +160,21 @@ func action(c *cli.Context) error { return err } - httpath, err := multiaddr.NewComponent("httpath", url.PathEscape(IndexerHandlerPath)) - if err != nil { - return err + announceAddr := frisbiiListenAddr.Maddr + ipniPath := config.IpniPath + if strings.HasPrefix("/ipni/v1/ad/", config.IpniPath) { + // if it's some subset of /ipni/v1/ad/ then we won't need to add it, + // ipnisync's ServeHTTP is agnostic and only cares about the path.Base() + // of the path. + ipniPath = "" + } + if ipniPath != "" { + httpath, err := multiaddr.NewComponent("httpath", url.PathEscape(ipniPath)) + if err != nil { + return err + } + announceAddr = multiaddr.Join(announceAddr, httpath) } - announceAddr := multiaddr.Join(frisbiiListenAddr.Maddr, httpath) engine, err := engine.New( engine.WithPrivateKey(privKey), @@ -171,7 +182,7 @@ func action(c *cli.Context) error { engine.WithDirectAnnounce(config.AnnounceUrl.String()), engine.WithPublisherKind(engine.HttpPublisher), engine.WithHttpPublisherWithoutServer(), - engine.WithHttpPublisherHandlerPath(IndexerHandlerPath), + engine.WithHttpPublisherHandlerPath(ipniPath), engine.WithHttpPublisherListenAddr(listenUrl.Host), engine.WithHttpPublisherAnnounceAddr(announceAddr.String()), ) @@ -187,7 +198,10 @@ func action(c *cli.Context) error { return err } - server.SetIndexerProvider(IndexerHandlerPath, engine) + // use config.IpniPath here, but the adjusted ipniPath above for setting up + // the engine; here we set our local mount expectations and it can't be + // "" + server.SetIndexerProvider(config.IpniPath, engine) if err := server.Announce(); err != nil { return err diff --git a/frisbii.go b/frisbii.go index 82ad469..29f2d1a 100644 --- a/frisbii.go +++ b/frisbii.go @@ -83,7 +83,7 @@ func (fs *FrisbiiServer) SetIndexerProvider(handlerPath string, indexerProvider if err != nil { return err } - if handlerPath[len(handlerPath)-1] != '/' { + if handlerPath == "" || handlerPath[len(handlerPath)-1] != '/' { handlerPath += "/" } fs.mux.HandleFunc(handlerPath, handlerFunc) diff --git a/internal/integration/ipni_test.go b/internal/integration/ipni_test.go index eb28d96..34fe6a2 100644 --- a/internal/integration/ipni_test.go +++ b/internal/integration/ipni_test.go @@ -22,106 +22,125 @@ import ( const rseed = 1234 func TestIpni(t *testing.T) { - req := require.New(t) - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - defer cancel() - - indexerReady := test.NewIndexerReadyWatcher() - frisbiiReady := test.NewStdoutWatcher("frisbii", "Announce() complete") - - tr := test.NewTestIndexerRunner(t, ctx, t.TempDir(), indexerReady, frisbiiReady) - - t.Log("Running in test directory:", tr.Dir) - - // install the frisbii cmd, when done in tr.Run() will use the GOPATH/GOBIN - // in the test directory, so we get a localised `frisbii` executable - frisbii := filepath.Join(tr.Dir, "frisbii") - tr.Run("go", "install", "../../cmd/frisbii/") - - cwd, err := os.Getwd() - req.NoError(err) - err = os.Chdir(tr.Dir) - req.NoError(err) - - // install the indexer to announce to - indexer := filepath.Join(tr.Dir, "storetheindex") - tr.Run("go", "install", "github.com/ipni/storetheindex@e56485343dd8e235581191b4668b5bfc0cea0781") // TODO: use @latest when we have a release - // install the ipni cli to inspect the indexer - ipni := filepath.Join(tr.Dir, "ipni") - tr.Run("go", "install", "github.com/ipni/ipni-cli/cmd/ipni@latest") - - err = os.Chdir(cwd) - req.NoError(err) - - // initialise and start the indexer and adjust the config - tr.Run(indexer, "init", "--store", "pebble", "--pubsub-topic", "/indexer/ingest/mainnet", "--no-bootstrap") - cmdIndexer := tr.Start(indexer, "daemon") - select { - case <-indexerReady.Signal: - case <-ctx.Done(): - t.Fatal("timed out waiting for indexer to start") - } - - /* - We don't seem to need to give it explicit permission, but if we do, here it is + for _, testCase := range []struct { + name string + frisbiiFlags []string + }{ + { + name: "vanilla", + }, + { + name: "ipni-path=/___ipni___", + frisbiiFlags: []string{ + "--ipni-path", "/___ipni___", + }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + req := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + indexerReady := test.NewIndexerReadyWatcher() + frisbiiReady := test.NewStdoutWatcher("frisbii", "Announce() complete") + + tr := test.NewTestIndexerRunner(t, ctx, t.TempDir(), indexerReady, frisbiiReady) + + t.Log("Running in test directory:", tr.Dir) + + // install the frisbii cmd, when done in tr.Run() will use the GOPATH/GOBIN + // in the test directory, so we get a localised `frisbii` executable + frisbii := filepath.Join(tr.Dir, "frisbii") + tr.Run("go", "install", "../../cmd/frisbii/") + + cwd, err := os.Getwd() + req.NoError(err) + err = os.Chdir(tr.Dir) + req.NoError(err) + + // install the indexer to announce to + indexer := filepath.Join(tr.Dir, "storetheindex") + tr.Run("go", "install", "github.com/ipni/storetheindex@e56485343dd8e235581191b4668b5bfc0cea0781") // TODO: use @latest when we have a release + // install the ipni cli to inspect the indexer + ipni := filepath.Join(tr.Dir, "ipni") + tr.Run("go", "install", "github.com/ipni/ipni-cli/cmd/ipni@latest") + + err = os.Chdir(cwd) + req.NoError(err) + + // initialise and start the indexer and adjust the config + tr.Run(indexer, "init", "--store", "pebble", "--pubsub-topic", "/indexer/ingest/mainnet", "--no-bootstrap") + cmdIndexer := tr.Start(indexer, "daemon") + select { + case <-indexerReady.Signal: + case <-ctx.Done(): + t.Fatal("timed out waiting for indexer to start") + } - // loading a private key will generate an ID before we start frisbii - confDir := filepath.Join(tr.Dir, util.FrisbiiConfigDir) - if _, err := os.Stat(confDir); os.IsNotExist(err) { - req.NoError(os.Mkdir(confDir, 0700)) - } - _, id, err := util.LoadPrivKey(confDir) - req.NoError(err) + /* + We don't seem to need to give it explicit permission, but if we do, here it is + + // loading a private key will generate an ID before we start frisbii + confDir := filepath.Join(tr.Dir, util.FrisbiiConfigDir) + if _, err := os.Stat(confDir); os.IsNotExist(err) { + req.NoError(os.Mkdir(confDir, 0700)) + } + _, id, err := util.LoadPrivKey(confDir) + req.NoError(err) + + // Allow provider advertisements, regardless of default policy. + tr.Run(indexer, "admin", "allow", "-i", "http://localhost:3002", "--peer", id.String()) + */ + + // setup the frisbii CLI args + args := []string{ + "--listen", "localhost:37471", + "--announce", "roots", + "--announce-url", "http://localhost:3001/announce", + "--verbose", + } - // Allow provider advertisements, regardless of default policy. - tr.Run(indexer, "admin", "allow", "-i", "http://localhost:3002", "--peer", id.String()) - */ + // make some CARs to announce and put them in the args + cars := mkCars(t, 4) + for _, carPath := range cars { + args = append(args, "--car", carPath) + } - // setup the frisbii CLI args - args := []string{ - "--listen", "localhost:37471", - "--announce", "roots", - "--announce-url", "http://localhost:3001/announce", - "--verbose", - } + args = append(args, testCase.frisbiiFlags...) - // make some CARs to announce and put them in the args - cars := mkCars(t, 4) - for _, carPath := range cars { - args = append(args, "--car", carPath) - } + // start frisbii + cmdFrisbii := tr.Start(frisbii, args...) - // start frisbii - cmdFrisbii := tr.Start(frisbii, args...) + select { + case <-frisbiiReady.Signal: + case <-ctx.Done(): + t.Fatal("timed out waiting for frisbii to announce") + } - select { - case <-frisbiiReady.Signal: - case <-ctx.Done(): - t.Fatal("timed out waiting for frisbii to announce") + // wait for the CARs to be indexed + req.Eventually(func() bool { + for mh := range cars { + findOutput := tr.Run(ipni, "find", "--no-priv", "-i", "http://localhost:3000", "-mh", mh) + t.Logf("import output:\n%s\n", findOutput) + + if bytes.Contains(findOutput, []byte("not found")) { + return false + } + if !bytes.Contains(findOutput, []byte("Provider:")) { + t.Logf("mh %s: unexpected error: %s", mh, findOutput) + return false + } + t.Logf("mh %s: found", mh) + } + return true + }, 10*time.Second, time.Second) + + // stop and clean up + tr.Stop(cmdIndexer, time.Second) + tr.Stop(cmdFrisbii, time.Second) + }) } - - // wait for the CARs to be indexed - req.Eventually(func() bool { - for mh := range cars { - findOutput := tr.Run(ipni, "find", "--no-priv", "-i", "http://localhost:3000", "-mh", mh) - t.Logf("import output:\n%s\n", findOutput) - - if bytes.Contains(findOutput, []byte("not found")) { - return false - } - if !bytes.Contains(findOutput, []byte("Provider:")) { - t.Logf("mh %s: unexpected error: %s", mh, findOutput) - return false - } - t.Logf("mh %s: found", mh) - } - return true - }, 10*time.Second, time.Second) - - // stop and clean up - tr.Stop(cmdIndexer, time.Second) - tr.Stop(cmdFrisbii, time.Second) } func mkCars(t *testing.T, count int) map[string]string {