Skip to content

Commit

Permalink
feat: add --ipni-path, remove /_ipni default, test variations
Browse files Browse the repository at this point in the history
  • Loading branch information
@rvagg
rvagg committed Sep 12, 2023
1 parent a1fc399 commit 78965ec
Show file tree
Hide file tree
Showing 4 changed files with 142 additions and 101 deletions.
8 changes: 8 additions & 0 deletions cmd/frisbii/flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ var Flags = []cli.Flag{
Usage: "announcement endpoint url for the indexer",
Value: IndexerAnnounceUrl,
},
&cli.StringFlag{
Name: "ipni-path",
Usage: "the local path to serve IPNI content from, requests will have /ipni/v1/ad/ automatically appended to it",
Value: IndexerHandlerPath,
},
&cli.StringFlag{
Name: "public-addr",
Usage: "multiaddr or URL of this server as seen by the indexer and other peers if it is different to the listen address",
Expand Down Expand Up @@ -68,6 +73,7 @@ type Config struct {
Listen string
Announce AnnounceType
AnnounceUrl *url.URL
IpniPath string
PublicAddr string
LogFile string
MaxResponseDuration time.Duration
Expand Down Expand Up @@ -102,6 +108,7 @@ func ToConfig(c *cli.Context) (Config, error) {
return Config{}, err
}

ipniPath := c.String("ipni-path")
listen := c.String("listen")
publicAddr := c.String("public-addr")
logFile := c.String("log-file")
Expand All @@ -122,6 +129,7 @@ func ToConfig(c *cli.Context) (Config, error) {
Listen: listen,
Announce: announceType,
AnnounceUrl: announceUrl,
IpniPath: ipniPath,
PublicAddr: publicAddr,
LogFile: logFile,
MaxResponseDuration: maxResponseDuration,
Expand Down
28 changes: 21 additions & 7 deletions cmd/frisbii/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"net/url"
"os"
"os/signal"
"strings"
"syscall"

"github.com/ipfs/go-log/v2"
Expand All @@ -22,7 +23,7 @@ import (
)

const (
IndexerHandlerPath = "/_ipni/"
IndexerHandlerPath = "/ipni/"
IndexerAnnounceUrl = "https://cid.contact/ingest/announce"
DefaultHttpPort = 3747
)
Expand Down Expand Up @@ -159,19 +160,29 @@ func action(c *cli.Context) error {
return err
}

httpath, err := multiaddr.NewComponent("httpath", url.PathEscape(IndexerHandlerPath))
if err != nil {
return err
announceAddr := frisbiiListenAddr.Maddr
ipniPath := config.IpniPath
if strings.HasPrefix("/ipni/v1/ad/", config.IpniPath) {
// if it's some subset of /ipni/v1/ad/ then we won't need to add it,
// ipnisync's ServeHTTP is agnostic and only cares about the path.Base()
// of the path.
ipniPath = ""
}
if ipniPath != "" {
httpath, err := multiaddr.NewComponent("httpath", url.PathEscape(ipniPath))
if err != nil {
return err
}
announceAddr = multiaddr.Join(announceAddr, httpath)
}
announceAddr := multiaddr.Join(frisbiiListenAddr.Maddr, httpath)

engine, err := engine.New(
engine.WithPrivateKey(privKey),
engine.WithProvider(peer.AddrInfo{ID: id, Addrs: []multiaddr.Multiaddr{frisbiiListenAddr.Maddr}}),
engine.WithDirectAnnounce(config.AnnounceUrl.String()),
engine.WithPublisherKind(engine.HttpPublisher),
engine.WithHttpPublisherWithoutServer(),
engine.WithHttpPublisherHandlerPath(IndexerHandlerPath),
engine.WithHttpPublisherHandlerPath(ipniPath),
engine.WithHttpPublisherListenAddr(listenUrl.Host),
engine.WithHttpPublisherAnnounceAddr(announceAddr.String()),
)
Expand All @@ -187,7 +198,10 @@ func action(c *cli.Context) error {
return err
}

server.SetIndexerProvider(IndexerHandlerPath, engine)
// use config.IpniPath here, but the adjusted ipniPath above for setting up
// the engine; here we set our local mount expectations and it can't be
// ""
server.SetIndexerProvider(config.IpniPath, engine)

if err := server.Announce(); err != nil {
return err
Expand Down
2 changes: 1 addition & 1 deletion frisbii.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ func (fs *FrisbiiServer) SetIndexerProvider(handlerPath string, indexerProvider
if err != nil {
return err
}
if handlerPath[len(handlerPath)-1] != '/' {
if handlerPath == "" || handlerPath[len(handlerPath)-1] != '/' {
handlerPath += "/"
}
fs.mux.HandleFunc(handlerPath, handlerFunc)
Expand Down
205 changes: 112 additions & 93 deletions internal/integration/ipni_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,106 +22,125 @@ import (
const rseed = 1234

func TestIpni(t *testing.T) {
req := require.New(t)

ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()

indexerReady := test.NewIndexerReadyWatcher()
frisbiiReady := test.NewStdoutWatcher("frisbii", "Announce() complete")

tr := test.NewTestIndexerRunner(t, ctx, t.TempDir(), indexerReady, frisbiiReady)

t.Log("Running in test directory:", tr.Dir)

// install the frisbii cmd, when done in tr.Run() will use the GOPATH/GOBIN
// in the test directory, so we get a localised `frisbii` executable
frisbii := filepath.Join(tr.Dir, "frisbii")
tr.Run("go", "install", "../../cmd/frisbii/")

cwd, err := os.Getwd()
req.NoError(err)
err = os.Chdir(tr.Dir)
req.NoError(err)

// install the indexer to announce to
indexer := filepath.Join(tr.Dir, "storetheindex")
tr.Run("go", "install", "github.com/ipni/storetheindex@e56485343dd8e235581191b4668b5bfc0cea0781") // TODO: use @latest when we have a release
// install the ipni cli to inspect the indexer
ipni := filepath.Join(tr.Dir, "ipni")
tr.Run("go", "install", "github.com/ipni/ipni-cli/cmd/ipni@latest")

err = os.Chdir(cwd)
req.NoError(err)

// initialise and start the indexer and adjust the config
tr.Run(indexer, "init", "--store", "pebble", "--pubsub-topic", "/indexer/ingest/mainnet", "--no-bootstrap")
cmdIndexer := tr.Start(indexer, "daemon")
select {
case <-indexerReady.Signal:
case <-ctx.Done():
t.Fatal("timed out waiting for indexer to start")
}

/*
We don't seem to need to give it explicit permission, but if we do, here it is
for _, testCase := range []struct {
name string
frisbiiFlags []string
}{
{
name: "vanilla",
},
{
name: "ipni-path=/___ipni___",
frisbiiFlags: []string{
"--ipni-path", "/___ipni___",
},
},
} {
t.Run(testCase.name, func(t *testing.T) {
req := require.New(t)

ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()

indexerReady := test.NewIndexerReadyWatcher()
frisbiiReady := test.NewStdoutWatcher("frisbii", "Announce() complete")

tr := test.NewTestIndexerRunner(t, ctx, t.TempDir(), indexerReady, frisbiiReady)

t.Log("Running in test directory:", tr.Dir)

// install the frisbii cmd, when done in tr.Run() will use the GOPATH/GOBIN
// in the test directory, so we get a localised `frisbii` executable
frisbii := filepath.Join(tr.Dir, "frisbii")
tr.Run("go", "install", "../../cmd/frisbii/")

cwd, err := os.Getwd()
req.NoError(err)
err = os.Chdir(tr.Dir)
req.NoError(err)

// install the indexer to announce to
indexer := filepath.Join(tr.Dir, "storetheindex")
tr.Run("go", "install", "github.com/ipni/storetheindex@e56485343dd8e235581191b4668b5bfc0cea0781") // TODO: use @latest when we have a release
// install the ipni cli to inspect the indexer
ipni := filepath.Join(tr.Dir, "ipni")
tr.Run("go", "install", "github.com/ipni/ipni-cli/cmd/ipni@latest")

err = os.Chdir(cwd)
req.NoError(err)

// initialise and start the indexer and adjust the config
tr.Run(indexer, "init", "--store", "pebble", "--pubsub-topic", "/indexer/ingest/mainnet", "--no-bootstrap")
cmdIndexer := tr.Start(indexer, "daemon")
select {
case <-indexerReady.Signal:
case <-ctx.Done():
t.Fatal("timed out waiting for indexer to start")
}

// loading a private key will generate an ID before we start frisbii
confDir := filepath.Join(tr.Dir, util.FrisbiiConfigDir)
if _, err := os.Stat(confDir); os.IsNotExist(err) {
req.NoError(os.Mkdir(confDir, 0700))
}
_, id, err := util.LoadPrivKey(confDir)
req.NoError(err)
/*
We don't seem to need to give it explicit permission, but if we do, here it is
// loading a private key will generate an ID before we start frisbii
confDir := filepath.Join(tr.Dir, util.FrisbiiConfigDir)
if _, err := os.Stat(confDir); os.IsNotExist(err) {
req.NoError(os.Mkdir(confDir, 0700))
}
_, id, err := util.LoadPrivKey(confDir)
req.NoError(err)
// Allow provider advertisements, regardless of default policy.
tr.Run(indexer, "admin", "allow", "-i", "http://localhost:3002", "--peer", id.String())
*/

// setup the frisbii CLI args
args := []string{
"--listen", "localhost:37471",
"--announce", "roots",
"--announce-url", "http://localhost:3001/announce",
"--verbose",
}

// Allow provider advertisements, regardless of default policy.
tr.Run(indexer, "admin", "allow", "-i", "http://localhost:3002", "--peer", id.String())
*/
// make some CARs to announce and put them in the args
cars := mkCars(t, 4)
for _, carPath := range cars {
args = append(args, "--car", carPath)
}

// setup the frisbii CLI args
args := []string{
"--listen", "localhost:37471",
"--announce", "roots",
"--announce-url", "http://localhost:3001/announce",
"--verbose",
}
args = append(args, testCase.frisbiiFlags...)

// make some CARs to announce and put them in the args
cars := mkCars(t, 4)
for _, carPath := range cars {
args = append(args, "--car", carPath)
}
// start frisbii
cmdFrisbii := tr.Start(frisbii, args...)

// start frisbii
cmdFrisbii := tr.Start(frisbii, args...)
select {
case <-frisbiiReady.Signal:
case <-ctx.Done():
t.Fatal("timed out waiting for frisbii to announce")
}

select {
case <-frisbiiReady.Signal:
case <-ctx.Done():
t.Fatal("timed out waiting for frisbii to announce")
// wait for the CARs to be indexed
req.Eventually(func() bool {
for mh := range cars {
findOutput := tr.Run(ipni, "find", "--no-priv", "-i", "http://localhost:3000", "-mh", mh)
t.Logf("import output:\n%s\n", findOutput)

if bytes.Contains(findOutput, []byte("not found")) {
return false
}
if !bytes.Contains(findOutput, []byte("Provider:")) {
t.Logf("mh %s: unexpected error: %s", mh, findOutput)
return false
}
t.Logf("mh %s: found", mh)
}
return true
}, 10*time.Second, time.Second)

// stop and clean up
tr.Stop(cmdIndexer, time.Second)
tr.Stop(cmdFrisbii, time.Second)
})
}

// wait for the CARs to be indexed
req.Eventually(func() bool {
for mh := range cars {
findOutput := tr.Run(ipni, "find", "--no-priv", "-i", "http://localhost:3000", "-mh", mh)
t.Logf("import output:\n%s\n", findOutput)

if bytes.Contains(findOutput, []byte("not found")) {
return false
}
if !bytes.Contains(findOutput, []byte("Provider:")) {
t.Logf("mh %s: unexpected error: %s", mh, findOutput)
return false
}
t.Logf("mh %s: found", mh)
}
return true
}, 10*time.Second, time.Second)

// stop and clean up
tr.Stop(cmdIndexer, time.Second)
tr.Stop(cmdFrisbii, time.Second)
}

func mkCars(t *testing.T, count int) map[string]string {
Expand Down

0 comments on commit 78965ec

Please sign in to comment.