From d09c1f970436715ec2eb402a298eedecffa7ef2d Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 3 Oct 2023 14:37:16 +0200 Subject: [PATCH 01/20] plugin: Add support for content blocking in Kubo Fixes #8492. This introduces "nopfs" as a preloaded plugin into Kubo. It automatically make Kubo watch *.deny files found in: - /etc/ipfs/denylists - $XDG_CONFIG_HOME/ipfs/denylists - $IPFS_PATH/denylists (files need to be present before boot in order to be watched). Debug logging can be enabled with `GOLOG_LOG_LEVEL="nopfs=debug"`. All blocks are logged to "nopfs-blocks", so logging requests for blocked content can be achieved with `GOLOG_LOG_LEVEL="nopfs-blocks=warn"`: ``` WARN (...) QmRFniDxwxoG2n4AcnGhRdjqDjCM5YeUcBE75K8WXmioH3: blocked (test.deny:9) ``` Interactive/gateway users will also receive errors as responses but with less details: ``` Error: /ipfs/QmQvjk82hPkSaZsyJ8vNER5cmzKW7HyGX5XVusK7EAenCN is blocked and cannot be provided ``` One particularity to keep in mind is that GetMany() will silently drop blocked blocks from the response (a warnings are logged). AddMany() will act similarly and avoid adding blocked blocks. The code implementing all this is actually in nopfs: - https://github.com/ipfs-shipyard/nopfs (main library) - https://github.com/ipfs-shipyard/nopfs/tree/master/ipfs (wrappers) The interpretation of the list rules and block detection is well tested, but a general review might be in order. --- docs/environment-variables.md | 6 +++ go.mod | 2 + go.sum | 4 ++ plugin/loader/preload.go | 2 + plugin/loader/preload_list | 1 + plugin/plugins/nopfs/nopfs.go | 83 +++++++++++++++++++++++++++++++++++ 6 files changed, 98 insertions(+) create mode 100644 plugin/plugins/nopfs/nopfs.go diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 4113be09b82..f0f6b3f183a 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -131,6 +131,12 @@ The above will replace implicit HTTP routers with single one, allowing for inspection/debug of HTTP requests sent by Kubo via `while true ; do nc -l 7423; done` or more advanced tools like [mitmproxy](https://docs.mitmproxy.org/stable/#mitmproxy). + +## `IPFS_CONTENT_BLOCKING_DISABLE` + +Disables the content-blocking subsystem. No denylists will be watched and no +content will be blocked. + ## `LIBP2P_TCP_REUSEPORT` Kubo tries to reuse the same source port for all connections to improve NAT diff --git a/go.mod b/go.mod index e9280f9967b..55abe4b0886 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,8 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 + github.com/ipfs-shipyard/nopfs v0.0.11 + github.com/ipfs-shipyard/nopfs/ipfs v0.13.1-4 github.com/ipfs/boxo v0.13.2-0.20231002142647-c28c847582f0 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 diff --git a/go.sum b/go.sum index 441c156ae45..4f445eb97b8 100644 --- a/go.sum +++ b/go.sum @@ -333,6 +333,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ipfs-shipyard/nopfs v0.0.11 h1:+JrOqVs4Za0eu4jntdAqxoWolWcmtJnTQXxn2Y/Io2E= +github.com/ipfs-shipyard/nopfs v0.0.11/go.mod h1:zCmxRNQBYwTRYemK6zcKBGLKN9DkDJYNzmNMqw9GtIA= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.1-4 h1:r4bv6eGurmJie8pmmqxcyogYRpajEUPp2p6z9udCl2o= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.1-4/go.mod h1:9IaQXXcza109YjbOJeUCc7Ha9+GWDlBX4wx6BZkNTDE= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.13.2-0.20231002142647-c28c847582f0 h1:oss04OCg1/QW0h3OfSCZJiUQErpYPOsz7+X4tpgwODs= diff --git a/plugin/loader/preload.go b/plugin/loader/preload.go index 4304862119d..2ad84e59489 100644 --- a/plugin/loader/preload.go +++ b/plugin/loader/preload.go @@ -7,6 +7,7 @@ import ( pluginfxtest "github.com/ipfs/kubo/plugin/plugins/fxtest" pluginipldgit "github.com/ipfs/kubo/plugin/plugins/git" pluginlevelds "github.com/ipfs/kubo/plugin/plugins/levelds" + pluginnopfs "github.com/ipfs/kubo/plugin/plugins/nopfs" pluginpeerlog "github.com/ipfs/kubo/plugin/plugins/peerlog" ) @@ -22,4 +23,5 @@ func init() { Preload(pluginlevelds.Plugins...) Preload(pluginpeerlog.Plugins...) Preload(pluginfxtest.Plugins...) + Preload(pluginnopfs.Plugins...) } diff --git a/plugin/loader/preload_list b/plugin/loader/preload_list index c18ea80ccd5..462a3f39337 100644 --- a/plugin/loader/preload_list +++ b/plugin/loader/preload_list @@ -11,3 +11,4 @@ flatfs github.com/ipfs/kubo/plugin/plugins/flatfs * levelds github.com/ipfs/kubo/plugin/plugins/levelds * peerlog github.com/ipfs/kubo/plugin/plugins/peerlog * fxtest github.com/ipfs/kubo/plugin/plugins/fxtest * +nopfs github.com/ipfs/kubo/plugin/plugins/nopfs * \ No newline at end of file diff --git a/plugin/plugins/nopfs/nopfs.go b/plugin/plugins/nopfs/nopfs.go new file mode 100644 index 00000000000..a04734187ba --- /dev/null +++ b/plugin/plugins/nopfs/nopfs.go @@ -0,0 +1,83 @@ +package nopfs + +import ( + "os" + "path/filepath" + + "github.com/ipfs-shipyard/nopfs" + "github.com/ipfs-shipyard/nopfs/ipfs" + "github.com/ipfs/kubo/config" + "github.com/ipfs/kubo/core" + "github.com/ipfs/kubo/core/node" + "github.com/ipfs/kubo/plugin" + "go.uber.org/fx" +) + +// Plugins sets the list of plugins to be loaded. +var Plugins = []plugin.Plugin{ + &nopfsPlugin{}, +} + +// fxtestPlugin is used for testing the fx plugin. +// It merely adds an fx option that logs a debug statement, so we can verify that it works in tests. +type nopfsPlugin struct{} + +var _ plugin.PluginFx = (*nopfsPlugin)(nil) + +func (p *nopfsPlugin) Name() string { + return "nopfs" +} + +func (p *nopfsPlugin) Version() string { + return "0.0.10" +} + +func (p *nopfsPlugin) Init(env *plugin.Environment) error { + return nil +} + +// MakeBlocker is a factory for the blocker so that it can be provided with Fx. +func MakeBlocker() (*nopfs.Blocker, error) { + ipfsPath, err := config.PathRoot() + if err != nil { + return nil, err + } + + defaultFiles, err := nopfs.GetDenylistFiles() + if err != nil { + return nil, err + } + + kuboFiles, err := nopfs.GetDenylistFilesInDir(filepath.Join(ipfsPath, "denylists")) + if err != nil { + return nil, err + } + + files := append(defaultFiles, kuboFiles...) + + return nopfs.NewBlocker(files) +} + +// PathResolvers returns wrapped PathResolvers for Kubo. +func PathResolvers(fetchers node.FetchersIn, blocker *nopfs.Blocker) node.PathResolversOut { + res := node.PathResolverConfig(fetchers) + return node.PathResolversOut{ + IPLDPathResolver: ipfs.WrapResolver(res.IPLDPathResolver, blocker), + UnixFSPathResolver: ipfs.WrapResolver(res.UnixFSPathResolver, blocker), + } +} + +func (p *nopfsPlugin) Options(info core.FXNodeInfo) ([]fx.Option, error) { + if os.Getenv("IPFS_CONTENT_BLOCKING_DISABLE") != "" { + return info.FXOptions, nil + } + + opts := append( + info.FXOptions, + fx.Provide(MakeBlocker), + fx.Decorate(ipfs.WrapBlockService), + fx.Decorate(ipfs.WrapNameSystem), + fx.Decorate(PathResolvers), + ) + return opts, nil +} From bcae8a4a6a76a2766d1c7478869f986fb360f6cd Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 4 Oct 2023 09:43:13 +0200 Subject: [PATCH 02/20] chore: go mod tidy recursively --- docs/examples/kubo-as-a-library/go.mod | 4 ++++ docs/examples/kubo-as-a-library/go.sum | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index a465c7cc68c..89f02c4425e 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -40,6 +40,7 @@ require ( github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gabriel-vasile/mimetype v1.4.1 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -60,6 +61,8 @@ require ( github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect github.com/huin/goupnp v1.2.0 // indirect + github.com/ipfs-shipyard/nopfs v0.0.11 // indirect + github.com/ipfs-shipyard/nopfs/ipfs v0.13.1-4 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.1.2 // indirect @@ -194,5 +197,6 @@ require ( google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index c3538693296..241e64b217f 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -163,6 +163,7 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -298,6 +299,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ipfs-shipyard/nopfs v0.0.11 h1:+JrOqVs4Za0eu4jntdAqxoWolWcmtJnTQXxn2Y/Io2E= +github.com/ipfs-shipyard/nopfs v0.0.11/go.mod h1:zCmxRNQBYwTRYemK6zcKBGLKN9DkDJYNzmNMqw9GtIA= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.1-4 h1:r4bv6eGurmJie8pmmqxcyogYRpajEUPp2p6z9udCl2o= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.1-4/go.mod h1:9IaQXXcza109YjbOJeUCc7Ha9+GWDlBX4wx6BZkNTDE= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.13.2-0.20231002142647-c28c847582f0 h1:oss04OCg1/QW0h3OfSCZJiUQErpYPOsz7+X4tpgwODs= @@ -1001,6 +1006,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= From 51acfb98163e0b9ec3f939a3a0b4f3df4d19e28b Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 19 Oct 2023 16:37:09 +0200 Subject: [PATCH 03/20] docs: content blocking --- README.md | 1 + docs/changelogs/v0.24.md | 9 ++++++ docs/content-blocking.md | 62 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 docs/content-blocking.md diff --git a/README.md b/README.md index 066f0790ae0..17881a44f32 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Featureset - [HTTP Kubo RPC API](https://docs.ipfs.tech/reference/kubo/rpc/) (`/api/v0`) to access and control the daemon - [Command Line Interface](https://docs.ipfs.tech/reference/kubo/cli/) based on (`/api/v0`) RPC API - [WebUI](https://github.com/ipfs/ipfs-webui/#readme) to manage the Kubo node +- [Content blocking](/docs/content-blocking.md) support for operators of public nodes ### Other implementations diff --git a/docs/changelogs/v0.24.md b/docs/changelogs/v0.24.md index a19526ad1ac..33f305ba113 100644 --- a/docs/changelogs/v0.24.md +++ b/docs/changelogs/v0.24.md @@ -6,6 +6,7 @@ - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) + - [Support for content blocking](#support-for-content-blocking) - [Gateway: the root of the CARs are no longer meaningful](#gateway-the-root-of-the-cars-are-no-longer-meaningful) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -14,6 +15,14 @@ ### ๐Ÿ”ฆ Highlights +#### Support for content blocking + +This Kubo release ships with built-in content-blocking subsystem [announced earlier this year](https://blog.ipfs.tech/2023-content-blocking-for-the-ipfs-stack/). +Content blocking is an opt-in decision made by the operator of `ipfs daemon`. +The official build does not ship with any denylists. + +Learn more at [`/docs/content-blocking.md`](https://github.com/ipfs/kubo/blob/master/docs/content-blocking.md) + #### Gateway: the root of the CARs are no longer meaningful When requesting a CAR from the gateway, the root of the CAR might no longer be diff --git a/docs/content-blocking.md b/docs/content-blocking.md new file mode 100644 index 00000000000..ef5461132db --- /dev/null +++ b/docs/content-blocking.md @@ -0,0 +1,62 @@ +

+
+ content blocking logo +
+ Content Blocking in Kubo +
+

+ +Kubo ships with built-in support for denylist format from [IPIP-383](https://github.com/ipfs/specs/pull/383). + +## Default behavior + +Official Kubo build does not ship with any denylists enabled by default. + +Content blocking is an opt-in decision made by the operator of `ipfs daemon`. + +## How to enable blocking + +Place a `*.deny` file in one of directories: + +- `$IPFS_PATH/denylists/` (`$HOME/.ipfs/denylists/` if `IPFS_PATH` is not set) +- `$XDG_CONFIG_HOME/ipfs/denylists/` (`$HOME/.config/ipfs/denylists/` if `XDG_CONFIG_HOME` is not set) +- `/etc/ipfs/denylists/` (global) + +Files need to be present before starting the `ipfs daemon` in order to be watched for updates. + +If a new denylist file is added, `ipfs daemon` needs to be restarted. + +CLI and Gateway users will receive errors in response to request impacted by a blocklist: + +``` +Error: /ipfs/QmQvjk82hPkSaZsyJ8vNER5cmzKW7HyGX5XVusK7EAenCN is blocked and cannot be provided +``` + +End user is not informed about the exact reason, see [How to +debug](#how-to-debug) if you need to find out which line of which denylist +caused the request to be blocked. + +## Denylist file format + +[NOpfs](https://github.com/ipfs-shipyard/nopfs) supports the format from [IPIP-383](https://github.com/ipfs/specs/pull/383). + +Example: https://badbits.dwebops.pub/badbits.deny + +## How to suspend blocking without removing denylists + +Set `IPFS_CONTENT_BLOCKING_DISABLE` environment variable to `true` and restart the daemon. + + +## How to debug + +Debug logging of `nopfs` subsystem can be enabled with `GOLOG_LOG_LEVEL="nopfs=debug"` + +All block events are logged as warnings on a separate level named `nopfs-blocks`. + +To only log requests for blocked content set `GOLOG_LOG_LEVEL="nopfs-blocks=warn"`: + +``` +WARN (...) QmRFniDxwxoG2n4AcnGhRdjqDjCM5YeUcBE75K8WXmioH3: blocked (test.deny:9) +``` + + From 0a8f651fdf99864dfb82ae890c0aa1757d68eec2 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 19 Oct 2023 22:25:49 +0200 Subject: [PATCH 04/20] test: content_blocking_test.go Includes dependency update from https://github.com/ipfs-shipyard/nopfs/pull/25 --- docs/examples/kubo-as-a-library/go.mod | 6 +- docs/examples/kubo-as-a-library/go.sum | 12 +- go.mod | 6 +- go.sum | 12 +- test/cli/content_blocking_test.go | 206 +++++++++++++++++++++++++ test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 +- 7 files changed, 227 insertions(+), 21 deletions(-) create mode 100644 test/cli/content_blocking_test.go diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index a013bc87d66..7fc4a35d9ac 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.20 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd + github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.31.0 github.com/multiformats/go-multiaddr v0.11.0 @@ -61,8 +61,8 @@ require ( github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huin/goupnp v1.2.0 // indirect - github.com/ipfs-shipyard/nopfs v0.0.11 // indirect - github.com/ipfs-shipyard/nopfs/ipfs v0.13.1-4 // indirect + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231019201810-ca629a80a385 // indirect + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231019201810-ca629a80a385 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 6d04f82d22f..f41fe641639 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -299,14 +299,14 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs-shipyard/nopfs v0.0.11 h1:+JrOqVs4Za0eu4jntdAqxoWolWcmtJnTQXxn2Y/Io2E= -github.com/ipfs-shipyard/nopfs v0.0.11/go.mod h1:zCmxRNQBYwTRYemK6zcKBGLKN9DkDJYNzmNMqw9GtIA= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.1-4 h1:r4bv6eGurmJie8pmmqxcyogYRpajEUPp2p6z9udCl2o= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.1-4/go.mod h1:9IaQXXcza109YjbOJeUCc7Ha9+GWDlBX4wx6BZkNTDE= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231019201810-ca629a80a385 h1:ShsOXLWzQFpqrd/YHmOX+/NU9FbI2BbzwWQjkOvAE9o= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231019201810-ca629a80a385/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231019201810-ca629a80a385 h1:jAeuoctl0Bq6Wack8CHfyYQDFaIVpP10HmJoKB1GF7I= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231019201810-ca629a80a385/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd h1:CWz2mhz+cmkLRlKgQYlKXkDtx4oWYkCorSSF4ZWkH3o= -github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= +github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= +github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/go.mod b/go.mod index 3720088e27f..11786d377bb 100644 --- a/go.mod +++ b/go.mod @@ -15,9 +15,9 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/google/uuid v1.3.1 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs-shipyard/nopfs v0.0.11 - github.com/ipfs-shipyard/nopfs/ipfs v0.13.1-4 - github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231019201810-ca629a80a385 + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231019201810-ca629a80a385 + github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 08f2f9bc4c6..a1c7704c481 100644 --- a/go.sum +++ b/go.sum @@ -333,14 +333,14 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs-shipyard/nopfs v0.0.11 h1:+JrOqVs4Za0eu4jntdAqxoWolWcmtJnTQXxn2Y/Io2E= -github.com/ipfs-shipyard/nopfs v0.0.11/go.mod h1:zCmxRNQBYwTRYemK6zcKBGLKN9DkDJYNzmNMqw9GtIA= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.1-4 h1:r4bv6eGurmJie8pmmqxcyogYRpajEUPp2p6z9udCl2o= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.1-4/go.mod h1:9IaQXXcza109YjbOJeUCc7Ha9+GWDlBX4wx6BZkNTDE= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231019201810-ca629a80a385 h1:ShsOXLWzQFpqrd/YHmOX+/NU9FbI2BbzwWQjkOvAE9o= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231019201810-ca629a80a385/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231019201810-ca629a80a385 h1:jAeuoctl0Bq6Wack8CHfyYQDFaIVpP10HmJoKB1GF7I= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231019201810-ca629a80a385/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd h1:CWz2mhz+cmkLRlKgQYlKXkDtx4oWYkCorSSF4ZWkH3o= -github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= +github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= +github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go new file mode 100644 index 00000000000..190460d1fe3 --- /dev/null +++ b/test/cli/content_blocking_test.go @@ -0,0 +1,206 @@ +package cli + +import ( + "fmt" + "log" + "net/http" + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/ipfs/kubo/test/cli/harness" + "github.com/stretchr/testify/assert" +) + +func TestContentBlocking(t *testing.T) { + t.Parallel() + + const blockedMsg = "blocked and cannot be provided" + + h := harness.NewT(t) + + // Init IPFS_PATH + node := h.NewNode().Init("--empty-repo", "--profile=test") + + // Create CIDs we use in test + h.WriteFile("blocked-dir/indirectly-blocked-file.txt", "indirectly blocked file content") + blockedDirCID := node.IPFS("add", "-Q", "-r", filepath.Join(h.Dir, "blocked-dir")).Stdout.Trimmed() + // indirectlyBlockedFileCID := node.IPFS("add", "-Q", filepath.Join(h.Dir, "blocked-dir", "indirectly-blocked-file.txt")).Stderr.Trimmed() + + h.WriteFile("directly-blocked-file.txt", "directly blocked file content") + blockedCID := node.IPFS("add", "-Q", filepath.Join(h.Dir, "directly-blocked-file.txt")).Stdout.Trimmed() + + h.WriteFile("not-blocked-file.txt", "not blocked file content") + allowedCID := node.IPFS("add", "-Q", filepath.Join(h.Dir, "not-blocked-file.txt")).Stdout.Trimmed() + + // Create denylist at $IPFS_PATH/denylists/test.deny + denylistTmp := h.WriteToTemp(fmt.Sprintf( + "//QmX9dhRcQcKUw3Ws8485T5a9dtjrSCQaUAHnG4iK9i4ceM\n"+ // Double hash CID block: base58btc-sha256-multihash(QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR) + "//QmbK7LDv5NNBvYQzNfm2eED17SNLt1yNMapcUhSuNLgkqz\n"+ // Double hash Path block using blake3 hashing: base58btc-blake3-multihash(gW7Nhu4HrfDtphEivm3Z9NNE7gpdh5Tga8g6JNZc1S8E47/path) + "//d9d295bde21f422d471a90f2a37ec53049fdf3e5fa3ee2e8f20e10003da429e7\n"+ // Legacy CID double-hash block: sha256(bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/) + "//3f8b9febd851873b3774b937cce126910699ceac56e72e64b866f8e258d09572\n"+ // Legacy Path double-hash block: sha256(bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/path) + "/ipfs/%s/*\n"+ // block subpaths + "/ipfs/%s\n"+ // block specific CID + "/ipns/blocked-cid.example.com\n"+ + "/ipns/blocked-dnslink.example.com\n", + blockedDirCID, blockedCID)) + + if err := os.MkdirAll(filepath.Join(node.Dir, "denylists"), 0o777); err != nil { + log.Panicf("failed to create denylists dir: %s", err.Error()) + } + if err := os.Rename(denylistTmp, filepath.Join(node.Dir, "denylists", "test.deny")); err != nil { + log.Panicf("failed to create test denylist: %s", err.Error()) + } + + // Add two entries to namesys resolution cache + // /ipns/blocked-cid.example.com point at a blocked CID (to confirm blocking impacts /ipns resolution) + // /ipns/blocked-dnslink.example.com with safe CID (to test blocking of /ipns/ paths) + os.Setenv("IPFS_NS_MAP", "blocked-cid.example.com:/ipfs/"+blockedCID+",blocked-dnslink.example.com/ipns/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") + defer os.Unsetenv("IPFS_NS_MAP") + + // Start daemon, it should pick up denylist from $IPFS_PATH/denylists/test.deny + node.StartDaemon("--offline") + client := node.GatewayClient() + + // First, confirm gateway works + t.Run("Gateway Allows CID that is not blocked", func(t *testing.T) { + t.Parallel() + resp := client.Get(fmt.Sprintf("/ipfs/%s", allowedCID)) + assert.Equal(t, http.StatusOK, resp.StatusCode) + // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) + assert.Equal(t, "not blocked file content", resp.Body) + }) + + t.Run("Gateway Denies CID that is directly blocked", func(t *testing.T) { + t.Parallel() + resp := client.Get(fmt.Sprintf("/ipfs/%s", blockedCID)) + assert.NotEqual(t, http.StatusOK, resp.StatusCode) + // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) + assert.NotEqual(t, "directly blocked file content", resp.Body) + assert.Contains(t, resp.Body, blockedMsg) + }) + + /* TODO: this does not seem to work yet + t.Run("Gateway (path) Denies CID when it is indirectly blocked (via parent)", func(t *testing.T) { + t.Parallel() + resp := client.Get(fmt.Sprintf("/ipfs/%s/indirectly-blocked-file.txt", blockedDirCID)) + assert.NotEqual(t, http.StatusOK, resp.StatusCode) + // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) + assert.NotEqual(t, "indirectly blocked file content", resp.Body) + }) + */ + + t.Run("Gateway Denies CID that is directly blocked (double hash CID block: base58btc-sha256-multihash)", func(t *testing.T) { + t.Parallel() + resp := client.Get(fmt.Sprintf("/ipfs/QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR")) + assert.NotEqual(t, http.StatusOK, resp.StatusCode) + // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) + assert.Contains(t, resp.Body, blockedMsg) + }) + + /* TODO: this fails due to parent block missing, we need to add fixture so it works in offline mode + t.Run("Gateway Denies Path that is directly blocked (double hash Path block: base58btc-blake3-multihash)", func(t *testing.T) { + t.Parallel() + resp := client.Get(fmt.Sprintf("/ipfs/bafyb4ieqht3b2rssdmc7sjv2cy2gfdilxkfh7623nvndziyqnawkmo266a/path")) + assert.NotEqual(t, http.StatusOK, resp.StatusCode) + // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) + assert.Contains(t, resp.Body, blockedMsg) + }) + */ + + t.Run("Gateway Denies CID that is directly blocked (legacy CID double-hash block: sha256)", func(t *testing.T) { + t.Parallel() + resp := client.Get(fmt.Sprintf("/ipfs/bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e")) + assert.NotEqual(t, http.StatusOK, resp.StatusCode) + // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) + assert.Contains(t, resp.Body, blockedMsg) + }) + + /* TODO: this fails due to parent block missing, we need to add fixture so it works in offline mode + t.Run("Gateway Denies Path that is directly blocked (legacy Path double-hash: sha256)", func(t *testing.T) { + t.Parallel() + resp := client.Get(fmt.Sprintf("/ipfs/bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/path")) + assert.NotEqual(t, http.StatusOK, resp.StatusCode) + // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) + assert.Contains(t, resp.Body, blockedMsg) + }) + */ + + t.Run("Gateway Denies /ipns that resolves to a blocked CID", func(t *testing.T) { + t.Parallel() + resp := client.Get(fmt.Sprintf("/ipns/blocked-cid.example.com")) + assert.NotEqual(t, http.StatusOK, resp.StatusCode) + assert.Contains(t, resp.Body, blockedMsg) + }) + + t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name", func(t *testing.T) { + t.Parallel() + resp := client.Get(fmt.Sprintf("/ipns/blocked-dnslink.example.com")) + assert.NotEqual(t, http.StatusOK, resp.StatusCode) + // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) + assert.Contains(t, resp.Body, blockedMsg) + }) + + t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain redirect)", func(t *testing.T) { + t.Parallel() + + gwURL, _ := url.Parse(node.GatewayURL()) + resp := client.Get(fmt.Sprintf("/ipns/blocked-dnslink.example.com"), func(r *http.Request) { + r.Host = "localhost:" + gwURL.Port() + }) + + assert.NotEqual(t, http.StatusOK, resp.StatusCode) + // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) + assert.Contains(t, resp.Body, blockedMsg) + }) + + t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain, no TLS)", func(t *testing.T) { + t.Parallel() + + gwURL, _ := url.Parse(node.GatewayURL()) + resp := client.Get("/", func(r *http.Request) { + r.Host = "blocked-dnslink.example.com.ipns.localhost:" + gwURL.Port() + }) + + assert.NotEqual(t, http.StatusOK, resp.StatusCode) + // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) + assert.Contains(t, resp.Body, blockedMsg) + }) + + t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain, inlined for TLS)", func(t *testing.T) { + t.Parallel() + + gwURL, _ := url.Parse(node.GatewayURL()) + resp := client.Get("/", func(r *http.Request) { + // Inlined DNSLink to fit in single DNS label for TLS interop: + // https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header + r.Host = "blocked--dnslink-example-com.ipns.localhost:" + gwURL.Port() + }) + + assert.NotEqual(t, http.StatusOK, resp.StatusCode) + // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) + assert.Contains(t, resp.Body, blockedMsg) + }) + + // Basic tests to confirm blocking applies to CLI + + t.Run("CLI 'block get' Denies CID that is directly blocked", func(t *testing.T) { + t.Parallel() + errMsg := node.RunIPFS("block", "get", blockedCID).Stderr.Trimmed() + assert.Contains(t, errMsg, blockedMsg) + }) + + t.Run("CLI 'dag get' Denies CID that is directly blocked", func(t *testing.T) { + t.Parallel() + errMsg := node.RunIPFS("dag", "get", blockedCID).Stderr.Trimmed() + assert.Contains(t, errMsg, blockedMsg) + }) + + t.Run("CLI 'cat' Denies CID that is directly blocked", func(t *testing.T) { + t.Parallel() + errMsg := node.RunIPFS("cat", blockedCID).Stderr.Trimmed() + assert.Contains(t, errMsg, blockedMsg) + }) + +} diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index fd8c9458568..e79c7ff5ebd 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -7,7 +7,7 @@ replace github.com/ipfs/kubo => ../../ require ( github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd github.com/golangci/golangci-lint v1.54.1 - github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd + github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 github.com/ipfs/go-datastore v0.6.0 diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index d37a47fe0c1..cbb547d2bc9 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -398,8 +398,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd h1:CWz2mhz+cmkLRlKgQYlKXkDtx4oWYkCorSSF4ZWkH3o= -github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= +github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= +github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= From e689080c4be943e74002665b0f26c78d316a441c Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 19 Oct 2023 22:37:37 +0200 Subject: [PATCH 05/20] chore: fix go lint --- test/cli/content_blocking_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go index 190460d1fe3..c7bea470890 100644 --- a/test/cli/content_blocking_test.go +++ b/test/cli/content_blocking_test.go @@ -93,7 +93,7 @@ func TestContentBlocking(t *testing.T) { t.Run("Gateway Denies CID that is directly blocked (double hash CID block: base58btc-sha256-multihash)", func(t *testing.T) { t.Parallel() - resp := client.Get(fmt.Sprintf("/ipfs/QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR")) + resp := client.Get("/ipfs/QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR") assert.NotEqual(t, http.StatusOK, resp.StatusCode) // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) assert.Contains(t, resp.Body, blockedMsg) @@ -102,7 +102,7 @@ func TestContentBlocking(t *testing.T) { /* TODO: this fails due to parent block missing, we need to add fixture so it works in offline mode t.Run("Gateway Denies Path that is directly blocked (double hash Path block: base58btc-blake3-multihash)", func(t *testing.T) { t.Parallel() - resp := client.Get(fmt.Sprintf("/ipfs/bafyb4ieqht3b2rssdmc7sjv2cy2gfdilxkfh7623nvndziyqnawkmo266a/path")) + resp := client.Get("/ipfs/bafyb4ieqht3b2rssdmc7sjv2cy2gfdilxkfh7623nvndziyqnawkmo266a/path") assert.NotEqual(t, http.StatusOK, resp.StatusCode) // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) assert.Contains(t, resp.Body, blockedMsg) @@ -111,7 +111,7 @@ func TestContentBlocking(t *testing.T) { t.Run("Gateway Denies CID that is directly blocked (legacy CID double-hash block: sha256)", func(t *testing.T) { t.Parallel() - resp := client.Get(fmt.Sprintf("/ipfs/bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e")) + resp := client.Get("/ipfs/bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e") assert.NotEqual(t, http.StatusOK, resp.StatusCode) // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) assert.Contains(t, resp.Body, blockedMsg) @@ -120,7 +120,7 @@ func TestContentBlocking(t *testing.T) { /* TODO: this fails due to parent block missing, we need to add fixture so it works in offline mode t.Run("Gateway Denies Path that is directly blocked (legacy Path double-hash: sha256)", func(t *testing.T) { t.Parallel() - resp := client.Get(fmt.Sprintf("/ipfs/bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/path")) + resp := client.Get("/ipfs/bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/path") assert.NotEqual(t, http.StatusOK, resp.StatusCode) // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) assert.Contains(t, resp.Body, blockedMsg) @@ -129,14 +129,14 @@ func TestContentBlocking(t *testing.T) { t.Run("Gateway Denies /ipns that resolves to a blocked CID", func(t *testing.T) { t.Parallel() - resp := client.Get(fmt.Sprintf("/ipns/blocked-cid.example.com")) + resp := client.Get("/ipns/blocked-cid.example.com") assert.NotEqual(t, http.StatusOK, resp.StatusCode) assert.Contains(t, resp.Body, blockedMsg) }) t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name", func(t *testing.T) { t.Parallel() - resp := client.Get(fmt.Sprintf("/ipns/blocked-dnslink.example.com")) + resp := client.Get("/ipns/blocked-dnslink.example.com") assert.NotEqual(t, http.StatusOK, resp.StatusCode) // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) assert.Contains(t, resp.Body, blockedMsg) @@ -146,7 +146,7 @@ func TestContentBlocking(t *testing.T) { t.Parallel() gwURL, _ := url.Parse(node.GatewayURL()) - resp := client.Get(fmt.Sprintf("/ipns/blocked-dnslink.example.com"), func(r *http.Request) { + resp := client.Get("/ipns/blocked-dnslink.example.com", func(r *http.Request) { r.Host = "localhost:" + gwURL.Port() }) From 511b67099017caada41976bd61ea4efdd74a4f60 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 19 Oct 2023 23:39:17 +0200 Subject: [PATCH 06/20] test: /ipfs/cid/* rule on CLI and gateway CLI works as expected, gateway does not respect the rule (needs investigation) --- test/cli/content_blocking_test.go | 165 ++++++++++++++++-------------- 1 file changed, 86 insertions(+), 79 deletions(-) diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go index c7bea470890..e584be1f275 100644 --- a/test/cli/content_blocking_test.go +++ b/test/cli/content_blocking_test.go @@ -7,6 +7,7 @@ import ( "net/url" "os" "path/filepath" + "strings" "testing" "github.com/ipfs/kubo/test/cli/harness" @@ -40,7 +41,7 @@ func TestContentBlocking(t *testing.T) { "//QmbK7LDv5NNBvYQzNfm2eED17SNLt1yNMapcUhSuNLgkqz\n"+ // Double hash Path block using blake3 hashing: base58btc-blake3-multihash(gW7Nhu4HrfDtphEivm3Z9NNE7gpdh5Tga8g6JNZc1S8E47/path) "//d9d295bde21f422d471a90f2a37ec53049fdf3e5fa3ee2e8f20e10003da429e7\n"+ // Legacy CID double-hash block: sha256(bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/) "//3f8b9febd851873b3774b937cce126910699ceac56e72e64b866f8e258d09572\n"+ // Legacy Path double-hash block: sha256(bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/path) - "/ipfs/%s/*\n"+ // block subpaths + "/ipfs/%s/*\n"+ // block subpaths under a CID "/ipfs/%s\n"+ // block specific CID "/ipns/blocked-cid.example.com\n"+ "/ipns/blocked-dnslink.example.com\n", @@ -72,75 +73,101 @@ func TestContentBlocking(t *testing.T) { assert.Equal(t, "not blocked file content", resp.Body) }) - t.Run("Gateway Denies CID that is directly blocked", func(t *testing.T) { + // Then, does the most basic blocking case work? + t.Run("Gateway Denies directly blocked CID", func(t *testing.T) { t.Parallel() resp := client.Get(fmt.Sprintf("/ipfs/%s", blockedCID)) assert.NotEqual(t, http.StatusOK, resp.StatusCode) - // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) assert.NotEqual(t, "directly blocked file content", resp.Body) assert.Contains(t, resp.Body, blockedMsg) }) - /* TODO: this does not seem to work yet - t.Run("Gateway (path) Denies CID when it is indirectly blocked (via parent)", func(t *testing.T) { - t.Parallel() - resp := client.Get(fmt.Sprintf("/ipfs/%s/indirectly-blocked-file.txt", blockedDirCID)) - assert.NotEqual(t, http.StatusOK, resp.StatusCode) - // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) - assert.NotEqual(t, "indirectly blocked file content", resp.Body) - }) - */ - - t.Run("Gateway Denies CID that is directly blocked (double hash CID block: base58btc-sha256-multihash)", func(t *testing.T) { - t.Parallel() - resp := client.Get("/ipfs/QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR") - assert.NotEqual(t, http.StatusOK, resp.StatusCode) - // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) - assert.Contains(t, resp.Body, blockedMsg) - }) - - /* TODO: this fails due to parent block missing, we need to add fixture so it works in offline mode - t.Run("Gateway Denies Path that is directly blocked (double hash Path block: base58btc-blake3-multihash)", func(t *testing.T) { - t.Parallel() - resp := client.Get("/ipfs/bafyb4ieqht3b2rssdmc7sjv2cy2gfdilxkfh7623nvndziyqnawkmo266a/path") - assert.NotEqual(t, http.StatusOK, resp.StatusCode) - // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) - assert.Contains(t, resp.Body, blockedMsg) - }) - */ + // Ok, now the full list of test cases we want to cover in both CLI and Gateway + testCases := []struct { + name string + path string + }{ + { + name: "directly blocked CID", + path: fmt.Sprintf("/ipfs/%s", blockedCID), + }, + { + // TODO: this works for CLI but fails on Gateway + name: "indirectly blocked subpath", + path: fmt.Sprintf("/ipfs/%s/indirectly-blocked-file.txt", blockedDirCID), + }, + { + name: "/ipns path that resolves to a blocked CID", + path: "/ipns/blocked-cid.example.com", + }, + { + name: "/ipns Path that is blocked by DNSLink name", + path: "/ipns/blocked-dnslink.example.com", + }, + { + name: "double hash CID block: base58btc-sha256-multihash", + path: "/ipfs/QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR", + }, + /* TODO + { + name: "double hash Path block: base58btc-blake3-multihash", + path: "/ipfs/bafyb4ieqht3b2rssdmc7sjv2cy2gfdilxkfh7623nvndziyqnawkmo266a/path", + }, + */ + { + name: "legacy CID double-hash block: sha256", + path: "/ipfs/bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e", + }, + + /* TODO + { + name: "legacy Path double-hash: sha256", + path: "/ipfs/bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/path", + }, + */ + } - t.Run("Gateway Denies CID that is directly blocked (legacy CID double-hash block: sha256)", func(t *testing.T) { - t.Parallel() - resp := client.Get("/ipfs/bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e") - assert.NotEqual(t, http.StatusOK, resp.StatusCode) - // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) - assert.Contains(t, resp.Body, blockedMsg) - }) + // Which specific cliCmds we test against testCases + cliCmds := [][]string{ + {"block", "get"}, + {"block", "stat"}, + {"dag", "get"}, + {"dag", "export"}, + {"dag", "stat"}, + {"cat"}, + {"ls"}, + {"get"}, + {"refs"}, + } - /* TODO: this fails due to parent block missing, we need to add fixture so it works in offline mode - t.Run("Gateway Denies Path that is directly blocked (legacy Path double-hash: sha256)", func(t *testing.T) { - t.Parallel() - resp := client.Get("/ipfs/bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/path") - assert.NotEqual(t, http.StatusOK, resp.StatusCode) - // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) - assert.Contains(t, resp.Body, blockedMsg) - }) - */ + expectedMsg := blockedMsg + for _, testCase := range testCases { + + // Confirm that denylist is active for every command in 'cliCmds' x 'testCases' + for _, cmd := range cliCmds { + cliTestName := fmt.Sprintf("CLI '%s' denies %s", strings.Join(cmd, " "), testCase.name) + t.Run(cliTestName, func(t *testing.T) { + t.Parallel() + args := append(cmd, testCase.path) + errMsg := node.RunIPFS(args...).Stderr.Trimmed() + if !strings.Contains(errMsg, expectedMsg) { + t.Errorf("Expected STDERR error message %q, but got: %q", expectedMsg, errMsg) + } + }) + } + + // Confirm that denylist is active for every content path in 'testCases' + gwTestName := fmt.Sprintf("Gateway denies %s", testCase.name) + t.Run(gwTestName, func(t *testing.T) { + resp := client.Get(testCase.path) + // TODO we should require HTTP 410, not 5XX: assert.Equal(t, http.StatusGone, resp.StatusCode) + assert.NotEqual(t, http.StatusOK, resp.StatusCode) + assert.Contains(t, resp.Body, blockedMsg) + }) - t.Run("Gateway Denies /ipns that resolves to a blocked CID", func(t *testing.T) { - t.Parallel() - resp := client.Get("/ipns/blocked-cid.example.com") - assert.NotEqual(t, http.StatusOK, resp.StatusCode) - assert.Contains(t, resp.Body, blockedMsg) - }) + } - t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name", func(t *testing.T) { - t.Parallel() - resp := client.Get("/ipns/blocked-dnslink.example.com") - assert.NotEqual(t, http.StatusOK, resp.StatusCode) - // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) - assert.Contains(t, resp.Body, blockedMsg) - }) + // Extra edge cases on subdomain gateway t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain redirect)", func(t *testing.T) { t.Parallel() @@ -183,24 +210,4 @@ func TestContentBlocking(t *testing.T) { assert.Contains(t, resp.Body, blockedMsg) }) - // Basic tests to confirm blocking applies to CLI - - t.Run("CLI 'block get' Denies CID that is directly blocked", func(t *testing.T) { - t.Parallel() - errMsg := node.RunIPFS("block", "get", blockedCID).Stderr.Trimmed() - assert.Contains(t, errMsg, blockedMsg) - }) - - t.Run("CLI 'dag get' Denies CID that is directly blocked", func(t *testing.T) { - t.Parallel() - errMsg := node.RunIPFS("dag", "get", blockedCID).Stderr.Trimmed() - assert.Contains(t, errMsg, blockedMsg) - }) - - t.Run("CLI 'cat' Denies CID that is directly blocked", func(t *testing.T) { - t.Parallel() - errMsg := node.RunIPFS("cat", blockedCID).Stderr.Trimmed() - assert.Contains(t, errMsg, blockedMsg) - }) - } From 0c4725085aabc78aebd7bfc6c908e62736071981 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 20 Oct 2023 01:21:15 +0200 Subject: [PATCH 07/20] fix: accept paths in dag export --- core/commands/dag/export.go | 16 +++++++++++----- test/cli/content_blocking_test.go | 1 + 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/core/commands/dag/export.go b/core/commands/dag/export.go index d46fa6e21bf..d97718d200e 100644 --- a/core/commands/dag/export.go +++ b/core/commands/dag/export.go @@ -14,6 +14,7 @@ import ( cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/kubo/core/commands/cmdenv" + "github.com/ipfs/kubo/core/commands/cmdutils" cmds "github.com/ipfs/go-ipfs-cmds" gocar "github.com/ipld/go-car" @@ -21,12 +22,10 @@ import ( ) func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - c, err := cid.Decode(req.Arguments[0]) + // Accept CID or a content path + p, err := cmdutils.PathOrCidPath(req.Arguments[0]) if err != nil { - return fmt.Errorf( - "unable to parse root specification (currently only bare CIDs are supported): %s", - err, - ) + return err } api, err := cmdenv.GetApi(env, req) @@ -34,6 +33,13 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment return err } + // Resolve path and confirm the root block is available, fail fast if not + b, err := api.Block().Stat(req.Context, p) + if err != nil { + return err + } + c := b.Path().RootCid() + pipeR, pipeW := io.Pipe() errCh := make(chan error, 2) // we only report the 1st error diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go index e584be1f275..29e6d2ca352 100644 --- a/test/cli/content_blocking_test.go +++ b/test/cli/content_blocking_test.go @@ -145,6 +145,7 @@ func TestContentBlocking(t *testing.T) { // Confirm that denylist is active for every command in 'cliCmds' x 'testCases' for _, cmd := range cliCmds { + cmd := cmd cliTestName := fmt.Sprintf("CLI '%s' denies %s", strings.Join(cmd, " "), testCase.name) t.Run(cliTestName, func(t *testing.T) { t.Parallel() From 2eed42d78046d4acb4bd7e5d2e34a7ddff085ba3 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 20 Oct 2023 19:03:10 +0200 Subject: [PATCH 08/20] Update nopfs to latest commit --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 11786d377bb..c9d206a6cea 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,8 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/google/uuid v1.3.1 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs-shipyard/nopfs v0.0.12-0.20231019201810-ca629a80a385 - github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231019201810-ca629a80a385 + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020165528-0fc78f3af214 + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020165528-0fc78f3af214 github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 diff --git a/go.sum b/go.sum index a1c7704c481..1c1c51d55a2 100644 --- a/go.sum +++ b/go.sum @@ -333,10 +333,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231019201810-ca629a80a385 h1:ShsOXLWzQFpqrd/YHmOX+/NU9FbI2BbzwWQjkOvAE9o= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231019201810-ca629a80a385/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231019201810-ca629a80a385 h1:jAeuoctl0Bq6Wack8CHfyYQDFaIVpP10HmJoKB1GF7I= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231019201810-ca629a80a385/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020165528-0fc78f3af214 h1:awerQVtBHg8ielQL2BOuayV6ne9VNiPgGgx9DrHfkqQ= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020165528-0fc78f3af214/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020165528-0fc78f3af214 h1:RYSU/v4NCf4ifIkhsmyNJqZMEItfLE4A13kjO1m2eXs= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020165528-0fc78f3af214/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= From 04c1f68a9a753fc6ee29ce0a6486b29c793d3d44 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 20 Oct 2023 19:08:37 +0200 Subject: [PATCH 09/20] Initialize gateway with the wrapped Resolver --- core/corehttp/gateway.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index ec9abef8e2a..700a83cac2b 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -132,7 +132,11 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) { } } - backend, err := gateway.NewBlocksBackend(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys)) + backend, err := gateway.NewBlocksBackend(bserv, + gateway.WithValueStore(vsRouting), + gateway.WithNameSystem(nsys), + gateway.WithResolver(n.UnixFSPathResolver), + ) if err != nil { return nil, err } From e1689b54d1a04d6ae0866f77449e47030ad0387b Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 20 Oct 2023 19:14:38 +0200 Subject: [PATCH 10/20] go.mod tidy in kubo-as-a-library example --- docs/examples/kubo-as-a-library/go.mod | 4 ++-- docs/examples/kubo-as-a-library/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 7fc4a35d9ac..899e54348e8 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -61,8 +61,8 @@ require ( github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huin/goupnp v1.2.0 // indirect - github.com/ipfs-shipyard/nopfs v0.0.12-0.20231019201810-ca629a80a385 // indirect - github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231019201810-ca629a80a385 // indirect + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020165528-0fc78f3af214 // indirect + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020165528-0fc78f3af214 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index f41fe641639..404285973aa 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -299,10 +299,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231019201810-ca629a80a385 h1:ShsOXLWzQFpqrd/YHmOX+/NU9FbI2BbzwWQjkOvAE9o= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231019201810-ca629a80a385/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231019201810-ca629a80a385 h1:jAeuoctl0Bq6Wack8CHfyYQDFaIVpP10HmJoKB1GF7I= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231019201810-ca629a80a385/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020165528-0fc78f3af214 h1:awerQVtBHg8ielQL2BOuayV6ne9VNiPgGgx9DrHfkqQ= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020165528-0fc78f3af214/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020165528-0fc78f3af214 h1:RYSU/v4NCf4ifIkhsmyNJqZMEItfLE4A13kjO1m2eXs= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020165528-0fc78f3af214/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= From f7483a6b6928b5ddc041b4aa4e28b471386c976e Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 20 Oct 2023 21:44:42 +0200 Subject: [PATCH 11/20] test: improved /ipfs/cid/subpath* case --- test/cli/content_blocking_test.go | 41 +++++++++++++++++-------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go index 29e6d2ca352..4f0a71760d9 100644 --- a/test/cli/content_blocking_test.go +++ b/test/cli/content_blocking_test.go @@ -25,8 +25,8 @@ func TestContentBlocking(t *testing.T) { node := h.NewNode().Init("--empty-repo", "--profile=test") // Create CIDs we use in test - h.WriteFile("blocked-dir/indirectly-blocked-file.txt", "indirectly blocked file content") - blockedDirCID := node.IPFS("add", "-Q", "-r", filepath.Join(h.Dir, "blocked-dir")).Stdout.Trimmed() + h.WriteFile("blocked-dir/subdir/indirectly-blocked-file.txt", "indirectly blocked file content") + parentDirCID := node.IPFS("add", "-Q", "-r", filepath.Join(h.Dir, "blocked-dir")).Stdout.Trimmed() // indirectlyBlockedFileCID := node.IPFS("add", "-Q", filepath.Join(h.Dir, "blocked-dir", "indirectly-blocked-file.txt")).Stderr.Trimmed() h.WriteFile("directly-blocked-file.txt", "directly blocked file content") @@ -36,16 +36,15 @@ func TestContentBlocking(t *testing.T) { allowedCID := node.IPFS("add", "-Q", filepath.Join(h.Dir, "not-blocked-file.txt")).Stdout.Trimmed() // Create denylist at $IPFS_PATH/denylists/test.deny - denylistTmp := h.WriteToTemp(fmt.Sprintf( - "//QmX9dhRcQcKUw3Ws8485T5a9dtjrSCQaUAHnG4iK9i4ceM\n"+ // Double hash CID block: base58btc-sha256-multihash(QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR) - "//QmbK7LDv5NNBvYQzNfm2eED17SNLt1yNMapcUhSuNLgkqz\n"+ // Double hash Path block using blake3 hashing: base58btc-blake3-multihash(gW7Nhu4HrfDtphEivm3Z9NNE7gpdh5Tga8g6JNZc1S8E47/path) - "//d9d295bde21f422d471a90f2a37ec53049fdf3e5fa3ee2e8f20e10003da429e7\n"+ // Legacy CID double-hash block: sha256(bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/) - "//3f8b9febd851873b3774b937cce126910699ceac56e72e64b866f8e258d09572\n"+ // Legacy Path double-hash block: sha256(bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/path) - "/ipfs/%s/*\n"+ // block subpaths under a CID - "/ipfs/%s\n"+ // block specific CID - "/ipns/blocked-cid.example.com\n"+ - "/ipns/blocked-dnslink.example.com\n", - blockedDirCID, blockedCID)) + denylistTmp := h.WriteToTemp("name: test list\n---\n" + + "//QmX9dhRcQcKUw3Ws8485T5a9dtjrSCQaUAHnG4iK9i4ceM\n" + // Double hash CID block: base58btc-sha256-multihash(QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR) + "//QmbK7LDv5NNBvYQzNfm2eED17SNLt1yNMapcUhSuNLgkqz\n" + // Double hash Path block using blake3 hashing: base58btc-blake3-multihash(gW7Nhu4HrfDtphEivm3Z9NNE7gpdh5Tga8g6JNZc1S8E47/path) + "//d9d295bde21f422d471a90f2a37ec53049fdf3e5fa3ee2e8f20e10003da429e7\n" + // Legacy CID double-hash block: sha256(bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/) + "//3f8b9febd851873b3774b937cce126910699ceac56e72e64b866f8e258d09572\n" + // Legacy Path double-hash block: sha256(bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/path) + "/ipfs/" + blockedCID + "\n" + // block specific CID + "/ipfs/" + parentDirCID + "/subdir*\n" + // block only specific subpath + "/ipns/blocked-cid.example.com\n" + + "/ipns/blocked-dnslink.example.com\n") if err := os.MkdirAll(filepath.Join(node.Dir, "denylists"), 0o777); err != nil { log.Panicf("failed to create denylists dir: %s", err.Error()) @@ -67,7 +66,7 @@ func TestContentBlocking(t *testing.T) { // First, confirm gateway works t.Run("Gateway Allows CID that is not blocked", func(t *testing.T) { t.Parallel() - resp := client.Get(fmt.Sprintf("/ipfs/%s", allowedCID)) + resp := client.Get("/ipfs/" + allowedCID) assert.Equal(t, http.StatusOK, resp.StatusCode) // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) assert.Equal(t, "not blocked file content", resp.Body) @@ -76,12 +75,19 @@ func TestContentBlocking(t *testing.T) { // Then, does the most basic blocking case work? t.Run("Gateway Denies directly blocked CID", func(t *testing.T) { t.Parallel() - resp := client.Get(fmt.Sprintf("/ipfs/%s", blockedCID)) + resp := client.Get("/ipfs/" + blockedCID) assert.NotEqual(t, http.StatusOK, resp.StatusCode) assert.NotEqual(t, "directly blocked file content", resp.Body) assert.Contains(t, resp.Body, blockedMsg) }) + // Confirm parent of blocked subpath is not blocked + t.Run("Gateway Allows parent Path that is not blocked", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/" + parentDirCID) + assert.Equal(t, http.StatusOK, resp.StatusCode) + }) + // Ok, now the full list of test cases we want to cover in both CLI and Gateway testCases := []struct { name string @@ -89,12 +95,11 @@ func TestContentBlocking(t *testing.T) { }{ { name: "directly blocked CID", - path: fmt.Sprintf("/ipfs/%s", blockedCID), + path: "/ipfs/" + blockedCID, }, { - // TODO: this works for CLI but fails on Gateway - name: "indirectly blocked subpath", - path: fmt.Sprintf("/ipfs/%s/indirectly-blocked-file.txt", blockedDirCID), + name: "indirectly blocked file (on a blocked subpath)", + path: "/ipfs/" + parentDirCID + "/subdir/indirectly-blocked-file.txt", }, { name: "/ipns path that resolves to a blocked CID", From f516fc58629b33bc8dff096fd555633312ccc527 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 20 Oct 2023 22:28:27 +0200 Subject: [PATCH 12/20] test: double-hashed denylist entries --- test/cli/content_blocking_test.go | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go index 4f0a71760d9..511d439f16f 100644 --- a/test/cli/content_blocking_test.go +++ b/test/cli/content_blocking_test.go @@ -27,7 +27,6 @@ func TestContentBlocking(t *testing.T) { // Create CIDs we use in test h.WriteFile("blocked-dir/subdir/indirectly-blocked-file.txt", "indirectly blocked file content") parentDirCID := node.IPFS("add", "-Q", "-r", filepath.Join(h.Dir, "blocked-dir")).Stdout.Trimmed() - // indirectlyBlockedFileCID := node.IPFS("add", "-Q", filepath.Join(h.Dir, "blocked-dir", "indirectly-blocked-file.txt")).Stderr.Trimmed() h.WriteFile("directly-blocked-file.txt", "directly blocked file content") blockedCID := node.IPFS("add", "-Q", filepath.Join(h.Dir, "directly-blocked-file.txt")).Stdout.Trimmed() @@ -38,9 +37,9 @@ func TestContentBlocking(t *testing.T) { // Create denylist at $IPFS_PATH/denylists/test.deny denylistTmp := h.WriteToTemp("name: test list\n---\n" + "//QmX9dhRcQcKUw3Ws8485T5a9dtjrSCQaUAHnG4iK9i4ceM\n" + // Double hash CID block: base58btc-sha256-multihash(QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR) - "//QmbK7LDv5NNBvYQzNfm2eED17SNLt1yNMapcUhSuNLgkqz\n" + // Double hash Path block using blake3 hashing: base58btc-blake3-multihash(gW7Nhu4HrfDtphEivm3Z9NNE7gpdh5Tga8g6JNZc1S8E47/path) - "//d9d295bde21f422d471a90f2a37ec53049fdf3e5fa3ee2e8f20e10003da429e7\n" + // Legacy CID double-hash block: sha256(bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/) - "//3f8b9febd851873b3774b937cce126910699ceac56e72e64b866f8e258d09572\n" + // Legacy Path double-hash block: sha256(bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/path) + "//QmbK7LDv5NNBvYQzNfm2eED17SNLt1yNMapcUhSuNLgkqz\n" + // Double hash Path block under blake3 root CID: base58btc-sha256-multihash(gW7Nhu4HrfDtphEivm3Z9NNE7gpdh5Tga8g6JNZc1S8E47/path) + "//8526ba05eec55e28f8db5974cc891d0d92c8af69d386fc6464f1e9f372caf549\n" + // Legacy CID double-hash block: sha256(bafkqahtcnrxwg23fmqqgi33vmjwgk2dbonuca3dfm5qwg6jamnuwicq/) + "//e5b7d2ce2594e2e09901596d8e1f29fa249b74c8c9e32ea01eda5111e4d33f07\n" + // Legacy Path double-hash block: sha256(bafyaagyscufaqalqaacauaqiaejao43vmjygc5didacauaqiae/subpath) "/ipfs/" + blockedCID + "\n" + // block specific CID "/ipfs/" + parentDirCID + "/subdir*\n" + // block only specific subpath "/ipns/blocked-cid.example.com\n" + @@ -110,26 +109,22 @@ func TestContentBlocking(t *testing.T) { path: "/ipns/blocked-dnslink.example.com", }, { - name: "double hash CID block: base58btc-sha256-multihash", + name: "double-hash CID block (base58btc-sha256-multihash)", path: "/ipfs/QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR", }, - /* TODO { - name: "double hash Path block: base58btc-blake3-multihash", + name: "double-hash Path block under blake3 root CID (base58btc-sha256-multihash)", path: "/ipfs/bafyb4ieqht3b2rssdmc7sjv2cy2gfdilxkfh7623nvndziyqnawkmo266a/path", }, - */ { - name: "legacy CID double-hash block: sha256", - path: "/ipfs/bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e", + name: "legacy CID double-hash block (sha256)", + path: "/ipfs/bafkqahtcnrxwg23fmqqgi33vmjwgk2dbonuca3dfm5qwg6jamnuwicq", }, - /* TODO { - name: "legacy Path double-hash: sha256", - path: "/ipfs/bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/path", + name: "legacy Path double-hash block (sha256)", + path: "/ipfs/bafyaagyscufaqalqaacauaqiaejao43vmjygc5didacauaqiae/subpath", }, - */ } // Which specific cliCmds we test against testCases From 0129417ce47553cef1bb30a86a3c8121840e3046 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 20 Oct 2023 22:52:17 +0200 Subject: [PATCH 13/20] test: fix sharness of dag export cosmetic change to error message, this should be more robust --- test/sharness/t0054-dag-car-import-export.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh index 0482f24d92d..e277cc46687 100755 --- a/test/sharness/t0054-dag-car-import-export.sh +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -178,13 +178,11 @@ test_expect_success "basic offline export of 'getting started' dag works" ' ipfs dag export "$HASH_WELCOME_DOCS" >/dev/null ' - -echo "Error: block was not found locally (offline): ipld: could not find QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX (currently offline, perhaps retry after attaching to the network)" > offline_fetch_error_expected test_expect_success "basic offline export of nonexistent cid" ' ! ipfs dag export QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 2> offline_fetch_error_actual >/dev/null ' test_expect_success "correct error" ' - test_cmp_sorted offline_fetch_error_expected offline_fetch_error_actual + test_should_contain "Error: block was not found locally (offline): ipld: could not find QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" offline_fetch_error_actual ' cat >multiroot_import_json_stats_expected < Date: Wed, 25 Oct 2023 22:43:19 +0200 Subject: [PATCH 14/20] Latest nopfs --- docs/examples/kubo-as-a-library/go.mod | 4 ++-- docs/examples/kubo-as-a-library/go.sum | 8 ++++---- go.mod | 4 ++-- go.sum | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 899e54348e8..4a7241bb911 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -61,8 +61,8 @@ require ( github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huin/goupnp v1.2.0 // indirect - github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020165528-0fc78f3af214 // indirect - github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020165528-0fc78f3af214 // indirect + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80 // indirect + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 404285973aa..39e3913ef5f 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -299,10 +299,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020165528-0fc78f3af214 h1:awerQVtBHg8ielQL2BOuayV6ne9VNiPgGgx9DrHfkqQ= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020165528-0fc78f3af214/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020165528-0fc78f3af214 h1:RYSU/v4NCf4ifIkhsmyNJqZMEItfLE4A13kjO1m2eXs= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020165528-0fc78f3af214/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80 h1:fGhrSi5CgMQxw0OdTKjrYXMxKjbmCFOFh4GtD8CK6dA= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80 h1:O2fhQrtzG44UiGOH8YRVtAtfjCTu+/TubeFymhz5Brg= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= diff --git a/go.mod b/go.mod index c9d206a6cea..6bf184ebd48 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,8 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/google/uuid v1.3.1 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020165528-0fc78f3af214 - github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020165528-0fc78f3af214 + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80 + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80 github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 diff --git a/go.sum b/go.sum index 1c1c51d55a2..acde8951876 100644 --- a/go.sum +++ b/go.sum @@ -333,10 +333,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020165528-0fc78f3af214 h1:awerQVtBHg8ielQL2BOuayV6ne9VNiPgGgx9DrHfkqQ= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020165528-0fc78f3af214/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020165528-0fc78f3af214 h1:RYSU/v4NCf4ifIkhsmyNJqZMEItfLE4A13kjO1m2eXs= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020165528-0fc78f3af214/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80 h1:fGhrSi5CgMQxw0OdTKjrYXMxKjbmCFOFh4GtD8CK6dA= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80 h1:O2fhQrtzG44UiGOH8YRVtAtfjCTu+/TubeFymhz5Brg= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= From b4b7ca71df26c200da07fc3622883104de7cd1e5 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 25 Oct 2023 23:04:18 +0200 Subject: [PATCH 15/20] gateway: NoFetchGateway should use resolver with offline-blockservice fetcher --- core/corehttp/gateway.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index 700a83cac2b..e21df146c8d 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -111,6 +111,8 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) { bserv := n.Blocks var vsRouting routing.ValueStore = n.Routing nsys := n.Namesys + resolver := n.UnixFSPathResolver + if cfg.Gateway.NoFetch { bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore())) @@ -130,12 +132,16 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) { if err != nil { return nil, fmt.Errorf("error constructing namesys: %w", err) } + + // Let NewBlocksBackend setup the default resolver using the + // offline backend. + resolver = nil } backend, err := gateway.NewBlocksBackend(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys), - gateway.WithResolver(n.UnixFSPathResolver), + gateway.WithResolver(resolver), ) if err != nil { return nil, err From c3bdfffa10e28078a2648ddc7cb3b497bdb03f93 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 27 Oct 2023 01:57:55 +0200 Subject: [PATCH 16/20] test: double-hash Path block (blake3-multihash) this ensures we test something other than default sha256 Ref. https://github.com/ipfs-shipyard/nopfs/pull/28 --- test/cli/content_blocking_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go index 511d439f16f..93209ff2496 100644 --- a/test/cli/content_blocking_test.go +++ b/test/cli/content_blocking_test.go @@ -36,8 +36,8 @@ func TestContentBlocking(t *testing.T) { // Create denylist at $IPFS_PATH/denylists/test.deny denylistTmp := h.WriteToTemp("name: test list\n---\n" + - "//QmX9dhRcQcKUw3Ws8485T5a9dtjrSCQaUAHnG4iK9i4ceM\n" + // Double hash CID block: base58btc-sha256-multihash(QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR) - "//QmbK7LDv5NNBvYQzNfm2eED17SNLt1yNMapcUhSuNLgkqz\n" + // Double hash Path block under blake3 root CID: base58btc-sha256-multihash(gW7Nhu4HrfDtphEivm3Z9NNE7gpdh5Tga8g6JNZc1S8E47/path) + "//QmX9dhRcQcKUw3Ws8485T5a9dtjrSCQaUAHnG4iK9i4ceM\n" + // Double hash (sha256) CID block: base58btc(sha256-multihash(QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR)) + "//gW813G35CnLsy7gRYYHuf63hrz71U1xoLFDVeV7actx6oX\n" + // Double hash (blake3) Path block under blake3 root CID: base58btc(blake3-multihash(gW7Nhu4HrfDtphEivm3Z9NNE7gpdh5Tga8g6JNZc1S8E47/path)) "//8526ba05eec55e28f8db5974cc891d0d92c8af69d386fc6464f1e9f372caf549\n" + // Legacy CID double-hash block: sha256(bafkqahtcnrxwg23fmqqgi33vmjwgk2dbonuca3dfm5qwg6jamnuwicq/) "//e5b7d2ce2594e2e09901596d8e1f29fa249b74c8c9e32ea01eda5111e4d33f07\n" + // Legacy Path double-hash block: sha256(bafyaagyscufaqalqaacauaqiaejao43vmjygc5didacauaqiae/subpath) "/ipfs/" + blockedCID + "\n" + // block specific CID @@ -109,11 +109,11 @@ func TestContentBlocking(t *testing.T) { path: "/ipns/blocked-dnslink.example.com", }, { - name: "double-hash CID block (base58btc-sha256-multihash)", + name: "double-hash CID block (sha256-multihash)", path: "/ipfs/QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR", }, { - name: "double-hash Path block under blake3 root CID (base58btc-sha256-multihash)", + name: "double-hash Path block (blake3-multihash)", path: "/ipfs/bafyb4ieqht3b2rssdmc7sjv2cy2gfdilxkfh7623nvndziyqnawkmo266a/path", }, { From c99068e3a7bf0f77d657fdc7897182d1aa796565 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 27 Oct 2023 03:57:17 +0200 Subject: [PATCH 17/20] test: Gateway.NoFetch and GatewayOverLibp2p adds missing tests for "no fetch" gateways one can expose, in both cases the offline mode is done by passing custom blockservice/exchange into path resolver, which means global path resolver that has nopfs intercept is not used, and the content blocking does not happen on these gateways. needs to be fixed, but at least now we have tests that fail until it is fixed. --- test/cli/content_blocking_test.go | 124 ++++++++++++++++++++++++++---- 1 file changed, 109 insertions(+), 15 deletions(-) diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go index 93209ff2496..d8445986ac8 100644 --- a/test/cli/content_blocking_test.go +++ b/test/cli/content_blocking_test.go @@ -1,7 +1,9 @@ package cli import ( + "context" "fmt" + "io" "log" "net/http" "net/url" @@ -11,13 +13,19 @@ import ( "testing" "github.com/ipfs/kubo/test/cli/harness" + "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/peer" + libp2phttp "github.com/libp2p/go-libp2p/p2p/http" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestContentBlocking(t *testing.T) { t.Parallel() const blockedMsg = "blocked and cannot be provided" + const statusExpl = "HTTP error code is expected" + const bodyExpl = "Error message informing about content block is expected" h := harness.NewT(t) @@ -26,13 +34,13 @@ func TestContentBlocking(t *testing.T) { // Create CIDs we use in test h.WriteFile("blocked-dir/subdir/indirectly-blocked-file.txt", "indirectly blocked file content") - parentDirCID := node.IPFS("add", "-Q", "-r", filepath.Join(h.Dir, "blocked-dir")).Stdout.Trimmed() + parentDirCID := node.IPFS("add", "--raw-leaves", "-Q", "-r", filepath.Join(h.Dir, "blocked-dir")).Stdout.Trimmed() h.WriteFile("directly-blocked-file.txt", "directly blocked file content") - blockedCID := node.IPFS("add", "-Q", filepath.Join(h.Dir, "directly-blocked-file.txt")).Stdout.Trimmed() + blockedCID := node.IPFS("add", "--raw-leaves", "-Q", filepath.Join(h.Dir, "directly-blocked-file.txt")).Stdout.Trimmed() h.WriteFile("not-blocked-file.txt", "not blocked file content") - allowedCID := node.IPFS("add", "-Q", filepath.Join(h.Dir, "not-blocked-file.txt")).Stdout.Trimmed() + allowedCID := node.IPFS("add", "--raw-leaves", "-Q", filepath.Join(h.Dir, "not-blocked-file.txt")).Stdout.Trimmed() // Create denylist at $IPFS_PATH/denylists/test.deny denylistTmp := h.WriteToTemp("name: test list\n---\n" + @@ -58,16 +66,20 @@ func TestContentBlocking(t *testing.T) { os.Setenv("IPFS_NS_MAP", "blocked-cid.example.com:/ipfs/"+blockedCID+",blocked-dnslink.example.com/ipns/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") defer os.Unsetenv("IPFS_NS_MAP") + // Enable GatewayOverLibp2p as we want to test denylist there too + node.IPFS("config", "--json", "Experimental.GatewayOverLibp2p", "true") + // Start daemon, it should pick up denylist from $IPFS_PATH/denylists/test.deny - node.StartDaemon("--offline") + node.StartDaemon() // we need online mode for GatewayOverLibp2p tests client := node.GatewayClient() + // TODO: run matrix with NoFetch=false|true + // First, confirm gateway works t.Run("Gateway Allows CID that is not blocked", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/" + allowedCID) assert.Equal(t, http.StatusOK, resp.StatusCode) - // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) assert.Equal(t, "not blocked file content", resp.Body) }) @@ -75,9 +87,9 @@ func TestContentBlocking(t *testing.T) { t.Run("Gateway Denies directly blocked CID", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/" + blockedCID) - assert.NotEqual(t, http.StatusOK, resp.StatusCode) + assert.NotEqual(t, http.StatusOK, resp.StatusCode, statusExpl) assert.NotEqual(t, "directly blocked file content", resp.Body) - assert.Contains(t, resp.Body, blockedMsg) + assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) // Confirm parent of blocked subpath is not blocked @@ -162,8 +174,8 @@ func TestContentBlocking(t *testing.T) { t.Run(gwTestName, func(t *testing.T) { resp := client.Get(testCase.path) // TODO we should require HTTP 410, not 5XX: assert.Equal(t, http.StatusGone, resp.StatusCode) - assert.NotEqual(t, http.StatusOK, resp.StatusCode) - assert.Contains(t, resp.Body, blockedMsg) + assert.NotEqual(t, http.StatusOK, resp.StatusCode, statusExpl) + assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) } @@ -178,9 +190,9 @@ func TestContentBlocking(t *testing.T) { r.Host = "localhost:" + gwURL.Port() }) - assert.NotEqual(t, http.StatusOK, resp.StatusCode) + assert.NotEqual(t, http.StatusOK, resp.StatusCode, statusExpl) // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) - assert.Contains(t, resp.Body, blockedMsg) + assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain, no TLS)", func(t *testing.T) { @@ -191,9 +203,9 @@ func TestContentBlocking(t *testing.T) { r.Host = "blocked-dnslink.example.com.ipns.localhost:" + gwURL.Port() }) - assert.NotEqual(t, http.StatusOK, resp.StatusCode) + assert.NotEqual(t, http.StatusOK, resp.StatusCode, statusExpl) // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) - assert.Contains(t, resp.Body, blockedMsg) + assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain, inlined for TLS)", func(t *testing.T) { @@ -206,9 +218,91 @@ func TestContentBlocking(t *testing.T) { r.Host = "blocked--dnslink-example-com.ipns.localhost:" + gwURL.Port() }) - assert.NotEqual(t, http.StatusOK, resp.StatusCode) + assert.NotEqual(t, http.StatusOK, resp.StatusCode, statusExpl) // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) - assert.Contains(t, resp.Body, blockedMsg) + assert.Contains(t, resp.Body, blockedMsg, bodyExpl) + }) + + // We need to confirm denylist is active when gateway is run in NoFetch + // mode (which usually swaps blockservice to a read-only one, and that swap + // may cause denylists to not be applied, as it is a separate code path) + t.Run("GatewayNoFetch", func(t *testing.T) { + // NOTE: we don't run this in parallel, as it requires restart with different config + + // Switch gateway to NoFetch mode + node.StopDaemon() + node.IPFS("config", "--json", "Gateway.NoFetch", "true") + node.StartDaemon() + + // update client, as the port of test node might've changed after restart + client = node.GatewayClient() + + // First, confirm gateway works + t.Run("Allows CID that is not blocked", func(t *testing.T) { + resp := client.Get("/ipfs/" + allowedCID) + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "not blocked file content", resp.Body) + }) + + // Then, does the most basic blocking case work? + t.Run("Denies directly blocked CID", func(t *testing.T) { + resp := client.Get("/ipfs/" + blockedCID) + assert.NotEqual(t, http.StatusOK, resp.StatusCode, statusExpl) + assert.NotEqual(t, "directly blocked file content", resp.Body) + assert.Contains(t, resp.Body, blockedMsg, bodyExpl) + }) + + // Restore default + node.StopDaemon() + node.IPFS("config", "--json", "Gateway.NoFetch", "false") + node.StartDaemon() + client = node.GatewayClient() }) + // We need to confirm denylist is active on the + // trustless gateway exposed over libp2p + // when Experimental.GatewayOverLibp2p=true + // (https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#http-gateway-over-libp2p) + // NOTE: this type fo gateway is hardcoded to be NoFetch: it does not fetch + // data that is not in local store, so we only need to run it once: a + // simple smoke-test for allowed CID and blockedCID. + t.Run("GatewayOverLibp2p", func(t *testing.T) { + t.Parallel() + + // Create libp2p client that connects to our node over + // /http1.1 and then talks gateway semantics over the /ipfs/gateway sub-protocol + clientHost, err := libp2p.New(libp2p.NoListenAddrs) + require.NoError(t, err) + err = clientHost.Connect(context.Background(), peer.AddrInfo{ + ID: node.PeerID(), + Addrs: node.SwarmAddrs(), + }) + require.NoError(t, err) + + libp2pClient, err := (&libp2phttp.Host{StreamHost: clientHost}).NamespacedClient("/ipfs/gateway", peer.AddrInfo{ID: node.PeerID()}) + require.NoError(t, err) + + t.Run("Serves Allowed CID", func(t *testing.T) { + t.Parallel() + resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=raw", allowedCID)) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, string(body), "not blocked file content", bodyExpl) + }) + + t.Run("Denies Blocked CID", func(t *testing.T) { + t.Parallel() + resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=raw", blockedCID)) + require.NoError(t, err) + defer resp.Body.Close() + assert.NotEqual(t, http.StatusOK, resp.StatusCode, statusExpl) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.NotEqual(t, string(body), "directly blocked file content") + assert.Contains(t, string(body), blockedMsg, bodyExpl) + }) + }) } From da2dc470d9ad7929dadad4381fd31f3b7776e65d Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 28 Oct 2023 01:31:44 +0200 Subject: [PATCH 18/20] fix: use offline path resolvers where appropriate this fixes the problem described in https://github.com/ipfs/kubo/pull/10161#issuecomment-1782175955 by adding explicit offline path resolvers that are backed by offline exchange, and using them in NoFetch gateways instead of the default online ones --- core/core.go | 58 ++++++++++++++------------ core/corehttp/gateway.go | 16 ++++--- core/node/core.go | 51 +++++++++++++--------- docs/examples/kubo-as-a-library/go.mod | 4 +- docs/examples/kubo-as-a-library/go.sum | 8 ++-- go.mod | 4 +- go.sum | 8 ++-- plugin/plugins/nopfs/nopfs.go | 6 ++- test/cli/content_blocking_test.go | 3 +- 9 files changed, 91 insertions(+), 67 deletions(-) diff --git a/core/core.go b/core/core.go index c35d3e4457f..c9bec6b8b00 100644 --- a/core/core.go +++ b/core/core.go @@ -76,35 +76,39 @@ type IpfsNode struct { PNetFingerprint libp2p.PNetFingerprint `optional:"true"` // fingerprint of private network // Services - Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances - Blockstore bstore.GCBlockstore // the block store (lower level) - Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore - BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping - GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc - Blocks bserv.BlockService // the block service, get/add blocks. - DAG ipld.DAGService // the merkle dag service, get/add objects. - IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model - UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data - Reporter *metrics.BandwidthCounter `optional:"true"` - Discovery mdns.Service `optional:"true"` - FilesRoot *mfs.Root - RecordValidator record.Validator + Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances + Blockstore bstore.GCBlockstore // the block store (lower level) + Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore + BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping + GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc + Blocks bserv.BlockService // the block service, get/add blocks. + DAG ipld.DAGService // the merkle dag service, get/add objects. + IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model + UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data + OfflineIPLDFetcherFactory fetcher.Factory `name:"offlineIpldFetcher"` // fetcher that paths over the IPLD data model without fetching new blocks + OfflineUnixFSFetcherFactory fetcher.Factory `name:"offlineUnixfsFetcher"` // fetcher that interprets UnixFS data without fetching new blocks + Reporter *metrics.BandwidthCounter `optional:"true"` + Discovery mdns.Service `optional:"true"` + FilesRoot *mfs.Root + RecordValidator record.Validator // Online - PeerHost p2phost.Host `optional:"true"` // the network host (server+client) - Peering *peering.PeeringService `optional:"true"` - Filters *ma.Filters `optional:"true"` - Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper - Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht - DNSResolver *madns.Resolver // the DNS resolver - IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver - UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver - Exchange exchange.Interface // the block exchange + strategy (bitswap) - Namesys namesys.NameSystem // the name system, resolves paths to hashes - Provider provider.System // the value provider system - IpnsRepub *ipnsrp.Republisher `optional:"true"` - GraphExchange graphsync.GraphExchange `optional:"true"` - ResourceManager network.ResourceManager `optional:"true"` + PeerHost p2phost.Host `optional:"true"` // the network host (server+client) + Peering *peering.PeeringService `optional:"true"` + Filters *ma.Filters `optional:"true"` + Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper + Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht + DNSResolver *madns.Resolver // the DNS resolver + IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver + UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver + OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` // The IPLD path resolver that uses only locally available blocks + OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks + Exchange exchange.Interface // the block exchange + strategy (bitswap) + Namesys namesys.NameSystem // the name system, resolves paths to hashes + Provider provider.System // the value provider system + IpnsRepub *ipnsrp.Republisher `optional:"true"` + GraphExchange graphsync.GraphExchange `optional:"true"` + ResourceManager network.ResourceManager `optional:"true"` PubSub *pubsub.PubSub `optional:"true"` PSRouter *psrouter.PubsubValueStore `optional:"true"` diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index e21df146c8d..3e0380d5a3c 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -81,7 +81,11 @@ func Libp2pGatewayOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { bserv := blockservice.New(n.Blocks.Blockstore(), offline.Exchange(n.Blocks.Blockstore())) - backend, err := gateway.NewBlocksBackend(bserv) + backend, err := gateway.NewBlocksBackend(bserv, + // GatewayOverLibp2p only returns things that are in local blockstore + // (same as Gateway.NoFetch=true), we have to pass offline path resolver + gateway.WithResolver(n.OfflineUnixFSPathResolver), + ) if err != nil { return nil, err } @@ -111,7 +115,7 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) { bserv := n.Blocks var vsRouting routing.ValueStore = n.Routing nsys := n.Namesys - resolver := n.UnixFSPathResolver + pathResolver := n.UnixFSPathResolver if cfg.Gateway.NoFetch { bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore())) @@ -133,15 +137,15 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) { return nil, fmt.Errorf("error constructing namesys: %w", err) } - // Let NewBlocksBackend setup the default resolver using the - // offline backend. - resolver = nil + // Gateway.NoFetch=true requires offline path resolver + // to avoid fetching missing blocks during path traversal + pathResolver = n.OfflineUnixFSPathResolver } backend, err := gateway.NewBlocksBackend(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys), - gateway.WithResolver(resolver), + gateway.WithResolver(pathResolver), ) if err != nil { return nil, err diff --git a/core/node/core.go b/core/node/core.go index d2d2c63d749..9a2035a4c8c 100644 --- a/core/node/core.go +++ b/core/node/core.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/boxo/blockservice" blockstore "github.com/ipfs/boxo/blockstore" exchange "github.com/ipfs/boxo/exchange" + offline "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/fetcher" bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/filestore" @@ -21,9 +22,6 @@ import ( format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-unixfsnode" dagpb "github.com/ipld/go-codec-dagpb" - "github.com/ipld/go-ipld-prime" - basicnode "github.com/ipld/go-ipld-prime/node/basic" - "github.com/ipld/go-ipld-prime/schema" "go.uber.org/fx" "github.com/ipfs/kubo/core/node/helpers" @@ -87,43 +85,58 @@ func (s *syncDagService) Session(ctx context.Context) format.NodeGetter { // FetchersOut allows injection of fetchers. type FetchersOut struct { fx.Out - IPLDFetcher fetcher.Factory `name:"ipldFetcher"` - UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"` + IPLDFetcher fetcher.Factory `name:"ipldFetcher"` + UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"` + OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"` + OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"` } // FetchersIn allows using fetchers for other dependencies. type FetchersIn struct { fx.In - IPLDFetcher fetcher.Factory `name:"ipldFetcher"` - UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"` + IPLDFetcher fetcher.Factory `name:"ipldFetcher"` + UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"` + OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"` + OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"` } // FetcherConfig returns a fetcher config that can build new fetcher instances func FetcherConfig(bs blockservice.BlockService) FetchersOut { ipldFetcher := bsfetcher.NewFetcherConfig(bs) - ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) - + ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser) unixFSFetcher := ipldFetcher.WithReifier(unixfsnode.Reify) - return FetchersOut{IPLDFetcher: ipldFetcher, UnixfsFetcher: unixFSFetcher} + + // Construct offline versions which we can safely use in contexts where + // path resolution should not fetch new blocks via exchange. + offlineBs := blockservice.New(bs.Blockstore(), offline.Exchange(bs.Blockstore())) + offlineIpldFetcher := bsfetcher.NewFetcherConfig(offlineBs) + offlineIpldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser) + offlineUnixFSFetcher := offlineIpldFetcher.WithReifier(unixfsnode.Reify) + + return FetchersOut{ + IPLDFetcher: ipldFetcher, + UnixfsFetcher: unixFSFetcher, + OfflineIPLDFetcher: offlineIpldFetcher, + OfflineUnixfsFetcher: offlineUnixFSFetcher, + } } // PathResolversOut allows injection of path resolvers type PathResolversOut struct { fx.Out - IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` - UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` + IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` + UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` + OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` + OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` } // PathResolverConfig creates path resolvers with the given fetchers. func PathResolverConfig(fetchers FetchersIn) PathResolversOut { return PathResolversOut{ - IPLDPathResolver: pathresolver.NewBasicResolver(fetchers.IPLDFetcher), - UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher), + IPLDPathResolver: pathresolver.NewBasicResolver(fetchers.IPLDFetcher), + UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher), + OfflineIPLDPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineIPLDFetcher), + OfflineUnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineUnixfsFetcher), } } diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 4a7241bb911..674f5b4f0dc 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -61,8 +61,8 @@ require ( github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huin/goupnp v1.2.0 // indirect - github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80 // indirect - github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80 // indirect + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c // indirect + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 39e3913ef5f..c582628193a 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -299,10 +299,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80 h1:fGhrSi5CgMQxw0OdTKjrYXMxKjbmCFOFh4GtD8CK6dA= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80 h1:O2fhQrtzG44UiGOH8YRVtAtfjCTu+/TubeFymhz5Brg= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c h1:17FO7HnKiFhO7iadu3zCgII+EblpdRmJt5qg9FqQo8Y= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7UynTbtdlt+w08ggb1UGLGaGjp1mMaZhoTZSctpn5Ak= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= diff --git a/go.mod b/go.mod index 6bf184ebd48..d2584cec03e 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,8 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/google/uuid v1.3.1 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80 - github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80 + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 diff --git a/go.sum b/go.sum index acde8951876..245c74e457b 100644 --- a/go.sum +++ b/go.sum @@ -333,10 +333,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80 h1:fGhrSi5CgMQxw0OdTKjrYXMxKjbmCFOFh4GtD8CK6dA= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80 h1:O2fhQrtzG44UiGOH8YRVtAtfjCTu+/TubeFymhz5Brg= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c h1:17FO7HnKiFhO7iadu3zCgII+EblpdRmJt5qg9FqQo8Y= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7UynTbtdlt+w08ggb1UGLGaGjp1mMaZhoTZSctpn5Ak= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= diff --git a/plugin/plugins/nopfs/nopfs.go b/plugin/plugins/nopfs/nopfs.go index a04734187ba..64350830f94 100644 --- a/plugin/plugins/nopfs/nopfs.go +++ b/plugin/plugins/nopfs/nopfs.go @@ -62,8 +62,10 @@ func MakeBlocker() (*nopfs.Blocker, error) { func PathResolvers(fetchers node.FetchersIn, blocker *nopfs.Blocker) node.PathResolversOut { res := node.PathResolverConfig(fetchers) return node.PathResolversOut{ - IPLDPathResolver: ipfs.WrapResolver(res.IPLDPathResolver, blocker), - UnixFSPathResolver: ipfs.WrapResolver(res.UnixFSPathResolver, blocker), + IPLDPathResolver: ipfs.WrapResolver(res.IPLDPathResolver, blocker), + UnixFSPathResolver: ipfs.WrapResolver(res.UnixFSPathResolver, blocker), + OfflineIPLDPathResolver: ipfs.WrapResolver(res.OfflineIPLDPathResolver, blocker), + OfflineUnixFSPathResolver: ipfs.WrapResolver(res.OfflineUnixFSPathResolver, blocker), } } diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go index d8445986ac8..575c7530540 100644 --- a/test/cli/content_blocking_test.go +++ b/test/cli/content_blocking_test.go @@ -21,7 +21,8 @@ import ( ) func TestContentBlocking(t *testing.T) { - t.Parallel() + // NOTE: we can't run this with t.Parallel() because we set IPFS_NS_MAP + // and running in parallel could impact other tests const blockedMsg = "blocked and cannot be provided" const statusExpl = "HTTP error code is expected" From 3d8feacec135bd6b43f1dac1dcb5241bdc268ba7 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 28 Oct 2023 04:04:43 +0200 Subject: [PATCH 19/20] fix(gw): blocked content produces http error 410 requires https://github.com/ipfs/boxo/pull/497 which is based on top of the boxo already used in kubo master to avoid issues caused by ilater commits in boxo main --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/cli/content_blocking_test.go | 16 ++++++++-------- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 674f5b4f0dc..dfb7848b036 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.20 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 + github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.31.0 github.com/multiformats/go-multiaddr v0.11.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index c582628193a..49dbacfee11 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -305,8 +305,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= -github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= +github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b h1:0Qi1EhB82x3SZJOBieG51BtPvjU3kSy4H8OpMnxKvtk= +github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b/go.mod h1:pu8HsZvuyYeYJsqtLDCoYSvy8rHj6vI3dlh8P0f83Zs= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/go.mod b/go.mod index d2584cec03e..c94a251cba4 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c - github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 + github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 245c74e457b..928433eaaa2 100644 --- a/go.sum +++ b/go.sum @@ -339,8 +339,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= -github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= +github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b h1:0Qi1EhB82x3SZJOBieG51BtPvjU3kSy4H8OpMnxKvtk= +github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b/go.mod h1:pu8HsZvuyYeYJsqtLDCoYSvy8rHj6vI3dlh8P0f83Zs= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go index 575c7530540..98e65397f15 100644 --- a/test/cli/content_blocking_test.go +++ b/test/cli/content_blocking_test.go @@ -25,7 +25,7 @@ func TestContentBlocking(t *testing.T) { // and running in parallel could impact other tests const blockedMsg = "blocked and cannot be provided" - const statusExpl = "HTTP error code is expected" + const statusExpl = "specific HTTP error code is expected" const bodyExpl = "Error message informing about content block is expected" h := harness.NewT(t) @@ -88,7 +88,7 @@ func TestContentBlocking(t *testing.T) { t.Run("Gateway Denies directly blocked CID", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/" + blockedCID) - assert.NotEqual(t, http.StatusOK, resp.StatusCode, statusExpl) + assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) assert.NotEqual(t, "directly blocked file content", resp.Body) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) @@ -175,7 +175,7 @@ func TestContentBlocking(t *testing.T) { t.Run(gwTestName, func(t *testing.T) { resp := client.Get(testCase.path) // TODO we should require HTTP 410, not 5XX: assert.Equal(t, http.StatusGone, resp.StatusCode) - assert.NotEqual(t, http.StatusOK, resp.StatusCode, statusExpl) + assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) @@ -191,7 +191,7 @@ func TestContentBlocking(t *testing.T) { r.Host = "localhost:" + gwURL.Port() }) - assert.NotEqual(t, http.StatusOK, resp.StatusCode, statusExpl) + assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) @@ -204,7 +204,7 @@ func TestContentBlocking(t *testing.T) { r.Host = "blocked-dnslink.example.com.ipns.localhost:" + gwURL.Port() }) - assert.NotEqual(t, http.StatusOK, resp.StatusCode, statusExpl) + assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) @@ -219,7 +219,7 @@ func TestContentBlocking(t *testing.T) { r.Host = "blocked--dnslink-example-com.ipns.localhost:" + gwURL.Port() }) - assert.NotEqual(t, http.StatusOK, resp.StatusCode, statusExpl) + assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) @@ -248,7 +248,7 @@ func TestContentBlocking(t *testing.T) { // Then, does the most basic blocking case work? t.Run("Denies directly blocked CID", func(t *testing.T) { resp := client.Get("/ipfs/" + blockedCID) - assert.NotEqual(t, http.StatusOK, resp.StatusCode, statusExpl) + assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) assert.NotEqual(t, "directly blocked file content", resp.Body) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) @@ -299,7 +299,7 @@ func TestContentBlocking(t *testing.T) { resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=raw", blockedCID)) require.NoError(t, err) defer resp.Body.Close() - assert.NotEqual(t, http.StatusOK, resp.StatusCode, statusExpl) + assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) body, err := io.ReadAll(resp.Body) require.NoError(t, err) assert.NotEqual(t, string(body), "directly blocked file content") diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index e79c7ff5ebd..cbc577f9b40 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -7,7 +7,7 @@ replace github.com/ipfs/kubo => ../../ require ( github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd github.com/golangci/golangci-lint v1.54.1 - github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 + github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 github.com/ipfs/go-datastore v0.6.0 diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index cbb547d2bc9..90a00aebcca 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -398,8 +398,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= -github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= +github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b h1:0Qi1EhB82x3SZJOBieG51BtPvjU3kSy4H8OpMnxKvtk= +github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b/go.mod h1:pu8HsZvuyYeYJsqtLDCoYSvy8rHj6vI3dlh8P0f83Zs= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= From d4766b6cd82053a2c737d4ff91c5fba352716ec6 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 28 Oct 2023 05:12:20 +0200 Subject: [PATCH 20/20] docs: docs/content-blocking.md example --- docs/content-blocking.md | 13 ++++++++++++- test/cli/content_blocking_test.go | 6 ------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/content-blocking.md b/docs/content-blocking.md index ef5461132db..ebc84bba3ea 100644 --- a/docs/content-blocking.md +++ b/docs/content-blocking.md @@ -40,7 +40,18 @@ caused the request to be blocked. [NOpfs](https://github.com/ipfs-shipyard/nopfs) supports the format from [IPIP-383](https://github.com/ipfs/specs/pull/383). -Example: https://badbits.dwebops.pub/badbits.deny +Clear-text rules are simple: just put content paths to block, one per line. +Paths with unicode and whitespace need to be percend-encoded: + +``` +/ipfs/QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR +/ipfs/bafybeihfg3d7rdltd43u3tfvncx7n5loqofbsobojcadtmokrljfthuc7y/927%20-%20Standards/927%20-%20Standards.png +``` + +Sensitive content paths can be double-hashed to block without revealing them. +Double-hashed list example: https://badbits.dwebops.pub/badbits.deny + +See [IPIP-383](https://github.com/ipfs/specs/pull/383) for detailed format specification and more examples. ## How to suspend blocking without removing denylists diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go index 98e65397f15..ddb7c951e88 100644 --- a/test/cli/content_blocking_test.go +++ b/test/cli/content_blocking_test.go @@ -74,8 +74,6 @@ func TestContentBlocking(t *testing.T) { node.StartDaemon() // we need online mode for GatewayOverLibp2p tests client := node.GatewayClient() - // TODO: run matrix with NoFetch=false|true - // First, confirm gateway works t.Run("Gateway Allows CID that is not blocked", func(t *testing.T) { t.Parallel() @@ -174,7 +172,6 @@ func TestContentBlocking(t *testing.T) { gwTestName := fmt.Sprintf("Gateway denies %s", testCase.name) t.Run(gwTestName, func(t *testing.T) { resp := client.Get(testCase.path) - // TODO we should require HTTP 410, not 5XX: assert.Equal(t, http.StatusGone, resp.StatusCode) assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) @@ -192,7 +189,6 @@ func TestContentBlocking(t *testing.T) { }) assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) - // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) @@ -205,7 +201,6 @@ func TestContentBlocking(t *testing.T) { }) assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) - // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) @@ -220,7 +215,6 @@ func TestContentBlocking(t *testing.T) { }) assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) - // TODO assert.Equal(t, http.StatusGone, resp.StatusCode) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) })