From 021e4365712d02487c289fd861ef488f9587ec6a Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 8 May 2024 16:06:11 +0200 Subject: [PATCH] config: introduce Import section --- config/config.go | 1 + config/import.go | 17 +++++++++ core/commands/add.go | 33 +++++++++++++++- core/commands/block.go | 17 ++++++++- core/commands/dag/dag.go | 2 +- core/commands/dag/put.go | 15 ++++++++ core/commands/files.go | 19 ++++++++-- docs/config.md | 44 ++++++++++++++++++++++ test/cli/add_test.go | 81 ++++++++++++++++++++++++++++++++++++++++ 9 files changed, 221 insertions(+), 8 deletions(-) create mode 100644 config/import.go create mode 100644 test/cli/add_test.go diff --git a/config/config.go b/config/config.go index 1951784dd1d0..046c930be93e 100644 --- a/config/config.go +++ b/config/config.go @@ -36,6 +36,7 @@ type Config struct { Experimental Experiments Plugins Plugins Pinning Pinning + Import Import Internal Internal // experimental/unstable options } diff --git a/config/import.go b/config/import.go new file mode 100644 index 000000000000..10af4edfa4d3 --- /dev/null +++ b/config/import.go @@ -0,0 +1,17 @@ +package config + +const ( + DefaultCidVersion = 0 + DefaultUnixFSRawLeaves = false + DefaultUnixFSChunker = "size-262144" + DefaultHashFunction = "sha2-256" +) + +// Import configures the default options for ingesting data. This affects commands +// that ingest data, such as 'ipfs add', 'ipfs dag put, 'ipfs block put', 'ipfs files write'. +type Import struct { + CidVersion OptionalInteger + UnixFSRawLeaves Flag + UnixFSChunker OptionalString + HashFunction OptionalString +} diff --git a/core/commands/add.go b/core/commands/add.go index 4e59cc867582..94a5a0f51f5d 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -8,6 +8,7 @@ import ( gopath "path" "strings" + "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/cheggaaa/pb" @@ -155,12 +156,12 @@ See 'dag export' and 'dag import' for more information. cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."), cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."), - cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash").WithDefault("size-262144"), + cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash"), cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes."), cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"), cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"), cmds.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. Passing version 1 will cause the raw-leaves option to default to true."), - cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"), + cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)"), cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"), cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32), cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true), @@ -191,6 +192,16 @@ See 'dag export' and 'dag import' for more information. return err } + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + cfg, err := nd.Repo.Config() + if err != nil { + return err + } + progress, _ := req.Options[progressOptionName].(bool) trickle, _ := req.Options[trickleOptionName].(bool) wrap, _ := req.Options[wrapOptionName].(bool) @@ -207,6 +218,24 @@ See 'dag export' and 'dag import' for more information. inlineLimit, _ := req.Options[inlineLimitOptionName].(int) toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string) + if chunker == "" { + chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker) + } + + if hashFunStr == "" { + hashFunStr = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction) + } + + if !cidVerSet && !cfg.Import.CidVersion.IsDefault() { + cidVerSet = true + cidVer = int(cfg.Import.CidVersion.WithDefault(config.DefaultCidVersion)) + } + + if !rbset && cfg.Import.UnixFSRawLeaves != config.Default { + rbset = true + rawblks = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves) + } + if onlyHash && toFilesSet { return fmt.Errorf("%s and %s options are not compatible", onlyHashOptionName, toFilesOptionName) } diff --git a/core/commands/block.go b/core/commands/block.go index 6ceb258f62c0..b4b0fd204571 100644 --- a/core/commands/block.go +++ b/core/commands/block.go @@ -8,6 +8,7 @@ import ( "github.com/ipfs/boxo/files" + "github.com/ipfs/kubo/config" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" @@ -153,7 +154,7 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0). }, Options: []cmds.Option{ cmds.StringOption(blockCidCodecOptionName, "Multicodec to use in returned CID").WithDefault("raw"), - cmds.StringOption(mhtypeOptionName, "Multihash hash function").WithDefault("sha2-256"), + cmds.StringOption(mhtypeOptionName, "Multihash hash function"), cmds.IntOption(mhlenOptionName, "Multihash hash length").WithDefault(-1), cmds.BoolOption(pinOptionName, "Pin added blocks recursively").WithDefault(false), cmdutils.AllowBigBlockOption, @@ -165,7 +166,21 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0). return err } + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + cfg, err := nd.Repo.Config() + if err != nil { + return err + } + mhtype, _ := req.Options[mhtypeOptionName].(string) + if mhtype == "" { + mhtype = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction) + } + mhtval, ok := mh.Names[mhtype] if !ok { return fmt.Errorf("unrecognized multihash function: %s", mhtype) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 56aae4105daa..ce5edb64167f 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -87,7 +87,7 @@ into an object of the specified format. cmds.StringOption("store-codec", "Codec that the stored object will be encoded with").WithDefault("dag-cbor"), cmds.StringOption("input-codec", "Codec that the input object is encoded in").WithDefault("dag-json"), cmds.BoolOption("pin", "Pin this object when adding."), - cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"), + cmds.StringOption("hash", "Hash function to use"), cmdutils.AllowBigBlockOption, }, Run: dagPut, diff --git a/core/commands/dag/put.go b/core/commands/dag/put.go index c9c0b455b072..fb719916cdf6 100644 --- a/core/commands/dag/put.go +++ b/core/commands/dag/put.go @@ -7,6 +7,7 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" ipldlegacy "github.com/ipfs/go-ipld-legacy" + "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" "github.com/ipld/go-ipld-prime/multicodec" @@ -32,11 +33,25 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e return err } + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + cfg, err := nd.Repo.Config() + if err != nil { + return err + } + inputCodec, _ := req.Options["input-codec"].(string) storeCodec, _ := req.Options["store-codec"].(string) hash, _ := req.Options["hash"].(string) dopin, _ := req.Options["pin"].(bool) + if hash == "" { + hash = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction) + } + var icodec mc.Code if err := icodec.Set(inputCodec); err != nil { return err diff --git a/core/commands/files.go b/core/commands/files.go index 9a7ee639a2c2..12891a7301d9 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -11,6 +11,7 @@ import ( "strings" humanize "github.com/dustin/go-humanize" + "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/commands/cmdenv" @@ -802,18 +803,28 @@ See '--to-files' in 'ipfs add --help' for more information. return err } + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + cfg, err := nd.Repo.Config() + if err != nil { + return err + } + create, _ := req.Options[filesCreateOptionName].(bool) mkParents, _ := req.Options[filesParentsOptionName].(bool) trunc, _ := req.Options[filesTruncateOptionName].(bool) flush, _ := req.Options[filesFlushOptionName].(bool) rawLeaves, rawLeavesDef := req.Options[filesRawLeavesOptionName].(bool) - prefix, err := getPrefixNew(req) - if err != nil { - return err + if !rawLeavesDef && cfg.Import.UnixFSRawLeaves != config.Default { + rawLeavesDef = true + rawLeaves = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves) } - nd, err := cmdenv.GetNode(env) + prefix, err := getPrefixNew(req) if err != nil { return err } diff --git a/docs/config.md b/docs/config.md index e36ffebb73eb..450b40caaf49 100644 --- a/docs/config.md +++ b/docs/config.md @@ -175,6 +175,11 @@ config file at runtime. - [`DNS`](#dns) - [`DNS.Resolvers`](#dnsresolvers) - [`DNS.MaxCacheTTL`](#dnsmaxcachettl) + - [`Import`](#import) + - [`Import.CidVersion`](#importcidversion) + - [`Import.UnixFSRawLeaves`](#importunixfsrawleaves) + - [`Import.UnixFSChunker`](#importunixfschunker) + - [`Import.HashFunction`](#importhashfunction) ## Profiles @@ -2377,3 +2382,42 @@ Note: this does NOT work with Go's default DNS resolver. To make this a global s Default: Respect DNS Response TTL Type: `optionalDuration` + +## `Import` + +Options to configure the default options used for ingesting data, in commands such as `ipfs add` or `ipfs block put`. All affected commands are detailed per option. + +Note that using flags will override the options defined here. + +### `Import.CidVersion` + +The default CID version. Commands affected: `ipfs add`. + +Default: `0` + +Type: `optionalInteger` + +### `Import.UnixFSRawLeaves` + +The default UnixFS raw leaves option. Commands affected: `ipfs add`, `ipfs files write`. + +Default: `false` + +Type: `flag` + +### `Import.UnixFSChunker` + +The default UnixFS chunker. Commands affected: `ipfs add`. + +Default: `size-262144` + +Type: `optionalString` + + +### `Import.HashFunction` + +The default hash function. Commands affected: `ipfs add`, `ipfs block put`, `ipfs dag put`. + +Default: `sha2-256` + +Type: `optionalString` diff --git a/test/cli/add_test.go b/test/cli/add_test.go new file mode 100644 index 000000000000..349c046da63a --- /dev/null +++ b/test/cli/add_test.go @@ -0,0 +1,81 @@ +package cli + +import ( + "testing" + + "github.com/ipfs/kubo/config" + "github.com/ipfs/kubo/test/cli/harness" + "github.com/stretchr/testify/require" +) + +func TestAdd(t *testing.T) { + t.Parallel() + + var ( + shortString = "hello world" + shortStringCidV0 = "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD" + shortStringCidV1 = "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e" + shortStringCidV1Sha512 = "bafkrgqbqt3gerhas23vuzrapkdeqf4vu2dwxp3srdj6hvg6nhsug2tgyn6mj3u23yx7utftq3i2ckw2fwdh5qmhid5qf3t35yvkc5e5ottlw6" + ) + + t.Run("output cid version: default (CIDv0)", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + cidStr := node.IPFSAddStr(shortString) + require.Equal(t, shortStringCidV0, cidStr) + }) + + t.Run("output cid version: follows configuration (CIDv0)", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.CidVersion = *config.NewOptionalInteger(0) + }) + node.StartDaemon() + defer node.StopDaemon() + + cidStr := node.IPFSAddStr(shortString) + require.Equal(t, shortStringCidV0, cidStr) + }) + + t.Run("output cid version: follows configuration (hash function)", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.HashFunction = *config.NewOptionalString("sha2-512") + }) + node.StartDaemon() + defer node.StopDaemon() + + cidStr := node.IPFSAddStr(shortString) + require.Equal(t, shortStringCidV1Sha512, cidStr) + }) + + t.Run("output cid version: follows configuration (CIDv1)", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.CidVersion = *config.NewOptionalInteger(1) + }) + node.StartDaemon() + defer node.StopDaemon() + + cidStr := node.IPFSAddStr(shortString) + require.Equal(t, shortStringCidV1, cidStr) + }) + + t.Run("output cid version: flag overrides configuration", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.CidVersion = *config.NewOptionalInteger(1) + }) + node.StartDaemon() + defer node.StopDaemon() + + cidStr := node.IPFSAddStr(shortString, "--cid-version", "0") + require.Equal(t, shortStringCidV0, cidStr) + }) +}