Skip to content

Commit

Permalink
config: introduce Import section
Browse files Browse the repository at this point in the history
  • Loading branch information
hacdias committed May 8, 2024
1 parent 2841ec0 commit 021e436
Show file tree
Hide file tree
Showing 9 changed files with 221 additions and 8 deletions.
1 change: 1 addition & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ type Config struct {
Experimental Experiments
Plugins Plugins
Pinning Pinning
Import Import

Internal Internal // experimental/unstable options
}
Expand Down
17 changes: 17 additions & 0 deletions config/import.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package config

const (
DefaultCidVersion = 0
DefaultUnixFSRawLeaves = false
DefaultUnixFSChunker = "size-262144"
DefaultHashFunction = "sha2-256"
)

// Import configures the default options for ingesting data. This affects commands
// that ingest data, such as 'ipfs add', 'ipfs dag put, 'ipfs block put', 'ipfs files write'.
type Import struct {
CidVersion OptionalInteger
UnixFSRawLeaves Flag
UnixFSChunker OptionalString
HashFunction OptionalString
}
33 changes: 31 additions & 2 deletions core/commands/add.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
gopath "path"
"strings"

"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/core/commands/cmdenv"

"github.com/cheggaaa/pb"
Expand Down Expand Up @@ -155,12 +156,12 @@ See 'dag export' and 'dag import' for more information.
cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash").WithDefault("size-262144"),
cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash"),
cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes."),
cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"),
cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
cmds.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. Passing version 1 will cause the raw-leaves option to default to true."),
cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"),
cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)"),
cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"),
cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true),
Expand Down Expand Up @@ -191,6 +192,16 @@ See 'dag export' and 'dag import' for more information.
return err
}

nd, err := cmdenv.GetNode(env)
if err != nil {
return err
}

cfg, err := nd.Repo.Config()
if err != nil {
return err
}

progress, _ := req.Options[progressOptionName].(bool)
trickle, _ := req.Options[trickleOptionName].(bool)
wrap, _ := req.Options[wrapOptionName].(bool)
Expand All @@ -207,6 +218,24 @@ See 'dag export' and 'dag import' for more information.
inlineLimit, _ := req.Options[inlineLimitOptionName].(int)
toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string)

if chunker == "" {
chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker)
}

if hashFunStr == "" {
hashFunStr = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)
}

if !cidVerSet && !cfg.Import.CidVersion.IsDefault() {
cidVerSet = true
cidVer = int(cfg.Import.CidVersion.WithDefault(config.DefaultCidVersion))
}

if !rbset && cfg.Import.UnixFSRawLeaves != config.Default {
rbset = true
rawblks = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)
}

if onlyHash && toFilesSet {
return fmt.Errorf("%s and %s options are not compatible", onlyHashOptionName, toFilesOptionName)
}
Expand Down
17 changes: 16 additions & 1 deletion core/commands/block.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"github.com/ipfs/boxo/files"

"github.com/ipfs/kubo/config"
cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
"github.com/ipfs/kubo/core/commands/cmdutils"

Expand Down Expand Up @@ -153,7 +154,7 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0).
},
Options: []cmds.Option{
cmds.StringOption(blockCidCodecOptionName, "Multicodec to use in returned CID").WithDefault("raw"),
cmds.StringOption(mhtypeOptionName, "Multihash hash function").WithDefault("sha2-256"),
cmds.StringOption(mhtypeOptionName, "Multihash hash function"),
cmds.IntOption(mhlenOptionName, "Multihash hash length").WithDefault(-1),
cmds.BoolOption(pinOptionName, "Pin added blocks recursively").WithDefault(false),
cmdutils.AllowBigBlockOption,
Expand All @@ -165,7 +166,21 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0).
return err
}

nd, err := cmdenv.GetNode(env)
if err != nil {
return err
}

cfg, err := nd.Repo.Config()
if err != nil {
return err
}

mhtype, _ := req.Options[mhtypeOptionName].(string)
if mhtype == "" {
mhtype = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)
}

mhtval, ok := mh.Names[mhtype]
if !ok {
return fmt.Errorf("unrecognized multihash function: %s", mhtype)
Expand Down
2 changes: 1 addition & 1 deletion core/commands/dag/dag.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ into an object of the specified format.
cmds.StringOption("store-codec", "Codec that the stored object will be encoded with").WithDefault("dag-cbor"),
cmds.StringOption("input-codec", "Codec that the input object is encoded in").WithDefault("dag-json"),
cmds.BoolOption("pin", "Pin this object when adding."),
cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"),
cmds.StringOption("hash", "Hash function to use"),
cmdutils.AllowBigBlockOption,
},
Run: dagPut,
Expand Down
15 changes: 15 additions & 0 deletions core/commands/dag/put.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipldlegacy "github.com/ipfs/go-ipld-legacy"
"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/core/commands/cmdenv"
"github.com/ipfs/kubo/core/commands/cmdutils"
"github.com/ipld/go-ipld-prime/multicodec"
Expand All @@ -32,11 +33,25 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e
return err
}

nd, err := cmdenv.GetNode(env)
if err != nil {
return err
}

cfg, err := nd.Repo.Config()
if err != nil {
return err
}

inputCodec, _ := req.Options["input-codec"].(string)
storeCodec, _ := req.Options["store-codec"].(string)
hash, _ := req.Options["hash"].(string)
dopin, _ := req.Options["pin"].(bool)

if hash == "" {
hash = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)
}

var icodec mc.Code
if err := icodec.Set(inputCodec); err != nil {
return err
Expand Down
19 changes: 15 additions & 4 deletions core/commands/files.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"strings"

humanize "github.com/dustin/go-humanize"
"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/core"
"github.com/ipfs/kubo/core/commands/cmdenv"

Expand Down Expand Up @@ -802,18 +803,28 @@ See '--to-files' in 'ipfs add --help' for more information.
return err
}

nd, err := cmdenv.GetNode(env)
if err != nil {
return err
}

cfg, err := nd.Repo.Config()
if err != nil {
return err
}

create, _ := req.Options[filesCreateOptionName].(bool)
mkParents, _ := req.Options[filesParentsOptionName].(bool)
trunc, _ := req.Options[filesTruncateOptionName].(bool)
flush, _ := req.Options[filesFlushOptionName].(bool)
rawLeaves, rawLeavesDef := req.Options[filesRawLeavesOptionName].(bool)

prefix, err := getPrefixNew(req)
if err != nil {
return err
if !rawLeavesDef && cfg.Import.UnixFSRawLeaves != config.Default {
rawLeavesDef = true
rawLeaves = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)
}

nd, err := cmdenv.GetNode(env)
prefix, err := getPrefixNew(req)
if err != nil {
return err
}
Expand Down
44 changes: 44 additions & 0 deletions docs/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,11 @@ config file at runtime.
- [`DNS`](#dns)
- [`DNS.Resolvers`](#dnsresolvers)
- [`DNS.MaxCacheTTL`](#dnsmaxcachettl)
- [`Import`](#import)
- [`Import.CidVersion`](#importcidversion)
- [`Import.UnixFSRawLeaves`](#importunixfsrawleaves)
- [`Import.UnixFSChunker`](#importunixfschunker)
- [`Import.HashFunction`](#importhashfunction)

## Profiles

Expand Down Expand Up @@ -2377,3 +2382,42 @@ Note: this does NOT work with Go's default DNS resolver. To make this a global s
Default: Respect DNS Response TTL

Type: `optionalDuration`

## `Import`

Options to configure the default options used for ingesting data, in commands such as `ipfs add` or `ipfs block put`. All affected commands are detailed per option.

Note that using flags will override the options defined here.

### `Import.CidVersion`

The default CID version. Commands affected: `ipfs add`.

Default: `0`

Type: `optionalInteger`

### `Import.UnixFSRawLeaves`

The default UnixFS raw leaves option. Commands affected: `ipfs add`, `ipfs files write`.

Default: `false`

Type: `flag`

### `Import.UnixFSChunker`

The default UnixFS chunker. Commands affected: `ipfs add`.

Default: `size-262144`

Type: `optionalString`


### `Import.HashFunction`

The default hash function. Commands affected: `ipfs add`, `ipfs block put`, `ipfs dag put`.

Default: `sha2-256`

Type: `optionalString`
81 changes: 81 additions & 0 deletions test/cli/add_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package cli

import (
"testing"

"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/test/cli/harness"
"github.com/stretchr/testify/require"
)

func TestAdd(t *testing.T) {
t.Parallel()

var (
shortString = "hello world"
shortStringCidV0 = "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD"
shortStringCidV1 = "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e"
shortStringCidV1Sha512 = "bafkrgqbqt3gerhas23vuzrapkdeqf4vu2dwxp3srdj6hvg6nhsug2tgyn6mj3u23yx7utftq3i2ckw2fwdh5qmhid5qf3t35yvkc5e5ottlw6"
)

t.Run("output cid version: default (CIDv0)", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init().StartDaemon()
defer node.StopDaemon()

cidStr := node.IPFSAddStr(shortString)
require.Equal(t, shortStringCidV0, cidStr)
})

t.Run("output cid version: follows configuration (CIDv0)", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Import.CidVersion = *config.NewOptionalInteger(0)
})
node.StartDaemon()
defer node.StopDaemon()

cidStr := node.IPFSAddStr(shortString)
require.Equal(t, shortStringCidV0, cidStr)
})

t.Run("output cid version: follows configuration (hash function)", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Import.HashFunction = *config.NewOptionalString("sha2-512")
})
node.StartDaemon()
defer node.StopDaemon()

cidStr := node.IPFSAddStr(shortString)
require.Equal(t, shortStringCidV1Sha512, cidStr)
})

t.Run("output cid version: follows configuration (CIDv1)", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Import.CidVersion = *config.NewOptionalInteger(1)
})
node.StartDaemon()
defer node.StopDaemon()

cidStr := node.IPFSAddStr(shortString)
require.Equal(t, shortStringCidV1, cidStr)
})

t.Run("output cid version: flag overrides configuration", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Import.CidVersion = *config.NewOptionalInteger(1)
})
node.StartDaemon()
defer node.StopDaemon()

cidStr := node.IPFSAddStr(shortString, "--cid-version", "0")
require.Equal(t, shortStringCidV0, cidStr)
})
}

0 comments on commit 021e436

Please sign in to comment.