From 700d930079b6cedd9478981d11e063cc90bd6e8d Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Sun, 17 Nov 2024 18:24:39 +1000 Subject: [PATCH 1/2] move to toml --- cmd/slackdump/internal/apiconfig/apiconfig.go | 15 +--- .../internal/apiconfig/apiconfig_test.go | 87 ++++++++++--------- cmd/slackdump/internal/apiconfig/check.go | 4 +- cmd/slackdump/internal/apiconfig/new.go | 10 +-- cmd/slackdump/internal/apiconfig/new_test.go | 22 ++--- go.mod | 1 + go.sum | 2 + 7 files changed, 72 insertions(+), 69 deletions(-) diff --git a/cmd/slackdump/internal/apiconfig/apiconfig.go b/cmd/slackdump/internal/apiconfig/apiconfig.go index e49be2fb..9cfd1821 100644 --- a/cmd/slackdump/internal/apiconfig/apiconfig.go +++ b/cmd/slackdump/internal/apiconfig/apiconfig.go @@ -6,19 +6,14 @@ import ( "io" "os" + "github.com/BurntSushi/toml" "github.com/go-playground/validator/v10" - "gopkg.in/yaml.v3" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/golang/base" "github.com/rusq/slackdump/v3/internal/network" ) -// schemaJSONpath is the path to the schema JSON file for the limits yaml -// configuration file. -// TODO: update once released -const schemaJSONpath = "https://raw.githubusercontent.com/rusq/slackdump/v3/cmd/slackdump/internal/apiconfig/schema.json" - var CmdConfig = &base.Command{ UsageLine: "slackdump config", Short: "API configuration", @@ -62,9 +57,8 @@ func Save(filename string, limits network.Limits) error { // the global config. func applyLimits(r io.Reader) (network.Limits, error) { var limits network.Limits - dec := yaml.NewDecoder(r) - dec.KnownFields(true) - if err := dec.Decode(&limits); err != nil { + dec := toml.NewDecoder(r) + if _, err := dec.Decode(&limits); err != nil { return network.Limits{}, err } @@ -78,8 +72,7 @@ func applyLimits(r io.Reader) (network.Limits, error) { } func writeLimits(w io.Writer, cfg network.Limits) error { - fmt.Fprintf(w, "# yaml-language-server: $schema=%s\n", schemaJSONpath) - return yaml.NewEncoder(w).Encode(cfg) + return toml.NewEncoder(w).Encode(cfg) } // printErrors prints configuration errors, if error is not nill and is of diff --git a/cmd/slackdump/internal/apiconfig/apiconfig_test.go b/cmd/slackdump/internal/apiconfig/apiconfig_test.go index 44c2cde1..3c5939b7 100644 --- a/cmd/slackdump/internal/apiconfig/apiconfig_test.go +++ b/cmd/slackdump/internal/apiconfig/apiconfig_test.go @@ -11,45 +11,52 @@ import ( ) const ( - sampleLimitsYaml = `# yaml-language-server: $schema=https://raw.githubusercontent.com/rusq/slackdump/v3/cmd/slackdump/internal/apiconfig/schema.json -workers: 4 -download_retries: 3 -tier_2: - boost: 20 - burst: 3 - retries: 20 -tier_3: - boost: 120 - burst: 5 - retries: 3 -tier_4: - boost: 10 - burst: 7 - retries: 3 -per_request: - conversations: 100 - channels: 100 - replies: 200 + sampleLimitsYaml = `workers = 4 +download_retries = 3 + +[tier_2] + boost = 20 + burst = 3 + retries = 20 + +[tier_3] + boost = 120 + burst = 5 + retries = 3 + +[tier_4] + boost = 10 + burst = 7 + retries = 3 + +[per_request] + conversations = 100 + channels = 100 + replies = 200 ` // workers set to 55 in this one, tier2.retries to 330 - updatedConfigYaml = `workers: 55 -download_retries: 3 -tier_2: - boost: 20 - burst: 1 - retries: 330 -tier_3: - boost: 120 - burst: 1 - retries: 3 -tier_4: - boost: 10 - burst: 1 - retries: 3 -per_request: - conversations: 100 - channels: 100 - replies: 200 + updatedConfigYaml = `workers = 55 +download_retries = 3 + +[tier_2] + boost = 20 + burst = 3 + retries = 330 + +[tier_3] + boost = 120 + burst = 5 + retries = 3 + +[tier_4] + boost = 10 + burst = 7 + retries = 3 + +[per_request] + conversations = 100 + channels = 100 + replies = 200 ` ) @@ -108,17 +115,17 @@ func Test_readConfig(t *testing.T) { DownloadRetries: 3, Tier2: network.TierLimit{ Boost: 20, - Burst: 1, + Burst: 3, Retries: 330, }, Tier3: network.TierLimit{ Boost: 120, - Burst: 1, + Burst: 5, Retries: 3, }, Tier4: network.TierLimit{ Boost: 10, - Burst: 1, + Burst: 7, Retries: 3, }, Request: network.RequestLimit{ diff --git a/cmd/slackdump/internal/apiconfig/check.go b/cmd/slackdump/internal/apiconfig/check.go index e8092000..2798343f 100644 --- a/cmd/slackdump/internal/apiconfig/check.go +++ b/cmd/slackdump/internal/apiconfig/check.go @@ -24,7 +24,7 @@ Allows to check the config for errors and invalid values. Example: - slackdump config check myconfig.yaml + slackdump config check myconfig.toml It will check for duplicate and unknown keys, and also ensure that values are within the allowed boundaries. @@ -58,7 +58,7 @@ func CheckFile(filename string) error { } func wizConfigCheck(ctx context.Context, cmd *base.Command, args []string) error { - f := filemgr.New(os.DirFS("."), ".", 15, "*.yaml", "*.yml") + f := filemgr.New(os.DirFS("."), ".", 15, "*.toml", "*.tml") f.Focus() f.ShowHelp = true f.Style = filemgr.Style{ diff --git a/cmd/slackdump/internal/apiconfig/new.go b/cmd/slackdump/internal/apiconfig/new.go index 8e52bbe4..bb7fb2f5 100644 --- a/cmd/slackdump/internal/apiconfig/new.go +++ b/cmd/slackdump/internal/apiconfig/new.go @@ -23,9 +23,9 @@ var CmdConfigNew = &base.Command{ Creates a new API configuration file containing default values. You will need to specify the filename, for example: - slackdump config new myconfig.yaml + slackdump config new myconfig.toml -If the extension is omitted, ".yaml" is automatically appended to the filename. +If the extension is omitted, ".toml" is automatically appended to the filename. `, FlagMask: cfg.OmitAll, PrintFlags: true, @@ -100,11 +100,11 @@ func shouldOverwrite(filename string, override bool) bool { return err != nil || override } -// maybeFixExt checks if the extension is one of .yaml or .yml, and if not +// maybeFixExt checks if the extension is one of .toml or .tml, and if not // appends it to the file. func maybeFixExt(filename string) string { - if ext := filepath.Ext(filename); !(ext == ".yaml" || ext == ".yml") { - return maybeAppendExt(filename, ".yaml") + if ext := filepath.Ext(filename); !(ext == ".toml" || ext == ".tml") { + return maybeAppendExt(filename, ".toml") } return filename } diff --git a/cmd/slackdump/internal/apiconfig/new_test.go b/cmd/slackdump/internal/apiconfig/new_test.go index fe4ef584..44510e58 100644 --- a/cmd/slackdump/internal/apiconfig/new_test.go +++ b/cmd/slackdump/internal/apiconfig/new_test.go @@ -62,24 +62,24 @@ func Test_maybeFixExt(t *testing.T) { want string }{ { - "already yaml", - args{filename: "lol.yaml"}, - "lol.yaml", + "already toml", + args{filename: "lol.toml"}, + "lol.toml", }, { - "already yml", - args{filename: "lol.yml"}, - "lol.yml", + "already tml", + args{filename: "lol.tml"}, + "lol.tml", }, { "no extension", args{filename: "foo"}, - "foo.yaml", + "foo.toml", }, { "different extension", args{filename: "foo.bar"}, - "foo.bar.yaml", + "foo.bar.toml", }, } for _, tt := range tests { @@ -154,7 +154,7 @@ func Test_shouldOverwrite(t *testing.T) { func Test_runConfigNew(t *testing.T) { dir := t.TempDir() - existingDir := filepath.Join(dir, "test.yaml") + existingDir := filepath.Join(dir, "test.toml") if err := os.MkdirAll(existingDir, 0777); err != nil { t.Fatal(err) } @@ -175,12 +175,12 @@ func Test_runConfigNew(t *testing.T) { }, { "file is created", - args{[]string{filepath.Join(dir, "sample.yml")}}, + args{[]string{filepath.Join(dir, "sample.tml")}}, false, true, }, { - "directory test.yaml", + "directory test.toml", args{[]string{existingDir}}, true, true, diff --git a/go.mod b/go.mod index 0e7152d9..66521cbf 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/rusq/slackdump/v3 go 1.23 require ( + github.com/BurntSushi/toml v1.4.0 github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403 github.com/ProtonMail/go-crypto v1.1.2 github.com/charmbracelet/bubbles v0.20.0 diff --git a/go.sum b/go.sum index c79a3e56..15df88e4 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403 h1:EtZwYyLbkEcIt+B//6sujwRCnHuTEK3qiSypAX5aJeM= From e70aa42cf80c654d9c687a9f2f5fbbfa19fa519d Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Sun, 17 Nov 2024 18:37:30 +1000 Subject: [PATCH 2/2] update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 79d1487b..90ac06f6 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ __debug_bin *.json *.yml *.yaml +*.toml json/* examples/* experiments/*