From 70bc042fbd3112fee75c4c2a79296fa0c57789c8 Mon Sep 17 00:00:00 2001 From: datelier <57349093+datelier@users.noreply.github.com> Date: Tue, 13 Jun 2023 15:22:57 +0900 Subject: [PATCH 1/7] add decode kvsdb tool --- hack/tools/kvsdb/main.go | 89 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 hack/tools/kvsdb/main.go diff --git a/hack/tools/kvsdb/main.go b/hack/tools/kvsdb/main.go new file mode 100644 index 0000000000..c714592bb8 --- /dev/null +++ b/hack/tools/kvsdb/main.go @@ -0,0 +1,89 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "context" + "encoding/gob" + "flag" + "fmt" + "io/fs" + "os" + + "github.com/vdaas/vald/internal/file" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/pkg/agent/core/ngt/service/kvs" +) + +var ( + format = flag.String("format", "csv", "file format(csv,tsv)") + kvsFileName = flag.String("file", "ngt-meta.kvsdb", "kvsdb file name") + kvsTimestampFileName = flag.String("timestamp-file", "ngt-timestamp.kvsdb", "kvsdb timestamp file name") + path = flag.String("path", ".", "kvsdb file path") +) + +func main() { + flag.Parse() + log.Init() + + kvsdb := kvs.New() + + // value + m := make(map[string]uint32) + gob.Register(map[string]uint32{}) + var f *os.File + defer f.Close() + f, _ = file.Open( + file.Join(*path, *kvsFileName), + os.O_RDONLY|os.O_SYNC, + fs.ModePerm, + ) + _ = gob.NewDecoder(f).Decode(&m) + + // timestamp + mt := make(map[string]int64) + gob.Register(map[string]int64{}) + var ft *os.File + defer ft.Close() + ft, _ = file.Open( + file.Join(*path, *kvsTimestampFileName), + os.O_RDONLY|os.O_SYNC, + fs.ModePerm, + ) + _= gob.NewDecoder(ft).Decode(&mt) + + // kvs load + for k, id := range m { + if ts, ok := mt[k]; ok { + kvsdb.Set(k, id, ts) + } else { + kvsdb.Set(k, id, 0) + } + } + + // print + kvsdb.Range(context.TODO(), func(uuid string, oid uint32, ts int64) bool { + if *format == "csv" { + fmt.Printf("%s,%d,%d\n", uuid, oid, ts) + } else if *format == "tsv" { + fmt.Printf("%s\t%d\t%d\n", uuid, oid, ts) + } else { + fmt.Println(uuid, oid, ts) + } + return true + }) +} From 7882c016541869b968afd7590ff7c25ee9480521 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Tue, 13 Jun 2023 06:35:30 +0000 Subject: [PATCH 2/7] style: Format code with prettier and gofumpt --- hack/tools/kvsdb/main.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hack/tools/kvsdb/main.go b/hack/tools/kvsdb/main.go index c714592bb8..35b240b90b 100644 --- a/hack/tools/kvsdb/main.go +++ b/hack/tools/kvsdb/main.go @@ -30,10 +30,10 @@ import ( ) var ( - format = flag.String("format", "csv", "file format(csv,tsv)") - kvsFileName = flag.String("file", "ngt-meta.kvsdb", "kvsdb file name") + format = flag.String("format", "csv", "file format(csv,tsv)") + kvsFileName = flag.String("file", "ngt-meta.kvsdb", "kvsdb file name") kvsTimestampFileName = flag.String("timestamp-file", "ngt-timestamp.kvsdb", "kvsdb timestamp file name") - path = flag.String("path", ".", "kvsdb file path") + path = flag.String("path", ".", "kvsdb file path") ) func main() { @@ -64,7 +64,7 @@ func main() { os.O_RDONLY|os.O_SYNC, fs.ModePerm, ) - _= gob.NewDecoder(ft).Decode(&mt) + _ = gob.NewDecoder(ft).Decode(&mt) // kvs load for k, id := range m { From 472e20a3e1834d51e8c7d7cc0b385567286f6ba8 Mon Sep 17 00:00:00 2001 From: datelier <57349093+datelier@users.noreply.github.com> Date: Tue, 13 Jun 2023 15:52:26 +0900 Subject: [PATCH 3/7] add csv header --- hack/tools/kvsdb/main.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hack/tools/kvsdb/main.go b/hack/tools/kvsdb/main.go index 35b240b90b..39406c21ed 100644 --- a/hack/tools/kvsdb/main.go +++ b/hack/tools/kvsdb/main.go @@ -76,6 +76,13 @@ func main() { } // print + if *format == "csv" { + fmt.Printf("uuid,oid,timestamp\n") + } else if *format == "tsv" { + fmt.Printf("uuid\toid\ttimestamp\n") + } else { + fmt.Println("uuid", "oid", "timestamp") + } kvsdb.Range(context.TODO(), func(uuid string, oid uint32, ts int64) bool { if *format == "csv" { fmt.Printf("%s,%d,%d\n", uuid, oid, ts) From c3ddfbaa8b158420cddb203f8bd43c3ab06f4880 Mon Sep 17 00:00:00 2001 From: datelier <57349093+datelier@users.noreply.github.com> Date: Wed, 14 Jun 2023 16:01:41 +0900 Subject: [PATCH 4/7] modify to strings.Builder --- hack/tools/kvsdb/main.go | 50 +++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/hack/tools/kvsdb/main.go b/hack/tools/kvsdb/main.go index 39406c21ed..e02aa82636 100644 --- a/hack/tools/kvsdb/main.go +++ b/hack/tools/kvsdb/main.go @@ -17,16 +17,17 @@ package main import ( - "context" "encoding/gob" "flag" "fmt" "io/fs" "os" + "strconv" + "strings" + "unsafe" "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/log" - "github.com/vdaas/vald/pkg/agent/core/ngt/service/kvs" ) var ( @@ -40,8 +41,6 @@ func main() { flag.Parse() log.Init() - kvsdb := kvs.New() - // value m := make(map[string]uint32) gob.Register(map[string]uint32{}) @@ -66,31 +65,30 @@ func main() { ) _ = gob.NewDecoder(ft).Decode(&mt) - // kvs load + // print + var ( + sb strings.Builder + sp string + ) + switch *format { + case "csv": + sp = "," + case "tsv": + sp = "\t" + default: + sp = " " + } + sb.WriteString("uuid"+sp+"oid"+sp+"timestamp\n") for k, id := range m { if ts, ok := mt[k]; ok { - kvsdb.Set(k, id, ts) + sb.WriteString(strings.Join([]string{k, strconv.FormatUint(uint64(id), 10), strconv.FormatInt(ts, 10), "\r\n"}, sp)) } else { - kvsdb.Set(k, id, 0) + sb.WriteString(strings.Join([]string{k, strconv.FormatUint(uint64(id), 10), "0", "\r\n"}, sp)) } - } - - // print - if *format == "csv" { - fmt.Printf("uuid,oid,timestamp\n") - } else if *format == "tsv" { - fmt.Printf("uuid\toid\ttimestamp\n") - } else { - fmt.Println("uuid", "oid", "timestamp") - } - kvsdb.Range(context.TODO(), func(uuid string, oid uint32, ts int64) bool { - if *format == "csv" { - fmt.Printf("%s,%d,%d\n", uuid, oid, ts) - } else if *format == "tsv" { - fmt.Printf("%s\t%d\t%d\n", uuid, oid, ts) - } else { - fmt.Println(uuid, oid, ts) + if sb.Len()*int(unsafe.Sizeof("")) > 4e+6 { + fmt.Print(sb.String()) + sb.Reset() } - return true - }) + } + fmt.Print(sb.String()) } From 71c0327a689733d9777fb84d525ac6a6a6a0f5fe Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Wed, 14 Jun 2023 07:05:05 +0000 Subject: [PATCH 5/7] style: Format code with prettier and gofumpt --- hack/tools/kvsdb/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/tools/kvsdb/main.go b/hack/tools/kvsdb/main.go index e02aa82636..259fe42833 100644 --- a/hack/tools/kvsdb/main.go +++ b/hack/tools/kvsdb/main.go @@ -78,7 +78,7 @@ func main() { default: sp = " " } - sb.WriteString("uuid"+sp+"oid"+sp+"timestamp\n") + sb.WriteString("uuid" + sp + "oid" + sp + "timestamp\n") for k, id := range m { if ts, ok := mt[k]; ok { sb.WriteString(strings.Join([]string{k, strconv.FormatUint(uint64(id), 10), strconv.FormatInt(ts, 10), "\r\n"}, sp)) From 7972aa493640b52d69f4881c1cd74b272b7fe08a Mon Sep 17 00:00:00 2001 From: datelier <57349093+datelier@users.noreply.github.com> Date: Mon, 19 Jun 2023 16:06:54 +0900 Subject: [PATCH 6/7] modify to encoding/csv --- hack/tools/kvsdb/main.go | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/hack/tools/kvsdb/main.go b/hack/tools/kvsdb/main.go index 259fe42833..94bc6a928d 100644 --- a/hack/tools/kvsdb/main.go +++ b/hack/tools/kvsdb/main.go @@ -17,13 +17,12 @@ package main import ( + "encoding/csv" "encoding/gob" "flag" - "fmt" "io/fs" "os" "strconv" - "strings" "unsafe" "github.com/vdaas/vald/internal/file" @@ -66,29 +65,27 @@ func main() { _ = gob.NewDecoder(ft).Decode(&mt) // print - var ( - sb strings.Builder - sp string - ) + var s [][]string + w := csv.NewWriter(os.Stdout) + defer w.Flush() switch *format { case "csv": - sp = "," case "tsv": - sp = "\t" + w.Comma = '\t' default: - sp = " " + w.Comma = ' ' } - sb.WriteString("uuid" + sp + "oid" + sp + "timestamp\n") + s = append(s, []string{"uuid", "oid", "timestamp"}) for k, id := range m { if ts, ok := mt[k]; ok { - sb.WriteString(strings.Join([]string{k, strconv.FormatUint(uint64(id), 10), strconv.FormatInt(ts, 10), "\r\n"}, sp)) + s = append(s, []string{k, strconv.FormatUint(uint64(id), 10), strconv.FormatInt(ts, 10)}) } else { - sb.WriteString(strings.Join([]string{k, strconv.FormatUint(uint64(id), 10), "0", "\r\n"}, sp)) + s = append(s, []string{k, strconv.FormatUint(uint64(id), 10), "0"}) } - if sb.Len()*int(unsafe.Sizeof("")) > 4e+6 { - fmt.Print(sb.String()) - sb.Reset() + if len(s)*int(unsafe.Sizeof("")) > 4e+6 { + w.WriteAll(s) + s = nil } } - fmt.Print(sb.String()) + w.WriteAll(s) } From 5e123073b889cb0c561f38ef080a048c9a14c79f Mon Sep 17 00:00:00 2001 From: datelier <57349093+datelier@users.noreply.github.com> Date: Wed, 21 Jun 2023 08:56:09 +0900 Subject: [PATCH 7/7] add error check --- hack/tools/kvsdb/main.go | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/hack/tools/kvsdb/main.go b/hack/tools/kvsdb/main.go index 94bc6a928d..9a1e2d664f 100644 --- a/hack/tools/kvsdb/main.go +++ b/hack/tools/kvsdb/main.go @@ -39,30 +39,41 @@ var ( func main() { flag.Parse() log.Init() + var err error // value m := make(map[string]uint32) gob.Register(map[string]uint32{}) var f *os.File defer f.Close() - f, _ = file.Open( + f, err = file.Open( file.Join(*path, *kvsFileName), os.O_RDONLY|os.O_SYNC, fs.ModePerm, ) - _ = gob.NewDecoder(f).Decode(&m) + if err != nil { + log.Fatal(err) + } + if err = gob.NewDecoder(f).Decode(&m); err != nil { + log.Fatal(err) + } // timestamp mt := make(map[string]int64) gob.Register(map[string]int64{}) var ft *os.File defer ft.Close() - ft, _ = file.Open( + ft, err = file.Open( file.Join(*path, *kvsTimestampFileName), os.O_RDONLY|os.O_SYNC, fs.ModePerm, ) - _ = gob.NewDecoder(ft).Decode(&mt) + if err != nil { + log.Fatal(err) + } + if err = gob.NewDecoder(ft).Decode(&mt); err != nil { + log.Fatal(err) + } // print var s [][]string @@ -83,9 +94,13 @@ func main() { s = append(s, []string{k, strconv.FormatUint(uint64(id), 10), "0"}) } if len(s)*int(unsafe.Sizeof("")) > 4e+6 { - w.WriteAll(s) + if err = w.WriteAll(s); err != nil { + log.Fatal(err) + } s = nil } } - w.WriteAll(s) + if err = w.WriteAll(s); err != nil { + log.Fatal(err) + } }