diff --git a/cmd/devtools/oplogexport/.gitignore b/cmd/devtools/oplogexport/.gitignore new file mode 100644 index 00000000..c8d06248 --- /dev/null +++ b/cmd/devtools/oplogexport/.gitignore @@ -0,0 +1 @@ +oplog.* \ No newline at end of file diff --git a/cmd/devtools/oplogexport/main.go b/cmd/devtools/oplogexport/main.go new file mode 100644 index 00000000..478a3407 --- /dev/null +++ b/cmd/devtools/oplogexport/main.go @@ -0,0 +1,79 @@ +package main + +import ( + "bytes" + "compress/gzip" + "errors" + "flag" + "os" + "path" + + v1 "github.com/garethgeorge/backrest/gen/go/v1" + "github.com/garethgeorge/backrest/internal/env" + "github.com/garethgeorge/backrest/internal/oplog" + "github.com/garethgeorge/backrest/internal/oplog/bboltstore" + "go.etcd.io/bbolt" + "go.uber.org/zap" + "google.golang.org/protobuf/encoding/prototext" +) + +var ( + outpath = flag.String("export-oplog-path", "", "path to export the oplog as a compressed textproto e.g. .textproto.gz") +) + +func main() { + flag.Parse() + + if *outpath == "" { + flag.Usage() + return + } + + oplogFile := path.Join(env.DataDir(), "oplog.boltdb") + opstore, err := bboltstore.NewBboltStore(oplogFile) + if err != nil { + if !errors.Is(err, bbolt.ErrTimeout) { + zap.S().Fatalf("timeout while waiting to open database, is the database open elsewhere?") + } + zap.S().Warnf("operation log may be corrupted, if errors recur delete the file %q and restart. Your backups stored in your repos are safe.", oplogFile) + zap.S().Fatalf("error creating oplog : %v", err) + } + defer opstore.Close() + + output := &v1.OperationList{} + + log := oplog.NewOpLog(opstore) + log.Query(oplog.Query{}, func(op *v1.Operation) error { + output.Operations = append(output.Operations, op) + return nil + }) + + bytes, err := prototext.MarshalOptions{Multiline: true}.Marshal(output) + if err != nil { + zap.S().Fatalf("error marshalling operations: %v", err) + } + + bytes, err = compress(bytes) + if err != nil { + zap.S().Fatalf("error compressing operations: %v", err) + } + + if err := os.WriteFile(*outpath, bytes, 0644); err != nil { + zap.S().Fatalf("error writing to file: %v", err) + } +} + +func compress(data []byte) ([]byte, error) { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + + if _, err := zw.Write(data); err != nil { + return nil, err + } + + if err := zw.Close(); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/cmd/devtools/oplogimport/main.go b/cmd/devtools/oplogimport/main.go new file mode 100644 index 00000000..0c7b67f4 --- /dev/null +++ b/cmd/devtools/oplogimport/main.go @@ -0,0 +1,70 @@ +package main + +import ( + "bytes" + "compress/gzip" + "flag" + "log" + "os" + "path" + + v1 "github.com/garethgeorge/backrest/gen/go/v1" + "github.com/garethgeorge/backrest/internal/env" + "github.com/garethgeorge/backrest/internal/oplog/bboltstore" + "go.uber.org/zap" + "google.golang.org/protobuf/encoding/prototext" +) + +var ( + outpath = flag.String("import-oplog-path", "", "path to import the oplog from compressed textproto e.g. .textproto.gz") +) + +func main() { + flag.Parse() + + if *outpath == "" { + flag.Usage() + return + } + + // create a reader from the file + f, err := os.Open(*outpath) + if err != nil { + log.Fatalf("error opening file: %v", err) + } + defer f.Close() + + cr, err := gzip.NewReader(f) + if err != nil { + log.Fatalf("error creating gzip reader: %v", err) + } + defer cr.Close() + + // read into a buffer + var buf bytes.Buffer + if _, err := buf.ReadFrom(cr); err != nil { + log.Printf("error reading from gzip reader: %v", err) + } + + log.Printf("importing operations from %q", *outpath) + + output := &v1.OperationList{} + if err := prototext.Unmarshal(buf.Bytes(), output); err != nil { + log.Fatalf("error unmarshalling operations: %v", err) + } + + zap.S().Infof("importing %d operations", len(output.Operations)) + + oplogFile := path.Join(env.DataDir(), "oplog.boltdb") + opstore, err := bboltstore.NewBboltStore(oplogFile) + if err != nil { + log.Fatalf("error creating oplog : %v", err) + } + defer opstore.Close() + + for _, op := range output.Operations { + if err := opstore.Add(op); err != nil { + log.Printf("error adding operation to oplog: %v", err) + } + } +} diff --git a/cmd/devtools/oplogimport/testdata/v1.4.0-config.json b/cmd/devtools/oplogimport/testdata/v1.4.0-config.json new file mode 100644 index 00000000..e69de29b diff --git a/cmd/devtools/oplogimport/testdata/v1.4.0-ops.v04log.textproto.gz b/cmd/devtools/oplogimport/testdata/v1.4.0-ops.v04log.textproto.gz new file mode 100644 index 00000000..ec77d318 Binary files /dev/null and b/cmd/devtools/oplogimport/testdata/v1.4.0-ops.v04log.textproto.gz differ diff --git a/scripts/testing/run-fresh.sh b/scripts/testing/run-fresh.sh new file mode 100755 index 00000000..1337350d --- /dev/null +++ b/scripts/testing/run-fresh.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +BASEDIR=$(dirname "$0") +TEMPDIR=$(mktemp -d) + +function cleanup { + echo "Removing temp dir: $TEMPDIR" + rm -rf $TEMPDIR +} + +trap cleanup EXIT + +echo "Temp dir: $TEMPDIR" + +go run $BASEDIR/../../cmd/backrest --config-file=$TEMPDIR/config.json --data-dir=$TEMPDIR/data diff --git a/scripts/testing/run-in-dir.sh b/scripts/testing/run-in-dir.sh new file mode 100755 index 00000000..fb143395 --- /dev/null +++ b/scripts/testing/run-in-dir.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +BASEDIR=$(dirname "$0") +RUNDIR=$1 + +if [ -z "$RUNDIR" ]; then + echo "Usage: $0 " + exit 1 +fi + +go run $BASEDIR/../../cmd/backrest --config-file=$RUNDIR/config.json --data-dir=$RUNDIR +