diff --git a/cmd/greenmask/cmd/dump/dump.go b/cmd/greenmask/cmd/dump/dump.go index 6c4782f2..21708fed 100644 --- a/cmd/greenmask/cmd/dump/dump.go +++ b/cmd/greenmask/cmd/dump/dump.go @@ -64,7 +64,7 @@ var ( ) // TODO: Check how does work mixed options - use-list + tables, etc. -// TODO: Option that currently does not implemented: +// TODO: Options currently are not implemented: // - encoding // - disable-triggers // - lock-wait-timeout diff --git a/cmd/greenmask/cmd/list_dump/list_dump.go b/cmd/greenmask/cmd/list_dumps/list_dumps.go similarity index 98% rename from cmd/greenmask/cmd/list_dump/list_dump.go rename to cmd/greenmask/cmd/list_dumps/list_dumps.go index a2c498d9..d30c6f85 100644 --- a/cmd/greenmask/cmd/list_dump/list_dump.go +++ b/cmd/greenmask/cmd/list_dumps/list_dumps.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package list_dump +package list_dumps import ( "context" @@ -43,7 +43,7 @@ var ( } if err := listDumps(); err != nil { - log.Err(err).Msg("") + log.Fatal().Err(err).Msg("") } }, } diff --git a/cmd/greenmask/cmd/list_transformers/list_transformers.go b/cmd/greenmask/cmd/list_transformers/list_transformers.go index 78a60f2c..d9f3cb41 100644 --- a/cmd/greenmask/cmd/list_transformers/list_transformers.go +++ b/cmd/greenmask/cmd/list_transformers/list_transformers.go @@ -20,9 +20,9 @@ import ( "fmt" "os" "slices" - "strconv" "strings" + "github.com/greenmaskio/greenmask/pkg/toolkit" "github.com/olekukonko/tablewriter" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -42,8 +42,8 @@ var ( log.Err(err).Msg("") } - if err := run(args); err != nil { - log.Err(err).Msg("") + if err := run(); err != nil { + log.Fatal().Err(err).Msg("") } }, } @@ -56,7 +56,20 @@ const ( TextFormatName = "text" ) -func run(transformerNames []string) error { +const anyTypesValue = "any" + +type parameter struct { + Name string `json:"name,omitempty"` + SupportedTypes []string `json:"supported_types,omitempty"` +} + +type jsonResponse struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Parameters []*parameter `json:"parameters,omitempty"` +} + +func run() error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() err := custom.BootstrapCustomTransformers(ctx, utils.DefaultTransformerRegistry, Config.CustomTransformers) @@ -64,11 +77,14 @@ func run(transformerNames []string) error { return fmt.Errorf("error registering custom transformer: %w", err) } + // TODO: Consider about listing format. The transformer can have one and more columns as an input + // and + switch format { case JsonFormatName: - err = listTransformersJson(utils.DefaultTransformerRegistry, transformerNames) + err = listTransformersJson(utils.DefaultTransformerRegistry) case TextFormatName: - err = listTransformersText(utils.DefaultTransformerRegistry, transformerNames) + err = listTransformersText(utils.DefaultTransformerRegistry) default: return fmt.Errorf(`unknown format %s`, format) } @@ -79,92 +95,73 @@ func run(transformerNames []string) error { return nil } -func listTransformersJson(registry *utils.TransformerRegistry, transformerNames []string) error { - var transformers []*utils.Definition - - if len(transformerNames) > 0 { +func listTransformersJson(registry *utils.TransformerRegistry) error { + var transformers []*jsonResponse - for _, name := range transformerNames { - def, ok := registry.M[name] - if ok { - transformers = append(transformers, def) - } else { - return fmt.Errorf("unknown transformer name \"%s\"", name) + for _, def := range registry.M { + var params []*parameter + for _, p := range def.Parameters { + if !p.IsColumn && !p.IsColumnContainer { + continue } + supportedTypes := getColumnTypes(p) + params = append(params, ¶meter{Name: p.Name, SupportedTypes: supportedTypes}) } - } else { - for _, def := range registry.M { - transformers = append(transformers, def) - } + transformers = append(transformers, &jsonResponse{ + Name: def.Properties.Name, + Description: def.Properties.Description, + Parameters: params, + }) } + slices.SortFunc(transformers, func(a, b *jsonResponse) int { + return strings.Compare(a.Name, b.Name) + }) + if err := json.NewEncoder(os.Stdout).Encode(transformers); err != nil { return err } return nil } -func listTransformersText(registry *utils.TransformerRegistry, transformerNames []string) error { +func listTransformersText(registry *utils.TransformerRegistry) error { var data [][]string table := tablewriter.NewWriter(os.Stdout) var names []string - if len(transformerNames) > 0 { - for _, name := range transformerNames { - _, ok := registry.M[name] - if ok { - names = append(names, name) - } else { - return fmt.Errorf("unknown transformer name \"%s\"", name) - } - } - - } else { - for name := range registry.M { - names = append(names, name) - } - slices.Sort(names) + for name := range registry.M { + names = append(names, name) } - + slices.Sort(names) + table.SetHeader([]string{"name", "description", "column parameter name", "supported types"}) for _, name := range names { def := registry.M[name] - data = append(data, []string{def.Properties.Name, "description", def.Properties.Description, "", "", ""}) + //allowedTypes := getAllowedTypesList(def) for _, p := range def.Parameters { - data = append(data, []string{def.Properties.Name, "parameters", p.Name, "description", p.Description, ""}) - data = append(data, []string{def.Properties.Name, "parameters", p.Name, "required", strconv.FormatBool(p.Required), ""}) - if p.DefaultValue != nil { - data = append(data, []string{def.Properties.Name, "parameters", p.Name, "default", string(p.DefaultValue), ""}) - } - if p.LinkParameter != "" { - data = append(data, []string{def.Properties.Name, "parameters", p.Name, "linked_parameter", p.LinkParameter, ""}) - } - if p.CastDbType != "" { - data = append(data, []string{def.Properties.Name, "parameters", p.Name, "cast_to_db_type", p.CastDbType, ""}) + if !p.IsColumn && !p.IsColumnContainer { + continue } - if p.ColumnProperties != nil { - if len(p.ColumnProperties.AllowedTypes) > 0 { - allowedTypes := strings.Join(p.ColumnProperties.AllowedTypes, ", ") - data = append(data, []string{def.Properties.Name, "parameters", p.Name, "column_properties", "allowed_types", allowedTypes}) - } - isAffected := strconv.FormatBool(p.ColumnProperties.Affected) - data = append(data, []string{def.Properties.Name, "parameters", p.Name, "column_properties", "is_affected", isAffected}) - skipOriginalData := strconv.FormatBool(p.ColumnProperties.SkipOriginalData) - data = append(data, []string{def.Properties.Name, "parameters", p.Name, "column_properties", "skip_original_data", skipOriginalData}) - skipOnNull := strconv.FormatBool(p.ColumnProperties.SkipOnNull) - data = append(data, []string{def.Properties.Name, "parameters", p.Name, "column_properties", "skip_on_null", skipOnNull}) - } - + supportedTypes := getColumnTypes(p) + data = append(data, []string{def.Properties.Name, def.Properties.Description, p.Name, strings.Join(supportedTypes, ", ")}) } } + table.AppendBulk(data) table.SetRowLine(true) - table.SetAutoMergeCellsByColumnIndex([]int{0, 1, 2, 3}) + table.SetAutoMergeCellsByColumnIndex([]int{0, 1}) table.Render() return nil } +func getColumnTypes(p *toolkit.Parameter) []string { + if p.ColumnProperties != nil && len(p.ColumnProperties.AllowedTypes) > 0 { + return p.ColumnProperties.AllowedTypes + } + return []string{anyTypesValue} +} + func init() { Cmd.Flags().StringVarP(&format, "format", "f", TextFormatName, "output format [text|json]") } diff --git a/cmd/greenmask/cmd/restore/restore.go b/cmd/greenmask/cmd/restore/restore.go index 9b7b7541..704e7aca 100644 --- a/cmd/greenmask/cmd/restore/restore.go +++ b/cmd/greenmask/cmd/restore/restore.go @@ -20,6 +20,7 @@ import ( "path" "slices" + "github.com/greenmaskio/greenmask/internal/storages" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -36,7 +37,6 @@ var ( Args: cobra.ExactArgs(1), Short: "restore dump with ID or the latest to the target database", Run: func(cmd *cobra.Command, args []string) { - var dumpId string if err := logger.SetLogLevel(Config.Log.Level, Config.Log.Format); err != nil { log.Fatal().Err(err).Msg("fatal") @@ -49,41 +49,9 @@ var ( log.Fatal().Err(err).Msg("fatal") } - if args[0] == "latest" { - var backupNames []string - - _, dirs, err := st.ListDir(ctx) - if err != nil { - log.Fatal().Err(err).Msg("cannot walk through directory") - } - for _, dir := range dirs { - exists, err := dir.Exists(ctx, "metadata.json") - if err != nil { - log.Fatal().Err(err).Msg("cannot check file existence") - } - if exists { - backupNames = append(backupNames, dir.Dirname()) - } - } - - slices.SortFunc( - backupNames, func(a, b string) int { - if a > b { - return -1 - } - return 1 - }, - ) - dumpId = backupNames[0] - } else { - dumpId = args[0] - exists, err := st.Exists(ctx, path.Join(dumpId, "metadata.json")) - if err != nil { - log.Fatal().Err(err).Msg("cannot check file existence") - } - if !exists { - log.Fatal().Err(err).Msg("choose another dump is failed") - } + dumpId, err := getDumpId(ctx, st, args[0]) + if err != nil { + log.Fatal().Err(err).Msg("") } st = st.SubStorage(dumpId, true) @@ -104,7 +72,51 @@ var ( Config = pgDomains.NewConfig() ) -// TODO: Option that currently does not implemented: +func getDumpId(ctx context.Context, st storages.Storager, dumpId string) (string, error) { + if dumpId == "latest" { + var backupNames []string + + _, dirs, err := st.ListDir(ctx) + if err != nil { + log.Fatal().Err(err).Msg("cannot walk through directory") + } + for _, dir := range dirs { + exists, err := dir.Exists(ctx, "metadata.json") + if err != nil { + log.Fatal().Err(err).Msg("cannot check file existence") + } + if exists { + backupNames = append(backupNames, dir.Dirname()) + } + } + + slices.SortFunc( + backupNames, func(a, b string) int { + if a > b { + return -1 + } + return 1 + }, + ) + dumpId = backupNames[0] + } else { + exists, err := st.Exists(ctx, path.Join(dumpId, "metadata.json")) + if err != nil { + log.Fatal(). + Err(err). + Msg("cannot check file existence") + } + if !exists { + log.Fatal(). + Err(err). + Str("DumpId", dumpId). + Msg("dump with provided id is not found") + } + } + return dumpId, nil +} + +// TODO: Options currently are not implemented: // * data-only // * exit-on-error // * use-list diff --git a/cmd/greenmask/cmd/root.go b/cmd/greenmask/cmd/root.go index 34763879..429af065 100644 --- a/cmd/greenmask/cmd/root.go +++ b/cmd/greenmask/cmd/root.go @@ -26,10 +26,11 @@ import ( "github.com/greenmaskio/greenmask/cmd/greenmask/cmd/delete_backup" "github.com/greenmaskio/greenmask/cmd/greenmask/cmd/dump" - "github.com/greenmaskio/greenmask/cmd/greenmask/cmd/list_dump" + "github.com/greenmaskio/greenmask/cmd/greenmask/cmd/list_dumps" "github.com/greenmaskio/greenmask/cmd/greenmask/cmd/list_transformers" "github.com/greenmaskio/greenmask/cmd/greenmask/cmd/restore" "github.com/greenmaskio/greenmask/cmd/greenmask/cmd/show_dump" + "github.com/greenmaskio/greenmask/cmd/greenmask/cmd/show_transformer" "github.com/greenmaskio/greenmask/cmd/greenmask/cmd/validate" pgDomains "github.com/greenmaskio/greenmask/internal/domains" configUtils "github.com/greenmaskio/greenmask/internal/utils/config" @@ -90,12 +91,13 @@ func init() { ) RootCmd.AddCommand(dump.Cmd) - RootCmd.AddCommand(list_dump.Cmd) + RootCmd.AddCommand(list_dumps.Cmd) RootCmd.AddCommand(restore.Cmd) RootCmd.AddCommand(delete_backup.Cmd) RootCmd.AddCommand(show_dump.Cmd) RootCmd.AddCommand(list_transformers.Cmd) RootCmd.AddCommand(validate.Cmd) + RootCmd.AddCommand(show_transformer.Cmd) if err := RootCmd.MarkPersistentFlagRequired("config"); err != nil { log.Fatal().Err(err).Msg("") diff --git a/cmd/greenmask/cmd/show_transformer/show_transformer.go b/cmd/greenmask/cmd/show_transformer/show_transformer.go new file mode 100644 index 00000000..663e3b74 --- /dev/null +++ b/cmd/greenmask/cmd/show_transformer/show_transformer.go @@ -0,0 +1,159 @@ +// Copyright 2023 Greenmask +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package show_transformer + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + + "github.com/olekukonko/tablewriter" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + + "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/custom" + "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" + "github.com/greenmaskio/greenmask/internal/domains" + "github.com/greenmaskio/greenmask/internal/utils/logger" +) + +var ( + Cmd = &cobra.Command{ + Use: "show-transformer", + Args: cobra.ExactArgs(1), + Short: "show transformer details", + Run: func(cmd *cobra.Command, args []string) { + if err := logger.SetLogLevel(Config.Log.Level, Config.Log.Format); err != nil { + log.Err(err).Msg("") + } + + if err := run(args[0]); err != nil { + log.Fatal().Err(err).Msg("") + } + }, + } + Config = domains.NewConfig() + format string +) + +const ( + JsonFormatName = "json" + TextFormatName = "text" +) + +const anyTypesValue = "any" + +func run(name string) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err := custom.BootstrapCustomTransformers(ctx, utils.DefaultTransformerRegistry, Config.CustomTransformers) + if err != nil { + return fmt.Errorf("error registering custom transformer: %w", err) + } + + switch format { + case JsonFormatName: + err = showTransformerJson(utils.DefaultTransformerRegistry, name) + case TextFormatName: + err = showTransformerText(utils.DefaultTransformerRegistry, name) + default: + return fmt.Errorf(`unknown format \"%s\"`, format) + } + if err != nil { + return fmt.Errorf("error listing transformers: %w", err) + } + + return nil +} + +func showTransformerJson(registry *utils.TransformerRegistry, transformerName string) error { + var transformers []*utils.Definition + + def, ok := registry.M[transformerName] + if ok { + transformers = append(transformers, def) + } else { + return fmt.Errorf("unknown transformer with name \"%s\"", transformerName) + } + + if err := json.NewEncoder(os.Stdout).Encode(transformers); err != nil { + return err + } + return nil +} + +func showTransformerText(registry *utils.TransformerRegistry, name string) error { + + var data [][]string + table := tablewriter.NewWriter(os.Stdout) + + def, err := getTransformerDefinition(registry, name) + if err != nil { + return err + } + + data = append(data, []string{def.Properties.Name, "description", def.Properties.Description, "", "", ""}) + for _, p := range def.Parameters { + data = append(data, []string{def.Properties.Name, "parameters", p.Name, "description", p.Description, ""}) + data = append(data, []string{def.Properties.Name, "parameters", p.Name, "required", strconv.FormatBool(p.Required), ""}) + if p.DefaultValue != nil { + data = append(data, []string{def.Properties.Name, "parameters", p.Name, "default", string(p.DefaultValue), ""}) + } + if p.LinkParameter != "" { + data = append(data, []string{def.Properties.Name, "parameters", p.Name, "linked_parameter", p.LinkParameter, ""}) + } + if p.CastDbType != "" { + data = append(data, []string{def.Properties.Name, "parameters", p.Name, "cast_to_db_type", p.CastDbType, ""}) + } + if p.IsColumnContainer { + data = append(data, []string{def.Properties.Name, "parameters", p.Name, "column_properties", "allowed_types", anyTypesValue}) + } + if p.ColumnProperties != nil { + allowedTypes := []string{anyTypesValue} + if len(p.ColumnProperties.AllowedTypes) > 0 { + allowedTypes = p.ColumnProperties.AllowedTypes + } + data = append(data, []string{def.Properties.Name, "parameters", p.Name, "column_properties", "allowed_types", strings.Join(allowedTypes, ", ")}) + isAffected := strconv.FormatBool(p.ColumnProperties.Affected) + data = append(data, []string{def.Properties.Name, "parameters", p.Name, "column_properties", "is_affected", isAffected}) + skipOriginalData := strconv.FormatBool(p.ColumnProperties.SkipOriginalData) + data = append(data, []string{def.Properties.Name, "parameters", p.Name, "column_properties", "skip_original_data", skipOriginalData}) + skipOnNull := strconv.FormatBool(p.ColumnProperties.SkipOnNull) + data = append(data, []string{def.Properties.Name, "parameters", p.Name, "column_properties", "skip_on_null", skipOnNull}) + } + } + + table.AppendBulk(data) + table.SetRowLine(true) + table.SetAutoMergeCellsByColumnIndex([]int{0, 1, 2, 3}) + table.Render() + + return nil +} + +func getTransformerDefinition(registry *utils.TransformerRegistry, name string) (*utils.Definition, error) { + def, ok := registry.M[name] + if ok { + return def, nil + } + return nil, fmt.Errorf("unknown transformer \"%s\"", name) +} + +func init() { + Cmd.Flags().StringVarP(&format, "format", "f", TextFormatName, "output format [text|json]") +} diff --git a/internal/db/postgres/transformers/cmd.go b/internal/db/postgres/transformers/cmd.go index e978fb48..c5add24e 100644 --- a/internal/db/postgres/transformers/cmd.go +++ b/internal/db/postgres/transformers/cmd.go @@ -66,7 +66,8 @@ var CmdTransformerDefinition = utils.NewDefinition( `"skip_original_data": "type:bool, required:false, description: is original data required for transformer",`+ `"skip_on_null_input": "type:bool, required:false, description: skip transformation on null input"`+ `}`, - ).SetDefaultValue([]byte("[]")), + ).SetDefaultValue([]byte("[]")). + SetIsColumnContainer(true), toolkit.MustNewParameter( "executable", diff --git a/internal/db/postgres/transformers/real_address.go b/internal/db/postgres/transformers/real_address.go index d8108730..96d262c9 100644 --- a/internal/db/postgres/transformers/real_address.go +++ b/internal/db/postgres/transformers/real_address.go @@ -43,7 +43,8 @@ var RealAddressTransformerDefinition = utils.NewDefinition( `"template": "type:string, required:true, description: gotemplate with real address attributes injections",`+ `"keep_null": "type:bool, required:false, description: keep null values",`+ `}`, - ).SetRequired(true), + ).SetRequired(true). + SetIsColumnContainer(true), ) type RealAddressTransformer struct { diff --git a/pkg/toolkit/parameter.go b/pkg/toolkit/parameter.go index 81b98bd9..e4b12cdc 100644 --- a/pkg/toolkit/parameter.go +++ b/pkg/toolkit/parameter.go @@ -113,6 +113,8 @@ type Parameter struct { // IsColumn - shows is this parameter column related. If so ColumnProperties must be defined and assigned // otherwise it may cause an unhandled behaviour IsColumn bool `mapstructure:"is_column" json:"is_column"` + // IsColumnContainer - describe is parameter container map or list with multiple columns inside. It allows us to + IsColumnContainer bool `mapstructure:"is_column_container" json:"is_column_container"` // LinkParameter - link with parameter with provided name. This is required if performing raw value encoding // depends on the provided column type and/or relies on the database Driver LinkParameter string `mapstructure:"link_parameter" json:"link_parameter,omitempty"` @@ -297,6 +299,11 @@ func (p *Parameter) SetIsColumn(columnProperties *ColumnProperties) *Parameter { return p } +func (p *Parameter) SetIsColumnContainer(v bool) *Parameter { + p.IsColumnContainer = v + return p +} + func (p *Parameter) SetUnmarshaller(unmarshaller Unmarshaller) *Parameter { p.Unmarshaller = unmarshaller return p diff --git a/tests/integration/toc/backward_compatibility_test.go b/tests/integration/toc/backward_compatibility_test.go index 69e49881..9db86668 100644 --- a/tests/integration/toc/backward_compatibility_test.go +++ b/tests/integration/toc/backward_compatibility_test.go @@ -24,13 +24,14 @@ import ( "path" "time" + "github.com/jackc/pgx/v5" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/suite" + "github.com/greenmaskio/greenmask/internal/db/postgres/pgdump" "github.com/greenmaskio/greenmask/internal/domains" "github.com/greenmaskio/greenmask/internal/storages/directory" "github.com/greenmaskio/greenmask/pkg/toolkit" - "github.com/jackc/pgx/v5" - "github.com/rs/zerolog/log" - "github.com/stretchr/testify/suite" ) var config = &domains.Config{