From e9552dd24d5c7ae87d6c3a3081477b76cb9f29ca Mon Sep 17 00:00:00 2001 From: Daniel Stinson-Diess Date: Tue, 8 Aug 2023 16:07:01 -0500 Subject: [PATCH 1/5] feat: add number inspector --- build/config/substation.libsonnet | 13 +++ condition/condition.go | 2 + condition/number.go | 100 +++++++++++++++++++++++ condition/number_test.go | 127 ++++++++++++++++++++++++++++++ 4 files changed, 242 insertions(+) create mode 100644 condition/number.go create mode 100644 condition/number_test.go diff --git a/build/config/substation.libsonnet b/build/config/substation.libsonnet index eb19d596..06caf36e 100644 --- a/build/config/substation.libsonnet +++ b/build/config/substation.libsonnet @@ -16,6 +16,9 @@ length: { options: { type: null, value: null, measurement: 'byte' }, }, + number: { + options: { type: null, value: null }, + }, regexp: { options: { expression: null }, }, @@ -276,6 +279,16 @@ type: 'length', settings: std.mergePatch({ options: opt }, s), }, + number(options=$.defaults.inspector.number.options, + settings=$.interfaces.inspector.settings): { + local opt = std.mergePatch($.defaults.inspector.number.options, options), + + assert $.helpers.inspector.validate(settings) : 'invalid inspector settings', + local s = std.mergePatch($.interfaces.inspector.settings, settings), + + type: 'number', + settings: std.mergePatch({ options: opt }, s), + }, random: { type: 'random', }, diff --git a/condition/condition.go b/condition/condition.go index 9194cbd5..dbf2f149 100644 --- a/condition/condition.go +++ b/condition/condition.go @@ -58,6 +58,8 @@ func NewInspector(ctx context.Context, cfg config.Config) (Inspector, error) { return newInspJSONValid(ctx, cfg) case "length": return newInspLength(ctx, cfg) + case "number": + return newInspNumber(ctx, cfg) case "random": return newInspRandom(ctx, cfg) case "regexp": diff --git a/condition/number.go b/condition/number.go new file mode 100644 index 00000000..b478f8d3 --- /dev/null +++ b/condition/number.go @@ -0,0 +1,100 @@ +package condition + +import ( + "context" + "encoding/binary" + "fmt" + + "golang.org/x/exp/slices" + + "github.com/brexhq/substation/config" + "github.com/brexhq/substation/internal/errors" +) + +// number evaluates data using Types from the standard library's number package. +// +// This inspector supports the data and object handling patterns. +type inspNumber struct { + condition + Options inspNumberOptions `json:"options"` +} + +type inspNumberOptions struct { + // Type is the string evaluation Type used during inspection. + // + // Must be one of: + // + // - equals + // + // - greater_than + // + // - less_than + // + // - bitwise_and + Type string `json:"type"` + // Value is the length that is used for comparison during inspection. + Value int64 `json:"value"` +} + +// Creates a new number inspector. +func newInspNumber(_ context.Context, cfg config.Config) (c inspNumber, err error) { + if err = config.Decode(cfg.Settings, &c); err != nil { + return inspNumber{}, err + } + + // validate option.type + if !slices.Contains( + []string{ + "equals", + "greater_than", + "less_than", + "bitwise_and", + }, + c.Options.Type) { + return inspNumber{}, fmt.Errorf("condition: number: type %q: %v", c.Options.Type, errors.ErrInvalidOption) + } + + return c, nil +} + +func (c inspNumber) String() string { + return toString(c) +} + +// Inspect evaluates encapsulated data with the number inspector. +func (c inspNumber) Inspect(ctx context.Context, capsule config.Capsule) (output bool, err error) { + var check int64 + if c.Key == "" { + check = int64(binary.BigEndian.Uint64(capsule.Data())) + } else { + check = capsule.Get(c.Key).Int() + } + + var matched bool + switch s := c.Options.Type; s { + case "equals": + if check == c.Options.Value { + matched = true + } + case "greater_than": + if check > c.Options.Value { + matched = true + } + case "less_than": + if check < c.Options.Value { + matched = true + } + case "bitwise_and": + if check&c.Options.Value != 0 { + matched = true + } + default: + return false, fmt.Errorf("condition: strings: type %s: %v", c.Options.Type, errors.ErrInvalidOption) + } + + if c.Negate { + return !matched, nil + } + + return matched, nil +} diff --git a/condition/number_test.go b/condition/number_test.go new file mode 100644 index 00000000..93edba0e --- /dev/null +++ b/condition/number_test.go @@ -0,0 +1,127 @@ +package condition + +import ( + "context" + "testing" + + "github.com/brexhq/substation/config" +) + +var _ Inspector = inspNumber{} + +var numberTests = []struct { + name string + cfg config.Config + test []byte + expected bool +}{ + { + "pass equals", + config.Config{ + Type: "number", + Settings: map[string]interface{}{ + "key": "foo", + "options": map[string]interface{}{ + "type": "equals", + "value": 42, + }, + }, + }, + []byte(`{"foo":"42"}`), + true, + }, + { + "pass greater_than", + config.Config{ + Type: "number", + Settings: map[string]interface{}{ + "key": "foo", + "options": map[string]interface{}{ + "type": "greater_than", + "value": -1, + }, + }, + }, + []byte(`{"foo":"0"}`), + true, + }, + { + "pass less_than", + config.Config{ + Type: "number", + Settings: map[string]interface{}{ + "key": "foo", + "options": map[string]interface{}{ + "type": "less_than", + "value": 50, + }, + }, + }, + []byte(`{"foo":42}`), + true, + }, + { + "pass bitwise_and", + config.Config{ + Type: "number", + Settings: map[string]interface{}{ + "key": "foo", + "options": map[string]interface{}{ + "type": "bitwise_and", + "value": 0x0001, + }, + }, + }, + []byte(`{"foo":"570506001"}`), + true, + }, +} + +func TestNumber(t *testing.T) { + ctx := context.TODO() + capsule := config.NewCapsule() + + for _, test := range numberTests { + t.Run(test.name, func(t *testing.T) { + capsule.SetData(test.test) + + insp, err := newInspNumber(ctx, test.cfg) + if err != nil { + t.Fatal(err) + } + + check, err := insp.Inspect(ctx, capsule) + if err != nil { + t.Error(err) + } + + if test.expected != check { + t.Errorf("expected %v, got %v", test.expected, check) + } + }) + } +} + +func benchmarkNumberByte(b *testing.B, inspector inspNumber, capsule config.Capsule) { + ctx := context.TODO() + for i := 0; i < b.N; i++ { + _, _ = inspector.Inspect(ctx, capsule) + } +} + +func BenchmarkNumberByte(b *testing.B) { + capsule := config.NewCapsule() + for _, test := range numberTests { + insp, err := newInspNumber(context.TODO(), test.cfg) + if err != nil { + b.Fatal(err) + } + + b.Run(test.name, + func(b *testing.B) { + capsule.SetData(test.test) + benchmarkNumberByte(b, insp, capsule) + }, + ) + } +} From 9f9af1f4dbbf42a9fec9d0a513ef9608f791aeaf Mon Sep 17 00:00:00 2001 From: Daniel Stinson-Diess Date: Wed, 9 Aug 2023 13:17:31 -0500 Subject: [PATCH 2/5] test: add missing unit tests for negation, and data processing --- condition/number_test.go | 46 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/condition/number_test.go b/condition/number_test.go index 93edba0e..ef401c90 100644 --- a/condition/number_test.go +++ b/condition/number_test.go @@ -30,6 +30,22 @@ var numberTests = []struct { []byte(`{"foo":"42"}`), true, }, + { + "!fail equals", + config.Config{ + Type: "number", + Settings: map[string]interface{}{ + "key": "foo", + "negate": true, + "options": map[string]interface{}{ + "type": "equals", + "value": 42, + }, + }, + }, + []byte(`{"foo":"42"}`), + false, + }, { "pass greater_than", config.Config{ @@ -45,6 +61,22 @@ var numberTests = []struct { []byte(`{"foo":"0"}`), true, }, + { + "!pass greater_than", + config.Config{ + Type: "number", + Settings: map[string]interface{}{ + "key": "foo", + "negate": true, + "options": map[string]interface{}{ + "type": "greater_than", + "value": 1, + }, + }, + }, + []byte(`{"foo":"0"}`), + true, + }, { "pass less_than", config.Config{ @@ -75,6 +107,20 @@ var numberTests = []struct { []byte(`{"foo":"570506001"}`), true, }, + { + "pass data", + config.Config{ + Type: "number", + Settings: map[string]interface{}{ + "options": map[string]interface{}{ + "type": "equals", + "value": 1, + }, + }, + }, + []byte(`0001`), + true, + }, } func TestNumber(t *testing.T) { From 46b0b8d399aa857ade10eefeb1aae1c3219e88e8 Mon Sep 17 00:00:00 2001 From: Daniel Stinson-Diess Date: Wed, 9 Aug 2023 13:18:02 -0500 Subject: [PATCH 3/5] fix: looser interpretation when processing data --- condition/number.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/condition/number.go b/condition/number.go index b478f8d3..d8df111d 100644 --- a/condition/number.go +++ b/condition/number.go @@ -2,8 +2,8 @@ package condition import ( "context" - "encoding/binary" "fmt" + "strconv" "golang.org/x/exp/slices" @@ -65,13 +65,16 @@ func (c inspNumber) String() string { func (c inspNumber) Inspect(ctx context.Context, capsule config.Capsule) (output bool, err error) { var check int64 if c.Key == "" { - check = int64(binary.BigEndian.Uint64(capsule.Data())) + check, err = strconv.ParseInt(string(capsule.Data()), 10, 64) + if err != nil { + return false, fmt.Errorf("condition: number: invalid data processing value: %v", err) + } } else { check = capsule.Get(c.Key).Int() } var matched bool - switch s := c.Options.Type; s { + switch c.Options.Type { case "equals": if check == c.Options.Value { matched = true @@ -89,7 +92,7 @@ func (c inspNumber) Inspect(ctx context.Context, capsule config.Capsule) (output matched = true } default: - return false, fmt.Errorf("condition: strings: type %s: %v", c.Options.Type, errors.ErrInvalidOption) + return false, fmt.Errorf("condition: number: type %s: %v", c.Options.Type, errors.ErrInvalidOption) } if c.Negate { From 387b4ca24fc295d85d25c0cde52070e4ffcd06a1 Mon Sep 17 00:00:00 2001 From: Daniel Stinson-Diess Date: Wed, 9 Aug 2023 14:12:21 -0500 Subject: [PATCH 4/5] refactor: replace number inspector with bitmath --- build/config/substation.libsonnet | 26 ++--- condition/bitmath.go | 103 ++++++++++++++++++ condition/{number_test.go => bitmath_test.go} | 89 ++++++++------- condition/condition.go | 4 +- condition/number.go | 103 ------------------ 5 files changed, 170 insertions(+), 155 deletions(-) create mode 100644 condition/bitmath.go rename condition/{number_test.go => bitmath_test.go} (65%) delete mode 100644 condition/number.go diff --git a/build/config/substation.libsonnet b/build/config/substation.libsonnet index 06caf36e..08879d75 100644 --- a/build/config/substation.libsonnet +++ b/build/config/substation.libsonnet @@ -1,6 +1,9 @@ { defaults: { inspector: { + bitmath: { + options: { type: null, value: null }, + }, content: { options: { type: null }, }, @@ -16,9 +19,6 @@ length: { options: { type: null, value: null, measurement: 'byte' }, }, - number: { - options: { type: null, value: null }, - }, regexp: { options: { expression: null }, }, @@ -212,6 +212,16 @@ }, inspector: { settings: { key: null, negate: null }, + bitmath(options=$.defaults.inspector.bitmath.options, + settings=$.interfaces.inspector.settings): { + local opt = std.mergePatch($.defaults.inspector.bitmath.options, options), + + assert $.helpers.inspector.validate(settings) : 'invalid inspector settings', + local s = std.mergePatch($.interfaces.inspector.settings, settings), + + type: 'bitmath', + settings: std.mergePatch({ options: opt }, s), + }, condition(options=null, settings=$.interfaces.inspector.settings): { assert options != null : 'invalid inspector options', @@ -279,16 +289,6 @@ type: 'length', settings: std.mergePatch({ options: opt }, s), }, - number(options=$.defaults.inspector.number.options, - settings=$.interfaces.inspector.settings): { - local opt = std.mergePatch($.defaults.inspector.number.options, options), - - assert $.helpers.inspector.validate(settings) : 'invalid inspector settings', - local s = std.mergePatch($.interfaces.inspector.settings, settings), - - type: 'number', - settings: std.mergePatch({ options: opt }, s), - }, random: { type: 'random', }, diff --git a/condition/bitmath.go b/condition/bitmath.go new file mode 100644 index 00000000..e6a64ce8 --- /dev/null +++ b/condition/bitmath.go @@ -0,0 +1,103 @@ +package condition + +import ( + "context" + "fmt" + "strconv" + + "golang.org/x/exp/slices" + + "github.com/brexhq/substation/config" + "github.com/brexhq/substation/internal/errors" +) + +// bitmath evaluates data using bitwith math operations. +// +// This inspector supports the data and object handling patterns. +type inspBitmath struct { + condition + Options inspBitmathOptions `json:"options"` +} + +type inspBitmathOptions struct { + // Type is the string evaluation Type used during inspection. + // + // Must be one of: + // + // - and + // + // - or + // + // - not + // + // - xor + Type string `json:"type"` + // Value is the length that is used for comparison during inspection. + Value int64 `json:"value"` +} + +// Creates a new bitmath inspector. +func newInspBitmath(_ context.Context, cfg config.Config) (c inspBitmath, err error) { + if err = config.Decode(cfg.Settings, &c); err != nil { + return inspBitmath{}, err + } + + // validate option.type + if !slices.Contains( + []string{ + "and", + "or", + "not", + "xor", + }, + c.Options.Type) { + return inspBitmath{}, fmt.Errorf("condition: bitmath: type %q: %v", c.Options.Type, errors.ErrInvalidOption) + } + + return c, nil +} + +func (c inspBitmath) String() string { + return toString(c) +} + +// Inspect evaluates encapsulated data with the bitmath inspector. +func (c inspBitmath) Inspect(ctx context.Context, capsule config.Capsule) (output bool, err error) { + var check int64 + if c.Key == "" { + check, err = strconv.ParseInt(string(capsule.Data()), 10, 64) + if err != nil { + return false, fmt.Errorf("condition: bitmath: invalid data processing value: %v", err) + } + } else { + check = capsule.Get(c.Key).Int() + } + + var matched bool + switch c.Options.Type { + case "and": + if check&c.Options.Value != 0 { + matched = true + } + case "or": + if check|c.Options.Value != 0 { + matched = true + } + case "not": + if ^check != 0 { + matched = true + } + case "xor": + if check^c.Options.Value != 0 { + matched = true + } + default: + return false, fmt.Errorf("condition: bitmath: type %s: %v", c.Options.Type, errors.ErrInvalidOption) + } + + if c.Negate { + return !matched, nil + } + + return matched, nil +} diff --git a/condition/number_test.go b/condition/bitmath_test.go similarity index 65% rename from condition/number_test.go rename to condition/bitmath_test.go index ef401c90..e3476aef 100644 --- a/condition/number_test.go +++ b/condition/bitmath_test.go @@ -7,53 +7,68 @@ import ( "github.com/brexhq/substation/config" ) -var _ Inspector = inspNumber{} +var _ Inspector = inspBitmath{} -var numberTests = []struct { +var bitmathTests = []struct { name string cfg config.Config test []byte expected bool }{ { - "pass equals", + "pass xor", config.Config{ - Type: "number", + Type: "bitmath", Settings: map[string]interface{}{ "key": "foo", "options": map[string]interface{}{ - "type": "equals", + "type": "xor", + "value": 3, + }, + }, + }, + []byte(`{"foo":"0"}`), + true, + }, + { + "fail xor", + config.Config{ + Type: "bitmath", + Settings: map[string]interface{}{ + "key": "foo", + "options": map[string]interface{}{ + "type": "xor", "value": 42, }, }, }, []byte(`{"foo":"42"}`), - true, + false, }, { - "!fail equals", + "!fail xor", config.Config{ - Type: "number", + Type: "bitmath", Settings: map[string]interface{}{ "key": "foo", "negate": true, "options": map[string]interface{}{ - "type": "equals", + "type": "xor", "value": 42, }, }, }, []byte(`{"foo":"42"}`), - false, + true, }, { - "pass greater_than", + "pass or", config.Config{ - Type: "number", + Type: "bitmath", Settings: map[string]interface{}{ "key": "foo", "options": map[string]interface{}{ - "type": "greater_than", + "type": "or", "value": -1, }, }, @@ -62,58 +77,58 @@ var numberTests = []struct { true, }, { - "!pass greater_than", + "!pass or", config.Config{ - Type: "number", + Type: "bitmath", Settings: map[string]interface{}{ "key": "foo", "negate": true, "options": map[string]interface{}{ - "type": "greater_than", + "type": "or", "value": 1, }, }, }, []byte(`{"foo":"0"}`), - true, + false, }, { - "pass less_than", + "pass and", config.Config{ - Type: "number", + Type: "bitmath", Settings: map[string]interface{}{ "key": "foo", "options": map[string]interface{}{ - "type": "less_than", - "value": 50, + "type": "and", + "value": 0x0001, }, }, }, - []byte(`{"foo":42}`), + []byte(`{"foo":"570506001"}`), true, }, { - "pass bitwise_and", + "fail and", config.Config{ - Type: "number", + Type: "bitmath", Settings: map[string]interface{}{ "key": "foo", "options": map[string]interface{}{ - "type": "bitwise_and", - "value": 0x0001, + "type": "and", + "value": 0x0002, }, }, }, []byte(`{"foo":"570506001"}`), - true, + false, }, { "pass data", config.Config{ - Type: "number", + Type: "bitmath", Settings: map[string]interface{}{ "options": map[string]interface{}{ - "type": "equals", + "type": "or", "value": 1, }, }, @@ -123,15 +138,15 @@ var numberTests = []struct { }, } -func TestNumber(t *testing.T) { +func TestBitmath(t *testing.T) { ctx := context.TODO() capsule := config.NewCapsule() - for _, test := range numberTests { + for _, test := range bitmathTests { t.Run(test.name, func(t *testing.T) { capsule.SetData(test.test) - insp, err := newInspNumber(ctx, test.cfg) + insp, err := newInspBitmath(ctx, test.cfg) if err != nil { t.Fatal(err) } @@ -148,17 +163,17 @@ func TestNumber(t *testing.T) { } } -func benchmarkNumberByte(b *testing.B, inspector inspNumber, capsule config.Capsule) { +func benchmarkBitmathByte(b *testing.B, inspector inspBitmath, capsule config.Capsule) { ctx := context.TODO() for i := 0; i < b.N; i++ { _, _ = inspector.Inspect(ctx, capsule) } } -func BenchmarkNumberByte(b *testing.B) { +func BenchmarkBitmathByte(b *testing.B) { capsule := config.NewCapsule() - for _, test := range numberTests { - insp, err := newInspNumber(context.TODO(), test.cfg) + for _, test := range bitmathTests { + insp, err := newInspBitmath(context.TODO(), test.cfg) if err != nil { b.Fatal(err) } @@ -166,7 +181,7 @@ func BenchmarkNumberByte(b *testing.B) { b.Run(test.name, func(b *testing.B) { capsule.SetData(test.test) - benchmarkNumberByte(b, insp, capsule) + benchmarkBitmathByte(b, insp, capsule) }, ) } diff --git a/condition/condition.go b/condition/condition.go index dbf2f149..33bdcac7 100644 --- a/condition/condition.go +++ b/condition/condition.go @@ -44,6 +44,8 @@ type Inspector interface { // NewInspector returns a configured Inspector from an Inspector configuration. func NewInspector(ctx context.Context, cfg config.Config) (Inspector, error) { switch cfg.Type { + case "bitmath": + return newInspBitmath(ctx, cfg) case "condition": return newInspCondition(ctx, cfg) case "content": @@ -58,8 +60,6 @@ func NewInspector(ctx context.Context, cfg config.Config) (Inspector, error) { return newInspJSONValid(ctx, cfg) case "length": return newInspLength(ctx, cfg) - case "number": - return newInspNumber(ctx, cfg) case "random": return newInspRandom(ctx, cfg) case "regexp": diff --git a/condition/number.go b/condition/number.go deleted file mode 100644 index d8df111d..00000000 --- a/condition/number.go +++ /dev/null @@ -1,103 +0,0 @@ -package condition - -import ( - "context" - "fmt" - "strconv" - - "golang.org/x/exp/slices" - - "github.com/brexhq/substation/config" - "github.com/brexhq/substation/internal/errors" -) - -// number evaluates data using Types from the standard library's number package. -// -// This inspector supports the data and object handling patterns. -type inspNumber struct { - condition - Options inspNumberOptions `json:"options"` -} - -type inspNumberOptions struct { - // Type is the string evaluation Type used during inspection. - // - // Must be one of: - // - // - equals - // - // - greater_than - // - // - less_than - // - // - bitwise_and - Type string `json:"type"` - // Value is the length that is used for comparison during inspection. - Value int64 `json:"value"` -} - -// Creates a new number inspector. -func newInspNumber(_ context.Context, cfg config.Config) (c inspNumber, err error) { - if err = config.Decode(cfg.Settings, &c); err != nil { - return inspNumber{}, err - } - - // validate option.type - if !slices.Contains( - []string{ - "equals", - "greater_than", - "less_than", - "bitwise_and", - }, - c.Options.Type) { - return inspNumber{}, fmt.Errorf("condition: number: type %q: %v", c.Options.Type, errors.ErrInvalidOption) - } - - return c, nil -} - -func (c inspNumber) String() string { - return toString(c) -} - -// Inspect evaluates encapsulated data with the number inspector. -func (c inspNumber) Inspect(ctx context.Context, capsule config.Capsule) (output bool, err error) { - var check int64 - if c.Key == "" { - check, err = strconv.ParseInt(string(capsule.Data()), 10, 64) - if err != nil { - return false, fmt.Errorf("condition: number: invalid data processing value: %v", err) - } - } else { - check = capsule.Get(c.Key).Int() - } - - var matched bool - switch c.Options.Type { - case "equals": - if check == c.Options.Value { - matched = true - } - case "greater_than": - if check > c.Options.Value { - matched = true - } - case "less_than": - if check < c.Options.Value { - matched = true - } - case "bitwise_and": - if check&c.Options.Value != 0 { - matched = true - } - default: - return false, fmt.Errorf("condition: number: type %s: %v", c.Options.Type, errors.ErrInvalidOption) - } - - if c.Negate { - return !matched, nil - } - - return matched, nil -} From f209fec264f04728c819558d1c3f387246cb4eef Mon Sep 17 00:00:00 2001 From: Daniel Stinson-Diess Date: Wed, 9 Aug 2023 17:12:56 -0500 Subject: [PATCH 5/5] fix: update bitmath comments/errors --- condition/bitmath.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/condition/bitmath.go b/condition/bitmath.go index e6a64ce8..6673066d 100644 --- a/condition/bitmath.go +++ b/condition/bitmath.go @@ -20,7 +20,7 @@ type inspBitmath struct { } type inspBitmathOptions struct { - // Type is the string evaluation Type used during inspection. + // Type is the bitmath evaluation Type used during inspection. // // Must be one of: // @@ -32,7 +32,7 @@ type inspBitmathOptions struct { // // - xor Type string `json:"type"` - // Value is the length that is used for comparison during inspection. + // Value is the number that is used for comparison during inspection. Value int64 `json:"value"` } @@ -67,7 +67,7 @@ func (c inspBitmath) Inspect(ctx context.Context, capsule config.Capsule) (outpu if c.Key == "" { check, err = strconv.ParseInt(string(capsule.Data()), 10, 64) if err != nil { - return false, fmt.Errorf("condition: bitmath: invalid data processing value: %v", err) + return false, fmt.Errorf("condition: bitmath: invalid data: %v", err) } } else { check = capsule.Get(c.Key).Int()