Skip to content

Commit

Permalink
feat!: Migrate to Meta Processors (#7)
Browse files Browse the repository at this point in the history
* feat: add JSON support for byte content

* fix: improve JSON validation

* feat: init meta processors

* refactor!: remove array & add data support

* refactor!: remove array support

* refactor!: remove array support

* test: add required options test

* refactor: update error handling

* refactor: update error handling

* refactor!: remove array support

* refactor: update error handling

* refactor: update error handling

* refactor: update error handling

* refactor!: remove retain keys option

* refactor!: remove array support

* refactor: update error handling

* refactor: standardize style

* refactor!: update process lib
  • Loading branch information
jshlbrd committed Jul 18, 2022
1 parent 06d42de commit f0aabce
Show file tree
Hide file tree
Showing 53 changed files with 2,334 additions and 1,379 deletions.
45 changes: 21 additions & 24 deletions config/process.libsonnet
Original file line number Diff line number Diff line change
@@ -1,74 +1,73 @@
{
base64(input, output, direction,
alphabet='std',
condition_operator='', condition_inspectors=[]): {
type: 'base64',
settings: {
input_key: input, output_key: output,
options: { direction: direction, alphabet: alphabet },
options: { direction: direction },
condition: { operator: condition_operator, inspectors: condition_inspectors},
input_key: input, output_key: output,
},
},
capture(input, output, expression,
_function='find', count=-1,
condition_operator='', condition_inspectors=[]): {
type: 'capture',
settings: {
input_key: input, output_key: output,
options: { expression: expression, 'function': _function, count: count },
condition: { operator: condition_operator, inspectors: condition_inspectors},
input_key: input, output_key: output,
},
},
case(input, output, case,
condition_operator='', condition_inspectors=[]): {
type: 'case',
settings: {
input_key: input, output_key: output,
options: { case: case },
condition: { operator: condition_operator, inspectors: condition_inspectors},
input_key: input, output_key: output,
},
},
concat(input, output, separator,
condition_operator='', condition_inspectors=[]): {
type: 'concat',
settings: {
input_key: input, output_key: output,
options: { separator: separator },
condition: { operator: condition_operator, inspectors: condition_inspectors},
input_key: input, output_key: output,
},
},
convert(input, output, type,
condition_operator='', condition_inspectors=[]): {
type: 'convert',
settings: {
input_key: input, output_key: output,
options: { type: type },
condition: { operator: condition_operator, inspectors: condition_inspectors},
input_key: input, output_key: output,
},
},
copy(input='', output='',
condition_operator='', condition_inspectors=[]): {
type: 'copy',
settings: {
input_key: input, output_key: output,
condition: { operator: condition_operator, inspectors: condition_inspectors},
input_key: input, output_key: output,
},
},
delete(input,
condition_operator='', condition_inspectors=[]): {
type: 'delete',
settings: {
input: input,
condition: { operator: condition_operator, inspectors: condition_inspectors},
input: input,
},
},
domain(input, output, _function,
condition_operator='', condition_inspectors=[]): {
type: 'domain',
settings: {
input_key: input, output_key: output,
options: { 'function': _function },
condition: { operator: condition_operator, inspectors: condition_inspectors},
input_key: input, output_key: output,
},
},
drop(condition_operator='', condition_inspectors=[]): {
Expand All @@ -82,39 +81,37 @@
condition_operator='', condition_inspectors=[]): {
type: 'dynamodb',
settings: {
input_key: input, output_key: output,
options: { table: table, key_condition_expression: key_condition_expression, limit: limit, scan_index_forward: scan_index_forward },
condition: { operator: condition_operator, inspectors: condition_inspectors},
input_key: input, output_key: output,
},
},
expand(input,
retain=[],
condition_operator='', condition_inspectors=[]): {
type: 'expand',
settings: {
input: input,
options: { retain: retain },
condition: { operator: condition_operator, inspectors: condition_inspectors},
input: input,
},
},
flatten(input, output,
deep=true,
condition_operator='', condition_inspectors=[]): {
type: 'flatten',
settings: {
input_key: input, output_key: output,
options: { deep: deep },
condition: { operator: condition_operator, inspectors: condition_inspectors},
input_key: input, output_key: output,
},
},
group(input, output,
options_keys=[],
keys=[],
condition_operator='', condition_inspectors=[]): {
type: 'group',
settings: {
input_key: input, output_key: output,
options: { keys: options_keys },
options: { keys: keys },
condition: { operator: condition_operator, inspectors: condition_inspectors},
input_key: input, output_key: output,
},
},
gzip(direction,
Expand All @@ -130,57 +127,57 @@
condition_operator='', condition_inspectors=[]): {
type: 'hash',
settings: {
input_key: input, output_key: output,
options: { algorithm: algorithm },
condition: { operator: condition_operator, inspectors: condition_inspectors},
input_key: input, output_key: output,
},
},
insert(output, value,
condition_operator='', condition_inspectors=[]): {
type: 'insert',
settings: {
output: output,
options: { value: value },
condition: { operator: condition_operator, inspectors: condition_inspectors},
output: output,
},
},
lambda(input, output, _function,
error_on_failure=false,
condition_operator='', condition_inspectors=[]): {
type: 'lambda',
settings: {
input_key: input, output_key: output,
options: { 'function': _function, error_on_failure: error_on_failure },
condition: { operator: condition_operator, inspectors: condition_inspectors},
input_key: input, output_key: output,
},
},
math(input, output, operation,
condition_operator='', condition_inspectors=[]): {
type: 'math',
settings: {
input_key: input, output_key: output,
options: { operation: operation },
condition: { operator: condition_operator, inspectors: condition_inspectors},
input_key: input, output_key: output,
},
},
replace(input, output, old, new,
count=-1,
condition_operator='', condition_inspectors=[]): {
type: 'replace',
settings: {
input_key: input, output_key: output,
options: { old: old, new: new, count: count },
condition: { operator: condition_operator, inspectors: condition_inspectors},
input_key: input, output_key: output,
},
},
time(input, output, input_format,
input_location='', output_format='2006-01-02T15:04:05.000000Z', output_location='',
condition_operator='', condition_inspectors=[]): {
type: 'time',
settings: {
input_key: input, output_key: output,
options: { input_format: input_format, input_location: input_location, output_format: output_format, output_location: output_location },
condition: { operator: condition_operator, inspectors: condition_inspectors},
input_key: input, output_key: output,
},
},
}
25 changes: 25 additions & 0 deletions internal/base64/base64.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package base64

import (
"encoding/base64"
"fmt"
)

// Decode is a convenience wrapper for base64 decoding bytes.
func Decode(b []byte) ([]byte, error) {
decode := make([]byte, base64.StdEncoding.DecodedLen(len(b)))
n, err := base64.StdEncoding.Decode(decode, b)
if err != nil {
return nil, fmt.Errorf("decode: %v", err)
}

return decode[:n], nil
}

// Encode is a convenience wrapper for base64 encoding bytes.
func Encode(b []byte) []byte {
encode := make([]byte, base64.StdEncoding.EncodedLen(len(b)))
base64.StdEncoding.Encode(encode, b)

return encode
}
99 changes: 99 additions & 0 deletions internal/base64/base64_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package base64

import (
"bytes"
"testing"
)

var decodeTests = []struct {
name string
test []byte
expected []byte
}{
{
name: "foo",
test: []byte(`Zm9v`),
expected: []byte(`foo`),
},
{
name: "zlib",
test: []byte(`eJwFwCENAAAAgLC22Pd3LAYCggFF`),
expected: []byte{120, 156, 5, 192, 33, 13, 0, 0, 0, 128, 176, 182, 216, 247, 119, 44, 6, 2, 130, 1, 69},
},
}

func TestBase64Decode(t *testing.T) {
for _, test := range decodeTests {
result, err := Decode(test.test)
if err != nil {
t.Logf("got error %v", err)
t.Fail()
return
}

if c := bytes.Compare(result, test.expected); c != 0 {
t.Logf("expected %s, got %s", test.expected, result)
t.Fail()
}
}
}

func benchmarkBase64Decode(b *testing.B, test []byte) {
for i := 0; i < b.N; i++ {
Decode(test)
}
}

func BenchmarkBase64Decode(b *testing.B) {
for _, test := range encodeTests {
b.Run(string(test.name),
func(b *testing.B) {
benchmarkBase64Decode(b, test.test)
},
)
}
}

var encodeTests = []struct {
name string
test []byte
expected []byte
}{
{
name: "foo",
test: []byte(`foo`),
expected: []byte(`Zm9v`),
},
{
name: "zlib",
test: []byte{120, 156, 5, 192, 33, 13, 0, 0, 0, 128, 176, 182, 216, 247, 119, 44, 6, 2, 130, 1, 69},
expected: []byte(`eJwFwCENAAAAgLC22Pd3LAYCggFF`),
},
}

func TestBase64Encode(t *testing.T) {
for _, test := range encodeTests {
result := Encode(test.test)

if c := bytes.Compare(result, test.expected); c != 0 {
t.Logf("expected %s, got %s", test.expected, result)
t.Fail()
}
}
}

func benchmarkBase64Encode(b *testing.B, test []byte) {
for i := 0; i < b.N; i++ {
Encode(test)
}
}

func BenchmarkBase64Encode(b *testing.B) {
for _, test := range encodeTests {
b.Run(string(test.name),
func(b *testing.B) {
benchmarkBase64Encode(b, test.test)
},
)
}
}
Loading

0 comments on commit f0aabce

Please sign in to comment.