Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Implement TransformSchema support. #1838

Merged
merged 3 commits into from
Jul 31, 2024
Merged

feat: Implement TransformSchema support. #1838

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e80367d
feat: Implement TransformSchema support.
marianogappa Jul 30, 2024
a5ca6a9
feat: Implement TransformSchema method.
marianogappa Jul 31, 2024
e8d951e
Merge branch 'main' into mariano/add-transform-schema-support
marianogappa Jul 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/simple_plugin/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ require (
github.com/buger/jsonparser v1.1.1 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cloudquery/cloudquery-api-go v1.12.4 // indirect
github.com/cloudquery/plugin-pb-go v1.21.1 // indirect
github.com/cloudquery/plugin-pb-go v1.21.2 // indirect
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions examples/simple_plugin/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/cloudquery/cloudquery-api-go v1.12.4 h1:48zJRUONRb0AJD/l4u5QZtGsVBq1QUU3M9+/+sCU3xo=
github.com/cloudquery/cloudquery-api-go v1.12.4/go.mod h1:5oo8HHnv2Y7NgcVvZn59xFlYKJUyeP0tcN8JH3IP2Aw=
github.com/cloudquery/plugin-pb-go v1.21.1 h1:Fm4FyGrgLUTcllaEI6jdk1Q1NQ6VEbaSl8UCN3AxFOg=
github.com/cloudquery/plugin-pb-go v1.21.1/go.mod h1:Gv336j2QCqGlUABUA6gBcrqWNjLpzv9NOmAJAHC3xg0=
github.com/cloudquery/plugin-pb-go v1.21.2 h1:WfuiZbunZBhcVBNW/FwJyv7RrSen9XNYH4ZbVl+4vzk=
github.com/cloudquery/plugin-pb-go v1.21.2/go.mod h1:Gv336j2QCqGlUABUA6gBcrqWNjLpzv9NOmAJAHC3xg0=
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
Expand Down
5 changes: 5 additions & 0 deletions examples/simple_plugin/plugin/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,11 @@ func (*Client) Transform(_ context.Context, _ <-chan arrow.Record, _ chan<- arro
return nil
}

func (*Client) TransformSchema(_ context.Context, _ *arrow.Schema) (*arrow.Schema, error) {
// Not implemented, just used for testing destination packaging
return nil, nil
}

func Configure(_ context.Context, logger zerolog.Logger, spec []byte, opts plugin.NewClientOptions) (plugin.Client, error) {
if opts.NoConnection {
return &Client{
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/apache/arrow/go/v17 v17.0.0
github.com/bradleyjkemp/cupaloy/v2 v2.8.0
github.com/cloudquery/cloudquery-api-go v1.12.4
github.com/cloudquery/plugin-pb-go v1.21.1
github.com/cloudquery/plugin-pb-go v1.21.2
github.com/cloudquery/plugin-sdk/v2 v2.7.0
github.com/goccy/go-json v0.10.3
github.com/google/go-cmp v0.6.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ github.com/cloudquery/cloudquery-api-go v1.12.4 h1:48zJRUONRb0AJD/l4u5QZtGsVBq1Q
github.com/cloudquery/cloudquery-api-go v1.12.4/go.mod h1:5oo8HHnv2Y7NgcVvZn59xFlYKJUyeP0tcN8JH3IP2Aw=
github.com/cloudquery/plugin-pb-go v1.21.1 h1:Fm4FyGrgLUTcllaEI6jdk1Q1NQ6VEbaSl8UCN3AxFOg=
github.com/cloudquery/plugin-pb-go v1.21.1/go.mod h1:Gv336j2QCqGlUABUA6gBcrqWNjLpzv9NOmAJAHC3xg0=
github.com/cloudquery/plugin-pb-go v1.21.2 h1:WfuiZbunZBhcVBNW/FwJyv7RrSen9XNYH4ZbVl+4vzk=
github.com/cloudquery/plugin-pb-go v1.21.2/go.mod h1:Gv336j2QCqGlUABUA6gBcrqWNjLpzv9NOmAJAHC3xg0=
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
Expand Down
4 changes: 4 additions & 0 deletions internal/memdb/memdb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,10 @@ func (*client) Transform(_ context.Context, _ <-chan arrow.Record, _ chan<- arro
return nil
}

func (*client) TransformSchema(_ context.Context, _ *arrow.Schema) (*arrow.Schema, error) {
return nil, nil
}

func evaluatePredicate(pred message.Predicate, record arrow.Record) bool {
sc := record.Schema()
indices := sc.FieldIndices(pred.Column)
Expand Down
4 changes: 4 additions & 0 deletions internal/reversertransformer/reversertransformer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ func (c *client) Transform(ctx context.Context, recvRecords <-chan arrow.Record,
}
}

func (*client) TransformSchema(_ context.Context, old *arrow.Schema) (*arrow.Schema, error) {
return old, nil
}

func (*client) reverseStrings(record arrow.Record) (arrow.Record, error) {
for i, column := range record.Columns() {
if column.DataType().ID() != arrow.STRING {
Expand Down
3 changes: 3 additions & 0 deletions plugin/plugin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ type UnimplementedTransformer struct{}
func (UnimplementedTransformer) Transform(context.Context, <-chan arrow.Record, chan<- arrow.Record) error {
return ErrNotImplemented
}
func (UnimplementedTransformer) TransformSchema(context.Context, *arrow.Schema) (*arrow.Schema, error) {
return nil, ErrNotImplemented
}

// Plugin is the base structure required to pass to sdk.serve
// We take a declarative approach to API here similar to Cobra
Expand Down
3 changes: 3 additions & 0 deletions plugin/plugin_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@
func (*testPluginClient) Transform(context.Context, <-chan arrow.Record, chan<- arrow.Record) error {
return nil
}
func (*testPluginClient) TransformSchema(ctx context.Context, old *arrow.Schema) (*arrow.Schema, error) {

Check failure on line 62 in plugin/plugin_test.go

View workflow job for this annotation

GitHub Actions / Lint with GolangCI 

unused-parameter: parameter 'ctx' seems to be unused, consider removing or renaming it as _ (revive)
return nil, nil
}

func TestPluginSuccess(t *testing.T) {
ctx := context.Background()
Expand Down
4 changes: 4 additions & 0 deletions plugin/plugin_transformer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,12 @@ import (

type TransformerClient interface {
Transform(context.Context, <-chan arrow.Record, chan<- arrow.Record) error
TransformSchema(context.Context, *arrow.Schema) (*arrow.Schema, error)
}

func (p *Plugin) Transform(ctx context.Context, recvRecords <-chan arrow.Record, sendRecords chan<- arrow.Record) error {
return p.client.Transform(ctx, recvRecords, sendRecords)
}
func (p *Plugin) TransformSchema(ctx context.Context, old *arrow.Schema) (*arrow.Schema, error) {
return p.client.TransformSchema(ctx, old)
}
Loading