Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement Discover mode #1

Merged
merged 14 commits into from
Jun 22, 2022
Merged

Implement Discover mode #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
25331c3
simple cli to implement singer tap interface
Jun 21, 2022
aa87096
Implement Discover mode
Jun 22, 2022
cc2b280
Add buildkite CI
Jun 22, 2022
abb49f5
catalog messages need a TYPE
Jun 22, 2022
567e53c
Add key_properties and bookmark_properties
Jun 22, 2022
397e1e5
remove keyProperties as an argument to GenerateMetadata
Jun 22, 2022
aa7c9ab
Add licensing container
Jun 22, 2022
9ab7930
Add DockerFile
Jun 22, 2022
3771ac8
can build docker container
Jun 22, 2022
fa5d553
Fix docker image in CI
Jun 22, 2022
8970e3b
remove unused variables
Jun 22, 2022
57a0efd
approve dependencies
Jun 22, 2022
2ad5311
replace all comments referencing Airbyte, with Singer
Jun 22, 2022
eaec167
CR feedback
Jun 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .buildkite/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
version: '3.8'

services:
ci:
build:
context: ../
38 changes: 38 additions & 0 deletions .buildkite/pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
agents:
queue: "private"

steps:
- name: "Tests"
label: "Tests"
command: "go test -race -v ./..."
env:
DOCKER_BUILDKIT: 1
plugins:
- docker-compose#v3.9.0:
run: ci

- name: "go vet"
label: "Tests"
command: "go vet ./..."
env:
DOCKER_BUILDKIT: 1
plugins:
- docker-compose#v3.9.0:
run: ci

- name: "staticcheck"
label: "Tests"
command: "go install honnef.co/go/tools/cmd/staticcheck@latest && staticcheck ./..."
env:
DOCKER_BUILDKIT: 1
plugins:
- docker-compose#v3.9.0:
run: ci

- name: "Verify dependency licenses %n"
command: "go get -v ./... && license_finder"
env:
DOCKER_BUILDKIT: 1
plugins:
- docker-compose#v3.9.0:
run: ci
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
sources/*
19 changes: 19 additions & 0 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# syntax=docker/dockerfile:1

ARG GO_VERSION=1.18.3
FROM golang:${GO_VERSION}-bullseye AS build

WORKDIR /singer-tap
COPY . .

RUN go mod download
RUN go build ./cmd/singer-tap

FROM debian:bullseye-slim

RUN apt-get update && apt-get upgrade -y && \
apt-get install -y default-mysql-client ca-certificates && \
rm -rf /var/lib/apt/lists/*

COPY --from=build /singer-tap /usr/local/bin/
ENTRYPOINT ["/usr/local/bin/singer-tap"]
17 changes: 17 additions & 0 deletions Dockerfile.ci
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# syntax=docker/dockerfile:1

ARG GO_VERSION=1.18.3
FROM golang:${GO_VERSION}-bullseye AS build

RUN apt-get update && apt-get upgrade -y && \
apt-get install -y default-mysql-client ca-certificates && \
apt-get install -y ruby && \
gem install license_finder && \
rm -rf /var/lib/apt/lists/*


WORKDIR /singer-tap
COPY . .

RUN go mod download
RUN go build ./cmd/singer-tap
224 changes: 224 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,226 @@
# singer-tap
Singer.io tap for extracting PlanetScale data

Singer specification for building a tap is available [here](https://github.com/singer-io/getting-started/blob/master/docs/SPEC.md#singer-specification)

Since this is a tap, it can run [Discover mode](https://github.com/singer-io/getting-started/blob/master/docs/DISCOVERY_MODE.md#discovery-mode) or [Sync mode](https://github.com/singer-io/getting-started/blob/master/docs/SYNC_MODE.md#sync-mode)

## Local Development
**1. Generate config file to connect to your PlanetScale database.**
1. Opt-in to Connect for your organization.
2. Generate a password to your PlanetScale database.
3. Make a copy of the `sample/config.json` available in this repo and edit values,
lets call this `sample/employee_database.json`

### Running in Discover Mode

Singer taps are a binary executable with a single entry point.
We signal that we want to discover the source by passing in the `--discover` flag.

**Example invocation**
``` bash
go run cmd/singer-tap/main.go --config sources/demo/employee_database.json --discover
```

**Example output**
``` bash
go run cmd/singer-tap/main.go --config sources/demo/employee_database.json --discover | jq .
PlanetScale Tap : INFO : Discovering Schema for PlanetScale database : employees
{
"streams":
[
{
"stream": "departments",
"tap_stream_id": "employees:departments",
"schema":
{
"type":
[
"null",
"object"
],
"additionalProperties": false,
"properties":
{
"dept_name":
{
"type":
[
"null",
"string"
]
},
"dept_no":
{
"type":
[
"null",
"string"
]
}
}
},
"table-name": "departments",
"metadata":
[
{
"metadata":
{
"inclusion": "available",
"valid-replication-keys":
[
"dept_no"
],
"table-key-properties":
[
"dept_no"
],
"breadcrumb":
[]
}
},
{
"metadata":
{
"inclusion": "available",
"breadcrumb":
[
"properties",
"dept_no"
]
}
},
{
"metadata":
{
"inclusion": "available",
"breadcrumb":
[
"properties",
"dept_name"
]
}
}
]
},
{
"stream": "dept_emp",
"tap_stream_id": "employees:dept_emp",
"schema":
{
"type":
[
"null",
"object"
],
"additionalProperties": false,
"properties":
{
"dept_no":
{
"type":
[
"null",
"string"
]
},
"emp_no":
{
"type":
[
"null",
"integer"
]
},
"from_date":
{
"type":
[
"null",
"date"
]
},
"to_date":
{
"type":
[
"null",
"date"
]
}
}
},
"table-name": "dept_emp",
"metadata":
[
{
"metadata":
{
"inclusion": "available",
"valid-replication-keys":
[
"emp_no",
"dept_no"
],
"table-key-properties":
[
"emp_no",
"dept_no"
],
"breadcrumb":
[]
}
},
{
"metadata":
{
"inclusion": "available",
"breadcrumb":
[
"properties",
"to_date"
]
}
},
{
"metadata":
{
"inclusion": "available",
"breadcrumb":
[
"properties",
"emp_no"
]
}
},
{
"metadata":
{
"inclusion": "available",
"breadcrumb":
[
"properties",
"dept_no"
]
}
},
{
"metadata":
{
"inclusion": "available",
"breadcrumb":
[
"properties",
"from_date"
]
}
}
]
}
]
}
```

### Running in Sync Mode

**Coming soon**
50 changes: 50 additions & 0 deletions cmd/internal/discover.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package internal

import (
"context"
"fmt"
"github.com/pkg/errors"
)

func Discover(ctx context.Context, source PlanetScaleSource) (Catalog, error) {
var c Catalog
mysql, err := NewMySQL(&source)
if err != nil {
return c, errors.Wrap(err, "unable to open mysql connection to PlanetScale Database")
}
defer mysql.Close()

tableNames, err := mysql.GetTableNames(ctx, source)
if err != nil {
return c, errors.Wrap(err, "unable to retrieve table names")
}

for _, name := range tableNames {
table := Stream{
Name: name,
ID: fmt.Sprintf("%s:%s", source.Database, name),
TableName: name,
}

tableSchema, err := mysql.GetTableSchema(ctx, source, name)
if err != nil {
return c, errors.Wrapf(err, "unable to retrieve schema for table : %v , failed with : %q", name, err)
}

table.Schema = StreamSchema{
Type: []string{"null", "object"},
Properties: tableSchema,
}
keyProperties, err := mysql.GetTablePrimaryKeys(ctx, source, name)
if err != nil {
return c, errors.Wrapf(err, "unable to retrieve primary keys for table : %v , failed with : %q", name, err)
}
table.KeyProperties = keyProperties
table.CursorProperties = keyProperties
table.GenerateMetadata(keyProperties)

c.Streams = append(c.Streams, table)
}

return c, nil
}
1 change: 1 addition & 0 deletions cmd/internal/discover_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
package internal
Loading