Skip to content

Commit

Permalink
fix tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@flarco
flarco committed Aug 27, 2024
1 parent d14949f commit fad6ad7
Show file tree
Hide file tree
Showing 7 changed files with 59 additions and 7 deletions.
Binary file modified cmd/sling/tests/files/parquet/test1.1.parquet
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion cmd/sling/tests/replications/r.07.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
source: local://
source: LOCAL
target: SQLITE

defaults:
Expand Down
27 changes: 27 additions & 0 deletions core/dbio/connection/connection_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ func TestConnectionDiscover(t *testing.T) {
expected: []string{
"sling_test/csv/",
"sling_test/files/",
"sling_test/lineitem_iceberg/",
"sling_test/delta/",
},
},
{
Expand All @@ -105,6 +107,8 @@ func TestConnectionDiscover(t *testing.T) {
expected: []string{
"sling_test/csv/",
"sling_test/files/",
"sling_test/lineitem_iceberg/",
"sling_test/delta/",
},
},
{
Expand All @@ -115,6 +119,14 @@ func TestConnectionDiscover(t *testing.T) {
connName: "aws_s3",
pattern: "sling_test/**",
expected: []string{
"sling_test/delta/",
"sling_test/delta/_delta_log/",
"sling_test/delta/country=Argentina/",
"sling_test/delta/country=China/",
"sling_test/delta/country=Germany/",
"sling_test/lineitem_iceberg/",
"sling_test/lineitem_iceberg/data/",
"sling_test/lineitem_iceberg/metadata/",
"sling_test/csv/part.01.0001.csv",
"sling_test/csv/part.01.0002.csv",
"sling_test/csv/part.01.0003.csv",
Expand All @@ -126,9 +138,24 @@ func TestConnectionDiscover(t *testing.T) {
"sling_test/csv/part.01.0009.csv",
"sling_test/csv/part.01.0010.csv",
"sling_test/csv/part.01.0011.csv",
"sling_test/delta/_delta_log/00000000000000000000.json",
"sling_test/delta/country=Argentina/part-00000-8d0390a3-f797-4265-b9c2-da1c941680a3.c000.snappy.parquet",
"sling_test/delta/country=China/part-00000-88fba1af-b28d-4303-9c85-9a97be631d40.c000.snappy.parquet",
"sling_test/delta/country=Germany/part-00000-030076e1-5ec9-47c2-830a-1569f823b6ee.c000.snappy.parquet",
"sling_test/files/test1k_s3.csv",
"sling_test/files/test1k_s3.json",
"sling_test/files/test1k_s3.parquet",
"sling_test/lineitem_iceberg/README.md",
"sling_test/lineitem_iceberg/data/00000-411-0792dcfe-4e25-4ca3-8ada-175286069a47-00001.parquet",
"sling_test/lineitem_iceberg/data/00041-414-f3c73457-bbd6-4b92-9c15-17b241171b16-00001.parquet",
"sling_test/lineitem_iceberg/metadata/10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m0.avro",
"sling_test/lineitem_iceberg/metadata/10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m1.avro",
"sling_test/lineitem_iceberg/metadata/cf3d0be5-cf70-453d-ad8f-48fdc412e608-m0.avro",
"sling_test/lineitem_iceberg/metadata/snap-3776207205136740581-1-cf3d0be5-cf70-453d-ad8f-48fdc412e608.avro",
"sling_test/lineitem_iceberg/metadata/snap-7635660646343998149-1-10eaca8a-1e1c-421e-ad6d-b232e5ee23d3.avro",
"sling_test/lineitem_iceberg/metadata/v1.metadata.json",
"sling_test/lineitem_iceberg/metadata/v2.metadata.json",
"sling_test/lineitem_iceberg/metadata/version-hint.text",
},
},
{
Expand Down
2 changes: 1 addition & 1 deletion core/dbio/iop/duckdb_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ func TestDuckDb(t *testing.T) {

t.Run("Stream", func(t *testing.T) {

duck := NewDuckDb(context.Background(), "path=/tmp/test.db")
duck := NewDuckDb(context.Background(), "path=/tmp/test.duckdb")

// Create a test table and insert some data
_, err := duck.ExecMultiContext(
Expand Down
2 changes: 1 addition & 1 deletion core/dbio/scripts/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ go test -v -run 'TestConnection'
cd -

cd iop
go test -v -run 'TestParseDate|TestDetectDelimiter|TestFIX|TestConstraints|TestDuckDb|TestParquetDuckDb|TestIcebergReader|TestDeltaReader'
go test -timeout 5m -v -run 'TestParseDate|TestDetectDelimiter|TestFIX|TestConstraints|TestDuckDb|TestParquetDuckDb|TestIcebergReader|TestDeltaReader'
cd -

cd database
Expand Down
25 changes: 25 additions & 0 deletions core/sling/replication_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ func TestReplicationWildcards(t *testing.T) {
expected: []string{
"sling_test/csv/",
"sling_test/files/",
"sling_test/lineitem_iceberg/",
"sling_test/delta/",
},
},
{
Expand All @@ -81,6 +83,14 @@ func TestReplicationWildcards(t *testing.T) {
"sling_test/**",
},
expected: []string{
"sling_test/delta/",
"sling_test/delta/_delta_log/",
"sling_test/delta/country=Argentina/",
"sling_test/delta/country=China/",
"sling_test/delta/country=Germany/",
"sling_test/lineitem_iceberg/",
"sling_test/lineitem_iceberg/data/",
"sling_test/lineitem_iceberg/metadata/",
"sling_test/csv/part.01.0001.csv",
"sling_test/csv/part.01.0002.csv",
"sling_test/csv/part.01.0003.csv",
Expand All @@ -92,9 +102,24 @@ func TestReplicationWildcards(t *testing.T) {
"sling_test/csv/part.01.0009.csv",
"sling_test/csv/part.01.0010.csv",
"sling_test/csv/part.01.0011.csv",
"sling_test/delta/_delta_log/00000000000000000000.json",
"sling_test/delta/country=Argentina/part-00000-8d0390a3-f797-4265-b9c2-da1c941680a3.c000.snappy.parquet",
"sling_test/delta/country=China/part-00000-88fba1af-b28d-4303-9c85-9a97be631d40.c000.snappy.parquet",
"sling_test/delta/country=Germany/part-00000-030076e1-5ec9-47c2-830a-1569f823b6ee.c000.snappy.parquet",
"sling_test/files/test1k_s3.csv",
"sling_test/files/test1k_s3.json",
"sling_test/files/test1k_s3.parquet",
"sling_test/lineitem_iceberg/README.md",
"sling_test/lineitem_iceberg/data/00000-411-0792dcfe-4e25-4ca3-8ada-175286069a47-00001.parquet",
"sling_test/lineitem_iceberg/data/00041-414-f3c73457-bbd6-4b92-9c15-17b241171b16-00001.parquet",
"sling_test/lineitem_iceberg/metadata/10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m0.avro",
"sling_test/lineitem_iceberg/metadata/10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m1.avro",
"sling_test/lineitem_iceberg/metadata/cf3d0be5-cf70-453d-ad8f-48fdc412e608-m0.avro",
"sling_test/lineitem_iceberg/metadata/snap-3776207205136740581-1-cf3d0be5-cf70-453d-ad8f-48fdc412e608.avro",
"sling_test/lineitem_iceberg/metadata/snap-7635660646343998149-1-10eaca8a-1e1c-421e-ad6d-b232e5ee23d3.avro",
"sling_test/lineitem_iceberg/metadata/v1.metadata.json",
"sling_test/lineitem_iceberg/metadata/v2.metadata.json",
"sling_test/lineitem_iceberg/metadata/version-hint.text",
},
},
{
Expand Down
8 changes: 4 additions & 4 deletions scripts/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@ shopt -s expand_aliases
# export _DEBUG_CALLER_LEVEL=2
cd cmd/sling
go test -v -run 'TestReplicationDefaults'
go test -v -parallel 3 -run 'TestSuiteFile|TestSuiteDatabaseClickhouse'
SKIP_CLICKHOUSE=TRUE go test -v -parallel 4 -timeout 15m -run TestSuiteDatabase
# go test -v -parallel 3 -run 'TestSuiteFile|TestSuiteDatabaseClickhouse'
# SKIP_CLICKHOUSE=TRUE go test -v -parallel 4 -timeout 15m -run TestSuiteDatabase
cd -

cd core/sling
go test -v -run 'TestTransformMsUUID'
go test -v -run 'TestReplication'
go test -v -run 'TestCheck'
go test -run 'TestCheck'
cd -

## test cli commands
Expand Down Expand Up @@ -78,7 +78,7 @@ sling run -r cmd/sling/tests/replications/r.05.yaml
sling run -r cmd/sling/tests/replications/r.05.yaml --streams 's3://ocral/mlo.community.test/channels.json,s3://ocral/mlo.community.test/random/'

SLING_STREAM_CNT=3 sling run -r cmd/sling/tests/replications/r.06.yaml
SLING_STREAM_CNT=14 sling run -r cmd/sling/tests/replications/r.07.yaml
SLING_STREAM_CNT=17 sling run -r cmd/sling/tests/replications/r.07.yaml
SLING_STREAM_CNT=4 sling run -r cmd/sling/tests/replications/r.08.yaml
# SLING_CONSTRAINT_FAILS=2 SLING_STREAM_CNT=">1" sling run -r cmd/sling/tests/replications/r.09.yaml
SLING_STREAM_CNT=">1" sling run -r cmd/sling/tests/replications/r.09.yaml
Expand Down

0 comments on commit fad6ad7

Please sign in to comment.