Skip to content

Commit

Permalink
rename new to inline
Browse files Browse the repository at this point in the history
Signed-off-by: xxchan <xxchan22f@gmail.com>
  • Loading branch information
xxchan committed Apr 11, 2024
1 parent cf50f5e commit 6dddbf3
Show file tree
Hide file tree
Showing 5 changed files with 27 additions and 31 deletions.
52 changes: 23 additions & 29 deletions ci/scripts/e2e-source-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,6 @@ set -euo pipefail

source ci/scripts/common.sh

# Arguments:
# $1: subject name
# $2: schema file path
function register_schema_registry() {
curl -X POST http://message_queue:8081/subjects/$1/versions \
-H ‘Content-Type: application/vnd.schemaregistry.v1+json’ \
--data-binary @<(jq -n --arg schema “$(cat $2)” ‘{schemaType: “PROTOBUF”, schema: $schema}’)
}

# prepare environment
export CONNECTOR_LIBS_PATH="./connector-node/libs"

Expand Down Expand Up @@ -73,26 +64,26 @@ echo 'db.runCommand({ping: 1})' | mongo mongodb://mongodb:27017
echo '> rs config'
echo 'rs.conf()' | mongo mongodb://mongodb:27017
echo '> run test..'
sqllogictest -p 4566 -d dev './e2e_test/source/cdc/mongodb/**/*.slt'
risedev slt './e2e_test/source/cdc/mongodb/**/*.slt'

echo "--- inline cdc test"
export MYSQL_HOST=mysql MYSQL_TCP_PORT=3306 MYSQL_PWD=123456
sqllogictest -p 4566 -d dev './e2e_test/source/cdc_inline/**/*.slt'
risedev slt './e2e_test/source/cdc_inline/**/*.slt'

echo "--- opendal source test"
sqllogictest -p 4566 -d dev './e2e_test/source/opendal/**/*.slt'
risedev slt './e2e_test/source/opendal/**/*.slt'

echo "--- mysql & postgres cdc validate test"
sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.validate.mysql.slt'
sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.validate.postgres.slt'
risedev slt './e2e_test/source/cdc/cdc.validate.mysql.slt'
risedev slt './e2e_test/source/cdc/cdc.validate.postgres.slt'

echo "--- cdc share source test"
# cdc share stream test cases
export MYSQL_HOST=mysql MYSQL_TCP_PORT=3306 MYSQL_PWD=123456
sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.share_stream.slt'
risedev slt './e2e_test/source/cdc/cdc.share_stream.slt'

# create a share source and check whether heartbeat message is received
sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.create_source_job.slt'
risedev slt './e2e_test/source/cdc/cdc.create_source_job.slt'
table_id=`psql -U root -h localhost -p 4566 -d dev -t -c "select id from rw_internal_tables where name like '%mysql_source%';" | xargs`;
table_count=`psql -U root -h localhost -p 4566 -d dev -t -c "select count(*) from rw_table(${table_id}, public);" | xargs`;
if [ $table_count -eq 0 ]; then
Expand All @@ -101,10 +92,10 @@ if [ $table_count -eq 0 ]; then
fi

echo "--- mysql & postgres load and check"
sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.load.slt'
risedev slt './e2e_test/source/cdc/cdc.load.slt'
# wait for cdc loading
sleep 10
sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.check.slt'
risedev slt './e2e_test/source/cdc/cdc.check.slt'

# kill cluster
risedev kill
Expand Down Expand Up @@ -135,10 +126,10 @@ echo "> wait for cluster recovery finish"
sleep 20
echo "> check mviews after cluster recovery"
# check results
sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.check_new_rows.slt'
risedev slt './e2e_test/source/cdc/cdc.check_new_rows.slt'

# drop relations
sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc_share_stream_drop.slt'
risedev slt './e2e_test/source/cdc/cdc_share_stream_drop.slt'

echo "--- Kill cluster"
risedev ci-kill
Expand All @@ -153,8 +144,8 @@ echo "make sure google/protobuf/source_context.proto is NOT in schema registry"
curl --silent 'http://message_queue:8081/subjects'; echo
# curl --silent --head -X GET 'http://message_queue:8081/subjects/google%2Fprotobuf%2Fsource_context.proto/versions' | grep 404
curl --silent 'http://message_queue:8081/subjects' | grep -v 'google/protobuf/source_context.proto'
sqllogictest -p 4566 -d dev './e2e_test/schema_registry/pb.slt'
sqllogictest -p 4566 -d dev './e2e_test/schema_registry/alter_sr.slt'
risedev slt './e2e_test/schema_registry/pb.slt'
risedev slt './e2e_test/schema_registry/alter_sr.slt'

echo "--- Kill cluster"
risedev ci-kill
Expand All @@ -164,19 +155,22 @@ RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=i
risedev ci-start ci-pubsub-kafka
./scripts/source/prepare_ci_kafka.sh
cargo run --bin prepare_ci_pubsub
sqllogictest -p 4566 -d dev './e2e_test/source/basic/*.slt'
sqllogictest -p 4566 -d dev './e2e_test/source/basic/old_row_format_syntax/*.slt'
sqllogictest -p 4566 -d dev './e2e_test/source/basic/alter/kafka.slt'
risedev slt './e2e_test/source/basic/*.slt'
risedev slt './e2e_test/source/basic/old_row_format_syntax/*.slt'
risedev slt './e2e_test/source/basic/alter/kafka.slt'

echo "--- e2e, kafka alter source"
chmod +x ./scripts/source/prepare_data_after_alter.sh
./scripts/source/prepare_data_after_alter.sh 2
sqllogictest -p 4566 -d dev './e2e_test/source/basic/alter/kafka_after_new_data.slt'
risedev slt './e2e_test/source/basic/alter/kafka_after_new_data.slt'

echo "--- e2e, kafka alter source again"
./scripts/source/prepare_data_after_alter.sh 3
sqllogictest -p 4566 -d dev './e2e_test/source/basic/alter/kafka_after_new_data_2.slt'
risedev slt './e2e_test/source/basic/alter/kafka_after_new_data_2.slt'

echo "--- e2e, inline test"
risedev slt './e2e_test/ource_inline/**/*.slt'

echo "--- Run CH-benCHmark"
risedev slt -p 4566 -d dev './e2e_test/ch_benchmark/batch/ch_benchmark.slt'
risedev slt -p 4566 -d dev './e2e_test/ch_benchmark/streaming/*.slt'
risedev slt './e2e_test/ch_benchmark/batch/ch_benchmark.slt'
risedev slt './e2e_test/ch_benchmark/streaming/*.slt'
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
# "Inline" style source e2e tests

Compared with prior source tests (`e2e_test/source`), tests in this directory are expected to be easy to run locally and easy to write.

To run locally, use `risdev d` to start services (including external systems like Kafka and Postgres).
To run locally, use `risdev d` to start services (including external systems like Kafka and Postgres, or specify `user-managed` to use your own service).
Then use `risedev slt` to run the tests, which will load the environment variables (ports, etc.)
according to the services started by `risdev d`.

```sh
risedev slt 'e2e_test/source-new/**/*.slt'
risedev slt 'e2e_test/source-inline/**/*.slt'
```

To write tests, please ensure each file is self-contained and does not depend on external scripts to setup the environment.
Expand Down
File renamed without changes.
File renamed without changes.

0 comments on commit 6dddbf3

Please sign in to comment.