test: add Kafka consumer-group, add-partition tests (inline style) #16244
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| # "Inline" style source e2e tests | ||
|
|
||
| Compared with prior source tests (`e2e_test/source`), tests in this directory are expected to be easy to run locally and easy to write. | ||
|
|
||
| To run locally, use `risedev d` to start services (including external systems like Kafka and Postgres, or specify `user-managed` to use your own service). | ||
| Then use `risedev slt` to run the tests, which will load the environment variables (ports, etc.) | ||
| according to the services started by `risedev d`. | ||
|
|
||
| ```sh | ||
| risedev slt 'e2e_test/source_inline/**/*.slt' | ||
| ``` | ||
|
|
||
| To write tests, please ensure each file is self-contained and does not depend on running external scripts to setup the environment. | ||
| Use `system` command to setup instead. | ||
|
|
||
| Refer to https://github.com/risingwavelabs/risingwave/issues/12451#issuecomment-2051861048 for more details. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| # This file contains commands used by the tests. | ||
|
|
||
| [tasks.source-test-hook] | ||
| private = true | ||
| dependencies = ["check-risedev-env-file"] | ||
| env_files = ["${PREFIX_CONFIG}/risedev-env"] | ||
|
|
||
| # Note about the Kafka CLI tooling: | ||
| # - Built-in Kafka console tools: | ||
| # Java based. | ||
| # Style example: kafka-topics.sh --bootstrap-server localhost:9092 --topic t --create | ||
| # Some limitations: cannot disable logging easily, cannot consume to end and then exit. | ||
| # - kcat: | ||
| # C based (rdkafka) | ||
| # Some limitations: cannot do admin operations, only consume/produce. | ||
| # - rpk: | ||
| # Golang based. | ||
| # Style example: RPK_BROKERS=localhost:9092 rpk topic create t | ||
| [tasks.kafka-hook] | ||
| private = true | ||
| description = "Check if Kafka is started by RiseDev" | ||
| dependencies = ["source-test-hook"] | ||
| script = ''' | ||
| #!/usr/bin/env sh | ||
| set -e | ||
|
|
||
| if [ ! -d "${PREFIX_BIN}/kafka" ]; then | ||
| echo "Kafka is not installed in ${PREFIX_BIN}/kafka. Did you enable Kafka using $(tput setaf 4)\`./risedev configure\`$(tput sgr0)?" | ||
| exit 1 | ||
| fi | ||
xxchan marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| if [ -z "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" ]; then | ||
| echo "RISEDEV_KAFKA_BOOTSTRAP_SERVERS is not set in risedev-env file. Did you start Kafka using $(tput setaf 4)\`./risedev d\`$(tput sgr0)?" | ||
| exit 1 | ||
| fi | ||
| ''' | ||
|
|
||
| [tasks.clean-kafka] | ||
| category = "RiseDev - Test - Source Test - Kafka" | ||
| description = "Delete all kafka topics." | ||
| dependencies = ["kafka-hook"] | ||
| command = "rpk" | ||
| args = ["topic", "delete", "-r", "*"] | ||
|
|
||
| [tasks.kafka-topics] | ||
| category = "RiseDev - Test - Source Test - Kafka" | ||
| dependencies = ["kafka-hook"] | ||
| script = """ | ||
| #!/usr/bin/env sh | ||
| set -e | ||
| ${PREFIX_BIN}/kafka/bin/kafka-topics.sh --bootstrap-server ${RISEDEV_KAFKA_BOOTSTRAP_SERVERS} "$@" | ||
| """ | ||
|
|
||
| [tasks.kafka-produce] | ||
| category = "RiseDev - Test - Source Test - Kafka" | ||
| dependencies = ["kafka-hook"] | ||
| script = """ | ||
| #!/usr/bin/env sh | ||
| set -e | ||
| ${PREFIX_BIN}/kafka/bin/kafka-console-producer.sh --bootstrap-server ${RISEDEV_KAFKA_BOOTSTRAP_SERVERS} "$@" | ||
| """ | ||
|
|
||
| [tasks.kafka-consume] | ||
| category = "RiseDev - Test - Source Test - Kafka" | ||
| dependencies = ["kafka-hook"] | ||
| script = """ | ||
| #!/usr/bin/env sh | ||
| set -e | ||
| ${PREFIX_BIN}/kafka/bin/kafka-console-consumer.sh --bootstrap-server ${RISEDEV_KAFKA_BOOTSTRAP_SERVERS} "$@" | ||
| """ | ||
|
|
||
| [tasks.kafka-consumer-groups] | ||
| category = "RiseDev - Test - Source Test - Kafka" | ||
| dependencies = ["kafka-hook"] | ||
| script = """ | ||
| #!/usr/bin/env sh | ||
| set -e | ||
| ${PREFIX_BIN}/kafka/bin/kafka-consumer-groups.sh --bootstrap-server ${RISEDEV_KAFKA_BOOTSTRAP_SERVERS} "$@" | ||
| """ | ||
|
|
||
| # rpk tools | ||
| [tasks.rpk] | ||
| category = "RiseDev - Test - Source Test - Kafka" | ||
| dependencies = ["kafka-hook"] | ||
| # check https://docs.redpanda.com/current/reference/rpk/rpk-x-options/ or rpk -X help/list for options | ||
| script = """ | ||
| #!/usr/bin/env sh | ||
| set -e | ||
|
|
||
| if [ -z "$(which rpk)" ]; then | ||
| echo "rpk is not installed. Install it via https://docs.redpanda.com/current/get-started/rpk-install/" | ||
| exit 1 | ||
| fi | ||
|
|
||
| rpk "$@" | ||
| """ | ||
|
|
||
| [tasks.redpanda-console] | ||
| category = "RiseDev - Test - Source Test - Kafka" | ||
| description = "Start Redpanda console (Kafka GUI) at localhost:8080." | ||
| dependencies = ["kafka-hook"] | ||
| script = ''' | ||
| #!/usr/bin/env sh | ||
| set -e | ||
| echo "$(tput setaf 2)Start Redpanda console at http://localhost:8080$(tput sgr0)" | ||
| docker run --network host -e KAFKA_BROKERS=$RPK_BROKERS docker.redpanda.com/redpandadata/console:latest | ||
| ''' | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| # Note: control substitution on will force us to use "\\n" instead of "\n" in commands | ||
| control substitution on | ||
|
|
||
| system ok | ||
| rpk topic create test_add_partition -p 3 | ||
|
There was a problem hiding this comment. This test case is for #15994 (incomplete yet, still need some more cases) |
||
|
|
||
| system ok | ||
| cat <<EOF | rpk topic produce test_add_partition -f "%p %v\\n" -p 0 | ||
| 0 {"x":"a"} | ||
| 1 {"x":"b"} | ||
| 2 {"x":"c"} | ||
| EOF | ||
|
|
||
| statement ok | ||
| CREATE SOURCE s(x varchar) | ||
| WITH( | ||
| ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON}, | ||
| topic = 'test_add_partition', | ||
| scan.startup.mode = 'earliest', | ||
| ) FORMAT PLAIN ENCODE JSON; | ||
|
|
||
| statement ok | ||
| CREATE MATERIALIZED VIEW mv AS SELECT * from s; | ||
|
|
||
| query ? | ||
| SELECT * FROM s order by x; | ||
| ---- | ||
| a | ||
| b | ||
| c | ||
|
|
||
| sleep 2s | ||
|
|
||
| query ? | ||
| SELECT * FROM mv order by x; | ||
| ---- | ||
| a | ||
| b | ||
| c | ||
|
|
||
| system ok | ||
| rpk topic add-partitions test_add_partition --num 1 | ||
|
|
||
| system ok | ||
| cat <<EOF | rpk topic produce test_add_partition -f "%p %v\\n" -p 0 | ||
| 3 {"x":"d"} | ||
| EOF | ||
|
|
||
|
|
||
| query ? | ||
| SELECT * FROM s order by x; | ||
| ---- | ||
| a | ||
| b | ||
| c | ||
| d | ||
|
|
||
| # It needs some time for the split change to be reflected in MV | ||
| sleep 40s | ||
|
|
||
| query ? | ||
| SELECT * FROM mv order by x; | ||
| ---- | ||
| a | ||
| b | ||
| c | ||
| d | ||
|
|
||
|
|
||
| statement ok | ||
| DROP SOURCE s CASCADE; | ||
|
|
||
| system ok | ||
| rpk topic delete test_add_partition | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The commands added here are actually not used in the
sltfile (although in theory they can). They are now mainly helper commands to run manually.e.g.,
risedev rpkis similar torisedev psql. It setsRPK_BROKERSenv var. So it's helpful for local testing.