Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Kafka 3.1.0 to CI matrix, migrate to bitnami kafka image #2124

Merged
merged 4 commits into from
Feb 1, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 15 additions & 10 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on: [push, pull_request]

jobs:
test:
name: Go ${{ matrix.go-version }} with Kafka ${{ matrix.kafka-version }} on Ubuntu
name: Go ${{ matrix.go-version }} with Kafka ${{ matrix.kafka-version }}
runs-on: ubuntu-latest
# We want to run on external PRs, but not on our own internal PRs as they'll be run
# by the push to the branch.
Expand All @@ -14,7 +14,7 @@ jobs:
fail-fast: false
matrix:
go-version: [1.16.x, 1.17.x]
kafka-version: [2.7.1, 2.8.1, 3.0.0]
kafka-version: [2.8.1, 3.0.0, 3.1.0]
platform: [ubuntu-latest]

env:
Expand Down Expand Up @@ -46,13 +46,18 @@ jobs:
restore-keys: |
${{ runner.os }}-go-

- name: Install dependencies
run: |
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.42.1
export REPOSITORY_ROOT=${GITHUB_WORKSPACE}
- name: Lint
uses: golangci/golangci-lint-action@v2
if: ${{ matrix.go-version == '1.17.x' }}
env:
GOFLAGS: -tags=functional
with:
version: v1.44.0

- name: Run test suite
run: make test_functional
- name: Test (Unit)
run: make test
if: ${{ matrix.go-version != '1.17.x' }}

- name: Run linter
run: make lint
- name: Test (Functional)
run: make test_functional
if: ${{ matrix.go-version == '1.17.x' }}
151 changes: 78 additions & 73 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
version: '3.7'
services:
zookeeper-1:
image: 'confluentinc/cp-zookeeper:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'confluentinc/cp-zookeeper:7.0.1'
Copy link
Contributor Author

@bai bai Feb 1, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't expect Zookeeper version requirement to change as often as Kafka versions change, so oped to hardcode Zookeeper version instead of adding an extra configuration parameter.

We might explore moving to non-proprietary zookeeper docker image in future.

restart: always
environment:
ZOOKEEPER_SERVER_ID: '1'
Expand All @@ -13,7 +13,7 @@ services:
ZOOKEEPER_SYNC_LIMIT: '5'
ZOOKEEPER_MAX_CLIENT_CONNS: '0'
zookeeper-2:
image: 'confluentinc/cp-zookeeper:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'confluentinc/cp-zookeeper:7.0.1'
restart: always
environment:
ZOOKEEPER_SERVER_ID: '2'
Expand All @@ -25,7 +25,7 @@ services:
ZOOKEEPER_SYNC_LIMIT: '5'
ZOOKEEPER_MAX_CLIENT_CONNS: '0'
zookeeper-3:
image: 'confluentinc/cp-zookeeper:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'confluentinc/cp-zookeeper:7.0.1'
restart: always
environment:
ZOOKEEPER_SERVER_ID: '3'
Expand All @@ -37,90 +37,95 @@ services:
ZOOKEEPER_SYNC_LIMIT: '5'
ZOOKEEPER_MAX_CLIENT_CONNS: '0'
kafka-1:
image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
restart: always
environment:
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091'
KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'
KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_BROKER_ID: '1'
KAFKA_BROKER_RACK: '1'
KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_CFG_BROKER_ID: '1'
KAFKA_CFG_BROKER_RACK: '1'
KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
ALLOW_PLAINTEXT_LISTENER: 'true'
kafka-2:
image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
restart: always
environment:
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092'
KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'
KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_BROKER_ID: '2'
KAFKA_BROKER_RACK: '2'
KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_CFG_BROKER_ID: '2'
KAFKA_CFG_BROKER_RACK: '2'
KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
ALLOW_PLAINTEXT_LISTENER: 'true'
kafka-3:
image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
restart: always
environment:
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093'
KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'
KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_BROKER_ID: '3'
KAFKA_BROKER_RACK: '3'
KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_CFG_BROKER_ID: '3'
KAFKA_CFG_BROKER_RACK: '3'
KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
ALLOW_PLAINTEXT_LISTENER: 'true'
kafka-4:
image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
restart: always
environment:
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094'
KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'
KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_BROKER_ID: '4'
KAFKA_BROKER_RACK: '4'
KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_CFG_BROKER_ID: '4'
KAFKA_CFG_BROKER_RACK: '4'
KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
ALLOW_PLAINTEXT_LISTENER: 'true'
kafka-5:
image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
restart: always
environment:
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095'
KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'
KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_BROKER_ID: '5'
KAFKA_BROKER_RACK: '5'
KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_CFG_BROKER_ID: '5'
KAFKA_CFG_BROKER_RACK: '5'
KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
ALLOW_PLAINTEXT_LISTENER: 'true'
toxiproxy:
image: 'ghcr.io/shopify/toxiproxy:2.3.0'
ports:
Expand Down
21 changes: 3 additions & 18 deletions functional_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,28 +136,13 @@ func prepareDockerTestEnvironment(ctx context.Context, env *testEnvironment) err
if version, ok := os.LookupEnv("KAFKA_VERSION"); ok {
env.KafkaVersion = version
} else {
// We have cp-7.0.0 as the default in the docker-compose file, so that's kafka 3.0.0.
env.KafkaVersion = "3.0.0"
}

// the mapping of confluent platform docker image versions -> kafka versions can be
// found here: https://docs.confluent.io/current/installation/versions-interoperability.html
var confluentPlatformVersion string
switch env.KafkaVersion {
case "3.0.0":
confluentPlatformVersion = "7.0.1"
case "2.8.1":
confluentPlatformVersion = "6.2.2"
case "2.7.1":
confluentPlatformVersion = "6.1.4"
default:
return fmt.Errorf("don't know what confluent platform version to use for kafka %s", env.KafkaVersion)
env.KafkaVersion = "3.1.0"
}

c := exec.Command("docker-compose", "up", "-d")
c.Stdout = os.Stdout
c.Stderr = os.Stderr
c.Env = append(os.Environ(), fmt.Sprintf("CONFLUENT_PLATFORM_VERSION=%s", confluentPlatformVersion))
c.Env = append(os.Environ(), fmt.Sprintf("KAFKA_VERSION=%s", env.KafkaVersion))
err := c.Run()
if err != nil {
return fmt.Errorf("failed to run docker-compose to start test environment: %w", err)
Expand All @@ -169,7 +154,7 @@ func prepareDockerTestEnvironment(ctx context.Context, env *testEnvironment) err

// Wait for the kafka broker to come up
allBrokersUp := false
for i := 0; i < 45 && !allBrokersUp; i++ {
for i := 0; i < 90 && !allBrokersUp; i++ {
Logger.Println("waiting for kafka brokers to come up")
time.Sleep(1 * time.Second)
config := NewTestConfig()
Expand Down
1 change: 1 addition & 0 deletions utils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ func TestVersionParsing(t *testing.T) {
"2.8.0",
"2.8.1",
"3.0.0",
"3.1.0",
}
for _, s := range validVersions {
v, err := ParseKafkaVersion(s)
Expand Down